fix: preserve background connection state and durable disconnect logs

This commit is contained in:
AttilaNet
2026-11-11 11:05:49 +08:00
parent e26d37e9c5
commit 654567caa3
14 changed files with 1622 additions and 48 deletions
+189 -15
View File
@@ -35,20 +35,23 @@ from __future__ import annotations
from collections import OrderedDict, deque
from copy import copy, deepcopy
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
import json
import math
import re
import threading
import uuid
from ranch_assistant.api import Client, Session, server_entries
import requests
from ranch_assistant.api import ApiError, Client, Session, SessionExpiredError, TemporaryApiError, server_entries
from ranch_assistant.battles import CancelledBeforeDispatch
from ranch_assistant.debug import TraceLog, sensitive
from ranch_assistant.engine import Engine
from ranch_assistant.login import PasswordLogin
from ranch_assistant.model import Catalog
from .activity import ActivityLog, ActivityLogError
from .policies import apply_config, default_config, validate_config
from .scheduler import AccountSchedule, FAST_KINDS, Scheduler
from .store import WebStore, session_identity
@@ -143,10 +146,19 @@ class AccountWorker:
self._secrets = set()
self._pending = deque()
self._jobs = OrderedDict()
self._logs = deque(maxlen=2000)
self._activity = ActivityLog(self.profile_root / "activity.json", max_entries=2000)
self._activity_warning = False
try:
self._logs = deque(self._activity.load(), maxlen=2000)
self._log_sequence = self._activity.sequence
except ActivityLogError:
self._activity_warning = True
self._log_sequence = 1
self._logs = deque([{"id": 1, "seq": 1, "debug": False, "level": "warning",
"time": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
"message": "首次读取运行日志失败,当前日志暂存内存;后续日志将重试保存"}], maxlen=2000)
self._debug_entries = OrderedDict()
self._debug_bytes = 0
self._log_sequence = 0
self._views = OrderedDict()
self._view_jobs = {}
self._view_epoch = 0
@@ -166,12 +178,21 @@ class AccountWorker:
self._restore_factory = getattr(manager.login_factory, "client_factory", Client)
self.active_identity = None
self.error = ""
self.connection_state = "offline"
self.connection_message = "离线演示" if self.demo else "尚未连接游戏,请登录并选择区服或手动恢复会话"
self.last_game_success_at = None
self.next_retry_at = None
self._retry_probe = False
self._last_game_error = None
self._game_failure_sequence = 0
self.schedule = AccountSchedule()
self.state_root = self.state = manager.store.role_root(self.id, demo=self.demo)
self.config, self.revision = self._load_config(self.state_root)
self.trace = WebTrace(self)
self.engine = self._engine()
self._summary = self._make_summary()
if self._activity_warning:
self.log("账号工作区已载入")
self.thread = threading.Thread(target=self._run, name="ranch-web-" + self.id[:8], daemon=True)
self.thread.start()
self.enqueue("__initialize", {}, internal=True)
@@ -240,8 +261,27 @@ class AccountWorker:
entry = {"time": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
"message": self.safe_text(message), "debug": bool(debug), "level": "debug" if debug else "info"}
with self.lock:
self._log_sequence += 1
entry["id"] = entry["seq"] = self._log_sequence
try:
if self._activity_warning:
# Confirm an actual durable write before declaring recovery;
# debug appends within a reserved block alone need no IO.
recovered = self._activity.append({"time": entry["time"], "debug": False, "level": "info",
"message": "运行日志持久化已恢复;降级期间的日志仅保留在内存"}, after_sequence=self._log_sequence)
self._logs.append(recovered)
self._log_sequence = recovered["seq"]
self._activity_warning = False
entry = self._activity.append(entry, after_sequence=self._log_sequence)
except ActivityLogError:
# Keep retrying the same writer on later events. Logging must
# never replace a dispatched game's result or original error.
if not self._activity_warning:
self._activity_warning = True
self._log_sequence += 1
self._logs.append({"id": self._log_sequence, "seq": self._log_sequence,
"time": entry["time"], "debug": False, "level": "warning",
"message": "运行日志无法持久保存,当前日志仅保留在内存;请检查 Web 数据目录"})
entry["id"] = entry["seq"] = self._log_sequence + 1
self._log_sequence = entry["seq"]
self._logs.append(entry)
return entry["id"]
@@ -275,9 +315,73 @@ class AccountWorker:
player[field] = master.get(field, 0)
player["name"] = master.get("roleName") or master.get("masterName") or master.get("name", "")
player["next_exp"] = self.catalog.data.get("farm_level", {}).get(str(int(master.get("level") or 1) + 1))
return self.public_json({"id": self.id, "label": self.label, "connected": bool(self.engine.client and snapshot),
return self.public_json({"id": self.id, "label": self.label, "connected": self.connection_state == "online",
"demo": self.demo, "running": self._running, "busy": False, "player": player,
"revision": self.revision, "error": self.error})
"revision": self.revision, "error": self.error, "connection_state": self.connection_state,
"connection_message": self.connection_message, "last_game_success_at": self.last_game_success_at,
"next_retry_at": self.next_retry_at})
def _require_game(self):
if self.connection_state != "online" and not (self._retry_probe and self.connection_state == "retrying"):
raise ValueError(self.connection_message)
def _game_failed(self, error, client):
# Candidate login/restore clients may fail while the previous role is
# still healthy. Only that client's verified activation replaces it.
if client is not self.engine.client or isinstance(error, CancelledBeforeDispatch):
return
if self._last_game_error is error:
return
cause = error.__cause__
code = str(error.code) if isinstance(error, ApiError) and error.code is not None else None
temporary = (isinstance(error, (TemporaryApiError, requests.RequestException))
or isinstance(cause, requests.RequestException) or code == "997")
expired = isinstance(error, SessionExpiredError) or code == "408"
if not (temporary or isinstance(error, ApiError)):
return
self._last_game_error = error
self._game_failure_sequence += 1
self.stop()
if code is not None and not (temporary or expired):
# A decoded business rejection proves neither disconnection nor
# expiry. Keep monitoring, but never retry its rejected write.
self.error = self._last_game_error_message = self.safe_text(str(error))
self.log(self.error)
return
self.engine.heartbeat_enabled = False
delay = self.schedule.game_failed(retry=temporary and not expired and client is not None)
with self.lock:
self.next_retry_at = None
if expired:
self.connection_state = "expired"
message = "游戏会话已失效(408);策略已停止,游戏请求已暂停,请手动恢复会话或重新登录"
elif temporary and client is not None:
self.connection_state = "retrying"
reason = "游戏网络请求超时" if isinstance(error, requests.Timeout) or isinstance(cause, requests.Timeout) else "游戏网络暂时不可用"
message = f"{reason};策略已停止,{delay} 秒后只读重试,未确认记录保留"
self.next_retry_at = (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat(timespec="milliseconds")
elif temporary:
self.connection_state = "offline"
message = "游戏网络请求未确认完成;会话尚未恢复,请手动再次恢复会话或重新登录,策略保持停止"
else:
self.connection_state = "offline"
message = "游戏请求失败,已停止策略和游戏请求,请手动恢复会话:" + str(error)
self.error = self.connection_message = self.safe_text(message)
self._last_game_error_message = self.error
self.log(self.connection_message)
def _game_succeeded(self):
previous = self.connection_state
with self.lock:
self.connection_state = "online"
self.connection_message = "游戏连接正常"
self.last_game_success_at = datetime.now(timezone.utc).isoformat(timespec="milliseconds")
self.next_retry_at = None
self.error = ""
self.schedule.game_resumed()
self.engine.heartbeat_enabled = True
if previous != "online":
self.log("游戏连接已确认;策略保持停止")
def status(self):
with self.lock:
@@ -386,18 +490,23 @@ class AccountWorker:
self._active = self._execution = job
job.status = "running"
final_status, final_error, public_result = "succeeded", "", None
failure_sequence = self._game_failure_sequence
try:
result = self._execute(job)
public_result = self.public_json(result)
with self.lock:
self.error = ""
if failure_sequence != self._game_failure_sequence:
final_status, final_error = "failed", self._last_game_error_message
except CancelledBeforeDispatch:
final_status, final_error = "cancelled", "未发送的操作已取消;已发请求保留核对记录"
except Exception as error:
message = self.safe_text(str(error)) or type(error).__name__
if self.engine.client is None and job.operation in {"enter_server", "restore_session"}:
self._game_failed(error, None)
message = self.error if self._last_game_error is error else self.safe_text(str(error)) or type(error).__name__
final_status, final_error = "failed", message
with self.lock:
self.error = message
if self.connection_state not in {"expired", "retrying"} and not (
self.engine.client is not None and job.operation in {"login", "enter_server", "restore_session"}):
self.error = message
if job.consuming:
self.stop()
self.log(message)
@@ -443,6 +552,8 @@ class AccountWorker:
self.config, self.revision = self._load_config(self.state_root)
self.active_identity = saved
self.engine = self._engine()
self.connection_message = "服务已启动或重启;已保留保存的会话,请手动恢复会话或重新登录,策略保持停止"
self.log(self.connection_message)
if self.demo:
self.result(*self.engine.poll())
self.schedule.monitor(True)
@@ -462,8 +573,9 @@ class AccountWorker:
return getattr(self, "_" + operation)(payload)
if job.workspace != self._workspace():
raise RevisionConflict("账号角色已切换,原角色的操作未执行;请刷新后重试")
if not self.demo and not self.engine.client and operation not in LOCAL_ACTIONS:
raise ValueError("尚未连接游戏角色,请先登录并选择区服或恢复会话")
local = operation in LOCAL_ACTIONS or operation == "refresh" and payload.get("page") in {"statistics", "yields", "plans"}
if not self.demo and not local:
self._require_game()
from .actions import execute_action
# A manual refresh previews state while the chosen execution switch stays on.
self.engine.auto = False
@@ -573,6 +685,17 @@ class AccountWorker:
return None
auto = False
try:
if kind == "retry":
# A raw role read cannot replay writes or reconcile pending
# battle records. Engine.poll() can perform both side effects.
self._retry_probe = True
try:
self.engine.snapshot = self.engine.client.snapshot()
finally:
self._retry_probe = False
self.schedule.poll_finished(self.config)
self._invalidate_views()
return {"kind": kind}
if kind in FAST_KINDS:
if not self._set_auto(True):
return None
@@ -616,11 +739,13 @@ class AccountWorker:
if not isinstance(client.session, Session):
raise ValueError("登录客户端未提供受支持的会话")
self.remember(asdict(client.session))
self._share_session_counters(client.session)
client.trace = self.trace
self._session_write_error = ""
def persist(current):
try:
self._share_session_counters(current)
self.securestore.save(current, self._session_label(current))
self._last_persisted_session = self._session_stamp(current)
self._session_write_error = ""
@@ -634,13 +759,42 @@ class AccountWorker:
def guarded_call(*args, **kwargs):
self.check_intent()
# This precedes Client.call's wire payload construction. A
# saved candidate can lag the still-online client's counters.
self._share_session_counters(client.session)
active = client is self.engine.client
if active:
self._require_game()
# A repaired directory may be retried. Client.persist_session
# sets this before Trace.begin blocks an unsafe dispatch.
self._session_write_error = ""
return original(*args, **kwargs)
try:
response = original(*args, **kwargs)
except Exception as error:
self._game_failed(error, client)
raise
path = args[0] if args else kwargs.get("path")
if active and path == "heartBeatService/keepAlive":
self._game_succeeded()
return response
client.call = guarded_call
client._ranch_web_owner = self
original_snapshot = client.snapshot
def guarded_snapshot(*args, **kwargs):
if client is self.engine.client:
self._require_game()
try:
snapshot = original_snapshot(*args, **kwargs)
except Exception as error:
self._game_failed(error, client)
raise
if client is self.engine.client:
self._game_succeeded()
return snapshot
client.snapshot = guarded_snapshot
http = getattr(client, "http", None)
if http is not None:
def guarded_transport(method):
@@ -659,6 +813,18 @@ class AccountWorker:
setattr(http, name, guarded_transport(method))
return client
def _share_session_counters(self, session):
client = self.engine.client
if client is None or client.session is session:
return
current = client.session
protocol = ("token", "key", "server_url", "account_id", "server_id", "role_id", "channel", "platform")
if all(getattr(session, field) == getattr(current, field) for field in protocol):
for field in ("reqid", "reqsid"):
high = max(getattr(session, field), getattr(current, field))
setattr(session, field, high)
setattr(current, field, high)
def _session_label(self, session):
role = ""
if self.engine.client is not None and self.engine.client.session is session and self.engine.snapshot:
@@ -729,6 +895,7 @@ class AccountWorker:
self.manager._release_identity(self, previous_identity)
if old_client is not None and old_client is not client:
old_client.close()
self._game_succeeded()
return {"connected": True, "running": False, "has_saved_password": self._has_saved_password}
def _login(self, payload):
@@ -843,6 +1010,11 @@ class AccountWorker:
with self.lock:
self._servers = []
self.schedule.monitor(False)
self.schedule.game_failed()
self.connection_state = "offline"
self.connection_message = "已手动退出游戏;请登录或手动恢复保存的会话,策略保持停止"
self.next_retry_at = None
self.error = ""
self._refresh_login_metadata()
self._invalidate_views(clear=True)
return {"connected": False, "running": False}
@@ -856,6 +1028,8 @@ class AccountWorker:
def _start(self, payload):
self.check_intent()
if not self.demo:
self._require_game()
if self.engine.snapshot is None or not (self.demo or self.engine.client):
raise ValueError("请先登录并选择区服或打开离线演示")
value = validate_config(self.config)