fix: preserve background connection state and durable disconnect logs
This commit is contained in:
+189
-15
@@ -35,20 +35,23 @@ from __future__ import annotations
|
||||
from collections import OrderedDict, deque
|
||||
from copy import copy, deepcopy
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
from ranch_assistant.api import Client, Session, server_entries
|
||||
import requests
|
||||
|
||||
from ranch_assistant.api import ApiError, Client, Session, SessionExpiredError, TemporaryApiError, server_entries
|
||||
from ranch_assistant.battles import CancelledBeforeDispatch
|
||||
from ranch_assistant.debug import TraceLog, sensitive
|
||||
from ranch_assistant.engine import Engine
|
||||
from ranch_assistant.login import PasswordLogin
|
||||
from ranch_assistant.model import Catalog
|
||||
|
||||
from .activity import ActivityLog, ActivityLogError
|
||||
from .policies import apply_config, default_config, validate_config
|
||||
from .scheduler import AccountSchedule, FAST_KINDS, Scheduler
|
||||
from .store import WebStore, session_identity
|
||||
@@ -143,10 +146,19 @@ class AccountWorker:
|
||||
self._secrets = set()
|
||||
self._pending = deque()
|
||||
self._jobs = OrderedDict()
|
||||
self._logs = deque(maxlen=2000)
|
||||
self._activity = ActivityLog(self.profile_root / "activity.json", max_entries=2000)
|
||||
self._activity_warning = False
|
||||
try:
|
||||
self._logs = deque(self._activity.load(), maxlen=2000)
|
||||
self._log_sequence = self._activity.sequence
|
||||
except ActivityLogError:
|
||||
self._activity_warning = True
|
||||
self._log_sequence = 1
|
||||
self._logs = deque([{"id": 1, "seq": 1, "debug": False, "level": "warning",
|
||||
"time": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
|
||||
"message": "首次读取运行日志失败,当前日志暂存内存;后续日志将重试保存"}], maxlen=2000)
|
||||
self._debug_entries = OrderedDict()
|
||||
self._debug_bytes = 0
|
||||
self._log_sequence = 0
|
||||
self._views = OrderedDict()
|
||||
self._view_jobs = {}
|
||||
self._view_epoch = 0
|
||||
@@ -166,12 +178,21 @@ class AccountWorker:
|
||||
self._restore_factory = getattr(manager.login_factory, "client_factory", Client)
|
||||
self.active_identity = None
|
||||
self.error = ""
|
||||
self.connection_state = "offline"
|
||||
self.connection_message = "离线演示" if self.demo else "尚未连接游戏,请登录并选择区服或手动恢复会话"
|
||||
self.last_game_success_at = None
|
||||
self.next_retry_at = None
|
||||
self._retry_probe = False
|
||||
self._last_game_error = None
|
||||
self._game_failure_sequence = 0
|
||||
self.schedule = AccountSchedule()
|
||||
self.state_root = self.state = manager.store.role_root(self.id, demo=self.demo)
|
||||
self.config, self.revision = self._load_config(self.state_root)
|
||||
self.trace = WebTrace(self)
|
||||
self.engine = self._engine()
|
||||
self._summary = self._make_summary()
|
||||
if self._activity_warning:
|
||||
self.log("账号工作区已载入")
|
||||
self.thread = threading.Thread(target=self._run, name="ranch-web-" + self.id[:8], daemon=True)
|
||||
self.thread.start()
|
||||
self.enqueue("__initialize", {}, internal=True)
|
||||
@@ -240,8 +261,27 @@ class AccountWorker:
|
||||
entry = {"time": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
|
||||
"message": self.safe_text(message), "debug": bool(debug), "level": "debug" if debug else "info"}
|
||||
with self.lock:
|
||||
self._log_sequence += 1
|
||||
entry["id"] = entry["seq"] = self._log_sequence
|
||||
try:
|
||||
if self._activity_warning:
|
||||
# Confirm an actual durable write before declaring recovery;
|
||||
# debug appends within a reserved block alone need no IO.
|
||||
recovered = self._activity.append({"time": entry["time"], "debug": False, "level": "info",
|
||||
"message": "运行日志持久化已恢复;降级期间的日志仅保留在内存"}, after_sequence=self._log_sequence)
|
||||
self._logs.append(recovered)
|
||||
self._log_sequence = recovered["seq"]
|
||||
self._activity_warning = False
|
||||
entry = self._activity.append(entry, after_sequence=self._log_sequence)
|
||||
except ActivityLogError:
|
||||
# Keep retrying the same writer on later events. Logging must
|
||||
# never replace a dispatched game's result or original error.
|
||||
if not self._activity_warning:
|
||||
self._activity_warning = True
|
||||
self._log_sequence += 1
|
||||
self._logs.append({"id": self._log_sequence, "seq": self._log_sequence,
|
||||
"time": entry["time"], "debug": False, "level": "warning",
|
||||
"message": "运行日志无法持久保存,当前日志仅保留在内存;请检查 Web 数据目录"})
|
||||
entry["id"] = entry["seq"] = self._log_sequence + 1
|
||||
self._log_sequence = entry["seq"]
|
||||
self._logs.append(entry)
|
||||
return entry["id"]
|
||||
|
||||
@@ -275,9 +315,73 @@ class AccountWorker:
|
||||
player[field] = master.get(field, 0)
|
||||
player["name"] = master.get("roleName") or master.get("masterName") or master.get("name", "")
|
||||
player["next_exp"] = self.catalog.data.get("farm_level", {}).get(str(int(master.get("level") or 1) + 1))
|
||||
return self.public_json({"id": self.id, "label": self.label, "connected": bool(self.engine.client and snapshot),
|
||||
return self.public_json({"id": self.id, "label": self.label, "connected": self.connection_state == "online",
|
||||
"demo": self.demo, "running": self._running, "busy": False, "player": player,
|
||||
"revision": self.revision, "error": self.error})
|
||||
"revision": self.revision, "error": self.error, "connection_state": self.connection_state,
|
||||
"connection_message": self.connection_message, "last_game_success_at": self.last_game_success_at,
|
||||
"next_retry_at": self.next_retry_at})
|
||||
|
||||
def _require_game(self):
|
||||
if self.connection_state != "online" and not (self._retry_probe and self.connection_state == "retrying"):
|
||||
raise ValueError(self.connection_message)
|
||||
|
||||
def _game_failed(self, error, client):
|
||||
# Candidate login/restore clients may fail while the previous role is
|
||||
# still healthy. Only that client's verified activation replaces it.
|
||||
if client is not self.engine.client or isinstance(error, CancelledBeforeDispatch):
|
||||
return
|
||||
if self._last_game_error is error:
|
||||
return
|
||||
cause = error.__cause__
|
||||
code = str(error.code) if isinstance(error, ApiError) and error.code is not None else None
|
||||
temporary = (isinstance(error, (TemporaryApiError, requests.RequestException))
|
||||
or isinstance(cause, requests.RequestException) or code == "997")
|
||||
expired = isinstance(error, SessionExpiredError) or code == "408"
|
||||
if not (temporary or isinstance(error, ApiError)):
|
||||
return
|
||||
self._last_game_error = error
|
||||
self._game_failure_sequence += 1
|
||||
self.stop()
|
||||
if code is not None and not (temporary or expired):
|
||||
# A decoded business rejection proves neither disconnection nor
|
||||
# expiry. Keep monitoring, but never retry its rejected write.
|
||||
self.error = self._last_game_error_message = self.safe_text(str(error))
|
||||
self.log(self.error)
|
||||
return
|
||||
self.engine.heartbeat_enabled = False
|
||||
delay = self.schedule.game_failed(retry=temporary and not expired and client is not None)
|
||||
with self.lock:
|
||||
self.next_retry_at = None
|
||||
if expired:
|
||||
self.connection_state = "expired"
|
||||
message = "游戏会话已失效(408);策略已停止,游戏请求已暂停,请手动恢复会话或重新登录"
|
||||
elif temporary and client is not None:
|
||||
self.connection_state = "retrying"
|
||||
reason = "游戏网络请求超时" if isinstance(error, requests.Timeout) or isinstance(cause, requests.Timeout) else "游戏网络暂时不可用"
|
||||
message = f"{reason};策略已停止,{delay} 秒后只读重试,未确认记录保留"
|
||||
self.next_retry_at = (datetime.now(timezone.utc) + timedelta(seconds=delay)).isoformat(timespec="milliseconds")
|
||||
elif temporary:
|
||||
self.connection_state = "offline"
|
||||
message = "游戏网络请求未确认完成;会话尚未恢复,请手动再次恢复会话或重新登录,策略保持停止"
|
||||
else:
|
||||
self.connection_state = "offline"
|
||||
message = "游戏请求失败,已停止策略和游戏请求,请手动恢复会话:" + str(error)
|
||||
self.error = self.connection_message = self.safe_text(message)
|
||||
self._last_game_error_message = self.error
|
||||
self.log(self.connection_message)
|
||||
|
||||
def _game_succeeded(self):
|
||||
previous = self.connection_state
|
||||
with self.lock:
|
||||
self.connection_state = "online"
|
||||
self.connection_message = "游戏连接正常"
|
||||
self.last_game_success_at = datetime.now(timezone.utc).isoformat(timespec="milliseconds")
|
||||
self.next_retry_at = None
|
||||
self.error = ""
|
||||
self.schedule.game_resumed()
|
||||
self.engine.heartbeat_enabled = True
|
||||
if previous != "online":
|
||||
self.log("游戏连接已确认;策略保持停止")
|
||||
|
||||
def status(self):
|
||||
with self.lock:
|
||||
@@ -386,18 +490,23 @@ class AccountWorker:
|
||||
self._active = self._execution = job
|
||||
job.status = "running"
|
||||
final_status, final_error, public_result = "succeeded", "", None
|
||||
failure_sequence = self._game_failure_sequence
|
||||
try:
|
||||
result = self._execute(job)
|
||||
public_result = self.public_json(result)
|
||||
with self.lock:
|
||||
self.error = ""
|
||||
if failure_sequence != self._game_failure_sequence:
|
||||
final_status, final_error = "failed", self._last_game_error_message
|
||||
except CancelledBeforeDispatch:
|
||||
final_status, final_error = "cancelled", "未发送的操作已取消;已发请求保留核对记录"
|
||||
except Exception as error:
|
||||
message = self.safe_text(str(error)) or type(error).__name__
|
||||
if self.engine.client is None and job.operation in {"enter_server", "restore_session"}:
|
||||
self._game_failed(error, None)
|
||||
message = self.error if self._last_game_error is error else self.safe_text(str(error)) or type(error).__name__
|
||||
final_status, final_error = "failed", message
|
||||
with self.lock:
|
||||
self.error = message
|
||||
if self.connection_state not in {"expired", "retrying"} and not (
|
||||
self.engine.client is not None and job.operation in {"login", "enter_server", "restore_session"}):
|
||||
self.error = message
|
||||
if job.consuming:
|
||||
self.stop()
|
||||
self.log(message)
|
||||
@@ -443,6 +552,8 @@ class AccountWorker:
|
||||
self.config, self.revision = self._load_config(self.state_root)
|
||||
self.active_identity = saved
|
||||
self.engine = self._engine()
|
||||
self.connection_message = "服务已启动或重启;已保留保存的会话,请手动恢复会话或重新登录,策略保持停止"
|
||||
self.log(self.connection_message)
|
||||
if self.demo:
|
||||
self.result(*self.engine.poll())
|
||||
self.schedule.monitor(True)
|
||||
@@ -462,8 +573,9 @@ class AccountWorker:
|
||||
return getattr(self, "_" + operation)(payload)
|
||||
if job.workspace != self._workspace():
|
||||
raise RevisionConflict("账号角色已切换,原角色的操作未执行;请刷新后重试")
|
||||
if not self.demo and not self.engine.client and operation not in LOCAL_ACTIONS:
|
||||
raise ValueError("尚未连接游戏角色,请先登录并选择区服或恢复会话")
|
||||
local = operation in LOCAL_ACTIONS or operation == "refresh" and payload.get("page") in {"statistics", "yields", "plans"}
|
||||
if not self.demo and not local:
|
||||
self._require_game()
|
||||
from .actions import execute_action
|
||||
# A manual refresh previews state while the chosen execution switch stays on.
|
||||
self.engine.auto = False
|
||||
@@ -573,6 +685,17 @@ class AccountWorker:
|
||||
return None
|
||||
auto = False
|
||||
try:
|
||||
if kind == "retry":
|
||||
# A raw role read cannot replay writes or reconcile pending
|
||||
# battle records. Engine.poll() can perform both side effects.
|
||||
self._retry_probe = True
|
||||
try:
|
||||
self.engine.snapshot = self.engine.client.snapshot()
|
||||
finally:
|
||||
self._retry_probe = False
|
||||
self.schedule.poll_finished(self.config)
|
||||
self._invalidate_views()
|
||||
return {"kind": kind}
|
||||
if kind in FAST_KINDS:
|
||||
if not self._set_auto(True):
|
||||
return None
|
||||
@@ -616,11 +739,13 @@ class AccountWorker:
|
||||
if not isinstance(client.session, Session):
|
||||
raise ValueError("登录客户端未提供受支持的会话")
|
||||
self.remember(asdict(client.session))
|
||||
self._share_session_counters(client.session)
|
||||
client.trace = self.trace
|
||||
self._session_write_error = ""
|
||||
|
||||
def persist(current):
|
||||
try:
|
||||
self._share_session_counters(current)
|
||||
self.securestore.save(current, self._session_label(current))
|
||||
self._last_persisted_session = self._session_stamp(current)
|
||||
self._session_write_error = ""
|
||||
@@ -634,13 +759,42 @@ class AccountWorker:
|
||||
|
||||
def guarded_call(*args, **kwargs):
|
||||
self.check_intent()
|
||||
# This precedes Client.call's wire payload construction. A
|
||||
# saved candidate can lag the still-online client's counters.
|
||||
self._share_session_counters(client.session)
|
||||
active = client is self.engine.client
|
||||
if active:
|
||||
self._require_game()
|
||||
# A repaired directory may be retried. Client.persist_session
|
||||
# sets this before Trace.begin blocks an unsafe dispatch.
|
||||
self._session_write_error = ""
|
||||
return original(*args, **kwargs)
|
||||
try:
|
||||
response = original(*args, **kwargs)
|
||||
except Exception as error:
|
||||
self._game_failed(error, client)
|
||||
raise
|
||||
path = args[0] if args else kwargs.get("path")
|
||||
if active and path == "heartBeatService/keepAlive":
|
||||
self._game_succeeded()
|
||||
return response
|
||||
|
||||
client.call = guarded_call
|
||||
client._ranch_web_owner = self
|
||||
original_snapshot = client.snapshot
|
||||
|
||||
def guarded_snapshot(*args, **kwargs):
|
||||
if client is self.engine.client:
|
||||
self._require_game()
|
||||
try:
|
||||
snapshot = original_snapshot(*args, **kwargs)
|
||||
except Exception as error:
|
||||
self._game_failed(error, client)
|
||||
raise
|
||||
if client is self.engine.client:
|
||||
self._game_succeeded()
|
||||
return snapshot
|
||||
|
||||
client.snapshot = guarded_snapshot
|
||||
http = getattr(client, "http", None)
|
||||
if http is not None:
|
||||
def guarded_transport(method):
|
||||
@@ -659,6 +813,18 @@ class AccountWorker:
|
||||
setattr(http, name, guarded_transport(method))
|
||||
return client
|
||||
|
||||
def _share_session_counters(self, session):
|
||||
client = self.engine.client
|
||||
if client is None or client.session is session:
|
||||
return
|
||||
current = client.session
|
||||
protocol = ("token", "key", "server_url", "account_id", "server_id", "role_id", "channel", "platform")
|
||||
if all(getattr(session, field) == getattr(current, field) for field in protocol):
|
||||
for field in ("reqid", "reqsid"):
|
||||
high = max(getattr(session, field), getattr(current, field))
|
||||
setattr(session, field, high)
|
||||
setattr(current, field, high)
|
||||
|
||||
def _session_label(self, session):
|
||||
role = ""
|
||||
if self.engine.client is not None and self.engine.client.session is session and self.engine.snapshot:
|
||||
@@ -729,6 +895,7 @@ class AccountWorker:
|
||||
self.manager._release_identity(self, previous_identity)
|
||||
if old_client is not None and old_client is not client:
|
||||
old_client.close()
|
||||
self._game_succeeded()
|
||||
return {"connected": True, "running": False, "has_saved_password": self._has_saved_password}
|
||||
|
||||
def _login(self, payload):
|
||||
@@ -843,6 +1010,11 @@ class AccountWorker:
|
||||
with self.lock:
|
||||
self._servers = []
|
||||
self.schedule.monitor(False)
|
||||
self.schedule.game_failed()
|
||||
self.connection_state = "offline"
|
||||
self.connection_message = "已手动退出游戏;请登录或手动恢复保存的会话,策略保持停止"
|
||||
self.next_retry_at = None
|
||||
self.error = ""
|
||||
self._refresh_login_metadata()
|
||||
self._invalidate_views(clear=True)
|
||||
return {"connected": False, "running": False}
|
||||
@@ -856,6 +1028,8 @@ class AccountWorker:
|
||||
|
||||
def _start(self, payload):
|
||||
self.check_intent()
|
||||
if not self.demo:
|
||||
self._require_game()
|
||||
if self.engine.snapshot is None or not (self.demo or self.engine.client):
|
||||
raise ValueError("请先登录并选择区服或打开离线演示")
|
||||
value = validate_config(self.config)
|
||||
|
||||
Reference in New Issue
Block a user