fix: preserve background connection state and durable disconnect logs

This commit is contained in:
AttilaNet
2026-11-11 11:05:49 +08:00
parent e26d37e9c5
commit 654567caa3
14 changed files with 1622 additions and 48 deletions
+442
View File
@@ -0,0 +1,442 @@
"""Connection regressions through real workers and an entirely offline transport."""
from copy import deepcopy
from types import SimpleNamespace
import threading
import unittest
from unittest.mock import patch
import requests
from ranch_assistant.api import ApiError, encode
from ranch_web.activity import ActivityLogError
from ranch_web.service import WebManager
from ranch_web.scheduler import Scheduler
from ranch_web.tests import test_service as fixtures
class ConnectionStateTests(unittest.TestCase):
setUp = fixtures.ServiceTests.setUp
account = fixtures.ServiceTests.account
idle = fixtures.ServiceTests.idle
job = fixtures.ServiceTests.job
login = fixtures.ServiceTests.login
enable = fixtures.ServiceTests.enable
def connected(self, *, running=False):
identifier = self.account(False)
self.login(identifier)
worker = self.manager._worker(identifier)
self.now = 1000.0
worker.schedule.clock = lambda: self.now
if running:
self.enable(identifier)
self.job(identifier, self.manager.submit(identifier, "start"))
return identifier, worker
def tick(self, identifier, worker, *, status="succeeded"):
return self.job(identifier, worker.enqueue("__tick"), status=status)
def expiry(self, worker):
return SimpleNamespace(status_code=200, headers={},
content=encode({"resCode": 408}, worker.engine.client.session.key))
def test_expired_state_survives_idle_view_config_and_blocks_game_requests(self):
identifier, worker = self.connected(running=True)
original_client, original_snapshot = worker.engine.client, worker.engine.snapshot
with patch.object(original_client.http, "post", return_value=self.expiry(worker)) as post:
worker.engine.last_heartbeat = 0
self.tick(identifier, worker, status="failed")
state = self.manager.status(identifier)
self.assertEqual(state.get("connection_state"), "expired")
self.assertIn("失效", state["connection_message"])
self.assertFalse(state["connected"])
self.assertFalse(state["running"])
reason = state["error"]
for _ in range(3):
self.now += 600
self.tick(identifier, worker)
view = self.manager.view(identifier, "bag")
self.job(identifier, {"id": view["job_id"]})
config = self.manager.get_config(identifier)
config["config"]["interval"] = 60
self.job(identifier, self.manager.set_config(identifier, config["config"])["job"])
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
self.job(identifier, self.manager.submit(identifier, "start"), status="failed")
self.assertEqual(post.call_count, 1)
self.assertEqual(self.manager.status(identifier)["error"], reason)
self.assertEqual(self.manager.status(identifier)["connection_state"], "expired")
self.assertIs(worker.engine.client, original_client)
self.assertIs(worker.engine.snapshot, original_snapshot)
self.assertIsNotNone(worker.securestore.load(worker.active_identity))
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}))
self.assertEqual(self.manager.status(identifier)["connection_state"], "online")
self.assertFalse(self.manager.status(identifier)["running"])
def test_timeout_retries_only_snapshot_and_keeps_uncertain_write_evidence(self):
identifier, worker = self.connected(running=True)
journal = worker.engine.battle_journal
pending = journal.begin(worker.engine.snapshot, {"kind": "battle", "description": "未确认的战斗",
"path": "stageService/stageStart", "payload": {}, "stage_id": "0101", "difficulty": "1"})
journal.update(pending, "unknown", message="未确认的战斗写请求")
evidence = journal.path
before = evidence.read_bytes()
client = worker.engine.client
def write(runtime, operation, payload):
runtime.engine.client.call("monsterService/monsterLock", {"monsterId": 1})
with patch("ranch_web.actions.execute_action", side_effect=write):
with patch.object(client.http, "post", side_effect=requests.ReadTimeout("synthetic timeout")) as post:
self.job(identifier, self.manager.submit(identifier, "lock"), status="failed")
self.assertEqual(post.call_count, 1)
state = self.manager.status(identifier)
self.assertEqual(state.get("connection_state"), "retrying")
self.assertNotIn("失效", state["connection_message"])
self.assertFalse(state["running"])
self.assertIsNotNone(state["next_retry_at"])
calls = len(fixtures.FakeClient.calls)
self.tick(identifier, worker)
self.assertEqual(len(fixtures.FakeClient.calls), calls)
self.now += 61
self.tick(identifier, worker)
paths = [url.rsplit("/", 2)[-2:] for _, url, _ in fixtures.FakeClient.calls[calls:]]
self.assertEqual(paths, [["masterService", "masterQueryDetail"]])
self.assertEqual(self.manager.status(identifier)["connection_state"], "online")
self.assertEqual(self.manager.status(identifier)["error"], "")
self.assertIsNone(self.manager.status(identifier)["next_retry_at"])
self.assertIsNotNone(self.manager.status(identifier)["last_game_success_at"])
self.assertFalse(worker.running)
self.assertEqual(journal.pending(worker.engine.snapshot)["attempt_id"], pending["attempt_id"])
self.assertEqual(evidence.read_bytes(), before)
# Continue through the ordinary monitor/reconcile path after recovery.
# The existing Engine barrier must still prevent the unknown write.
worker.engine.last_heartbeat = self.now
worker.schedule.next_poll = 0
self.now += 121
self.tick(identifier, worker)
paths = [url.rsplit("/", 2)[-2:] for _, url, _ in fixtures.FakeClient.calls[calls:]]
self.assertEqual(paths, [["masterService", "masterQueryDetail"]] * 2)
self.assertEqual(journal.pending(worker.engine.snapshot)["attempt_id"], pending["attempt_id"])
self.assertEqual(journal.pending(worker.engine.snapshot)["phase"], "unknown")
self.assertFalse(worker.running)
def test_business_rejection_keeps_connection_and_idle_does_not_clear_error(self):
identifier, worker = self.connected(running=True)
client = worker.engine.client
response = SimpleNamespace(status_code=200, headers={},
content=encode({"resCode": 403}, client.session.key))
with patch.object(client.http, "post", return_value=response):
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
state = self.manager.status(identifier)
self.assertEqual(state["connection_state"], "online")
self.assertTrue(state["connected"])
self.assertFalse(state["running"])
self.assertFalse(worker.schedule.game_paused)
self.assertTrue(worker.schedule.monitoring)
reason = state["error"]
self.assertIn("403", reason)
worker.engine.last_heartbeat = self.now
worker.schedule.next_poll = float("inf")
with patch.object(worker.engine, "master_sync_due", return_value=False), patch.object(worker.engine, "growth_sync_due", return_value=False):
self.tick(identifier, worker)
view = self.manager.view(identifier, "bag")
self.job(identifier, {"id": view["job_id"]})
self.assertEqual(self.manager.status(identifier)["error"], reason)
def test_997_is_retrying_without_expiry_and_backoff_is_bounded(self):
identifier, worker = self.connected()
response = SimpleNamespace(status_code=200, headers={},
content=encode({"resCode": 997}, worker.engine.client.session.key))
with patch.object(worker.engine.client.http, "post", return_value=response) as post:
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
for delay in (5, 10, 20, 40, 60, 60):
self.assertEqual(worker.schedule.retry_deadline - self.now, delay)
state = self.manager.status(identifier)
self.assertEqual(state["connection_state"], "retrying")
self.assertNotIn("失效", state["connection_message"])
calls = post.call_count
self.tick(identifier, worker)
self.assertEqual(post.call_count, calls)
self.now += delay
self.tick(identifier, worker, status="failed")
self.assertEqual(post.call_count, calls + 1)
def test_swallowed_expiry_still_stops_strategy_and_marks_job_failed(self):
identifier, worker = self.connected(running=True)
def swallowed(runtime, operation, payload):
try:
runtime.engine.client.call("monsterService/monsterLock")
except ApiError:
return {"message": "Engine caught the error"}
with patch("ranch_web.actions.execute_action", side_effect=swallowed):
with patch.object(worker.engine.client.http, "post", return_value=self.expiry(worker)):
result = self.job(identifier, self.manager.submit(identifier, "lock"), status="failed")
self.assertIn("失效", result["error"])
self.assertEqual(self.manager.status(identifier)["connection_state"], "expired")
self.assertFalse(worker.running)
def test_scheduler_runs_fake_heartbeat_and_poll_without_any_browser_requests(self):
identifier, worker = self.connected()
heartbeat, poll = threading.Event(), threading.Event()
original = worker.engine.client.http.post
def post(url, *args, **kwargs):
result = original(url, *args, **kwargs)
if url.endswith("heartBeatService/keepAlive"):
heartbeat.set()
if url.endswith("masterService/masterQueryDetail"):
poll.set()
return result
worker.engine.last_heartbeat = 0
worker.schedule.next_poll = 0
with patch.object(worker.engine.client.http, "post", side_effect=post):
scheduler = Scheduler(self.manager._worker_list, interval=.01)
try:
self.assertTrue(heartbeat.wait(3), "No heartbeat without browser polling")
self.assertTrue(poll.wait(3), "No game poll without browser polling")
finally:
scheduler.close()
self.idle(identifier)
self.assertFalse(worker.running)
def test_idle_local_success_does_not_clear_game_error(self):
identifier, worker = self.connected()
with patch.object(worker.engine.client.http, "post", side_effect=requests.Timeout("offline")):
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
reason = self.manager.status(identifier)["error"]
self.assertTrue(reason)
self.tick(identifier, worker)
view = self.manager.view(identifier, "bag")
self.job(identifier, {"id": view["job_id"]})
self.job(identifier, self.manager.submit(identifier, "refresh", {"page": "statistics"}))
self.assertEqual(self.manager.status(identifier)["error"], reason)
self.assertFalse(self.manager.status(identifier)["connected"])
def test_failed_restore_candidate_does_not_poison_online_client(self):
identifier, worker = self.connected()
client = worker.engine.client
original = deepcopy(self.manager.status(identifier))
response = self.expiry(worker)
with patch.object(fixtures.FakeClient, "_post", return_value=response):
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}), status="failed")
self.assertIs(worker.engine.client, client)
self.assertFalse(client.closed)
state = self.manager.status(identifier)
self.assertEqual(state.get("connection_state"), "online")
self.assertEqual(state["last_game_success_at"], original["last_game_success_at"])
self.assertEqual(state["connection_message"], original["connection_message"])
self.assertTrue(state["connected"])
def test_failed_same_protocol_restore_preserves_counter_high_water_before_wire_and_finally(self):
identifier, worker = self.connected()
for active_reqid, active_reqsid in ((400, 900), (500, 1200)):
with self.subTest(active_reqid=active_reqid):
client = worker.engine.client
saved = deepcopy(client.session)
saved.reqid, saved.reqsid = 400, 900
worker.securestore.save(saved, "离线高水位回归")
client.session.reqid, client.session.reqsid = active_reqid, active_reqsid
wire = []
def post(url, data, **kwargs):
wire.append(fixtures.decode(data, client.session.key))
return self.expiry(worker)
with patch.object(fixtures.FakeClient, "_post", side_effect=post):
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}), status="failed")
self.assertEqual((wire[0]["reqid"], wire[0]["reqsid"]), (active_reqid, active_reqsid))
persisted = worker.securestore.load(worker.active_identity)
self.assertEqual((persisted.reqid, persisted.reqsid), (active_reqid + 1, active_reqsid + 1))
self.assertEqual((client.session.reqid, client.session.reqsid), (persisted.reqid, persisted.reqsid))
client.call("heartBeatService/keepAlive")
payload = fixtures.FakeClient.calls[-1][2]
self.assertEqual((payload["reqid"], payload["reqsid"]), (active_reqid + 1, active_reqsid + 1))
def test_counter_merge_requires_exact_protocol_session(self):
identifier, worker = self.connected()
current = worker.engine.client.session
current.reqid, current.reqsid = 500, 900
changes = {"token": "different", "key": "different", "server_url": "https://other.invalid/",
"account_id": "different", "server_id": 9, "role_id": 999, "channel": "other", "platform": "other"}
for field, value in changes.items():
candidate = deepcopy(current)
candidate.reqid, candidate.reqsid = 800, 1800
setattr(candidate, field, value)
worker._share_session_counters(candidate)
self.assertEqual((current.reqid, current.reqsid), (500, 900), field)
self.assertEqual((candidate.reqid, candidate.reqsid), (800, 1800), field)
def test_first_restore_timeout_without_client_requires_manual_restore(self):
identifier, worker = self.connected()
identity = worker.active_identity
self.manager.close()
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
self.idle(identifier)
worker = self.manager._worker(identifier)
with patch.object(fixtures.FakeClient, "_post", side_effect=requests.ReadTimeout("offline")) as post:
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": identity}), status="failed")
for _ in range(3):
self.tick(identifier, worker)
self.assertEqual(post.call_count, 1)
state = self.manager.status(identifier)
self.assertEqual(state["connection_state"], "offline")
self.assertIn("手动", state["connection_message"])
self.assertIsNone(state["next_retry_at"])
self.assertIsNone(worker.schedule.retry_deadline)
self.assertIsNone(worker.engine.client)
def test_activity_restart_cursor_survives_debug_only_tail_and_records_reason(self):
identifier, worker = self.connected()
worker.log("可持久保存的运行事件")
worker.log("不持久保存的调试内容", debug=True)
old_cursor = self.manager.logs(identifier, debug=True)["next"]
self.manager.close()
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
self.idle(identifier)
logs = self.manager.logs(identifier, after=old_cursor)
self.assertGreater(logs["next"], old_cursor)
self.assertTrue(any("重启" in row["message"] and "恢复" in row["message"] for row in logs["entries"]))
all_logs = self.manager.logs(identifier, debug=True)["entries"]
self.assertTrue(any("可持久保存的运行事件" == row["message"] for row in all_logs))
self.assertFalse(any("不持久保存的调试内容" == row["message"] for row in all_logs))
def test_activity_write_failure_keeps_memory_logs_and_original_game_outcome(self):
identifier, worker = self.connected()
with patch.object(worker._activity, "append", side_effect=ActivityLogError("Unable to save activity storage")):
with patch.object(worker.engine.client.http, "post", return_value=self.expiry(worker)):
result = self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
self.assertIn("失效", result["error"])
self.assertEqual(self.manager.status(identifier)["connection_state"], "expired")
logs = self.manager.logs(identifier, debug=True)["entries"]
self.assertTrue(any("日志无法持久保存" in entry["message"] for entry in logs))
self.assertEqual(len({entry["seq"] for entry in logs}), len(logs))
def test_activity_idle_tick_and_view_do_not_write_log_storage(self):
identifier, worker = self.connected()
worker.engine.last_heartbeat = self.now
worker.schedule.next_poll = float("inf")
with patch.object(worker.engine, "master_sync_due", return_value=False), patch.object(worker.engine, "growth_sync_due", return_value=False):
with patch.object(worker._activity, "append", wraps=worker._activity.append) as append:
self.tick(identifier, worker)
view = self.manager.view(identifier, "bag")
self.job(identifier, {"id": view["job_id"]})
append.assert_not_called()
def test_activity_failure_does_not_turn_successful_game_read_into_failure(self):
identifier, worker = self.connected()
with patch.object(worker._activity, "append", side_effect=ActivityLogError("Unable to save activity storage")):
self.job(identifier, self.manager.submit(identifier, "refresh"))
self.assertEqual(self.manager.status(identifier)["connection_state"], "online")
self.assertEqual(self.manager.status(identifier)["error"], "")
self.assertTrue(any("日志无法持久保存" in entry["message"] for entry in self.manager.logs(identifier)["entries"]))
def test_activity_recovers_after_long_memory_fallback_without_reusing_cursor(self):
identifier, worker = self.connected()
activity = worker._activity
with patch.object(activity, "_write", side_effect=ActivityLogError("Unable to save activity storage")):
worker.log("临时无法保存的运行事件")
for _ in range(300):
worker.log("磁盘只读期间的调试事件", debug=True)
memory_cursor = self.manager.logs(identifier, debug=True)["next"]
for _ in range(300):
worker.log("磁盘恢复后的调试事件", debug=True)
self.assertIs(worker._activity, activity)
logs = self.manager.logs(identifier, debug=True)
old_cursor = logs["next"]
self.assertGreater(old_cursor, memory_cursor)
self.assertEqual(len({entry["seq"] for entry in logs["entries"]}), len(logs["entries"]))
self.assertEqual(sum("日志无法持久保存" in entry["message"] for entry in logs["entries"]), 1)
self.assertEqual(sum("日志持久化已恢复" in entry["message"] for entry in logs["entries"]), 1)
self.manager.close()
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
self.idle(identifier)
after_restart = self.manager.logs(identifier, after=old_cursor)
self.assertTrue(after_restart["entries"])
self.assertTrue(all(entry["seq"] > old_cursor for entry in after_restart["entries"]))
def test_activity_initial_load_failure_retries_on_later_events(self):
self.manager.close()
with patch("ranch_web.service.ActivityLog.load", side_effect=ActivityLogError("Unable to read activity storage")), \
patch("ranch_web.activity.ActivityLog._read", side_effect=ActivityLogError("Unable to read activity storage")):
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
identifier = self.account(False)
worker = self.manager._worker(identifier)
self.assertTrue(any("首次读取运行日志失败" in entry["message"] for entry in self.manager.logs(identifier)["entries"]))
before = self.manager.logs(identifier)["next"]
identifier = worker.log("日志存储重新可用")
self.assertGreater(identifier, before)
self.assertIsNotNone(worker._activity)
self.assertTrue(any("日志持久化已恢复" in entry["message"] for entry in worker._activity.load()))
def test_activity_initial_load_failure_then_immediate_recovery_has_ordered_messages(self):
self.manager.close()
with patch("ranch_web.service.ActivityLog.load", side_effect=ActivityLogError("Unable to read activity storage")):
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
identifier = self.account(False)
messages = [entry["message"] for entry in self.manager.logs(identifier)["entries"]]
self.assertIn("首次读取运行日志失败", messages[0])
self.assertIn("日志持久化已恢复", messages[1])
self.assertEqual(messages[2], "账号工作区已载入")
self.assertFalse(any("无法持久保存" in message for message in messages[1:]))
def test_restart_and_explicit_logout_require_manual_restore(self):
identifier, worker = self.connected()
self.manager.close()
calls = len(fixtures.FakeClient.calls)
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
self.idle(identifier)
worker = self.manager._worker(identifier)
state = self.manager.status(identifier)
self.assertEqual(state.get("connection_state"), "offline")
self.assertIn("重启", state["connection_message"])
self.assertIn("恢复", state["connection_message"])
self.assertFalse(state["running"])
self.assertEqual(len(fixtures.FakeClient.calls), calls)
self.tick(identifier, worker)
self.assertEqual(len(fixtures.FakeClient.calls), calls)
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}))
self.job(identifier, self.manager.submit(identifier, "logout"))
calls = len(fixtures.FakeClient.calls)
self.now += 10000
self.tick(identifier, worker)
state = self.manager.status(identifier)
self.assertEqual(state["connection_state"], "offline")
self.assertIn("退出", state["connection_message"])
self.assertFalse(state["running"])
self.assertEqual(len(fixtures.FakeClient.calls), calls)
class ErrorClassificationTests(unittest.TestCase):
def test_failed_heartbeat_does_not_advance_its_success_clock(self):
def fail(*args):
raise ApiError("offline")
engine = fixtures.Engine(fixtures.Catalog(), SimpleNamespace(call=fail))
engine.last_heartbeat = 123
with self.assertRaises(ApiError):
engine.keep_alive()
self.assertEqual(engine.last_heartbeat, 123)
def test_wire_expiry_and_transport_timeout_have_distinct_types(self):
client = fixtures.FakeClient(fixtures.Session("https://offline.invalid/game/", "synthetic-token"))
self.addCleanup(client.close)
response = SimpleNamespace(status_code=200, headers={}, content=encode({"resCode": 408}, client.session.key))
with patch.object(client.http, "post", return_value=response):
with self.assertRaises(ApiError) as expired:
client.call("masterService/masterQueryDetail")
with patch.object(client.http, "post", side_effect=requests.Timeout("offline")) as post:
with self.assertRaises(ApiError) as temporary:
client.call("monsterService/monsterLock")
self.assertEqual(post.call_count, 1)
self.assertEqual(getattr(expired.exception, "code", None), 408)
self.assertNotEqual(type(expired.exception), type(temporary.exception))
self.assertIsInstance(temporary.exception.__cause__, requests.Timeout)