"""Connection regressions through real workers and an entirely offline transport.""" from copy import deepcopy from types import SimpleNamespace import threading import unittest from unittest.mock import patch import requests from ranch_assistant.api import ApiError, encode from ranch_web.activity import ActivityLogError from ranch_web.service import WebManager from ranch_web.scheduler import Scheduler from ranch_web.tests import test_service as fixtures from ranch_web.tests.test_auto_relogin import AutoLogin class ConnectionStateTests(unittest.TestCase): setUp = fixtures.ServiceTests.setUp account = fixtures.ServiceTests.account idle = fixtures.ServiceTests.idle job = fixtures.ServiceTests.job login = fixtures.ServiceTests.login enable = fixtures.ServiceTests.enable def connected(self, *, running=False): identifier = self.account(False) self.login(identifier) worker = self.manager._worker(identifier) self.now = 1000.0 worker.schedule.clock = lambda: self.now if running: self.enable(identifier) self.job(identifier, self.manager.submit(identifier, "start")) else: self.manager.submit(identifier, "stop") return identifier, worker def tick(self, identifier, worker, *, status="succeeded"): return self.job(identifier, worker.enqueue("__tick"), status=status) def expiry(self, worker): return SimpleNamespace(status_code=200, headers={}, content=encode({"resCode": 408}, worker.engine.client.session.key)) def test_expired_state_survives_idle_view_config_and_blocks_game_requests(self): identifier, worker = self.connected(running=True) original_client, original_snapshot = worker.engine.client, worker.engine.snapshot with patch.object(original_client.http, "post", return_value=self.expiry(worker)) as post: worker.engine.last_heartbeat = 0 self.tick(identifier, worker, status="failed") state = self.manager.status(identifier) self.assertEqual(state.get("connection_state"), "expired") self.assertIn("失效", state["connection_message"]) self.assertFalse(state["connected"]) self.assertFalse(state["running"]) reason = state["error"] for _ in range(3): self.now += 600 self.tick(identifier, worker) view = self.manager.view(identifier, "bag") self.job(identifier, {"id": view["job_id"]}) config = self.manager.get_config(identifier) config["config"]["interval"] = 60 self.job(identifier, self.manager.set_config(identifier, config["config"])["job"]) self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed") self.job(identifier, self.manager.submit(identifier, "start"), status="failed") self.assertEqual(post.call_count, 1) self.assertEqual(self.manager.status(identifier)["error"], reason) self.assertEqual(self.manager.status(identifier)["connection_state"], "expired") self.assertIs(worker.engine.client, original_client) self.assertIs(worker.engine.snapshot, original_snapshot) self.assertIsNotNone(worker.securestore.load(worker.active_identity)) self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity})) self.assertEqual(self.manager.status(identifier)["connection_state"], "online") self.assertTrue(self.manager.status(identifier)["running"]) self.assertFalse(worker.engine.auto, "Session verification must not authorize selected writes inline") def test_explicit_stop_survives_heartbeat_refresh_monitor_and_relogin(self): identifier, worker = self.connected(running=True) self.manager.submit(identifier, "stop") worker.engine.last_heartbeat = 0 result = self.tick(identifier, worker)["result"] self.assertEqual(result["kind"], "heartbeat") self.job(identifier, self.manager.submit(identifier, "refresh", {"page": "statistics"})) self.job(identifier, self.manager.submit(identifier, "refresh")) worker.schedule.next_poll = 0 self.tick(identifier, worker) self.assertFalse(worker.running) self.assertFalse(worker.engine.auto) self.assertTrue(self.manager.status(identifier)["connected"]) result = self.job(identifier, self.manager.submit(identifier, "restore_session", { "session_id": worker.active_identity}))["result"] self.assertFalse(result["running"]) self.assertFalse(worker.engine.auto) self.assertTrue(self.manager.status(identifier)["manual_stop"]) self.assertFalse(self.manager.status(identifier)["strategy_requested"]) self.login(identifier) self.assertFalse(worker.running, "Password login erased the same role's explicit Stop") self.assertFalse(worker.engine.auto) self.job(identifier, self.manager.submit(identifier, "start")) self.assertTrue(worker.running) self.assertFalse(self.manager.status(identifier)["manual_stop"]) def test_timeout_retries_only_snapshot_and_keeps_uncertain_write_evidence(self): identifier, worker = self.connected(running=True) journal = worker.engine.battle_journal pending = journal.begin(worker.engine.snapshot, {"kind": "battle", "description": "未确认的战斗", "path": "stageService/stageStart", "payload": {}, "stage_id": "0101", "difficulty": "1"}) journal.update(pending, "unknown", message="未确认的战斗写请求") evidence = journal.path before = evidence.read_bytes() client = worker.engine.client def write(runtime, operation, payload): runtime.engine.client.call("monsterService/monsterLock", {"monsterId": 1}) with patch("ranch_web.actions.execute_action", side_effect=write): with patch.object(client.http, "post", side_effect=requests.ReadTimeout("synthetic timeout")) as post: self.job(identifier, self.manager.submit(identifier, "lock"), status="failed") self.assertEqual(post.call_count, 1) state = self.manager.status(identifier) self.assertEqual(state.get("connection_state"), "retrying") self.assertNotIn("失效", state["connection_message"]) self.assertFalse(state["running"]) self.assertIsNotNone(state["next_retry_at"]) calls = len(fixtures.FakeClient.calls) self.tick(identifier, worker) self.assertEqual(len(fixtures.FakeClient.calls), calls) self.now += 61 self.tick(identifier, worker) paths = [url.rsplit("/", 2)[-2:] for _, url, _ in fixtures.FakeClient.calls[calls:]] self.assertEqual(paths, [["masterService", "masterQueryDetail"]]) self.assertEqual(self.manager.status(identifier)["connection_state"], "online") self.assertEqual(self.manager.status(identifier)["error"], "") self.assertIsNone(self.manager.status(identifier)["next_retry_at"]) self.assertIsNotNone(self.manager.status(identifier)["last_game_success_at"]) self.assertFalse(worker.running) self.assertFalse(worker.engine.auto) self.assertTrue(self.manager.status(identifier)["strategy_requested"]) self.assertEqual(self.manager.status(identifier)["strategy_state"], "blocked") self.assertFalse(self.manager.status(identifier)["manual_stop"]) self.assertEqual(journal.pending(worker.engine.snapshot)["attempt_id"], pending["attempt_id"]) self.assertEqual(evidence.read_bytes(), before) # Continue through the ordinary monitor/reconcile path after recovery. # The existing Engine barrier must still prevent the unknown write. worker.engine.last_heartbeat = self.now worker.schedule.next_poll = 0 self.now += 121 self.tick(identifier, worker) paths = [url.rsplit("/", 2)[-2:] for _, url, _ in fixtures.FakeClient.calls[calls:]] self.assertEqual(paths, [["masterService", "masterQueryDetail"]] * 2) self.assertEqual(journal.pending(worker.engine.snapshot)["attempt_id"], pending["attempt_id"]) self.assertEqual(journal.pending(worker.engine.snapshot)["phase"], "unknown") self.assertFalse(worker.running) self.assertFalse(worker.engine.auto) self.assertTrue(self.manager.status(identifier)["strategy_requested"]) self.assertEqual(evidence.read_bytes(), before) def test_business_rejection_keeps_connection_and_idle_does_not_clear_error(self): identifier, worker = self.connected(running=True) client = worker.engine.client response = SimpleNamespace(status_code=200, headers={}, content=encode({"resCode": 403}, client.session.key)) with patch.object(client.http, "post", return_value=response): self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed") state = self.manager.status(identifier) self.assertEqual(state["connection_state"], "online") self.assertTrue(state["connected"]) self.assertFalse(state["running"]) self.assertFalse(worker.schedule.game_paused) self.assertTrue(worker.schedule.monitoring) reason = state["error"] self.assertIn("403", reason) worker.engine.last_heartbeat = self.now worker.schedule.next_poll = float("inf") with patch.object(worker.engine, "master_sync_due", return_value=False), patch.object(worker.engine, "growth_sync_due", return_value=False): self.tick(identifier, worker) view = self.manager.view(identifier, "bag") self.job(identifier, {"id": view["job_id"]}) self.assertEqual(self.manager.status(identifier)["error"], reason) def test_997_is_retrying_without_expiry_and_backoff_is_bounded(self): identifier, worker = self.connected() response = SimpleNamespace(status_code=200, headers={}, content=encode({"resCode": 997}, worker.engine.client.session.key)) with patch.object(worker.engine.client.http, "post", return_value=response) as post: self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed") for delay in (5, 10, 20, 40, 60, 60): self.assertEqual(worker.schedule.retry_deadline - self.now, delay) state = self.manager.status(identifier) self.assertEqual(state["connection_state"], "retrying") self.assertNotIn("失效", state["connection_message"]) calls = post.call_count self.tick(identifier, worker) self.assertEqual(post.call_count, calls) self.now += delay self.tick(identifier, worker, status="failed") self.assertEqual(post.call_count, calls + 1) def test_swallowed_expiry_still_stops_strategy_and_marks_job_failed(self): identifier, worker = self.connected(running=True) def swallowed(runtime, operation, payload): try: runtime.engine.client.call("monsterService/monsterLock") except ApiError: return {"message": "Engine caught the error"} with patch("ranch_web.actions.execute_action", side_effect=swallowed): with patch.object(worker.engine.client.http, "post", return_value=self.expiry(worker)): result = self.job(identifier, self.manager.submit(identifier, "lock"), status="failed") self.assertIn("失效", result["error"]) self.assertEqual(self.manager.status(identifier)["connection_state"], "expired") self.assertFalse(worker.running) def test_scheduler_runs_fake_heartbeat_and_poll_without_any_browser_requests(self): identifier, worker = self.connected() heartbeat, poll = threading.Event(), threading.Event() original = worker.engine.client.http.post def post(url, *args, **kwargs): result = original(url, *args, **kwargs) if url.endswith("heartBeatService/keepAlive"): heartbeat.set() if url.endswith("masterService/masterQueryDetail"): poll.set() return result worker.engine.last_heartbeat = 0 worker.schedule.next_poll = 0 with patch.object(worker.engine.client.http, "post", side_effect=post): scheduler = Scheduler(self.manager._worker_list, interval=.01) try: self.assertTrue(heartbeat.wait(3), "No heartbeat without browser polling") self.assertTrue(poll.wait(3), "No game poll without browser polling") finally: scheduler.close() self.idle(identifier) self.assertFalse(worker.running) def test_idle_local_success_does_not_clear_game_error(self): identifier, worker = self.connected() with patch.object(worker.engine.client.http, "post", side_effect=requests.Timeout("offline")): self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed") reason = self.manager.status(identifier)["error"] self.assertTrue(reason) self.tick(identifier, worker) view = self.manager.view(identifier, "bag") self.job(identifier, {"id": view["job_id"]}) self.job(identifier, self.manager.submit(identifier, "refresh", {"page": "statistics"})) self.assertEqual(self.manager.status(identifier)["error"], reason) self.assertFalse(self.manager.status(identifier)["connected"]) def test_failed_restore_and_login_candidates_preserve_online_client_and_monitor(self): identifier, worker = self.connected(running=True) client = worker.engine.client original = deepcopy(self.manager.status(identifier)) response = self.expiry(worker) with patch.object(fixtures.FakeClient, "_post", return_value=response): self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}), status="failed") self.assertIs(worker.engine.client, client) self.assertFalse(client.closed) state = self.manager.status(identifier) self.assertEqual(state.get("connection_state"), "online") self.assertEqual(state["last_game_success_at"], original["last_game_success_at"]) self.assertEqual(state["connection_message"], original["connection_message"]) self.assertTrue(state["connected"]) for fail in (True, False): with self.subTest(login_failure=fail): before = len(fixtures.FakeClient.calls) if fail: with patch.object(fixtures.FakeLogin, "authenticate", side_effect=ApiError("Candidate login failed")): self.job(identifier, self.manager.submit(identifier, "login", { "account": "synthetic-owner", "password": "synthetic-password"}), status="failed") else: self.job(identifier, self.manager.submit(identifier, "login", { "account": "synthetic-owner", "password": "synthetic-password"})) self.assertIs(worker.engine.client, client) self.assertFalse(client.closed) self.assertTrue(self.manager.status(identifier)["connected"]) self.assertTrue(self.manager.status(identifier)["strategy_requested"]) self.assertEqual(len(fixtures.FakeClient.calls), before, "Server discovery issued old-role strategy writes inline") self.now += 61 worker.engine.last_heartbeat = self.now worker.schedule.next_poll = 0 with patch.object(worker.engine, "poll", wraps=worker.engine.poll) as poll: self.tick(identifier, worker) self.assertEqual(poll.call_count, 1, "An online role lost automatic polling after candidate login") self.assertTrue(worker.schedule.monitoring) self.assertTrue(worker.running) def test_failed_same_protocol_restore_preserves_counter_high_water_before_wire_and_finally(self): identifier, worker = self.connected() for active_reqid, active_reqsid in ((400, 900), (500, 1200)): with self.subTest(active_reqid=active_reqid): client = worker.engine.client saved = deepcopy(client.session) saved.reqid, saved.reqsid = 400, 900 worker.securestore.save(saved, "离线高水位回归") client.session.reqid, client.session.reqsid = active_reqid, active_reqsid wire = [] def post(url, data, **kwargs): wire.append(fixtures.decode(data, client.session.key)) return self.expiry(worker) with patch.object(fixtures.FakeClient, "_post", side_effect=post): self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}), status="failed") self.assertEqual((wire[0]["reqid"], wire[0]["reqsid"]), (active_reqid, active_reqsid)) persisted = worker.securestore.load(worker.active_identity) self.assertEqual((persisted.reqid, persisted.reqsid), (active_reqid + 1, active_reqsid + 1)) self.assertEqual((client.session.reqid, client.session.reqsid), (persisted.reqid, persisted.reqsid)) client.call("heartBeatService/keepAlive") payload = fixtures.FakeClient.calls[-1][2] self.assertEqual((payload["reqid"], payload["reqsid"]), (active_reqid + 1, active_reqsid + 1)) def test_counter_merge_requires_exact_protocol_session(self): identifier, worker = self.connected() current = worker.engine.client.session current.reqid, current.reqsid = 500, 900 changes = {"token": "different", "key": "different", "server_url": "https://other.invalid/", "account_id": "different", "server_id": 9, "role_id": 999, "channel": "other", "platform": "other"} for field, value in changes.items(): candidate = deepcopy(current) candidate.reqid, candidate.reqsid = 800, 1800 setattr(candidate, field, value) worker._share_session_counters(candidate) self.assertEqual((current.reqid, current.reqsid), (500, 900), field) self.assertEqual((candidate.reqid, candidate.reqsid), (800, 1800), field) def test_first_restore_timeout_without_client_requires_manual_restore(self): identifier, worker = self.connected() identity = worker.active_identity self.manager.close() self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin) self.addCleanup(self.manager.close) self.idle(identifier) worker = self.manager._worker(identifier) with patch.object(fixtures.FakeClient, "_post", side_effect=requests.ReadTimeout("offline")) as post: self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": identity}), status="failed") for _ in range(3): self.tick(identifier, worker) self.assertEqual(post.call_count, 1) state = self.manager.status(identifier) self.assertEqual(state["connection_state"], "offline") self.assertIn("手动", state["connection_message"]) self.assertIsNone(state["next_retry_at"]) self.assertIsNone(worker.schedule.retry_deadline) self.assertIsNone(worker.engine.client) def test_activity_restart_cursor_survives_debug_only_tail_and_records_reason(self): identifier, worker = self.connected() worker.log("可持久保存的运行事件") worker.log("不持久保存的调试内容", debug=True) old_cursor = self.manager.logs(identifier, debug=True)["next"] self.manager.close() self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin) self.addCleanup(self.manager.close) self.idle(identifier) logs = self.manager.logs(identifier, after=old_cursor) self.assertGreater(logs["next"], old_cursor) self.assertTrue(any("重启" in row["message"] and "重新登录" in row["message"] for row in logs["entries"])) all_logs = self.manager.logs(identifier, debug=True)["entries"] self.assertTrue(any("可持久保存的运行事件" == row["message"] for row in all_logs)) self.assertFalse(any("不持久保存的调试内容" == row["message"] for row in all_logs)) def test_activity_write_failure_keeps_memory_logs_and_original_game_outcome(self): identifier, worker = self.connected() with patch.object(worker._activity, "append", side_effect=ActivityLogError("Unable to save activity storage")): with patch.object(worker.engine.client.http, "post", return_value=self.expiry(worker)): result = self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed") self.assertIn("失效", result["error"]) self.assertEqual(self.manager.status(identifier)["connection_state"], "expired") logs = self.manager.logs(identifier, debug=True)["entries"] self.assertTrue(any("日志无法持久保存" in entry["message"] for entry in logs)) self.assertEqual(len({entry["seq"] for entry in logs}), len(logs)) def test_activity_idle_tick_and_view_do_not_write_log_storage(self): identifier, worker = self.connected() worker.engine.last_heartbeat = self.now worker.schedule.next_poll = float("inf") with patch.object(worker.engine, "master_sync_due", return_value=False), patch.object(worker.engine, "growth_sync_due", return_value=False): with patch.object(worker._activity, "append", wraps=worker._activity.append) as append: self.tick(identifier, worker) view = self.manager.view(identifier, "bag") self.job(identifier, {"id": view["job_id"]}) append.assert_not_called() def test_activity_failure_does_not_turn_successful_game_read_into_failure(self): identifier, worker = self.connected() with patch.object(worker._activity, "append", side_effect=ActivityLogError("Unable to save activity storage")): self.job(identifier, self.manager.submit(identifier, "refresh")) self.assertEqual(self.manager.status(identifier)["connection_state"], "online") self.assertEqual(self.manager.status(identifier)["error"], "") self.assertTrue(any("日志无法持久保存" in entry["message"] for entry in self.manager.logs(identifier)["entries"])) def test_activity_recovers_after_long_memory_fallback_without_reusing_cursor(self): identifier, worker = self.connected() activity = worker._activity with patch.object(activity, "_write", side_effect=ActivityLogError("Unable to save activity storage")): worker.log("临时无法保存的运行事件") for _ in range(300): worker.log("磁盘只读期间的调试事件", debug=True) memory_cursor = self.manager.logs(identifier, debug=True)["next"] for _ in range(300): worker.log("磁盘恢复后的调试事件", debug=True) self.assertIs(worker._activity, activity) logs = self.manager.logs(identifier, debug=True) old_cursor = logs["next"] self.assertGreater(old_cursor, memory_cursor) self.assertEqual(len({entry["seq"] for entry in logs["entries"]}), len(logs["entries"])) self.assertEqual(sum("日志无法持久保存" in entry["message"] for entry in logs["entries"]), 1) self.assertEqual(sum("日志持久化已恢复" in entry["message"] for entry in logs["entries"]), 1) self.manager.close() self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin) self.addCleanup(self.manager.close) self.idle(identifier) after_restart = self.manager.logs(identifier, after=old_cursor) self.assertTrue(after_restart["entries"]) self.assertTrue(all(entry["seq"] > old_cursor for entry in after_restart["entries"])) def test_activity_initial_load_failure_retries_on_later_events(self): self.manager.close() with patch("ranch_web.service.ActivityLog.load", side_effect=ActivityLogError("Unable to read activity storage")), \ patch("ranch_web.activity.ActivityLog._read", side_effect=ActivityLogError("Unable to read activity storage")): self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin) self.addCleanup(self.manager.close) identifier = self.account(False) worker = self.manager._worker(identifier) self.assertTrue(any("首次读取运行日志失败" in entry["message"] for entry in self.manager.logs(identifier)["entries"])) before = self.manager.logs(identifier)["next"] identifier = worker.log("日志存储重新可用") self.assertGreater(identifier, before) self.assertIsNotNone(worker._activity) self.assertTrue(any("日志持久化已恢复" in entry["message"] for entry in worker._activity.load())) def test_activity_initial_load_failure_then_immediate_recovery_has_ordered_messages(self): self.manager.close() with patch("ranch_web.service.ActivityLog.load", side_effect=ActivityLogError("Unable to read activity storage")): self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin) self.addCleanup(self.manager.close) identifier = self.account(False) messages = [entry["message"] for entry in self.manager.logs(identifier)["entries"]] self.assertIn("首次读取运行日志失败", messages[0]) self.assertIn("日志持久化已恢复", messages[1]) self.assertEqual(messages[2], "账号工作区已载入") self.assertFalse(any("无法持久保存" in message for message in messages[1:])) def test_restart_and_logout_retain_explicit_stop_and_saved_login(self): identifier, worker = self.connected() self.manager.close() calls = len(fixtures.FakeClient.calls) self.manager = WebManager(self.root, start_scheduler=False, login_factory=AutoLogin) self.addCleanup(self.manager.close) self.idle(identifier) worker = self.manager._worker(identifier) state = self.manager.status(identifier) self.assertEqual(state.get("connection_state"), "offline") self.assertIn("重启", state["connection_message"]) self.assertIn("重新登录", state["connection_message"]) self.assertFalse(state["running"]) self.assertEqual(len(fixtures.FakeClient.calls), calls) self.assertTrue(state["manual_stop"]) self.assertFalse(state["strategy_requested"]) self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity})) self.assertFalse(worker.running) identity = worker.active_identity self.job(identifier, self.manager.submit(identifier, "logout")) self.assertEqual(worker.active_identity, identity) self.assertEqual(worker.securestore.active(), identity) self.assertTrue(worker.securestore.load_credentials()[0]) self.assertEqual(self.manager.status(identifier)["auto_relogin_status"]["state"], "waiting") # The delay is anchored in the restarted scheduler's own clock. self.now = worker.schedule.disconnected_at + worker.config["auto_relogin"]["delay_minutes"] * 60 worker.schedule.clock = lambda: self.now attempts = len(AutoLogin.calls) self.tick(identifier, worker) state = self.manager.status(identifier) self.assertEqual(len(AutoLogin.calls), attempts + 1) self.assertTrue(state["connected"]) self.assertEqual(state["auto_relogin_status"]["state"], "idle") self.assertTrue(state["manual_stop"]) self.assertFalse(state["running"]) class ErrorClassificationTests(unittest.TestCase): def test_failed_heartbeat_does_not_advance_its_success_clock(self): def fail(*args): raise ApiError("offline") engine = fixtures.Engine(fixtures.Catalog(), SimpleNamespace(call=fail)) engine.last_heartbeat = 123 with self.assertRaises(ApiError): engine.keep_alive() self.assertEqual(engine.last_heartbeat, 123) def test_wire_expiry_and_transport_timeout_have_distinct_types(self): client = fixtures.FakeClient(fixtures.Session("https://offline.invalid/game/", "synthetic-token")) self.addCleanup(client.close) response = SimpleNamespace(status_code=200, headers={}, content=encode({"resCode": 408}, client.session.key)) with patch.object(client.http, "post", return_value=response): with self.assertRaises(ApiError) as expired: client.call("masterService/masterQueryDetail") with patch.object(client.http, "post", side_effect=requests.Timeout("offline")) as post: with self.assertRaises(ApiError) as temporary: client.call("monsterService/monsterLock") self.assertEqual(post.call_count, 1) self.assertEqual(getattr(expired.exception, "code", None), 408) self.assertNotEqual(type(expired.exception), type(temporary.exception)) self.assertIsInstance(temporary.exception.__cause__, requests.Timeout)