"""Web recovery through real Engine barriers, temporary stores and offline HTTP.""" from copy import deepcopy import json import math import threading import time import unittest from unittest.mock import patch import requests from ranch_assistant.game_data import ProgressionData from ranch_assistant.model import Snapshot from ranch_assistant.tests.test_battle_orchestration import ExperimentalRuntime from ranch_web.service import WebManager from ranch_web.tests import test_service as fixtures class LifecycleRecoveryTests(unittest.TestCase): setUp = fixtures.ServiceTests.setUp account = fixtures.ServiceTests.account idle = fixtures.ServiceTests.idle job = fixtures.ServiceTests.job login = fixtures.ServiceTests.login def connected(self): identifier = self.account(False) self.login(identifier) worker = self.manager._worker(identifier) self.now = 1000.0 worker.schedule.clock = lambda: self.now worker.schedule.uniform = lambda low, high: high worker.engine.last_heartbeat = self.now self.enterContext(patch.object(worker.engine, "master_sync_due", return_value=False)) self.enterContext(patch.object(worker.engine, "growth_sync_due", return_value=False)) return identifier, worker def battle(self, identifier, worker, *, timed_out=False): """Small task fixture already used by Engine's orchestration regressions.""" engine = worker.engine raw = engine.client.raw raw["master"].update(ti=1000, money=1000, timeTi=0) raw["currentTasks"] = [{"id": 51, "typeId": "100", "curStep": 1, "stepValue": 0}] raw["history"] = {"unlockCity": {"1": True}, "unlockStage": {"0101": [1, 2, 3]}} raw["teams"] = [{"teamName": "teamA", "teamType": "battleTeam", "monsters": [raw["monsters"][0]["id"]]}] raw["taskStepValue"] = 0 engine.snapshot = Snapshot(deepcopy(raw)) engine.game_data = ProgressionData({ "task_info": {"100": {"type": 1, "name": "离线任务", "steps": [ {"statusType": "passStage", "statusValue": {"stage": "0101", "level": 1}}]}}, "game_setting": {"cityInfo": {"1": {"name": "离线城市"}}}, "stage_info": {"0101": {"city": 1, "cost": {"ti": 6}, "level": { str(n): {"cost": 0, "battle": {"fixture": True}} for n in (1, 2, 3)}}}}) runtime = ExperimentalRuntime() original = runtime._run if timed_out: runtime.run.return_value = {"outcome": "timeout", "execution_mode": "experimental", "execution_status": "timed_out", "terminal_reason": "worker_wall_clock_limit"} runtime.run.side_effect = None else: runtime.run.side_effect = lambda request, cancel_event: original(request, cancel_event=cancel_event) | { "requires_realtime_wait": True, "model_elapsed_ms": 10000} engine._battle_runtimes["experimental"] = runtime engine.battles.request_builder = None config = self.manager.get_config(identifier) config["config"]["progression_policy"].update(enabled=True, battle_mode="experimental", reserve_ti=0) self.job(identifier, self.manager.set_config(identifier, config["config"], config["revision"])["job"]) worker.schedule.next_poll = 0 return runtime def tick(self, identifier, worker): return self.job(identifier, worker.enqueue("__tick")) def paths(self, start=0): return ["/".join(url.rsplit("/", 2)[-2:]) for _, url, _ in fixtures.FakeClient.calls[start:]] def interrupted_clock(self): identifier, worker = self.connected() runtime = self.battle(identifier, worker) entered = threading.Event() original = worker.engine.battles.wait_for_battle_clock def computed_wait(record, request, result, data, started, cancel): wait = cancel.wait def clock_wait(timeout): entered.set() return wait(timeout) with patch.object(cancel, "wait", side_effect=clock_wait): return original(record, request, result, data, started, cancel) with patch.object(worker.engine.battles, "wait_for_battle_clock", side_effect=computed_wait): tick = worker.enqueue("__tick") self.assertTrue(entered.wait(3), "Offline battle never entered the computed clock wait") config = self.manager.get_config(identifier) config["config"]["policy"]["keep_money"] += 1 changed = self.manager.set_config(identifier, config["config"], config["revision"]) self.job(identifier, tick) self.job(identifier, changed["job"]) record = worker.engine.battle_journal.pending(worker.engine.snapshot) self.assertTrue(worker.engine.battles.resumable_task_clock(record), record) self.assertTrue(worker.engine._progression_cancel.is_set()) return identifier, worker, runtime, record def ready(self, worker): journal = worker.engine.battle_journal record = journal.pending(worker.engine.snapshot) journal.update(record, record["phase"], battle_ready_at=1) worker.engine.last_pending_check = time.monotonic() - 121 worker.schedule.next_poll = math.inf def test_config_cancelled_computed_clock_resumes_on_reconcile_without_new_start(self): identifier, worker, runtime, record = self.interrupted_clock() saved = deepcopy(record["battle_data"]) self.ready(worker) before = len(fixtures.FakeClient.calls) self.assertEqual(self.tick(identifier, worker)["result"]["kind"], "reconcile") self.assertIsNone(worker.engine.battle_journal.pending(worker.engine.snapshot)) writes = [path for path in self.paths(before) if path != "masterService/masterQueryDetail"] self.assertEqual(writes, ["taskService/taskCompleteStep"]) payload = next(payload for _, url, payload in fixtures.FakeClient.calls[before:] if url.endswith("taskService/taskCompleteStep")) self.assertEqual(payload["battleData"], saved) self.assertEqual(runtime.run.call_count, 1) self.assertEqual(self.paths().count("battleService/battleStart"), 1) self.assertTrue(self.manager.status(identifier)["strategy_requested"]) self.assertTrue(worker.running) def test_start_accepts_saved_computed_result_but_does_not_settle_inline(self): identifier, worker, _, _ = self.interrupted_clock() self.manager.submit(identifier, "stop") before = len(fixtures.FakeClient.calls) self.job(identifier, self.manager.submit(identifier, "start")) self.assertTrue(worker.running) self.assertFalse(worker.engine.auto) self.assertEqual(len(fixtures.FakeClient.calls), before) self.assertEqual(self.manager.status(identifier)["strategy_state"], "recovering") def test_computed_result_waits_for_deadline_and_manual_refresh_stays_read_only(self): identifier, worker, runtime, _ = self.interrupted_clock() journal = worker.engine.battle_journal record = journal.pending(worker.engine.snapshot) journal.update(record, record["phase"], battle_ready_at=time.time() + 3600) worker.engine.last_pending_check = time.monotonic() - 121 worker.schedule.next_poll = math.inf before = len(fixtures.FakeClient.calls) self.tick(identifier, worker) self.job(identifier, self.manager.submit(identifier, "refresh")) self.assertTrue(all(path == "masterService/masterQueryDetail" for path in self.paths(before))) self.assertIsNotNone(journal.pending(worker.engine.snapshot)) self.assertEqual(runtime.run.call_count, 1) self.assertTrue(worker.running) self.assertFalse(worker.engine.auto) self.assertEqual(self.manager.status(identifier)["strategy_state"], "recovering") def test_lost_resumed_settlement_reply_becomes_query_only_without_replay(self): identifier, worker, runtime, _ = self.interrupted_clock() self.ready(worker) post = worker.engine.client.http.post attempted = [] def lost_reply(url, *args, **kwargs): if url.endswith("taskService/taskCompleteStep"): attempted.append(url) raise requests.ReadTimeout("offline lost settlement reply") return post(url, *args, **kwargs) with patch.object(worker.engine.client.http, "post", side_effect=lost_reply): self.job(identifier, worker.enqueue("__tick"), status="failed") record = worker.engine.battle_journal.pending(worker.engine.snapshot) self.assertTrue(record["settlement_dispatched"]) self.assertFalse(worker.engine.battles.resumable_task_clock(record)) for _ in range(3): self.now += 121 worker.engine.last_heartbeat = self.now worker.engine.last_pending_check = time.monotonic() - 121 self.tick(identifier, worker) self.assertEqual(len(attempted), 1) self.assertEqual(runtime.run.call_count, 1) self.assertEqual(self.paths().count("battleService/battleStart"), 1) self.assertFalse(worker.running) self.assertTrue(worker.strategy_requested) def test_manual_stop_keeps_computed_result_unsent_through_config_and_restart(self): identifier, worker, _, _ = self.interrupted_clock() self.manager.submit(identifier, "stop") self.ready(worker) journal_path = worker.engine.battle_journal.path evidence = journal_path.read_bytes() config = self.manager.get_config(identifier) config["config"]["policy"]["keep_money"] += 1 self.job(identifier, self.manager.set_config(identifier, config["config"], config["revision"])["job"]) before = len(fixtures.FakeClient.calls) self.tick(identifier, worker) self.assertEqual(journal_path.read_bytes(), evidence) self.assertTrue(all(path == "masterService/masterQueryDetail" for path in self.paths(before))) self.assertTrue(self.manager.status(identifier)["manual_stop"]) self.manager.close() self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin) self.addCleanup(self.manager.close) self.idle(identifier) self.assertTrue(self.manager.status(identifier)["manual_stop"]) self.assertEqual(journal_path.read_bytes(), evidence) def test_stop_racing_authorized_reconcile_cancels_settlement_before_dispatch(self): identifier, worker, _, _ = self.interrupted_clock() self.ready(worker) entered, release = threading.Event(), threading.Event() poll = worker.engine.poll journal_path = worker.engine.battle_journal.path evidence = journal_path.read_bytes() def held_poll(**kwargs): self.assertFalse(worker.engine.auto) self.assertIs(kwargs.get("resume_pending"), True) entered.set() self.assertTrue(release.wait(3)) return poll(**kwargs) before = len(fixtures.FakeClient.calls) with patch.object(worker.engine, "poll", side_effect=held_poll): tick = worker.enqueue("__tick") try: self.assertTrue(entered.wait(3), self.manager.job(identifier, tick["id"])) self.manager.submit(identifier, "stop") finally: release.set() self.job(identifier, tick, status="cancelled") self.assertEqual(len(fixtures.FakeClient.calls), before) self.assertEqual(journal_path.read_bytes(), evidence) self.assertFalse(worker.running) self.assertTrue(self.manager.status(identifier)["manual_stop"]) def test_runtime_recovery_timeout_stays_blocked_after_single_local_retry(self): identifier, worker = self.connected() runtime = self.battle(identifier, worker, timed_out=True) self.tick(identifier, worker) record = worker.engine.battle_journal.pending(worker.engine.snapshot) self.assertEqual(record["execution_status"], "timed_out") self.assertFalse(worker.engine.battles.resumable_task_clock(record)) self.assertNotIn("battle_data", record) self.assertTrue(worker.engine.pending_battle_recovery()["resumable"]) self.job(identifier, self.manager.submit(identifier, "start")) worker.engine.last_pending_check = time.monotonic() - 121 worker.schedule.next_poll = math.inf self.tick(identifier, worker) record = worker.engine.battle_journal.pending(worker.engine.snapshot) self.assertEqual(record["runtime_recovery_attempts"], 1) self.assertFalse(worker.engine.pending_battle_recovery()["resumable"]) evidence = worker.engine.battle_journal.path.read_bytes() before = len(fixtures.FakeClient.calls) self.job(identifier, self.manager.submit(identifier, "start"), status="failed") for _ in range(3): self.now += 121 worker.engine.last_heartbeat = self.now worker.engine.last_pending_check = time.monotonic() - 121 self.tick(identifier, worker) self.assertTrue(all(path == "masterService/masterQueryDetail" for path in self.paths(before))) self.assertEqual(worker.engine.battle_journal.path.read_bytes(), evidence) self.assertEqual(runtime.run.call_count, 2) self.assertEqual(self.paths().count("battleService/battleStart"), 1) state = self.manager.status(identifier) self.assertFalse(state["running"]) self.assertTrue(state["strategy_requested"]) self.assertFalse(state["manual_stop"]) self.assertIn("本地战斗计算超时", state["strategy_message"]) self.assertIn("尚未提交结算", state["strategy_message"]) def test_stage_farming_timeout_reports_its_operation_without_promising_resolution(self): identifier, worker = self.connected() fixtures.ServiceTests.enable(self, identifier) self.job(identifier, self.manager.submit(identifier, "start")) journal = worker.engine.battle_journal record = journal.begin(worker.engine.snapshot, {"kind": "stage_battle", "stage_id": "0102", "difficulty": 1, "path": "stageService/stageStart", "task_id": 0}) journal.update(record, "unknown", start_response={"resCode": 0, "triggerEvent": False, "stageEvent": None}, execution_status="timed_out", terminal_reason="worker_wall_clock_limit", message="战斗已取消、超时或结果不确定,等待核对") self.tick(identifier, worker) state = self.manager.status(identifier) self.assertTrue(state["connected"]) self.assertFalse(state["running"]) self.assertIn("0102", state["strategy_message"]) self.assertIn("本地战斗计算超时", state["strategy_message"]) worker.schedule.monitor(False) reason = worker._pending_execution_reason() self.assertNotIn("后台继续", reason) self.assertNotIn("自动恢复", reason) config = self.manager.get_config(identifier) config["config"]["policy"]["keep_money"] += 1 self.job(identifier, self.manager.set_config(identifier, config["config"], config["revision"])["job"]) self.assertEqual(worker.config["policy"]["keep_money"], config["config"]["policy"]["keep_money"]) self.assertTrue(worker.strategy_requested) intent = json.loads((worker.profile_root / "run_intent.json").read_text(encoding="utf-8")) self.assertTrue(intent["roles"][worker.active_identity]) def test_config_cancellation_before_start_dispatch_replans_without_unknown_barrier(self): identifier, worker = self.connected() runtime = self.battle(identifier, worker, timed_out=True) update = worker.engine.battle_journal.update changed = [] def cancel_before_send(record, phase, **fields): update(record, phase, **fields) if phase == "start_pending": config = self.manager.get_config(identifier) config["config"]["policy"]["keep_money"] += 1 changed.append(self.manager.set_config(identifier, config["config"], config["revision"])["job"]) with patch.object(worker.engine.battle_journal, "update", side_effect=cancel_before_send): self.tick(identifier, worker) self.job(identifier, changed[0]) self.assertNotIn("battleService/battleStart", self.paths()) self.assertFalse(worker.engine._progression_pending()) self.assertIsNone(worker.engine.battle_journal.pending(worker.engine.snapshot)) self.assertIsNone(worker.engine.progression_planner.pending(worker.engine.snapshot)) runtime.run.assert_not_called() self.assertTrue(worker.running) self.assertTrue(worker.strategy_requested) worker.schedule.next_poll = 0 self.tick(identifier, worker) self.assertEqual(self.paths().count("battleService/battleStart"), 1) def test_internal_pending_pause_does_not_revoke_generation_or_persist_manual_stop(self): identifier, worker = self.connected() worker.schedule.next_poll = 0 generation = worker._generation def paused(**kwargs): journal = worker.engine.battle_journal record = journal.begin(worker.engine.snapshot, {"kind": "stage_battle", "stage_id": "0102"}) journal.update(record, "unknown", message="请求结果未知") worker.engine.auto = False return worker.engine.snapshot, [], "请求结果未知" with patch.object(worker.engine, "poll", side_effect=paused): self.tick(identifier, worker) self.assertEqual(worker._generation, generation) self.assertTrue(worker.strategy_requested) self.assertFalse(self.manager.status(identifier)["manual_stop"]) if __name__ == "__main__": unittest.main()