Files
myrzx_bot/ranch_web/tests/test_lifecycle_recovery.py

346 lines
18 KiB
Python

"""Web recovery through real Engine barriers, temporary stores and offline HTTP."""
from copy import deepcopy
import json
import math
import threading
import time
import unittest
from unittest.mock import patch
import requests
from ranch_assistant.game_data import ProgressionData
from ranch_assistant.model import Snapshot
from ranch_assistant.tests.test_battle_orchestration import ExperimentalRuntime
from ranch_web.service import WebManager
from ranch_web.tests import test_service as fixtures
class LifecycleRecoveryTests(unittest.TestCase):
setUp = fixtures.ServiceTests.setUp
account = fixtures.ServiceTests.account
idle = fixtures.ServiceTests.idle
job = fixtures.ServiceTests.job
login = fixtures.ServiceTests.login
def connected(self):
identifier = self.account(False)
self.login(identifier)
worker = self.manager._worker(identifier)
self.now = 1000.0
worker.schedule.clock = lambda: self.now
worker.schedule.uniform = lambda low, high: high
worker.engine.last_heartbeat = self.now
self.enterContext(patch.object(worker.engine, "master_sync_due", return_value=False))
self.enterContext(patch.object(worker.engine, "growth_sync_due", return_value=False))
return identifier, worker
def battle(self, identifier, worker, *, timed_out=False):
"""Small task fixture already used by Engine's orchestration regressions."""
engine = worker.engine
raw = engine.client.raw
raw["master"].update(ti=1000, money=1000, timeTi=0)
raw["currentTasks"] = [{"id": 51, "typeId": "100", "curStep": 1, "stepValue": 0}]
raw["history"] = {"unlockCity": {"1": True}, "unlockStage": {"0101": [1, 2, 3]}}
raw["teams"] = [{"teamName": "teamA", "teamType": "battleTeam", "monsters": [raw["monsters"][0]["id"]]}]
raw["taskStepValue"] = 0
engine.snapshot = Snapshot(deepcopy(raw))
engine.game_data = ProgressionData({
"task_info": {"100": {"type": 1, "name": "离线任务", "steps": [
{"statusType": "passStage", "statusValue": {"stage": "0101", "level": 1}}]}},
"game_setting": {"cityInfo": {"1": {"name": "离线城市"}}},
"stage_info": {"0101": {"city": 1, "cost": {"ti": 6}, "level": {
str(n): {"cost": 0, "battle": {"fixture": True}} for n in (1, 2, 3)}}}})
runtime = ExperimentalRuntime()
original = runtime._run
if timed_out:
runtime.run.return_value = {"outcome": "timeout", "execution_mode": "experimental",
"execution_status": "timed_out", "terminal_reason": "worker_wall_clock_limit"}
runtime.run.side_effect = None
else:
runtime.run.side_effect = lambda request, cancel_event: original(request, cancel_event=cancel_event) | {
"requires_realtime_wait": True, "model_elapsed_ms": 10000}
engine._battle_runtimes["experimental"] = runtime
engine.battles.request_builder = None
config = self.manager.get_config(identifier)
config["config"]["progression_policy"].update(enabled=True, battle_mode="experimental", reserve_ti=0)
self.job(identifier, self.manager.set_config(identifier, config["config"], config["revision"])["job"])
worker.schedule.next_poll = 0
return runtime
def tick(self, identifier, worker):
return self.job(identifier, worker.enqueue("__tick"))
def paths(self, start=0):
return ["/".join(url.rsplit("/", 2)[-2:]) for _, url, _ in fixtures.FakeClient.calls[start:]]
def interrupted_clock(self):
identifier, worker = self.connected()
runtime = self.battle(identifier, worker)
entered = threading.Event()
original = worker.engine.battles.wait_for_battle_clock
def computed_wait(record, request, result, data, started, cancel):
wait = cancel.wait
def clock_wait(timeout):
entered.set()
return wait(timeout)
with patch.object(cancel, "wait", side_effect=clock_wait):
return original(record, request, result, data, started, cancel)
with patch.object(worker.engine.battles, "wait_for_battle_clock", side_effect=computed_wait):
tick = worker.enqueue("__tick")
self.assertTrue(entered.wait(3), "Offline battle never entered the computed clock wait")
config = self.manager.get_config(identifier)
config["config"]["policy"]["keep_money"] += 1
changed = self.manager.set_config(identifier, config["config"], config["revision"])
self.job(identifier, tick)
self.job(identifier, changed["job"])
record = worker.engine.battle_journal.pending(worker.engine.snapshot)
self.assertTrue(worker.engine.battles.resumable_task_clock(record), record)
self.assertTrue(worker.engine._progression_cancel.is_set())
return identifier, worker, runtime, record
def ready(self, worker):
journal = worker.engine.battle_journal
record = journal.pending(worker.engine.snapshot)
journal.update(record, record["phase"], battle_ready_at=1)
worker.engine.last_pending_check = time.monotonic() - 121
worker.schedule.next_poll = math.inf
def test_config_cancelled_computed_clock_resumes_on_reconcile_without_new_start(self):
identifier, worker, runtime, record = self.interrupted_clock()
saved = deepcopy(record["battle_data"])
self.ready(worker)
before = len(fixtures.FakeClient.calls)
self.assertEqual(self.tick(identifier, worker)["result"]["kind"], "reconcile")
self.assertIsNone(worker.engine.battle_journal.pending(worker.engine.snapshot))
writes = [path for path in self.paths(before) if path != "masterService/masterQueryDetail"]
self.assertEqual(writes, ["taskService/taskCompleteStep"])
payload = next(payload for _, url, payload in fixtures.FakeClient.calls[before:]
if url.endswith("taskService/taskCompleteStep"))
self.assertEqual(payload["battleData"], saved)
self.assertEqual(runtime.run.call_count, 1)
self.assertEqual(self.paths().count("battleService/battleStart"), 1)
self.assertTrue(self.manager.status(identifier)["strategy_requested"])
self.assertTrue(worker.running)
def test_start_accepts_saved_computed_result_but_does_not_settle_inline(self):
identifier, worker, _, _ = self.interrupted_clock()
self.manager.submit(identifier, "stop")
before = len(fixtures.FakeClient.calls)
self.job(identifier, self.manager.submit(identifier, "start"))
self.assertTrue(worker.running)
self.assertFalse(worker.engine.auto)
self.assertEqual(len(fixtures.FakeClient.calls), before)
self.assertEqual(self.manager.status(identifier)["strategy_state"], "recovering")
def test_computed_result_waits_for_deadline_and_manual_refresh_stays_read_only(self):
identifier, worker, runtime, _ = self.interrupted_clock()
journal = worker.engine.battle_journal
record = journal.pending(worker.engine.snapshot)
journal.update(record, record["phase"], battle_ready_at=time.time() + 3600)
worker.engine.last_pending_check = time.monotonic() - 121
worker.schedule.next_poll = math.inf
before = len(fixtures.FakeClient.calls)
self.tick(identifier, worker)
self.job(identifier, self.manager.submit(identifier, "refresh"))
self.assertTrue(all(path == "masterService/masterQueryDetail" for path in self.paths(before)))
self.assertIsNotNone(journal.pending(worker.engine.snapshot))
self.assertEqual(runtime.run.call_count, 1)
self.assertTrue(worker.running)
self.assertFalse(worker.engine.auto)
self.assertEqual(self.manager.status(identifier)["strategy_state"], "recovering")
def test_lost_resumed_settlement_reply_becomes_query_only_without_replay(self):
identifier, worker, runtime, _ = self.interrupted_clock()
self.ready(worker)
post = worker.engine.client.http.post
attempted = []
def lost_reply(url, *args, **kwargs):
if url.endswith("taskService/taskCompleteStep"):
attempted.append(url)
raise requests.ReadTimeout("offline lost settlement reply")
return post(url, *args, **kwargs)
with patch.object(worker.engine.client.http, "post", side_effect=lost_reply):
self.job(identifier, worker.enqueue("__tick"), status="failed")
record = worker.engine.battle_journal.pending(worker.engine.snapshot)
self.assertTrue(record["settlement_dispatched"])
self.assertFalse(worker.engine.battles.resumable_task_clock(record))
for _ in range(3):
self.now += 121
worker.engine.last_heartbeat = self.now
worker.engine.last_pending_check = time.monotonic() - 121
self.tick(identifier, worker)
self.assertEqual(len(attempted), 1)
self.assertEqual(runtime.run.call_count, 1)
self.assertEqual(self.paths().count("battleService/battleStart"), 1)
self.assertFalse(worker.running)
self.assertTrue(worker.strategy_requested)
def test_manual_stop_keeps_computed_result_unsent_through_config_and_restart(self):
identifier, worker, _, _ = self.interrupted_clock()
self.manager.submit(identifier, "stop")
self.ready(worker)
journal_path = worker.engine.battle_journal.path
evidence = journal_path.read_bytes()
config = self.manager.get_config(identifier)
config["config"]["policy"]["keep_money"] += 1
self.job(identifier, self.manager.set_config(identifier, config["config"], config["revision"])["job"])
before = len(fixtures.FakeClient.calls)
self.tick(identifier, worker)
self.assertEqual(journal_path.read_bytes(), evidence)
self.assertTrue(all(path == "masterService/masterQueryDetail" for path in self.paths(before)))
self.assertTrue(self.manager.status(identifier)["manual_stop"])
self.manager.close()
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
self.idle(identifier)
self.assertTrue(self.manager.status(identifier)["manual_stop"])
self.assertEqual(journal_path.read_bytes(), evidence)
def test_stop_racing_authorized_reconcile_cancels_settlement_before_dispatch(self):
identifier, worker, _, _ = self.interrupted_clock()
self.ready(worker)
entered, release = threading.Event(), threading.Event()
poll = worker.engine.poll
journal_path = worker.engine.battle_journal.path
evidence = journal_path.read_bytes()
def held_poll(**kwargs):
self.assertFalse(worker.engine.auto)
self.assertIs(kwargs.get("resume_pending"), True)
entered.set()
self.assertTrue(release.wait(3))
return poll(**kwargs)
before = len(fixtures.FakeClient.calls)
with patch.object(worker.engine, "poll", side_effect=held_poll):
tick = worker.enqueue("__tick")
try:
self.assertTrue(entered.wait(3), self.manager.job(identifier, tick["id"]))
self.manager.submit(identifier, "stop")
finally:
release.set()
self.job(identifier, tick, status="cancelled")
self.assertEqual(len(fixtures.FakeClient.calls), before)
self.assertEqual(journal_path.read_bytes(), evidence)
self.assertFalse(worker.running)
self.assertTrue(self.manager.status(identifier)["manual_stop"])
def test_runtime_recovery_timeout_stays_blocked_after_single_local_retry(self):
identifier, worker = self.connected()
runtime = self.battle(identifier, worker, timed_out=True)
self.tick(identifier, worker)
record = worker.engine.battle_journal.pending(worker.engine.snapshot)
self.assertEqual(record["execution_status"], "timed_out")
self.assertFalse(worker.engine.battles.resumable_task_clock(record))
self.assertNotIn("battle_data", record)
self.assertTrue(worker.engine.pending_battle_recovery()["resumable"])
self.job(identifier, self.manager.submit(identifier, "start"))
worker.engine.last_pending_check = time.monotonic() - 121
worker.schedule.next_poll = math.inf
self.tick(identifier, worker)
record = worker.engine.battle_journal.pending(worker.engine.snapshot)
self.assertEqual(record["runtime_recovery_attempts"], 1)
self.assertFalse(worker.engine.pending_battle_recovery()["resumable"])
evidence = worker.engine.battle_journal.path.read_bytes()
before = len(fixtures.FakeClient.calls)
self.job(identifier, self.manager.submit(identifier, "start"), status="failed")
for _ in range(3):
self.now += 121
worker.engine.last_heartbeat = self.now
worker.engine.last_pending_check = time.monotonic() - 121
self.tick(identifier, worker)
self.assertTrue(all(path == "masterService/masterQueryDetail" for path in self.paths(before)))
self.assertEqual(worker.engine.battle_journal.path.read_bytes(), evidence)
self.assertEqual(runtime.run.call_count, 2)
self.assertEqual(self.paths().count("battleService/battleStart"), 1)
state = self.manager.status(identifier)
self.assertFalse(state["running"])
self.assertTrue(state["strategy_requested"])
self.assertFalse(state["manual_stop"])
self.assertIn("本地战斗计算超时", state["strategy_message"])
self.assertIn("尚未提交结算", state["strategy_message"])
def test_stage_farming_timeout_reports_its_operation_without_promising_resolution(self):
identifier, worker = self.connected()
fixtures.ServiceTests.enable(self, identifier)
self.job(identifier, self.manager.submit(identifier, "start"))
journal = worker.engine.battle_journal
record = journal.begin(worker.engine.snapshot, {"kind": "stage_battle", "stage_id": "0102",
"difficulty": 1, "path": "stageService/stageStart", "task_id": 0})
journal.update(record, "unknown", start_response={"resCode": 0, "triggerEvent": False, "stageEvent": None},
execution_status="timed_out", terminal_reason="worker_wall_clock_limit",
message="战斗已取消、超时或结果不确定,等待核对")
self.tick(identifier, worker)
state = self.manager.status(identifier)
self.assertTrue(state["connected"])
self.assertFalse(state["running"])
self.assertIn("0102", state["strategy_message"])
self.assertIn("本地战斗计算超时", state["strategy_message"])
worker.schedule.monitor(False)
reason = worker._pending_execution_reason()
self.assertNotIn("后台继续", reason)
self.assertNotIn("自动恢复", reason)
config = self.manager.get_config(identifier)
config["config"]["policy"]["keep_money"] += 1
self.job(identifier, self.manager.set_config(identifier, config["config"], config["revision"])["job"])
self.assertEqual(worker.config["policy"]["keep_money"], config["config"]["policy"]["keep_money"])
self.assertTrue(worker.strategy_requested)
intent = json.loads((worker.profile_root / "run_intent.json").read_text(encoding="utf-8"))
self.assertTrue(intent["roles"][worker.active_identity])
def test_config_cancellation_before_start_dispatch_replans_without_unknown_barrier(self):
identifier, worker = self.connected()
runtime = self.battle(identifier, worker, timed_out=True)
update = worker.engine.battle_journal.update
changed = []
def cancel_before_send(record, phase, **fields):
update(record, phase, **fields)
if phase == "start_pending":
config = self.manager.get_config(identifier)
config["config"]["policy"]["keep_money"] += 1
changed.append(self.manager.set_config(identifier, config["config"], config["revision"])["job"])
with patch.object(worker.engine.battle_journal, "update", side_effect=cancel_before_send):
self.tick(identifier, worker)
self.job(identifier, changed[0])
self.assertNotIn("battleService/battleStart", self.paths())
self.assertFalse(worker.engine._progression_pending())
self.assertIsNone(worker.engine.battle_journal.pending(worker.engine.snapshot))
self.assertIsNone(worker.engine.progression_planner.pending(worker.engine.snapshot))
runtime.run.assert_not_called()
self.assertTrue(worker.running)
self.assertTrue(worker.strategy_requested)
worker.schedule.next_poll = 0
self.tick(identifier, worker)
self.assertEqual(self.paths().count("battleService/battleStart"), 1)
def test_internal_pending_pause_does_not_revoke_generation_or_persist_manual_stop(self):
identifier, worker = self.connected()
worker.schedule.next_poll = 0
generation = worker._generation
def paused(**kwargs):
journal = worker.engine.battle_journal
record = journal.begin(worker.engine.snapshot, {"kind": "stage_battle", "stage_id": "0102"})
journal.update(record, "unknown", message="请求结果未知")
worker.engine.auto = False
return worker.engine.snapshot, [], "请求结果未知"
with patch.object(worker.engine, "poll", side_effect=paused):
self.tick(identifier, worker)
self.assertEqual(worker._generation, generation)
self.assertTrue(worker.strategy_requested)
self.assertFalse(self.manager.status(identifier)["manual_stop"])
if __name__ == "__main__":
unittest.main()