346 lines
18 KiB
Python
346 lines
18 KiB
Python
"""Web recovery through real Engine barriers, temporary stores and offline HTTP."""
|
|
from copy import deepcopy
|
|
import json
|
|
import math
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import requests
|
|
|
|
from ranch_assistant.game_data import ProgressionData
|
|
from ranch_assistant.model import Snapshot
|
|
from ranch_assistant.tests.test_battle_orchestration import ExperimentalRuntime
|
|
from ranch_web.service import WebManager
|
|
from ranch_web.tests import test_service as fixtures
|
|
|
|
|
|
class LifecycleRecoveryTests(unittest.TestCase):
|
|
setUp = fixtures.ServiceTests.setUp
|
|
account = fixtures.ServiceTests.account
|
|
idle = fixtures.ServiceTests.idle
|
|
job = fixtures.ServiceTests.job
|
|
login = fixtures.ServiceTests.login
|
|
|
|
def connected(self):
|
|
identifier = self.account(False)
|
|
self.login(identifier)
|
|
worker = self.manager._worker(identifier)
|
|
self.now = 1000.0
|
|
worker.schedule.clock = lambda: self.now
|
|
worker.schedule.uniform = lambda low, high: high
|
|
worker.engine.last_heartbeat = self.now
|
|
self.enterContext(patch.object(worker.engine, "master_sync_due", return_value=False))
|
|
self.enterContext(patch.object(worker.engine, "growth_sync_due", return_value=False))
|
|
return identifier, worker
|
|
|
|
def battle(self, identifier, worker, *, timed_out=False):
|
|
"""Small task fixture already used by Engine's orchestration regressions."""
|
|
engine = worker.engine
|
|
raw = engine.client.raw
|
|
raw["master"].update(ti=1000, money=1000, timeTi=0)
|
|
raw["currentTasks"] = [{"id": 51, "typeId": "100", "curStep": 1, "stepValue": 0}]
|
|
raw["history"] = {"unlockCity": {"1": True}, "unlockStage": {"0101": [1, 2, 3]}}
|
|
raw["teams"] = [{"teamName": "teamA", "teamType": "battleTeam", "monsters": [raw["monsters"][0]["id"]]}]
|
|
raw["taskStepValue"] = 0
|
|
engine.snapshot = Snapshot(deepcopy(raw))
|
|
engine.game_data = ProgressionData({
|
|
"task_info": {"100": {"type": 1, "name": "离线任务", "steps": [
|
|
{"statusType": "passStage", "statusValue": {"stage": "0101", "level": 1}}]}},
|
|
"game_setting": {"cityInfo": {"1": {"name": "离线城市"}}},
|
|
"stage_info": {"0101": {"city": 1, "cost": {"ti": 6}, "level": {
|
|
str(n): {"cost": 0, "battle": {"fixture": True}} for n in (1, 2, 3)}}}})
|
|
runtime = ExperimentalRuntime()
|
|
original = runtime._run
|
|
if timed_out:
|
|
runtime.run.return_value = {"outcome": "timeout", "execution_mode": "experimental",
|
|
"execution_status": "timed_out", "terminal_reason": "worker_wall_clock_limit"}
|
|
runtime.run.side_effect = None
|
|
else:
|
|
runtime.run.side_effect = lambda request, cancel_event: original(request, cancel_event=cancel_event) | {
|
|
"requires_realtime_wait": True, "model_elapsed_ms": 10000}
|
|
engine._battle_runtimes["experimental"] = runtime
|
|
engine.battles.request_builder = None
|
|
config = self.manager.get_config(identifier)
|
|
config["config"]["progression_policy"].update(enabled=True, battle_mode="experimental", reserve_ti=0)
|
|
self.job(identifier, self.manager.set_config(identifier, config["config"], config["revision"])["job"])
|
|
worker.schedule.next_poll = 0
|
|
return runtime
|
|
|
|
def tick(self, identifier, worker):
|
|
return self.job(identifier, worker.enqueue("__tick"))
|
|
|
|
def paths(self, start=0):
|
|
return ["/".join(url.rsplit("/", 2)[-2:]) for _, url, _ in fixtures.FakeClient.calls[start:]]
|
|
|
|
def interrupted_clock(self):
|
|
identifier, worker = self.connected()
|
|
runtime = self.battle(identifier, worker)
|
|
entered = threading.Event()
|
|
original = worker.engine.battles.wait_for_battle_clock
|
|
|
|
def computed_wait(record, request, result, data, started, cancel):
|
|
wait = cancel.wait
|
|
|
|
def clock_wait(timeout):
|
|
entered.set()
|
|
return wait(timeout)
|
|
|
|
with patch.object(cancel, "wait", side_effect=clock_wait):
|
|
return original(record, request, result, data, started, cancel)
|
|
|
|
with patch.object(worker.engine.battles, "wait_for_battle_clock", side_effect=computed_wait):
|
|
tick = worker.enqueue("__tick")
|
|
self.assertTrue(entered.wait(3), "Offline battle never entered the computed clock wait")
|
|
config = self.manager.get_config(identifier)
|
|
config["config"]["policy"]["keep_money"] += 1
|
|
changed = self.manager.set_config(identifier, config["config"], config["revision"])
|
|
self.job(identifier, tick)
|
|
self.job(identifier, changed["job"])
|
|
record = worker.engine.battle_journal.pending(worker.engine.snapshot)
|
|
self.assertTrue(worker.engine.battles.resumable_task_clock(record), record)
|
|
self.assertTrue(worker.engine._progression_cancel.is_set())
|
|
return identifier, worker, runtime, record
|
|
|
|
def ready(self, worker):
|
|
journal = worker.engine.battle_journal
|
|
record = journal.pending(worker.engine.snapshot)
|
|
journal.update(record, record["phase"], battle_ready_at=1)
|
|
worker.engine.last_pending_check = time.monotonic() - 121
|
|
worker.schedule.next_poll = math.inf
|
|
|
|
def test_config_cancelled_computed_clock_resumes_on_reconcile_without_new_start(self):
|
|
identifier, worker, runtime, record = self.interrupted_clock()
|
|
saved = deepcopy(record["battle_data"])
|
|
self.ready(worker)
|
|
before = len(fixtures.FakeClient.calls)
|
|
self.assertEqual(self.tick(identifier, worker)["result"]["kind"], "reconcile")
|
|
self.assertIsNone(worker.engine.battle_journal.pending(worker.engine.snapshot))
|
|
writes = [path for path in self.paths(before) if path != "masterService/masterQueryDetail"]
|
|
self.assertEqual(writes, ["taskService/taskCompleteStep"])
|
|
payload = next(payload for _, url, payload in fixtures.FakeClient.calls[before:]
|
|
if url.endswith("taskService/taskCompleteStep"))
|
|
self.assertEqual(payload["battleData"], saved)
|
|
self.assertEqual(runtime.run.call_count, 1)
|
|
self.assertEqual(self.paths().count("battleService/battleStart"), 1)
|
|
self.assertTrue(self.manager.status(identifier)["strategy_requested"])
|
|
self.assertTrue(worker.running)
|
|
|
|
def test_start_accepts_saved_computed_result_but_does_not_settle_inline(self):
|
|
identifier, worker, _, _ = self.interrupted_clock()
|
|
self.manager.submit(identifier, "stop")
|
|
before = len(fixtures.FakeClient.calls)
|
|
self.job(identifier, self.manager.submit(identifier, "start"))
|
|
self.assertTrue(worker.running)
|
|
self.assertFalse(worker.engine.auto)
|
|
self.assertEqual(len(fixtures.FakeClient.calls), before)
|
|
self.assertEqual(self.manager.status(identifier)["strategy_state"], "recovering")
|
|
|
|
def test_computed_result_waits_for_deadline_and_manual_refresh_stays_read_only(self):
|
|
identifier, worker, runtime, _ = self.interrupted_clock()
|
|
journal = worker.engine.battle_journal
|
|
record = journal.pending(worker.engine.snapshot)
|
|
journal.update(record, record["phase"], battle_ready_at=time.time() + 3600)
|
|
worker.engine.last_pending_check = time.monotonic() - 121
|
|
worker.schedule.next_poll = math.inf
|
|
before = len(fixtures.FakeClient.calls)
|
|
self.tick(identifier, worker)
|
|
self.job(identifier, self.manager.submit(identifier, "refresh"))
|
|
self.assertTrue(all(path == "masterService/masterQueryDetail" for path in self.paths(before)))
|
|
self.assertIsNotNone(journal.pending(worker.engine.snapshot))
|
|
self.assertEqual(runtime.run.call_count, 1)
|
|
self.assertTrue(worker.running)
|
|
self.assertFalse(worker.engine.auto)
|
|
self.assertEqual(self.manager.status(identifier)["strategy_state"], "recovering")
|
|
|
|
def test_lost_resumed_settlement_reply_becomes_query_only_without_replay(self):
|
|
identifier, worker, runtime, _ = self.interrupted_clock()
|
|
self.ready(worker)
|
|
post = worker.engine.client.http.post
|
|
attempted = []
|
|
|
|
def lost_reply(url, *args, **kwargs):
|
|
if url.endswith("taskService/taskCompleteStep"):
|
|
attempted.append(url)
|
|
raise requests.ReadTimeout("offline lost settlement reply")
|
|
return post(url, *args, **kwargs)
|
|
|
|
with patch.object(worker.engine.client.http, "post", side_effect=lost_reply):
|
|
self.job(identifier, worker.enqueue("__tick"), status="failed")
|
|
record = worker.engine.battle_journal.pending(worker.engine.snapshot)
|
|
self.assertTrue(record["settlement_dispatched"])
|
|
self.assertFalse(worker.engine.battles.resumable_task_clock(record))
|
|
for _ in range(3):
|
|
self.now += 121
|
|
worker.engine.last_heartbeat = self.now
|
|
worker.engine.last_pending_check = time.monotonic() - 121
|
|
self.tick(identifier, worker)
|
|
self.assertEqual(len(attempted), 1)
|
|
self.assertEqual(runtime.run.call_count, 1)
|
|
self.assertEqual(self.paths().count("battleService/battleStart"), 1)
|
|
self.assertFalse(worker.running)
|
|
self.assertTrue(worker.strategy_requested)
|
|
|
|
def test_manual_stop_keeps_computed_result_unsent_through_config_and_restart(self):
|
|
identifier, worker, _, _ = self.interrupted_clock()
|
|
self.manager.submit(identifier, "stop")
|
|
self.ready(worker)
|
|
journal_path = worker.engine.battle_journal.path
|
|
evidence = journal_path.read_bytes()
|
|
config = self.manager.get_config(identifier)
|
|
config["config"]["policy"]["keep_money"] += 1
|
|
self.job(identifier, self.manager.set_config(identifier, config["config"], config["revision"])["job"])
|
|
before = len(fixtures.FakeClient.calls)
|
|
self.tick(identifier, worker)
|
|
self.assertEqual(journal_path.read_bytes(), evidence)
|
|
self.assertTrue(all(path == "masterService/masterQueryDetail" for path in self.paths(before)))
|
|
self.assertTrue(self.manager.status(identifier)["manual_stop"])
|
|
self.manager.close()
|
|
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
|
|
self.addCleanup(self.manager.close)
|
|
self.idle(identifier)
|
|
self.assertTrue(self.manager.status(identifier)["manual_stop"])
|
|
self.assertEqual(journal_path.read_bytes(), evidence)
|
|
|
|
def test_stop_racing_authorized_reconcile_cancels_settlement_before_dispatch(self):
|
|
identifier, worker, _, _ = self.interrupted_clock()
|
|
self.ready(worker)
|
|
entered, release = threading.Event(), threading.Event()
|
|
poll = worker.engine.poll
|
|
journal_path = worker.engine.battle_journal.path
|
|
evidence = journal_path.read_bytes()
|
|
|
|
def held_poll(**kwargs):
|
|
self.assertFalse(worker.engine.auto)
|
|
self.assertIs(kwargs.get("resume_pending"), True)
|
|
entered.set()
|
|
self.assertTrue(release.wait(3))
|
|
return poll(**kwargs)
|
|
|
|
before = len(fixtures.FakeClient.calls)
|
|
with patch.object(worker.engine, "poll", side_effect=held_poll):
|
|
tick = worker.enqueue("__tick")
|
|
try:
|
|
self.assertTrue(entered.wait(3), self.manager.job(identifier, tick["id"]))
|
|
self.manager.submit(identifier, "stop")
|
|
finally:
|
|
release.set()
|
|
self.job(identifier, tick, status="cancelled")
|
|
self.assertEqual(len(fixtures.FakeClient.calls), before)
|
|
self.assertEqual(journal_path.read_bytes(), evidence)
|
|
self.assertFalse(worker.running)
|
|
self.assertTrue(self.manager.status(identifier)["manual_stop"])
|
|
|
|
def test_runtime_recovery_timeout_stays_blocked_after_single_local_retry(self):
|
|
identifier, worker = self.connected()
|
|
runtime = self.battle(identifier, worker, timed_out=True)
|
|
self.tick(identifier, worker)
|
|
record = worker.engine.battle_journal.pending(worker.engine.snapshot)
|
|
self.assertEqual(record["execution_status"], "timed_out")
|
|
self.assertFalse(worker.engine.battles.resumable_task_clock(record))
|
|
self.assertNotIn("battle_data", record)
|
|
self.assertTrue(worker.engine.pending_battle_recovery()["resumable"])
|
|
self.job(identifier, self.manager.submit(identifier, "start"))
|
|
worker.engine.last_pending_check = time.monotonic() - 121
|
|
worker.schedule.next_poll = math.inf
|
|
self.tick(identifier, worker)
|
|
record = worker.engine.battle_journal.pending(worker.engine.snapshot)
|
|
self.assertEqual(record["runtime_recovery_attempts"], 1)
|
|
self.assertFalse(worker.engine.pending_battle_recovery()["resumable"])
|
|
evidence = worker.engine.battle_journal.path.read_bytes()
|
|
before = len(fixtures.FakeClient.calls)
|
|
self.job(identifier, self.manager.submit(identifier, "start"), status="failed")
|
|
for _ in range(3):
|
|
self.now += 121
|
|
worker.engine.last_heartbeat = self.now
|
|
worker.engine.last_pending_check = time.monotonic() - 121
|
|
self.tick(identifier, worker)
|
|
self.assertTrue(all(path == "masterService/masterQueryDetail" for path in self.paths(before)))
|
|
self.assertEqual(worker.engine.battle_journal.path.read_bytes(), evidence)
|
|
self.assertEqual(runtime.run.call_count, 2)
|
|
self.assertEqual(self.paths().count("battleService/battleStart"), 1)
|
|
state = self.manager.status(identifier)
|
|
self.assertFalse(state["running"])
|
|
self.assertTrue(state["strategy_requested"])
|
|
self.assertFalse(state["manual_stop"])
|
|
self.assertIn("本地战斗计算超时", state["strategy_message"])
|
|
self.assertIn("尚未提交结算", state["strategy_message"])
|
|
|
|
def test_stage_farming_timeout_reports_its_operation_without_promising_resolution(self):
|
|
identifier, worker = self.connected()
|
|
fixtures.ServiceTests.enable(self, identifier)
|
|
self.job(identifier, self.manager.submit(identifier, "start"))
|
|
journal = worker.engine.battle_journal
|
|
record = journal.begin(worker.engine.snapshot, {"kind": "stage_battle", "stage_id": "0102",
|
|
"difficulty": 1, "path": "stageService/stageStart", "task_id": 0})
|
|
journal.update(record, "unknown", start_response={"resCode": 0, "triggerEvent": False, "stageEvent": None},
|
|
execution_status="timed_out", terminal_reason="worker_wall_clock_limit",
|
|
message="战斗已取消、超时或结果不确定,等待核对")
|
|
self.tick(identifier, worker)
|
|
state = self.manager.status(identifier)
|
|
self.assertTrue(state["connected"])
|
|
self.assertFalse(state["running"])
|
|
self.assertIn("0102", state["strategy_message"])
|
|
self.assertIn("本地战斗计算超时", state["strategy_message"])
|
|
worker.schedule.monitor(False)
|
|
reason = worker._pending_execution_reason()
|
|
self.assertNotIn("后台继续", reason)
|
|
self.assertNotIn("自动恢复", reason)
|
|
config = self.manager.get_config(identifier)
|
|
config["config"]["policy"]["keep_money"] += 1
|
|
self.job(identifier, self.manager.set_config(identifier, config["config"], config["revision"])["job"])
|
|
self.assertEqual(worker.config["policy"]["keep_money"], config["config"]["policy"]["keep_money"])
|
|
self.assertTrue(worker.strategy_requested)
|
|
intent = json.loads((worker.profile_root / "run_intent.json").read_text(encoding="utf-8"))
|
|
self.assertTrue(intent["roles"][worker.active_identity])
|
|
|
|
def test_config_cancellation_before_start_dispatch_replans_without_unknown_barrier(self):
|
|
identifier, worker = self.connected()
|
|
runtime = self.battle(identifier, worker, timed_out=True)
|
|
update = worker.engine.battle_journal.update
|
|
changed = []
|
|
|
|
def cancel_before_send(record, phase, **fields):
|
|
update(record, phase, **fields)
|
|
if phase == "start_pending":
|
|
config = self.manager.get_config(identifier)
|
|
config["config"]["policy"]["keep_money"] += 1
|
|
changed.append(self.manager.set_config(identifier, config["config"], config["revision"])["job"])
|
|
|
|
with patch.object(worker.engine.battle_journal, "update", side_effect=cancel_before_send):
|
|
self.tick(identifier, worker)
|
|
self.job(identifier, changed[0])
|
|
self.assertNotIn("battleService/battleStart", self.paths())
|
|
self.assertFalse(worker.engine._progression_pending())
|
|
self.assertIsNone(worker.engine.battle_journal.pending(worker.engine.snapshot))
|
|
self.assertIsNone(worker.engine.progression_planner.pending(worker.engine.snapshot))
|
|
runtime.run.assert_not_called()
|
|
self.assertTrue(worker.running)
|
|
self.assertTrue(worker.strategy_requested)
|
|
worker.schedule.next_poll = 0
|
|
self.tick(identifier, worker)
|
|
self.assertEqual(self.paths().count("battleService/battleStart"), 1)
|
|
|
|
def test_internal_pending_pause_does_not_revoke_generation_or_persist_manual_stop(self):
|
|
identifier, worker = self.connected()
|
|
worker.schedule.next_poll = 0
|
|
generation = worker._generation
|
|
|
|
def paused(**kwargs):
|
|
journal = worker.engine.battle_journal
|
|
record = journal.begin(worker.engine.snapshot, {"kind": "stage_battle", "stage_id": "0102"})
|
|
journal.update(record, "unknown", message="请求结果未知")
|
|
worker.engine.auto = False
|
|
return worker.engine.snapshot, [], "请求结果未知"
|
|
|
|
with patch.object(worker.engine, "poll", side_effect=paused):
|
|
self.tick(identifier, worker)
|
|
self.assertEqual(worker._generation, generation)
|
|
self.assertTrue(worker.strategy_requested)
|
|
self.assertFalse(self.manager.status(identifier)["manual_stop"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|