fix: preserve strategy intent and recover accounts automatically
This commit is contained in:
@@ -0,0 +1,346 @@
|
||||
"""Durable user intent and recovery, using temporary stores and offline clients."""
|
||||
from copy import deepcopy
|
||||
import json
|
||||
import math
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import requests
|
||||
|
||||
from ranch_assistant.api import ApiError, SessionExpiredError, encode
|
||||
from ranch_web.policies import export_config
|
||||
from ranch_web.service import WebManager
|
||||
from ranch_web.tests import test_service as fixtures
|
||||
from ranch_web.tests.test_auto_relogin import AutoLogin
|
||||
|
||||
|
||||
class RunIntentTests(unittest.TestCase):
|
||||
setUp = fixtures.ServiceTests.setUp
|
||||
account = fixtures.ServiceTests.account
|
||||
idle = fixtures.ServiceTests.idle
|
||||
job = fixtures.ServiceTests.job
|
||||
login = fixtures.ServiceTests.login
|
||||
enable = fixtures.ServiceTests.enable
|
||||
|
||||
def connected(self, *, keep_login=False):
|
||||
self.manager.login_factory = AutoLogin
|
||||
identifier = self.account(False)
|
||||
self.login(identifier, account="intent-owner-" + identifier[:8])
|
||||
worker = self.manager._worker(identifier)
|
||||
self.now = 1000.0
|
||||
worker.schedule.clock = lambda: self.now
|
||||
worker.schedule.uniform = lambda low, high: high
|
||||
worker.engine.last_heartbeat = self.now
|
||||
value = self.manager.get_config(identifier)["config"]
|
||||
value["auto_relogin"] = {"enabled": keep_login, "delay_minutes": 10}
|
||||
self.job(identifier, self.manager.set_config(identifier, value)["job"])
|
||||
worker.schedule.next_poll = math.inf
|
||||
return identifier, worker
|
||||
|
||||
def tick(self, identifier, worker, *, status="succeeded"):
|
||||
return self.job(identifier, worker.enqueue("__tick"), status=status)
|
||||
|
||||
def fail_read(self, identifier, worker, error=None):
|
||||
error = error or requests.ReadTimeout("offline synthetic timeout")
|
||||
with patch.object(worker.engine.client.http, "post", side_effect=error) as post:
|
||||
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
|
||||
self.assertEqual(post.call_count, 1)
|
||||
|
||||
def restart(self, identifier):
|
||||
self.manager.close()
|
||||
self.manager = WebManager(self.root, start_scheduler=False, login_factory=AutoLogin)
|
||||
self.addCleanup(self.manager.close)
|
||||
self.idle(identifier)
|
||||
worker = self.manager._worker(identifier)
|
||||
origin = worker.schedule.disconnected_at
|
||||
self.now = origin if origin is not None else self.now
|
||||
worker.schedule.clock = lambda: self.now
|
||||
return worker
|
||||
|
||||
def pending(self, worker):
|
||||
journal = worker.engine.battle_journal
|
||||
record = journal.begin(worker.engine.snapshot, {"kind": "battle", "description": "unknown synthetic battle",
|
||||
"path": "stageService/stageStart", "payload": {}, "stage_id": "0101", "difficulty": "1"})
|
||||
journal.update(record, "unknown", message="请求结果未知,禁止自动重发")
|
||||
return journal, journal.path.read_bytes()
|
||||
|
||||
def test_read_timeout_recovers_requested_execution_after_read_only_probe(self):
|
||||
identifier, worker = self.connected()
|
||||
self.assertTrue(worker.running)
|
||||
self.fail_read(identifier, worker)
|
||||
self.assertFalse(worker.running)
|
||||
before = len(fixtures.FakeClient.calls)
|
||||
self.now += 4
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(len(fixtures.FakeClient.calls), before)
|
||||
self.now += 1
|
||||
self.tick(identifier, worker)
|
||||
self.assertTrue(worker.running, "Successful retry restored online but permanently lost execution")
|
||||
self.assertTrue(self.manager.status(identifier)["strategy_requested"])
|
||||
self.assertFalse(worker.engine.auto, "Reconnect verification enabled writes in its own job")
|
||||
self.assertEqual([url.rsplit("/", 2)[-2:] for _, url, _ in fixtures.FakeClient.calls[before:]],
|
||||
[["masterService", "masterQueryDetail"]])
|
||||
worker.engine.last_heartbeat = self.now
|
||||
worker.schedule.next_poll = 0
|
||||
with patch.object(worker.engine, "master_sync_due", return_value=False), \
|
||||
patch.object(worker.engine, "growth_sync_due", return_value=False), \
|
||||
patch.object(worker.engine, "poll", wraps=worker.engine.poll) as poll:
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(poll.call_count, 1)
|
||||
self.assertTrue(worker.running)
|
||||
|
||||
def test_only_explicit_stop_persists_and_start_clears_it(self):
|
||||
identifier, worker = self.connected()
|
||||
identity = worker.active_identity
|
||||
worker.stop() # Desktop adapter / cancellation is not the Stop button.
|
||||
self.tick(identifier, worker)
|
||||
self.assertTrue(worker.running)
|
||||
self.manager.submit(identifier, "stop")
|
||||
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": identity}))
|
||||
self.assertFalse(worker.running, "Activation erased the user's explicit Stop")
|
||||
worker = self.restart(identifier)
|
||||
self.assertFalse(self.manager.status(identifier)["strategy_requested"])
|
||||
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": identity}))
|
||||
self.assertFalse(worker.running)
|
||||
self.enable(identifier)
|
||||
self.job(identifier, self.manager.submit(identifier, "start"))
|
||||
self.assertTrue(worker.running)
|
||||
self.assertFalse(self.manager.status(identifier)["manual_stop"])
|
||||
worker = self.restart(identifier)
|
||||
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": identity}))
|
||||
self.assertTrue(worker.running)
|
||||
|
||||
def test_stop_does_not_cancel_offline_password_reconnect(self):
|
||||
identifier, worker = self.connected(keep_login=True)
|
||||
self.fail_read(identifier, worker, SessionExpiredError("offline expired", code=408))
|
||||
self.manager.submit(identifier, "stop")
|
||||
before = len(AutoLogin.calls)
|
||||
self.now += 599
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(len(AutoLogin.calls), before)
|
||||
self.now += 1
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(len(AutoLogin.calls), before + 1)
|
||||
self.assertTrue(self.manager.status(identifier)["connected"])
|
||||
self.assertFalse(worker.running)
|
||||
self.assertFalse(self.manager.status(identifier)["strategy_requested"])
|
||||
|
||||
def test_logout_keeps_identity_credentials_and_reconnects_even_legacy_disabled(self):
|
||||
identifier, worker = self.connected()
|
||||
identity = worker.active_identity
|
||||
self.job(identifier, self.manager.submit(identifier, "logout"))
|
||||
self.assertEqual(worker.active_identity, identity)
|
||||
self.assertEqual(worker.securestore.active(), identity)
|
||||
self.assertTrue(worker.securestore.load_credentials()[0])
|
||||
before = len(AutoLogin.calls)
|
||||
self.now += 600
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(len(AutoLogin.calls), before + 1)
|
||||
self.assertTrue(worker.running)
|
||||
|
||||
def test_startup_migrates_saved_identity_and_reconnects_without_button(self):
|
||||
identifier, worker = self.connected()
|
||||
identity = worker.active_identity
|
||||
# Old installations have no intent file and could save enabled=False.
|
||||
local = self.manager.store.load_config(worker._login_settings_root)
|
||||
local["config"]["auto_relogin"]["enabled"] = False
|
||||
self.manager.store.save_config(worker._login_settings_root, local["config"], local["revision"])
|
||||
worker = self.restart(identifier)
|
||||
state = self.manager.status(identifier)
|
||||
self.assertTrue(state["strategy_requested"])
|
||||
self.assertTrue(worker.config["auto_relogin"]["enabled"])
|
||||
self.assertIsNotNone(state["auto_relogin_status"]["next_attempt_at"])
|
||||
self.now += 600
|
||||
self.tick(identifier, worker)
|
||||
self.assertTrue(worker.running)
|
||||
self.assertEqual(worker.active_identity, identity)
|
||||
|
||||
def test_pending_barrier_keeps_requested_intent_and_resumes_after_resolution(self):
|
||||
identifier, worker = self.connected()
|
||||
journal, evidence = self.pending(worker)
|
||||
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}))
|
||||
state = self.manager.status(identifier)
|
||||
self.assertFalse(state["running"])
|
||||
self.assertTrue(state["strategy_requested"])
|
||||
self.assertEqual(state["strategy_state"], "blocked")
|
||||
self.assertIn("核对", state["strategy_message"])
|
||||
self.assertEqual(journal.path.read_bytes(), evidence)
|
||||
self.manager.submit(identifier, "stop")
|
||||
self.job(identifier, self.manager.submit(identifier, "start"), status="failed")
|
||||
self.assertTrue(self.manager.status(identifier)["strategy_requested"], "Start must clear Stop even while blocked")
|
||||
self.assertEqual(journal.path.read_bytes(), evidence)
|
||||
with patch.object(worker.engine, "_progression_pending", return_value=None):
|
||||
self.tick(identifier, worker)
|
||||
self.assertTrue(worker.running)
|
||||
|
||||
def test_intent_is_separate_from_portable_policy_and_scoped_to_role(self):
|
||||
identifier, worker = self.connected()
|
||||
first = worker.active_identity
|
||||
config = deepcopy(worker.config)
|
||||
self.manager.submit(identifier, "stop")
|
||||
self.assertEqual(worker.config, config)
|
||||
intent_path = worker.profile_root / "run_intent.json"
|
||||
self.assertTrue(intent_path.is_file())
|
||||
portable = json.dumps(export_config(worker.config))
|
||||
for value in ("strategy_requested", "manual_stop", "intent-owner", first):
|
||||
self.assertNotIn(value, portable)
|
||||
self.login(identifier, account=worker._saved_account, server=2)
|
||||
self.assertTrue(worker.running, "A different role inherited the first role's Stop")
|
||||
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": first}))
|
||||
self.assertFalse(worker.running)
|
||||
|
||||
def test_manual_refresh_does_not_enable_auto_inside_snapshot_callback(self):
|
||||
identifier, worker = self.connected()
|
||||
self.fail_read(identifier, worker)
|
||||
self.now += 5
|
||||
self.tick(identifier, worker)
|
||||
seen = []
|
||||
|
||||
def read_only(*args, **kwargs):
|
||||
worker.engine.read_snapshot(sync_master=False)
|
||||
seen.append(worker.engine.auto)
|
||||
return worker.engine.snapshot, [], "synthetic preview"
|
||||
|
||||
with patch.object(worker.engine, "poll", side_effect=read_only):
|
||||
self.job(identifier, self.manager.submit(identifier, "refresh"))
|
||||
self.assertEqual(seen, [False])
|
||||
self.assertTrue(worker.running)
|
||||
|
||||
def test_stop_racing_retry_callback_wins_without_disabling_keep_login(self):
|
||||
identifier, worker = self.connected()
|
||||
self.fail_read(identifier, worker)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
self.addCleanup(release.set)
|
||||
original = worker.engine.client.http.post
|
||||
|
||||
def held(*args, **kwargs):
|
||||
entered.set()
|
||||
if not release.wait(5):
|
||||
raise AssertionError("offline retry gate not released")
|
||||
return original(*args, **kwargs)
|
||||
|
||||
self.now += 5
|
||||
with patch.object(worker.engine.client.http, "post", side_effect=held):
|
||||
job = worker.enqueue("__tick")
|
||||
self.assertTrue(entered.wait(5))
|
||||
self.manager.submit(identifier, "stop")
|
||||
release.set()
|
||||
self.job(identifier, job)
|
||||
self.assertFalse(worker.running)
|
||||
self.assertFalse(worker.engine.auto)
|
||||
self.assertFalse(self.manager.status(identifier)["strategy_requested"])
|
||||
|
||||
def test_failed_candidate_login_rearms_existing_online_monitor(self):
|
||||
identifier, worker = self.connected()
|
||||
original = worker.engine.client
|
||||
with patch.object(AutoLogin, "authenticate", side_effect=ValueError("synthetic candidate failure")):
|
||||
self.job(identifier, self.manager.submit(identifier, "login", {
|
||||
"account": worker._saved_account, "password": "synthetic-password"}), status="failed")
|
||||
self.assertIs(worker.engine.client, original)
|
||||
self.assertTrue(worker.schedule.monitoring)
|
||||
self.assertTrue(self.manager.status(identifier)["strategy_requested"])
|
||||
self.now += 5
|
||||
with patch.object(worker.engine, "master_sync_due", return_value=False), \
|
||||
patch.object(worker.engine, "growth_sync_due", return_value=False), \
|
||||
patch.object(worker.engine, "poll", wraps=worker.engine.poll) as poll:
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(poll.call_count, 1)
|
||||
self.assertTrue(worker.running)
|
||||
|
||||
def test_transient_local_planner_error_has_bounded_backoff_without_stop_intent(self):
|
||||
identifier, worker = self.connected()
|
||||
worker.schedule.next_poll = 0
|
||||
with patch.object(worker.engine, "master_sync_due", return_value=False), \
|
||||
patch.object(worker.engine, "growth_sync_due", return_value=False), \
|
||||
patch.object(worker.engine, "poll", side_effect=ValueError("synthetic planner unavailable")) as poll:
|
||||
self.tick(identifier, worker, status="failed")
|
||||
state = self.manager.status(identifier)
|
||||
self.assertTrue(state["strategy_requested"])
|
||||
self.assertEqual(state["strategy_state"], "recovering")
|
||||
self.assertIsNotNone(state["strategy_retry_at"])
|
||||
self.assertEqual(worker._execution_retry_deadline, self.now + 5)
|
||||
for _ in range(3):
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(poll.call_count, 1)
|
||||
self.now += 60 # Scheduler also applies its per-kind failure cooldown.
|
||||
with patch.object(worker.engine, "master_sync_due", return_value=False), \
|
||||
patch.object(worker.engine, "growth_sync_due", return_value=False):
|
||||
self.tick(identifier, worker)
|
||||
self.assertTrue(worker.running)
|
||||
self.assertIsNone(self.manager.status(identifier)["strategy_retry_at"])
|
||||
|
||||
def test_decoded_read_business_rejection_gets_execution_backoff(self):
|
||||
identifier, worker = self.connected()
|
||||
worker.schedule.next_poll = 0
|
||||
with patch.object(worker.engine.client.http, "post", side_effect=ApiError("synthetic rejection", code=403)) as post:
|
||||
self.tick(identifier, worker, status="failed")
|
||||
self.assertEqual(post.call_count, 1)
|
||||
self.assertFalse(worker.running)
|
||||
state = self.manager.status(identifier)
|
||||
self.assertTrue(state["strategy_requested"])
|
||||
self.assertEqual(state["strategy_state"], "recovering")
|
||||
self.assertIsNotNone(state["strategy_retry_at"])
|
||||
self.assertEqual(state["connection_state"], "online")
|
||||
|
||||
def test_uncertain_journaled_write_is_never_replayed_after_recovery(self):
|
||||
identifier, worker = self.connected()
|
||||
journal, evidence = self.pending(worker)
|
||||
self.fail_read(identifier, worker)
|
||||
before = len(fixtures.FakeClient.calls)
|
||||
self.now += 5
|
||||
self.tick(identifier, worker)
|
||||
self.assertFalse(worker.running)
|
||||
self.assertTrue(self.manager.status(identifier)["strategy_requested"])
|
||||
self.assertEqual(self.manager.status(identifier)["strategy_state"], "blocked")
|
||||
self.assertEqual(journal.path.read_bytes(), evidence)
|
||||
for _ in range(3):
|
||||
self.now += 600
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(journal.path.read_bytes(), evidence)
|
||||
self.assertTrue(all(url.endswith(("masterService/masterQueryDetail", "heartBeatService/keepAlive"))
|
||||
for _, url, _ in fixtures.FakeClient.calls[before:]))
|
||||
|
||||
def test_legacy_gift_unchanged_mixed_timestamp_units_stay_blocked_until_confirmed(self):
|
||||
identifier, worker = self.connected()
|
||||
mid = next(iter(worker.engine.snapshot.monsters))
|
||||
original_time = 1_000_000_000
|
||||
worker.engine.snapshot.monsters[mid]["giftTime"] = original_time
|
||||
raw = next(mon for mon in worker.engine.client.raw["monsters"] if mon["id"] == mid)
|
||||
raw["giftTime"] = original_time * 1000
|
||||
|
||||
def gift(*args):
|
||||
worker.engine.client.call("monsterService/monsterSendGift", {"monsterId": mid, "itemId": 1})
|
||||
return {"message": "synthetic response lacks gift confirmation"}
|
||||
|
||||
with patch("ranch_web.actions.execute_action", side_effect=gift):
|
||||
self.job(identifier, self.manager.submit(identifier, "consume_probe"))
|
||||
self.job(identifier, self.manager.submit(identifier, "refresh"))
|
||||
self.assertEqual(self.manager.status(identifier)["strategy_state"], "blocked",
|
||||
"Unchanged milliseconds were mistaken for a later gift than cached seconds")
|
||||
raw["giftTime"] = (original_time + 60) * 1000
|
||||
self.job(identifier, self.manager.submit(identifier, "refresh"))
|
||||
self.assertTrue(worker.running)
|
||||
self.assertTrue(self.manager.status(identifier)["strategy_requested"])
|
||||
|
||||
def test_valid_gift_response_confirms_cooldown_without_waiting_for_role_read(self):
|
||||
identifier, worker = self.connected()
|
||||
mid = next(iter(worker.engine.snapshot.monsters))
|
||||
worker.engine.snapshot.monsters[mid]["giftTime"] = 1_000_000_000
|
||||
response = SimpleNamespace(status_code=200, headers={}, content=encode(
|
||||
{"resCode": 0, "feel": 20, "giftDate": 1_000_000_060_000}, worker.engine.client.session.key))
|
||||
|
||||
def gift(*args):
|
||||
worker.engine.client.call("monsterService/monsterSendGift", {"monsterId": mid, "itemId": 1})
|
||||
return {"message": "synthetic confirmed gift"}
|
||||
|
||||
with patch("ranch_web.actions.execute_action", side_effect=gift), \
|
||||
patch.object(worker.engine.client.http, "post", return_value=response):
|
||||
self.job(identifier, self.manager.submit(identifier, "consume_probe"))
|
||||
self.assertTrue(worker.running)
|
||||
self.assertNotIn(worker.active_identity, worker._legacy_pending)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user