Files
myrzx_bot/ranch_web/tests/test_connection_state.py
T

443 lines
25 KiB
Python

"""Connection regressions through real workers and an entirely offline transport."""
from copy import deepcopy
from types import SimpleNamespace
import threading
import unittest
from unittest.mock import patch
import requests
from ranch_assistant.api import ApiError, encode
from ranch_web.activity import ActivityLogError
from ranch_web.service import WebManager
from ranch_web.scheduler import Scheduler
from ranch_web.tests import test_service as fixtures
class ConnectionStateTests(unittest.TestCase):
setUp = fixtures.ServiceTests.setUp
account = fixtures.ServiceTests.account
idle = fixtures.ServiceTests.idle
job = fixtures.ServiceTests.job
login = fixtures.ServiceTests.login
enable = fixtures.ServiceTests.enable
def connected(self, *, running=False):
identifier = self.account(False)
self.login(identifier)
worker = self.manager._worker(identifier)
self.now = 1000.0
worker.schedule.clock = lambda: self.now
if running:
self.enable(identifier)
self.job(identifier, self.manager.submit(identifier, "start"))
return identifier, worker
def tick(self, identifier, worker, *, status="succeeded"):
return self.job(identifier, worker.enqueue("__tick"), status=status)
def expiry(self, worker):
return SimpleNamespace(status_code=200, headers={},
content=encode({"resCode": 408}, worker.engine.client.session.key))
def test_expired_state_survives_idle_view_config_and_blocks_game_requests(self):
identifier, worker = self.connected(running=True)
original_client, original_snapshot = worker.engine.client, worker.engine.snapshot
with patch.object(original_client.http, "post", return_value=self.expiry(worker)) as post:
worker.engine.last_heartbeat = 0
self.tick(identifier, worker, status="failed")
state = self.manager.status(identifier)
self.assertEqual(state.get("connection_state"), "expired")
self.assertIn("失效", state["connection_message"])
self.assertFalse(state["connected"])
self.assertFalse(state["running"])
reason = state["error"]
for _ in range(3):
self.now += 600
self.tick(identifier, worker)
view = self.manager.view(identifier, "bag")
self.job(identifier, {"id": view["job_id"]})
config = self.manager.get_config(identifier)
config["config"]["interval"] = 60
self.job(identifier, self.manager.set_config(identifier, config["config"])["job"])
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
self.job(identifier, self.manager.submit(identifier, "start"), status="failed")
self.assertEqual(post.call_count, 1)
self.assertEqual(self.manager.status(identifier)["error"], reason)
self.assertEqual(self.manager.status(identifier)["connection_state"], "expired")
self.assertIs(worker.engine.client, original_client)
self.assertIs(worker.engine.snapshot, original_snapshot)
self.assertIsNotNone(worker.securestore.load(worker.active_identity))
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}))
self.assertEqual(self.manager.status(identifier)["connection_state"], "online")
self.assertFalse(self.manager.status(identifier)["running"])
def test_timeout_retries_only_snapshot_and_keeps_uncertain_write_evidence(self):
identifier, worker = self.connected(running=True)
journal = worker.engine.battle_journal
pending = journal.begin(worker.engine.snapshot, {"kind": "battle", "description": "未确认的战斗",
"path": "stageService/stageStart", "payload": {}, "stage_id": "0101", "difficulty": "1"})
journal.update(pending, "unknown", message="未确认的战斗写请求")
evidence = journal.path
before = evidence.read_bytes()
client = worker.engine.client
def write(runtime, operation, payload):
runtime.engine.client.call("monsterService/monsterLock", {"monsterId": 1})
with patch("ranch_web.actions.execute_action", side_effect=write):
with patch.object(client.http, "post", side_effect=requests.ReadTimeout("synthetic timeout")) as post:
self.job(identifier, self.manager.submit(identifier, "lock"), status="failed")
self.assertEqual(post.call_count, 1)
state = self.manager.status(identifier)
self.assertEqual(state.get("connection_state"), "retrying")
self.assertNotIn("失效", state["connection_message"])
self.assertFalse(state["running"])
self.assertIsNotNone(state["next_retry_at"])
calls = len(fixtures.FakeClient.calls)
self.tick(identifier, worker)
self.assertEqual(len(fixtures.FakeClient.calls), calls)
self.now += 61
self.tick(identifier, worker)
paths = [url.rsplit("/", 2)[-2:] for _, url, _ in fixtures.FakeClient.calls[calls:]]
self.assertEqual(paths, [["masterService", "masterQueryDetail"]])
self.assertEqual(self.manager.status(identifier)["connection_state"], "online")
self.assertEqual(self.manager.status(identifier)["error"], "")
self.assertIsNone(self.manager.status(identifier)["next_retry_at"])
self.assertIsNotNone(self.manager.status(identifier)["last_game_success_at"])
self.assertFalse(worker.running)
self.assertEqual(journal.pending(worker.engine.snapshot)["attempt_id"], pending["attempt_id"])
self.assertEqual(evidence.read_bytes(), before)
# Continue through the ordinary monitor/reconcile path after recovery.
# The existing Engine barrier must still prevent the unknown write.
worker.engine.last_heartbeat = self.now
worker.schedule.next_poll = 0
self.now += 121
self.tick(identifier, worker)
paths = [url.rsplit("/", 2)[-2:] for _, url, _ in fixtures.FakeClient.calls[calls:]]
self.assertEqual(paths, [["masterService", "masterQueryDetail"]] * 2)
self.assertEqual(journal.pending(worker.engine.snapshot)["attempt_id"], pending["attempt_id"])
self.assertEqual(journal.pending(worker.engine.snapshot)["phase"], "unknown")
self.assertFalse(worker.running)
def test_business_rejection_keeps_connection_and_idle_does_not_clear_error(self):
identifier, worker = self.connected(running=True)
client = worker.engine.client
response = SimpleNamespace(status_code=200, headers={},
content=encode({"resCode": 403}, client.session.key))
with patch.object(client.http, "post", return_value=response):
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
state = self.manager.status(identifier)
self.assertEqual(state["connection_state"], "online")
self.assertTrue(state["connected"])
self.assertFalse(state["running"])
self.assertFalse(worker.schedule.game_paused)
self.assertTrue(worker.schedule.monitoring)
reason = state["error"]
self.assertIn("403", reason)
worker.engine.last_heartbeat = self.now
worker.schedule.next_poll = float("inf")
with patch.object(worker.engine, "master_sync_due", return_value=False), patch.object(worker.engine, "growth_sync_due", return_value=False):
self.tick(identifier, worker)
view = self.manager.view(identifier, "bag")
self.job(identifier, {"id": view["job_id"]})
self.assertEqual(self.manager.status(identifier)["error"], reason)
def test_997_is_retrying_without_expiry_and_backoff_is_bounded(self):
identifier, worker = self.connected()
response = SimpleNamespace(status_code=200, headers={},
content=encode({"resCode": 997}, worker.engine.client.session.key))
with patch.object(worker.engine.client.http, "post", return_value=response) as post:
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
for delay in (5, 10, 20, 40, 60, 60):
self.assertEqual(worker.schedule.retry_deadline - self.now, delay)
state = self.manager.status(identifier)
self.assertEqual(state["connection_state"], "retrying")
self.assertNotIn("失效", state["connection_message"])
calls = post.call_count
self.tick(identifier, worker)
self.assertEqual(post.call_count, calls)
self.now += delay
self.tick(identifier, worker, status="failed")
self.assertEqual(post.call_count, calls + 1)
def test_swallowed_expiry_still_stops_strategy_and_marks_job_failed(self):
identifier, worker = self.connected(running=True)
def swallowed(runtime, operation, payload):
try:
runtime.engine.client.call("monsterService/monsterLock")
except ApiError:
return {"message": "Engine caught the error"}
with patch("ranch_web.actions.execute_action", side_effect=swallowed):
with patch.object(worker.engine.client.http, "post", return_value=self.expiry(worker)):
result = self.job(identifier, self.manager.submit(identifier, "lock"), status="failed")
self.assertIn("失效", result["error"])
self.assertEqual(self.manager.status(identifier)["connection_state"], "expired")
self.assertFalse(worker.running)
def test_scheduler_runs_fake_heartbeat_and_poll_without_any_browser_requests(self):
identifier, worker = self.connected()
heartbeat, poll = threading.Event(), threading.Event()
original = worker.engine.client.http.post
def post(url, *args, **kwargs):
result = original(url, *args, **kwargs)
if url.endswith("heartBeatService/keepAlive"):
heartbeat.set()
if url.endswith("masterService/masterQueryDetail"):
poll.set()
return result
worker.engine.last_heartbeat = 0
worker.schedule.next_poll = 0
with patch.object(worker.engine.client.http, "post", side_effect=post):
scheduler = Scheduler(self.manager._worker_list, interval=.01)
try:
self.assertTrue(heartbeat.wait(3), "No heartbeat without browser polling")
self.assertTrue(poll.wait(3), "No game poll without browser polling")
finally:
scheduler.close()
self.idle(identifier)
self.assertFalse(worker.running)
def test_idle_local_success_does_not_clear_game_error(self):
identifier, worker = self.connected()
with patch.object(worker.engine.client.http, "post", side_effect=requests.Timeout("offline")):
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
reason = self.manager.status(identifier)["error"]
self.assertTrue(reason)
self.tick(identifier, worker)
view = self.manager.view(identifier, "bag")
self.job(identifier, {"id": view["job_id"]})
self.job(identifier, self.manager.submit(identifier, "refresh", {"page": "statistics"}))
self.assertEqual(self.manager.status(identifier)["error"], reason)
self.assertFalse(self.manager.status(identifier)["connected"])
def test_failed_restore_candidate_does_not_poison_online_client(self):
identifier, worker = self.connected()
client = worker.engine.client
original = deepcopy(self.manager.status(identifier))
response = self.expiry(worker)
with patch.object(fixtures.FakeClient, "_post", return_value=response):
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}), status="failed")
self.assertIs(worker.engine.client, client)
self.assertFalse(client.closed)
state = self.manager.status(identifier)
self.assertEqual(state.get("connection_state"), "online")
self.assertEqual(state["last_game_success_at"], original["last_game_success_at"])
self.assertEqual(state["connection_message"], original["connection_message"])
self.assertTrue(state["connected"])
def test_failed_same_protocol_restore_preserves_counter_high_water_before_wire_and_finally(self):
identifier, worker = self.connected()
for active_reqid, active_reqsid in ((400, 900), (500, 1200)):
with self.subTest(active_reqid=active_reqid):
client = worker.engine.client
saved = deepcopy(client.session)
saved.reqid, saved.reqsid = 400, 900
worker.securestore.save(saved, "离线高水位回归")
client.session.reqid, client.session.reqsid = active_reqid, active_reqsid
wire = []
def post(url, data, **kwargs):
wire.append(fixtures.decode(data, client.session.key))
return self.expiry(worker)
with patch.object(fixtures.FakeClient, "_post", side_effect=post):
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}), status="failed")
self.assertEqual((wire[0]["reqid"], wire[0]["reqsid"]), (active_reqid, active_reqsid))
persisted = worker.securestore.load(worker.active_identity)
self.assertEqual((persisted.reqid, persisted.reqsid), (active_reqid + 1, active_reqsid + 1))
self.assertEqual((client.session.reqid, client.session.reqsid), (persisted.reqid, persisted.reqsid))
client.call("heartBeatService/keepAlive")
payload = fixtures.FakeClient.calls[-1][2]
self.assertEqual((payload["reqid"], payload["reqsid"]), (active_reqid + 1, active_reqsid + 1))
def test_counter_merge_requires_exact_protocol_session(self):
identifier, worker = self.connected()
current = worker.engine.client.session
current.reqid, current.reqsid = 500, 900
changes = {"token": "different", "key": "different", "server_url": "https://other.invalid/",
"account_id": "different", "server_id": 9, "role_id": 999, "channel": "other", "platform": "other"}
for field, value in changes.items():
candidate = deepcopy(current)
candidate.reqid, candidate.reqsid = 800, 1800
setattr(candidate, field, value)
worker._share_session_counters(candidate)
self.assertEqual((current.reqid, current.reqsid), (500, 900), field)
self.assertEqual((candidate.reqid, candidate.reqsid), (800, 1800), field)
def test_first_restore_timeout_without_client_requires_manual_restore(self):
identifier, worker = self.connected()
identity = worker.active_identity
self.manager.close()
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
self.idle(identifier)
worker = self.manager._worker(identifier)
with patch.object(fixtures.FakeClient, "_post", side_effect=requests.ReadTimeout("offline")) as post:
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": identity}), status="failed")
for _ in range(3):
self.tick(identifier, worker)
self.assertEqual(post.call_count, 1)
state = self.manager.status(identifier)
self.assertEqual(state["connection_state"], "offline")
self.assertIn("手动", state["connection_message"])
self.assertIsNone(state["next_retry_at"])
self.assertIsNone(worker.schedule.retry_deadline)
self.assertIsNone(worker.engine.client)
def test_activity_restart_cursor_survives_debug_only_tail_and_records_reason(self):
identifier, worker = self.connected()
worker.log("可持久保存的运行事件")
worker.log("不持久保存的调试内容", debug=True)
old_cursor = self.manager.logs(identifier, debug=True)["next"]
self.manager.close()
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
self.idle(identifier)
logs = self.manager.logs(identifier, after=old_cursor)
self.assertGreater(logs["next"], old_cursor)
self.assertTrue(any("重启" in row["message"] and "恢复" in row["message"] for row in logs["entries"]))
all_logs = self.manager.logs(identifier, debug=True)["entries"]
self.assertTrue(any("可持久保存的运行事件" == row["message"] for row in all_logs))
self.assertFalse(any("不持久保存的调试内容" == row["message"] for row in all_logs))
def test_activity_write_failure_keeps_memory_logs_and_original_game_outcome(self):
identifier, worker = self.connected()
with patch.object(worker._activity, "append", side_effect=ActivityLogError("Unable to save activity storage")):
with patch.object(worker.engine.client.http, "post", return_value=self.expiry(worker)):
result = self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
self.assertIn("失效", result["error"])
self.assertEqual(self.manager.status(identifier)["connection_state"], "expired")
logs = self.manager.logs(identifier, debug=True)["entries"]
self.assertTrue(any("日志无法持久保存" in entry["message"] for entry in logs))
self.assertEqual(len({entry["seq"] for entry in logs}), len(logs))
def test_activity_idle_tick_and_view_do_not_write_log_storage(self):
identifier, worker = self.connected()
worker.engine.last_heartbeat = self.now
worker.schedule.next_poll = float("inf")
with patch.object(worker.engine, "master_sync_due", return_value=False), patch.object(worker.engine, "growth_sync_due", return_value=False):
with patch.object(worker._activity, "append", wraps=worker._activity.append) as append:
self.tick(identifier, worker)
view = self.manager.view(identifier, "bag")
self.job(identifier, {"id": view["job_id"]})
append.assert_not_called()
def test_activity_failure_does_not_turn_successful_game_read_into_failure(self):
identifier, worker = self.connected()
with patch.object(worker._activity, "append", side_effect=ActivityLogError("Unable to save activity storage")):
self.job(identifier, self.manager.submit(identifier, "refresh"))
self.assertEqual(self.manager.status(identifier)["connection_state"], "online")
self.assertEqual(self.manager.status(identifier)["error"], "")
self.assertTrue(any("日志无法持久保存" in entry["message"] for entry in self.manager.logs(identifier)["entries"]))
def test_activity_recovers_after_long_memory_fallback_without_reusing_cursor(self):
identifier, worker = self.connected()
activity = worker._activity
with patch.object(activity, "_write", side_effect=ActivityLogError("Unable to save activity storage")):
worker.log("临时无法保存的运行事件")
for _ in range(300):
worker.log("磁盘只读期间的调试事件", debug=True)
memory_cursor = self.manager.logs(identifier, debug=True)["next"]
for _ in range(300):
worker.log("磁盘恢复后的调试事件", debug=True)
self.assertIs(worker._activity, activity)
logs = self.manager.logs(identifier, debug=True)
old_cursor = logs["next"]
self.assertGreater(old_cursor, memory_cursor)
self.assertEqual(len({entry["seq"] for entry in logs["entries"]}), len(logs["entries"]))
self.assertEqual(sum("日志无法持久保存" in entry["message"] for entry in logs["entries"]), 1)
self.assertEqual(sum("日志持久化已恢复" in entry["message"] for entry in logs["entries"]), 1)
self.manager.close()
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
self.idle(identifier)
after_restart = self.manager.logs(identifier, after=old_cursor)
self.assertTrue(after_restart["entries"])
self.assertTrue(all(entry["seq"] > old_cursor for entry in after_restart["entries"]))
def test_activity_initial_load_failure_retries_on_later_events(self):
self.manager.close()
with patch("ranch_web.service.ActivityLog.load", side_effect=ActivityLogError("Unable to read activity storage")), \
patch("ranch_web.activity.ActivityLog._read", side_effect=ActivityLogError("Unable to read activity storage")):
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
identifier = self.account(False)
worker = self.manager._worker(identifier)
self.assertTrue(any("首次读取运行日志失败" in entry["message"] for entry in self.manager.logs(identifier)["entries"]))
before = self.manager.logs(identifier)["next"]
identifier = worker.log("日志存储重新可用")
self.assertGreater(identifier, before)
self.assertIsNotNone(worker._activity)
self.assertTrue(any("日志持久化已恢复" in entry["message"] for entry in worker._activity.load()))
def test_activity_initial_load_failure_then_immediate_recovery_has_ordered_messages(self):
self.manager.close()
with patch("ranch_web.service.ActivityLog.load", side_effect=ActivityLogError("Unable to read activity storage")):
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
identifier = self.account(False)
messages = [entry["message"] for entry in self.manager.logs(identifier)["entries"]]
self.assertIn("首次读取运行日志失败", messages[0])
self.assertIn("日志持久化已恢复", messages[1])
self.assertEqual(messages[2], "账号工作区已载入")
self.assertFalse(any("无法持久保存" in message for message in messages[1:]))
def test_restart_and_explicit_logout_require_manual_restore(self):
identifier, worker = self.connected()
self.manager.close()
calls = len(fixtures.FakeClient.calls)
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
self.addCleanup(self.manager.close)
self.idle(identifier)
worker = self.manager._worker(identifier)
state = self.manager.status(identifier)
self.assertEqual(state.get("connection_state"), "offline")
self.assertIn("重启", state["connection_message"])
self.assertIn("恢复", state["connection_message"])
self.assertFalse(state["running"])
self.assertEqual(len(fixtures.FakeClient.calls), calls)
self.tick(identifier, worker)
self.assertEqual(len(fixtures.FakeClient.calls), calls)
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}))
self.job(identifier, self.manager.submit(identifier, "logout"))
calls = len(fixtures.FakeClient.calls)
self.now += 10000
self.tick(identifier, worker)
state = self.manager.status(identifier)
self.assertEqual(state["connection_state"], "offline")
self.assertIn("退出", state["connection_message"])
self.assertFalse(state["running"])
self.assertEqual(len(fixtures.FakeClient.calls), calls)
class ErrorClassificationTests(unittest.TestCase):
def test_failed_heartbeat_does_not_advance_its_success_clock(self):
def fail(*args):
raise ApiError("offline")
engine = fixtures.Engine(fixtures.Catalog(), SimpleNamespace(call=fail))
engine.last_heartbeat = 123
with self.assertRaises(ApiError):
engine.keep_alive()
self.assertEqual(engine.last_heartbeat, 123)
def test_wire_expiry_and_transport_timeout_have_distinct_types(self):
client = fixtures.FakeClient(fixtures.Session("https://offline.invalid/game/", "synthetic-token"))
self.addCleanup(client.close)
response = SimpleNamespace(status_code=200, headers={}, content=encode({"resCode": 408}, client.session.key))
with patch.object(client.http, "post", return_value=response):
with self.assertRaises(ApiError) as expired:
client.call("masterService/masterQueryDetail")
with patch.object(client.http, "post", side_effect=requests.Timeout("offline")) as post:
with self.assertRaises(ApiError) as temporary:
client.call("monsterService/monsterLock")
self.assertEqual(post.call_count, 1)
self.assertEqual(getattr(expired.exception, "code", None), 408)
self.assertNotEqual(type(expired.exception), type(temporary.exception))
self.assertIsInstance(temporary.exception.__cause__, requests.Timeout)