fix: preserve background connection state and durable disconnect logs
This commit is contained in:
@@ -0,0 +1,249 @@
|
||||
"""Offline browser regressions for game and Web connection feedback.
|
||||
|
||||
All accounts are disposable demos. Game status/action responses are synthetic;
|
||||
Python HTTP and browser requests outside this test's loopback server are blocked.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from . import test_workbench_browser as fixtures
|
||||
|
||||
if fixtures.AVAILABLE:
|
||||
from playwright.sync_api import expect
|
||||
|
||||
|
||||
@unittest.skipUnless(fixtures.AVAILABLE, "Python Playwright and a system Chromium browser are required")
|
||||
class ConnectionBrowserTests(unittest.TestCase):
|
||||
setUpClass = classmethod(fixtures.WorkbenchBrowserTests.setUpClass.__func__)
|
||||
tearDownClass = classmethod(fixtures.WorkbenchBrowserTests.tearDownClass.__func__)
|
||||
tearDown = fixtures.WorkbenchBrowserTests.tearDown
|
||||
await_job = fixtures.WorkbenchBrowserTests.await_job
|
||||
open = fixtures.WorkbenchBrowserTests.open
|
||||
go = fixtures.WorkbenchBrowserTests.go
|
||||
fulfill = staticmethod(fixtures.WorkbenchBrowserTests.fulfill)
|
||||
|
||||
def setUp(self):
|
||||
guard = patch("requests.sessions.Session.request", side_effect=AssertionError("Live HTTP is forbidden"))
|
||||
guard.start()
|
||||
self.addCleanup(guard.stop)
|
||||
fixtures.WorkbenchBrowserTests.setUp(self)
|
||||
self.external_requests = []
|
||||
|
||||
def local_only(route):
|
||||
if route.request.url.startswith(self.base + "/"):
|
||||
route.continue_()
|
||||
else:
|
||||
self.external_requests.append(route.request.url)
|
||||
route.abort()
|
||||
|
||||
self.context.route("**/*", local_only)
|
||||
self.summary = copy.deepcopy(self.manager.status(self.first))
|
||||
self.summary.update(demo=False, connected=True, running=False, error="", connection_state="online",
|
||||
connection_message="游戏连接正常", last_game_success_at="2026-11-11T00:00:00Z", next_retry_at=None)
|
||||
self.calls = {"accounts": 0, "logs": 0, "views": 0}
|
||||
self.fail_accounts = False
|
||||
self.fail_logs = False
|
||||
|
||||
def accounts(route):
|
||||
self.calls["accounts"] += 1
|
||||
if self.fail_accounts:
|
||||
route.abort("connectionrefused")
|
||||
else:
|
||||
self.fulfill(route, [self.summary, self.manager.status(self.second)])
|
||||
|
||||
def logs(route):
|
||||
self.calls["logs"] += 1
|
||||
if self.fail_logs:
|
||||
route.abort("connectionrefused")
|
||||
else:
|
||||
self.fulfill(route, {"entries": [], "next": 0})
|
||||
|
||||
self.page.route("**/api/accounts", accounts)
|
||||
self.page.route(f"**/api/accounts/{self.first}/status", lambda route: self.fulfill(route, self.summary))
|
||||
self.page.route(f"**/api/accounts/{self.first}/logs?*", logs)
|
||||
self.page.on("request", lambda request: self.calls.__setitem__("views", self.calls["views"] + 1)
|
||||
if f"/api/accounts/{self.first}/views/" in request.url else None)
|
||||
|
||||
def test_expired_cached_snapshot_stops_view_polling_and_recovers(self):
|
||||
self.open()
|
||||
panel = self.go("ranch")
|
||||
expect(panel.locator("tbody tr").first).to_be_visible()
|
||||
row_count = panel.locator("tbody tr").count()
|
||||
self.summary.update(connection_state="expired", connection_message="登录凭据已失效,请重新登录。")
|
||||
expect(self.page.locator("#connection")).to_have_text("登录已过期")
|
||||
expect(self.page.locator("#data-summary")).to_contain_text(self.summary["connection_message"])
|
||||
expect(panel.locator(".page-loading")).to_contain_text("缓存")
|
||||
expect(panel.locator("[data-connection-status]")).to_contain_text("2026/11/11")
|
||||
expect(self.page.locator("#strategy-toggle")).to_be_disabled()
|
||||
expect(self.page.locator("#refresh")).to_be_disabled()
|
||||
expect(self.page.locator('[data-command="start"]')).to_be_disabled()
|
||||
count = dict(self.calls)
|
||||
self.page.wait_for_timeout(5200)
|
||||
self.assertEqual(self.calls["views"], count["views"])
|
||||
self.assertGreater(self.calls["accounts"], count["accounts"])
|
||||
self.assertGreater(self.calls["logs"], count["logs"])
|
||||
self.assertEqual(panel.locator("tbody tr").count(), row_count)
|
||||
self.assertTrue(panel.locator("input[data-config-path]").first.is_enabled())
|
||||
self.summary.update(connection_state="online", connection_message="游戏连接正常", error="", last_game_success_at="2026-11-11T00:01:00Z")
|
||||
expect(self.page.locator("#connection")).to_have_text("已登录")
|
||||
expect(self.page.locator("#strategy-toggle")).to_be_enabled()
|
||||
expect(panel.locator("[data-connection-status]")).to_be_hidden()
|
||||
expect(self.page.locator("#data-summary")).not_to_contain_text("凭据已失效")
|
||||
expect(self.page.locator("#status-text")).not_to_contain_text("凭据已失效")
|
||||
expect(panel.locator(".page-loading")).not_to_contain_text("缓存")
|
||||
self.assertEqual(self.page.locator("#page-tabs [role=tab]").count(), 15)
|
||||
self.assertEqual(self.external_requests, [])
|
||||
self.assertEqual(self.errors, [])
|
||||
|
||||
def test_retry_countdown_offline_reason_and_login_metadata(self):
|
||||
self.open()
|
||||
self.summary.update(connection_state="retrying", connection_message="游戏请求暂时失败(网络连接中断),即将重试。")
|
||||
self.summary["next_retry_at"] = self.page.evaluate("new Date(Date.now()+30000).toISOString()")
|
||||
expect(self.page.locator("#connection")).to_have_text("正在重试")
|
||||
panel = self.go("bag")
|
||||
expect(panel.locator("[data-connection-status]")).to_contain_text(self.summary["connection_message"])
|
||||
expect(self.page.locator("#data-summary")).to_contain_text("秒后重试")
|
||||
first = self.page.locator("#data-summary").inner_text()
|
||||
self.page.wait_for_timeout(2100)
|
||||
self.assertNotEqual(first, self.page.locator("#data-summary").inner_text())
|
||||
self.summary.update(connection_state="offline", connection_message="尚未连接游戏,请在登录页恢复会话。", next_retry_at=None)
|
||||
expect(self.page.locator("#connection")).to_have_text("游戏离线")
|
||||
expect(self.page.locator("#data-summary")).to_contain_text("尚未连接游戏")
|
||||
expect(self.page.locator("#data-summary")).not_to_contain_text("过期")
|
||||
expect(self.page.locator("#data-summary")).not_to_contain_text("秒后重试")
|
||||
login_calls = []
|
||||
self.page.on("request", lambda request: login_calls.append(request.url) if "/views/login?" in request.url else None)
|
||||
login = self.go("login")
|
||||
expect(login.get_by_role("button", name="恢复选中会话(无需重新输密码)")).to_be_visible()
|
||||
before = len(login_calls)
|
||||
self.page.wait_for_timeout(5200)
|
||||
self.assertGreater(len(login_calls), before, "Disconnected login metadata must continue refreshing")
|
||||
self.assertEqual(self.external_requests, [])
|
||||
self.assertEqual(self.errors, [])
|
||||
|
||||
def test_web_status_and_log_failures_clear_only_the_recovered_channel(self):
|
||||
self.open()
|
||||
self.go("ranch")
|
||||
self.fail_accounts = True
|
||||
expect(self.page.locator("#data-summary")).to_contain_text("Web 状态读取失败")
|
||||
expect(self.page.locator("#strategy-toggle")).to_be_disabled()
|
||||
expect(self.page.locator("#connection")).not_to_have_text("已登录")
|
||||
count = dict(self.calls)
|
||||
self.page.wait_for_timeout(5100)
|
||||
self.assertGreater(self.calls["accounts"], count["accounts"])
|
||||
self.assertGreater(self.calls["logs"], count["logs"])
|
||||
self.assertEqual(self.calls["views"], count["views"])
|
||||
self.fail_logs = True
|
||||
expect(self.page.locator("#data-summary")).to_contain_text("日志读取失败")
|
||||
self.fail_accounts = False
|
||||
expect(self.page.locator("#data-summary")).not_to_contain_text("Web 状态读取失败")
|
||||
expect(self.page.locator("#data-summary")).to_contain_text("日志读取失败")
|
||||
self.fail_logs = False
|
||||
expect(self.page.locator("#data-summary")).not_to_contain_text("日志读取失败")
|
||||
expect(self.page.locator("#status-text")).not_to_contain_text("读取失败")
|
||||
expect(self.page.locator("#strategy-toggle")).to_be_enabled()
|
||||
self.assertEqual(self.external_requests, [])
|
||||
|
||||
def test_successful_refresh_job_with_disconnected_summary_never_reports_success(self):
|
||||
self.open()
|
||||
def refresh(route):
|
||||
self.summary.update(connection_state="retrying", connection_message="游戏请求超时,稍后重试。", next_retry_at=None)
|
||||
self.fulfill(route, {"id": "fixture-refresh", "status": "succeeded", "operation": "refresh", "result": {"message": "刷新成功"}})
|
||||
self.page.route(f"**/api/accounts/{self.first}/actions/refresh", refresh)
|
||||
self.page.locator("#refresh").click()
|
||||
expect(self.page.locator("#connection")).to_have_text("正在重试")
|
||||
expect(self.page.locator("#status-text")).to_contain_text("游戏请求超时")
|
||||
expect(self.page.locator("#status-text")).not_to_contain_text("刷新成功")
|
||||
expect(self.page.locator("#status-text")).not_to_contain_text("刷新 / 预览已完成")
|
||||
expect(self.page.locator("#notice")).not_to_have_text("刷新成功")
|
||||
self.assertEqual(self.external_requests, [])
|
||||
|
||||
def test_closing_page_never_stops_or_logs_out_account(self):
|
||||
self.summary["running"] = True
|
||||
self.open()
|
||||
writes = []
|
||||
self.page.on("request", lambda request: writes.append(request.url) if request.method in ("POST", "PUT", "DELETE") else None)
|
||||
self.page.goto("about:blank")
|
||||
self.page.close()
|
||||
self.assertEqual(writes, [])
|
||||
self.assertEqual(self.external_requests, [])
|
||||
|
||||
def test_disconnected_loading_response_is_cached_and_config_remains_editable(self):
|
||||
self.open()
|
||||
self.summary.update(connection_state="offline", connection_message="Web 服务已重启,请手动恢复会话。", last_game_success_at=None)
|
||||
expect(self.page.locator("#connection")).to_have_text("游戏离线")
|
||||
response = {"loading": True, "data": {"page": "bag", "summary": "已保存的背包", "filters": [], "columns": [], "rows": [],
|
||||
"form": [{"path": ["policy", "collect"], "label": "收取魔物", "type": "checkbox"}], "actions": []}}
|
||||
view_calls, actions = [], []
|
||||
self.page.route(f"**/api/accounts/{self.first}/views/bag?*", lambda route: (view_calls.append(route.request.url), self.fulfill(route, response)))
|
||||
self.page.on("request", lambda request: actions.append(request.url) if "/actions/" in request.url else None)
|
||||
panel = self.go("bag")
|
||||
expect(panel.locator("[data-connection-status]")).to_contain_text("Web 服务已重启")
|
||||
expect(panel.locator(".page-loading")).to_contain_text("缓存")
|
||||
self.page.locator('[data-command="start"]').dispatch_event("click")
|
||||
self.page.locator("#refresh").dispatch_event("click")
|
||||
self.page.wait_for_timeout(1800)
|
||||
self.assertEqual(len(view_calls), 1, "Disconnected loading=true must not create a 650ms reload chain")
|
||||
self.assertEqual(actions, [], "Disconnected starts and refreshes must be guarded beyond button disabled state")
|
||||
control = panel.locator("input[data-config-path]").first
|
||||
expect(control).to_be_enabled()
|
||||
control.set_checked(not control.is_checked())
|
||||
expect(self.page.locator("#save-status")).to_have_text("策略已自动保存")
|
||||
self.page.wait_for_timeout(800)
|
||||
self.assertEqual(self.manager.get_config(self.first)["config"]["policy"]["collect"], control.is_checked())
|
||||
self.assertEqual(actions, [])
|
||||
self.assertEqual(self.external_requests, [])
|
||||
self.assertEqual(self.errors, [])
|
||||
|
||||
def test_job_transport_failure_retries_and_clears_on_job_recovery(self):
|
||||
self.open()
|
||||
fail_job = True
|
||||
job_calls = []
|
||||
self.page.route(f"**/api/accounts/{self.first}/actions/refresh", lambda route: self.fulfill(route,
|
||||
{"id": "fixture-job", "status": "queued", "operation": "refresh"}))
|
||||
def job(route):
|
||||
job_calls.append(route.request.url)
|
||||
if fail_job:
|
||||
route.abort("connectionrefused")
|
||||
else:
|
||||
self.fulfill(route, {"id": "fixture-job", "status": "succeeded", "operation": "refresh", "result": {}})
|
||||
self.page.route(f"**/api/accounts/{self.first}/jobs/fixture-job", job)
|
||||
self.page.locator("#refresh").click()
|
||||
expect(self.page.locator("#data-summary")).to_contain_text("任务状态读取失败")
|
||||
self.page.wait_for_timeout(1200)
|
||||
self.assertGreater(len(job_calls), 1)
|
||||
expect(self.page.locator("#data-summary")).to_contain_text("任务状态读取失败")
|
||||
fail_job = False
|
||||
expect(self.page.locator("#data-summary")).not_to_contain_text("任务状态读取失败")
|
||||
expect(self.page.locator("#status-text")).not_to_contain_text("任务状态读取失败")
|
||||
self.assertEqual(self.external_requests, [])
|
||||
|
||||
def test_view_failure_survives_successful_logs_and_clears_on_view_recovery(self):
|
||||
self.open()
|
||||
fail_view = True
|
||||
response = {"loading": False, "data": {"page": "bag", "summary": "已恢复页面数据", "form": [], "filters": [], "actions": [], "columns": [], "rows": []}}
|
||||
def view(route):
|
||||
if fail_view:
|
||||
route.abort("connectionrefused")
|
||||
else:
|
||||
self.fulfill(route, response)
|
||||
self.page.route(f"**/api/accounts/{self.first}/views/bag?*", view)
|
||||
self.page.locator('.page-tab[data-page="bag"]').click()
|
||||
panel = self.page.locator('.page-panel[data-page="bag"]:not([hidden])')
|
||||
expect(panel.locator(".page-error")).to_be_visible()
|
||||
expect(self.page.locator("#data-summary")).to_contain_text("页面数据读取失败")
|
||||
self.page.wait_for_timeout(1200)
|
||||
expect(self.page.locator("#data-summary")).to_contain_text("页面数据读取失败")
|
||||
fail_view = False
|
||||
panel.get_by_role("button", name="重新读取").click()
|
||||
expect(panel.locator(".page-error")).to_be_hidden()
|
||||
expect(self.page.locator("#data-summary")).not_to_contain_text("页面数据读取失败")
|
||||
expect(self.page.locator("#status-text")).not_to_contain_text("页面数据读取失败")
|
||||
self.assertEqual(self.external_requests, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -283,10 +283,17 @@ class WorkbenchBrowserTests(unittest.TestCase):
|
||||
|
||||
def test_fake_login_uses_servers_without_persisting_password(self):
|
||||
payloads = []
|
||||
login_result = {"servers": [{"id": 7, "name": "测试区", "role_name": "测试角色"}], "has_saved_password": True}
|
||||
def route_login(route):
|
||||
payloads.append(route.request.post_data_json)
|
||||
self.fulfill(route, {"id": "fake_login", "status": "succeeded", "operation": "login", "result": {"servers": [{"id": 7, "name": "测试区", "role_name": "测试角色"}], "has_saved_password": True}, "error": None})
|
||||
self.fulfill(route, {"id": "fake_login", "status": "succeeded", "operation": "login", "result": login_result, "error": None})
|
||||
def route_login_view(route):
|
||||
response = route.fetch().json()
|
||||
if payloads and response.get("data"):
|
||||
response["data"]["login"] = login_result
|
||||
self.fulfill(route, response)
|
||||
self.page.route(f"**/api/accounts/{self.first}/actions/login", route_login)
|
||||
self.page.route(f"**/api/accounts/{self.first}/views/login?*", route_login_view)
|
||||
self.open()
|
||||
panel = self.go("login")
|
||||
panel.get_by_label("游戏账号", exact=True).fill("offline-test")
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
"""Standalone activity tests: disposable files, stdlib, no service/game imports."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from ranch_web.activity import ActivityLog, ActivityLogError, MAX_MESSAGE_CHARS
|
||||
|
||||
|
||||
class ActivityTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.root = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="activity-test-")))
|
||||
self.path = self.root / "profiles" / "account-a" / "activity.json"
|
||||
self.watermark = self.path.with_name(self.path.name + ".seq")
|
||||
|
||||
def log(self, **kwargs):
|
||||
return ActivityLog(self.path, **kwargs)
|
||||
|
||||
def append(self, log, message="safe lifecycle event", **kwargs):
|
||||
return log.append({"message": message, **kwargs})
|
||||
|
||||
def test_creation_load_sequence_and_repeated_idle_reads_never_write(self):
|
||||
with patch("ranch_web.activity.os.replace") as replace, patch("ranch_web.activity.os.fsync") as fsync:
|
||||
log = self.log()
|
||||
for _ in range(100):
|
||||
self.assertEqual(log.load(), [])
|
||||
self.assertEqual(log.sequence, 0)
|
||||
replace.assert_not_called()
|
||||
fsync.assert_not_called()
|
||||
self.assertFalse(self.path.parent.exists())
|
||||
|
||||
def test_account_isolation_and_persistence(self):
|
||||
log = self.log()
|
||||
first = self.append(log, "account A operation", level="warning")
|
||||
other_path = self.root / "profiles" / "account-b" / "activity.json"
|
||||
other = ActivityLog(other_path)
|
||||
self.assertEqual(other.load(), [])
|
||||
self.assertEqual(other.sequence, 0)
|
||||
second = self.append(other, "account B start")
|
||||
self.assertEqual((first["seq"], second["seq"]), (1, 1))
|
||||
self.assertEqual(ActivityLog(self.path).load(), [first])
|
||||
self.assertEqual(ActivityLog(other_path).load(), [second])
|
||||
self.assertEqual(first["level"], "warning")
|
||||
|
||||
def test_debug_only_tail_restart_startup_exceeds_browser_cursor(self):
|
||||
log = self.log(sequence_block_size=8)
|
||||
persisted = self.append(log, "persisted operation")
|
||||
# More than one reservation block, with no final nondebug flush.
|
||||
for number in range(23):
|
||||
tail = self.append(log, "transient trace " + str(number), debug=True)
|
||||
browser_cursor = log.sequence
|
||||
self.assertEqual(browser_cursor, tail["seq"])
|
||||
self.assertGreater(browser_cursor, persisted["seq"])
|
||||
self.assertNotIn(b"transient trace", self.path.read_bytes())
|
||||
restored = self.log(sequence_block_size=8)
|
||||
self.assertEqual(restored.load(), [persisted])
|
||||
# This becomes worker._log_sequence and therefore manager.logs().next.
|
||||
self.assertGreaterEqual(restored.sequence, browser_cursor)
|
||||
startup = self.append(restored, "startup")
|
||||
self.assertGreater(startup["seq"], browser_cursor)
|
||||
page = {"entries": [e for e in restored.load() if e["seq"] > browser_cursor],
|
||||
"next": restored.sequence}
|
||||
self.assertEqual(page, {"entries": [startup], "next": startup["seq"]})
|
||||
following = self.append(restored, "following event")
|
||||
self.assertEqual(following["seq"], startup["seq"] + 1)
|
||||
|
||||
def test_entirely_debug_run_survives_restart_without_snapshot(self):
|
||||
log = self.log(sequence_block_size=8)
|
||||
for _ in range(19):
|
||||
self.append(log, "never persist this trace", debug=True)
|
||||
cursor = log.sequence
|
||||
self.assertFalse(self.path.exists())
|
||||
self.assertTrue(self.watermark.exists())
|
||||
restarted = self.log(sequence_block_size=8)
|
||||
self.assertEqual(restarted.load(), [])
|
||||
self.assertGreater(self.append(restarted, "startup")["seq"], cursor)
|
||||
|
||||
def test_caller_fallback_cursor_is_reserved_before_resuming_debug_events(self):
|
||||
log = self.log()
|
||||
self.append(log)
|
||||
recovered = log.append({"message": "debug after recovery", "debug": True}, after_sequence=1000)
|
||||
self.assertEqual(recovered["seq"], 1001)
|
||||
self.assertGreaterEqual(json.loads(self.watermark.read_bytes())["sequence"], recovered["seq"])
|
||||
self.assertGreater(self.append(self.log(), "startup")["seq"], recovered["seq"])
|
||||
|
||||
def test_failed_caller_cursor_reservation_can_retry_without_reusing_ids(self):
|
||||
log = self.log()
|
||||
self.append(log)
|
||||
with patch.object(log, "_write", side_effect=ActivityLogError("Unable to save activity storage")):
|
||||
with self.assertRaises(ActivityLogError):
|
||||
log.append({"message": "not published", "debug": True}, after_sequence=1000)
|
||||
self.assertEqual(log.sequence, 1)
|
||||
self.assertEqual(log.append({"message": "recovery"}, after_sequence=1001)["seq"], 1002)
|
||||
|
||||
def test_invalid_caller_cursor_is_rejected(self):
|
||||
for cursor in (True, -1, None, 1.0, 1 << 53):
|
||||
with self.subTest(cursor=cursor), self.assertRaises(ValueError):
|
||||
self.log().append({"message": "invalid cursor"}, after_sequence=cursor)
|
||||
|
||||
def test_debug_cost_is_small_watermark_per_block_no_snapshot_rewrite(self):
|
||||
log = self.log(sequence_block_size=8)
|
||||
self.append(log)
|
||||
snapshot = self.path.read_bytes()
|
||||
with patch("ranch_web.activity.os.replace", wraps=os.replace) as replace:
|
||||
for _ in range(31):
|
||||
self.append(log, "debug private detail", debug=True)
|
||||
self.assertEqual(replace.call_count, 3)
|
||||
self.assertTrue(all(Path(call.args[1]) == self.watermark for call in replace.call_args_list))
|
||||
self.assertEqual(self.path.read_bytes(), snapshot)
|
||||
self.assertLess(self.watermark.stat().st_size, 64)
|
||||
self.assertNotIn(b"private detail", self.watermark.read_bytes())
|
||||
|
||||
def test_max_entries_and_highwater_survive_eviction(self):
|
||||
log = self.log(max_entries=3, sequence_block_size=8)
|
||||
entries = [self.append(log, str(n)) for n in range(12)]
|
||||
self.assertEqual(log.load(), entries[-3:])
|
||||
restarted = self.log(max_entries=2, sequence_block_size=8)
|
||||
self.assertEqual(restarted.load(), entries[-2:])
|
||||
self.assertGreater(self.append(restarted)["seq"], entries[-1]["seq"])
|
||||
|
||||
def test_hard_2000_entry_bound(self):
|
||||
# Construct a valid oversized snapshot to test bounded recovery without
|
||||
# manufacturing 2001 synchronous fsync operations in this unit suite.
|
||||
self.path.parent.mkdir(parents=True)
|
||||
rows = [{"seq": seq, "message": "short"} for seq in range(1, 2006)]
|
||||
self.path.write_text(json.dumps({"schema": 1, "sequence": 2005, "entries": rows}), encoding="utf-8")
|
||||
log = self.log()
|
||||
self.assertEqual(len(log.load()), 2000)
|
||||
last = self.append(log, "last")
|
||||
rows = json.loads(self.path.read_bytes())["entries"]
|
||||
self.assertEqual(len(rows), 2000)
|
||||
self.assertEqual(rows[0]["seq"], 7)
|
||||
self.assertEqual(rows[-1], last)
|
||||
|
||||
def test_unicode_message_and_total_encoded_byte_limits(self):
|
||||
log = self.log(max_bytes=16384)
|
||||
for _ in range(12):
|
||||
last = self.append(log, "操作\n" * 16000)
|
||||
self.assertEqual(len(last["message"]), MAX_MESSAGE_CHARS)
|
||||
self.assertNotIn("\n", last["message"])
|
||||
self.assertLessEqual(self.path.stat().st_size, 16384)
|
||||
self.assertLess(len(log.load()), 12)
|
||||
self.assertEqual(self.log(max_bytes=16384).load(), log.load())
|
||||
self.assertEqual(log.load()[-1], last)
|
||||
|
||||
def test_caller_redaction_and_whitelist_exclude_credentials_payloads_paths(self):
|
||||
secret, password, local_path = "synthetic-token-value", "synthetic-password", str(self.root)
|
||||
|
||||
def safe_text(value):
|
||||
for sensitive in (secret, password, local_path):
|
||||
value = value.replace(sensitive, "<redacted>")
|
||||
return value
|
||||
|
||||
entry = {"message": safe_text("operation " + secret + " " + password + " " + local_path),
|
||||
"token": secret, "password": password, "path": local_path,
|
||||
"headers": {"Authorization": secret}, "request": {"body": password},
|
||||
"response": {"raw": secret}, "seq": 90000, "id": 90000}
|
||||
log = self.log()
|
||||
saved = log.append(entry)
|
||||
self.assertEqual(set(saved), {"time", "message", "debug", "level", "id", "seq"})
|
||||
self.assertEqual(saved["seq"], 1)
|
||||
for filename in (self.path, self.watermark):
|
||||
raw = filename.read_text(encoding="utf-8")
|
||||
for sensitive in (secret, password, local_path, '"headers"', '"request"', '"response"'):
|
||||
self.assertNotIn(sensitive, raw)
|
||||
|
||||
def test_output_copies_and_helper_integer_id(self):
|
||||
log = self.log()
|
||||
entry = self.append(log, "original")
|
||||
entry["message"] = "mutated"
|
||||
snapshot = log.load()
|
||||
snapshot[0]["message"] = "also mutated"
|
||||
self.assertEqual(log.load()[0]["message"], "original")
|
||||
identifier = log.log("second")
|
||||
self.assertIs(type(identifier), int)
|
||||
self.assertEqual(identifier, log.sequence)
|
||||
self.assertEqual(identifier, log.load()[-1]["id"])
|
||||
|
||||
def test_corrupt_snapshot_preserves_debug_cursor_via_watermark(self):
|
||||
log = self.log(sequence_block_size=8)
|
||||
self.append(log)
|
||||
for _ in range(20):
|
||||
self.append(log, debug=True)
|
||||
old_cursor = log.sequence
|
||||
for damaged in (b"{broken", b"\xff\xfe", b"[]", b"null", b'{"schema":99}'):
|
||||
with self.subTest(damaged=damaged):
|
||||
self.path.write_bytes(damaged)
|
||||
restored = self.log(sequence_block_size=8)
|
||||
self.assertEqual(restored.load(), [])
|
||||
self.assertGreater(self.append(restored, "recovered")["seq"], old_cursor)
|
||||
|
||||
def test_corrupt_rows_and_metadata_are_tolerated_and_scrubbed(self):
|
||||
self.path.parent.mkdir(parents=True)
|
||||
rows = [None, 17, {"seq": True, "message": "bad"}, {"seq": -1, "message": "bad"},
|
||||
{"seq": 5, "message": "five", "token": "discard-me"},
|
||||
{"seq": 4, "message": "four", "debug": False},
|
||||
{"seq": 6, "message": "debug", "debug": True},
|
||||
{"seq": 7, "message": "wrong flag", "debug": "false"},
|
||||
{"seq": 8, "message": []}, {"seq": 9, "message": "nine"}]
|
||||
self.path.write_text(json.dumps({"schema": 1, "sequence": 20, "entries": rows}), encoding="utf-8")
|
||||
self.watermark.write_text("broken", encoding="utf-8")
|
||||
log = self.log()
|
||||
self.assertEqual([entry["seq"] for entry in log.load()], [4, 5, 9])
|
||||
self.assertEqual(log.sequence, 20)
|
||||
self.assertEqual(self.append(log)["seq"], 21)
|
||||
self.assertNotIn(b"discard-me", self.path.read_bytes())
|
||||
|
||||
def test_oversized_or_deeply_nested_corrupt_files_are_tolerated(self):
|
||||
self.path.parent.mkdir(parents=True)
|
||||
for damaged in (b" " * (2 * 1024 * 1024 + 1), b"[" * 2000 + b"]" * 2000):
|
||||
self.path.write_bytes(damaged)
|
||||
self.assertEqual(self.log().load(), [])
|
||||
self.assertEqual(self.append(self.log())["seq"], 1)
|
||||
|
||||
def test_atomic_replace_failure_keeps_old_snapshot_and_hides_path(self):
|
||||
log = self.log()
|
||||
first = self.append(log)
|
||||
old = self.path.read_bytes()
|
||||
with patch("ranch_web.activity.os.replace", side_effect=OSError("PRIVATE " + str(self.path))):
|
||||
with self.assertRaises(ActivityLogError) as raised:
|
||||
self.append(log, "not published")
|
||||
self.assertNotIn(str(self.root), str(raised.exception))
|
||||
self.assertTrue(raised.exception.__suppress_context__)
|
||||
self.assertEqual(self.path.read_bytes(), old)
|
||||
self.assertEqual(log.sequence, first["seq"])
|
||||
self.assertEqual(log.load(), [first])
|
||||
self.assertEqual(list(self.path.parent.glob("*.tmp")), [])
|
||||
self.assertEqual(self.append(log, "retry")["seq"], first["seq"] + 1)
|
||||
|
||||
def test_reservation_failure_does_not_publish_debug_sequence(self):
|
||||
log = self.log()
|
||||
with patch("ranch_web.activity.os.fsync", side_effect=OSError("PRIVATE " + str(self.path))):
|
||||
with self.assertRaisesRegex(ActivityLogError, "Unable to save activity storage"):
|
||||
self.append(log, debug=True)
|
||||
self.assertEqual(log.sequence, 0)
|
||||
self.assertFalse(self.path.exists())
|
||||
self.assertFalse(self.watermark.exists())
|
||||
self.assertEqual(list(self.path.parent.glob("*.tmp")), [])
|
||||
|
||||
def test_atomic_temp_permissions_and_replacement(self):
|
||||
real_replace = os.replace
|
||||
destinations = []
|
||||
|
||||
def inspect_replace(source, destination):
|
||||
source, destination = Path(source), Path(destination)
|
||||
self.assertEqual(source.parent, destination.parent)
|
||||
json.loads(source.read_bytes()) # Always a complete JSON document.
|
||||
if os.name != "nt":
|
||||
self.assertEqual(stat.S_IMODE(source.stat().st_mode), 0o600)
|
||||
destinations.append(destination)
|
||||
return real_replace(source, destination)
|
||||
|
||||
with patch("ranch_web.activity.os.replace", side_effect=inspect_replace), \
|
||||
patch("ranch_web.activity.os.chmod", wraps=os.chmod) as chmod:
|
||||
self.append(self.log())
|
||||
self.assertEqual(destinations, [self.watermark, self.path])
|
||||
self.assertEqual([call.args[1] for call in chmod.call_args_list], [0o600, 0o600])
|
||||
if os.name != "nt":
|
||||
for path in (self.path, self.watermark):
|
||||
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600)
|
||||
|
||||
def test_trace_level_is_not_persisted_even_without_debug_flag(self):
|
||||
log = self.log()
|
||||
entry = self.append(log, "transient", level="trace")
|
||||
self.assertTrue(entry["debug"])
|
||||
self.assertEqual(entry["level"], "debug")
|
||||
self.assertEqual(log.load(), [])
|
||||
self.assertFalse(self.path.exists())
|
||||
|
||||
def test_thread_safe_unique_sequences(self):
|
||||
log = self.log(sequence_block_size=8)
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
entries = list(pool.map(lambda _: self.append(log, debug=True), range(160)))
|
||||
self.assertEqual(sorted(entry["seq"] for entry in entries), list(range(1, 161)))
|
||||
self.assertEqual(log.sequence, 160)
|
||||
self.assertGreater(self.append(self.log())["seq"], 160)
|
||||
|
||||
def test_validation_has_path_free_messages(self):
|
||||
for options in ({"max_entries": 0}, {"max_entries": 2001}, {"max_entries": True},
|
||||
{"max_bytes": 1}, {"max_bytes": 2097153}, {"sequence_block_size": 0}):
|
||||
with self.subTest(options=options), self.assertRaises(ValueError):
|
||||
self.log(**options)
|
||||
for entry in (None, {}, {"message": self.path}, {"message": "safe", "debug": "false"}):
|
||||
with self.subTest(entry=entry), self.assertRaises(ValueError) as raised:
|
||||
self.log().append(entry)
|
||||
self.assertNotIn(str(self.root), str(raised.exception))
|
||||
|
||||
def test_unreadable_storage_error_hides_path(self):
|
||||
with patch("pathlib.Path.open", side_effect=PermissionError("PRIVATE " + str(self.path))):
|
||||
with self.assertRaises(ActivityLogError) as raised:
|
||||
self.log().load()
|
||||
self.assertEqual(str(raised.exception), "Unable to read activity storage")
|
||||
self.assertTrue(raised.exception.__suppress_context__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -185,6 +185,16 @@ class ApiTests(unittest.TestCase):
|
||||
self.assertEqual(self.client.delete(f"/api/accounts/{response.json()['id']}").status_code, 204)
|
||||
self.assertEqual(self.client.get(f"/api/accounts/{response.json()['id']}/status").status_code, 404)
|
||||
|
||||
def test_connection_metadata_survives_summary_dto(self):
|
||||
metadata = {"connection_state": "retrying", "connection_message": "网络请求超时;策略已停止,正在只读重试",
|
||||
"last_game_success_at": "2026-11-11T02:05:36.000+00:00", "next_retry_at": "2026-11-11T02:05:41.000+00:00"}
|
||||
self.manager.accounts[self.account_id].update(metadata)
|
||||
for url in (self.url + "/status", "/api/accounts"):
|
||||
response = self.client.get(url)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
summary = response.json()[0] if url == "/api/accounts" else response.json()
|
||||
self.assertEqual({key: summary[key] for key in metadata}, metadata)
|
||||
|
||||
def test_game_login_is_queued_and_credentials_stay_server_side(self):
|
||||
response = self.client.post(self.url + "/actions/login", json={"account": "fake-account", "password": "GAME_PASSWORD", "remember": True})
|
||||
self.assertEqual(response.status_code, 202)
|
||||
|
||||
@@ -0,0 +1,442 @@
|
||||
"""Connection regressions through real workers and an entirely offline transport."""
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import requests
|
||||
|
||||
from ranch_assistant.api import ApiError, encode
|
||||
from ranch_web.activity import ActivityLogError
|
||||
from ranch_web.service import WebManager
|
||||
from ranch_web.scheduler import Scheduler
|
||||
from ranch_web.tests import test_service as fixtures
|
||||
|
||||
|
||||
class ConnectionStateTests(unittest.TestCase):
|
||||
setUp = fixtures.ServiceTests.setUp
|
||||
account = fixtures.ServiceTests.account
|
||||
idle = fixtures.ServiceTests.idle
|
||||
job = fixtures.ServiceTests.job
|
||||
login = fixtures.ServiceTests.login
|
||||
enable = fixtures.ServiceTests.enable
|
||||
|
||||
def connected(self, *, running=False):
|
||||
identifier = self.account(False)
|
||||
self.login(identifier)
|
||||
worker = self.manager._worker(identifier)
|
||||
self.now = 1000.0
|
||||
worker.schedule.clock = lambda: self.now
|
||||
if running:
|
||||
self.enable(identifier)
|
||||
self.job(identifier, self.manager.submit(identifier, "start"))
|
||||
return identifier, worker
|
||||
|
||||
def tick(self, identifier, worker, *, status="succeeded"):
|
||||
return self.job(identifier, worker.enqueue("__tick"), status=status)
|
||||
|
||||
def expiry(self, worker):
|
||||
return SimpleNamespace(status_code=200, headers={},
|
||||
content=encode({"resCode": 408}, worker.engine.client.session.key))
|
||||
|
||||
def test_expired_state_survives_idle_view_config_and_blocks_game_requests(self):
|
||||
identifier, worker = self.connected(running=True)
|
||||
original_client, original_snapshot = worker.engine.client, worker.engine.snapshot
|
||||
with patch.object(original_client.http, "post", return_value=self.expiry(worker)) as post:
|
||||
worker.engine.last_heartbeat = 0
|
||||
self.tick(identifier, worker, status="failed")
|
||||
state = self.manager.status(identifier)
|
||||
self.assertEqual(state.get("connection_state"), "expired")
|
||||
self.assertIn("失效", state["connection_message"])
|
||||
self.assertFalse(state["connected"])
|
||||
self.assertFalse(state["running"])
|
||||
reason = state["error"]
|
||||
for _ in range(3):
|
||||
self.now += 600
|
||||
self.tick(identifier, worker)
|
||||
view = self.manager.view(identifier, "bag")
|
||||
self.job(identifier, {"id": view["job_id"]})
|
||||
config = self.manager.get_config(identifier)
|
||||
config["config"]["interval"] = 60
|
||||
self.job(identifier, self.manager.set_config(identifier, config["config"])["job"])
|
||||
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
|
||||
self.job(identifier, self.manager.submit(identifier, "start"), status="failed")
|
||||
self.assertEqual(post.call_count, 1)
|
||||
self.assertEqual(self.manager.status(identifier)["error"], reason)
|
||||
self.assertEqual(self.manager.status(identifier)["connection_state"], "expired")
|
||||
self.assertIs(worker.engine.client, original_client)
|
||||
self.assertIs(worker.engine.snapshot, original_snapshot)
|
||||
self.assertIsNotNone(worker.securestore.load(worker.active_identity))
|
||||
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}))
|
||||
self.assertEqual(self.manager.status(identifier)["connection_state"], "online")
|
||||
self.assertFalse(self.manager.status(identifier)["running"])
|
||||
|
||||
def test_timeout_retries_only_snapshot_and_keeps_uncertain_write_evidence(self):
|
||||
identifier, worker = self.connected(running=True)
|
||||
journal = worker.engine.battle_journal
|
||||
pending = journal.begin(worker.engine.snapshot, {"kind": "battle", "description": "未确认的战斗",
|
||||
"path": "stageService/stageStart", "payload": {}, "stage_id": "0101", "difficulty": "1"})
|
||||
journal.update(pending, "unknown", message="未确认的战斗写请求")
|
||||
evidence = journal.path
|
||||
before = evidence.read_bytes()
|
||||
client = worker.engine.client
|
||||
|
||||
def write(runtime, operation, payload):
|
||||
runtime.engine.client.call("monsterService/monsterLock", {"monsterId": 1})
|
||||
|
||||
with patch("ranch_web.actions.execute_action", side_effect=write):
|
||||
with patch.object(client.http, "post", side_effect=requests.ReadTimeout("synthetic timeout")) as post:
|
||||
self.job(identifier, self.manager.submit(identifier, "lock"), status="failed")
|
||||
self.assertEqual(post.call_count, 1)
|
||||
state = self.manager.status(identifier)
|
||||
self.assertEqual(state.get("connection_state"), "retrying")
|
||||
self.assertNotIn("失效", state["connection_message"])
|
||||
self.assertFalse(state["running"])
|
||||
self.assertIsNotNone(state["next_retry_at"])
|
||||
calls = len(fixtures.FakeClient.calls)
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(len(fixtures.FakeClient.calls), calls)
|
||||
self.now += 61
|
||||
self.tick(identifier, worker)
|
||||
paths = [url.rsplit("/", 2)[-2:] for _, url, _ in fixtures.FakeClient.calls[calls:]]
|
||||
self.assertEqual(paths, [["masterService", "masterQueryDetail"]])
|
||||
self.assertEqual(self.manager.status(identifier)["connection_state"], "online")
|
||||
self.assertEqual(self.manager.status(identifier)["error"], "")
|
||||
self.assertIsNone(self.manager.status(identifier)["next_retry_at"])
|
||||
self.assertIsNotNone(self.manager.status(identifier)["last_game_success_at"])
|
||||
self.assertFalse(worker.running)
|
||||
self.assertEqual(journal.pending(worker.engine.snapshot)["attempt_id"], pending["attempt_id"])
|
||||
self.assertEqual(evidence.read_bytes(), before)
|
||||
# Continue through the ordinary monitor/reconcile path after recovery.
|
||||
# The existing Engine barrier must still prevent the unknown write.
|
||||
worker.engine.last_heartbeat = self.now
|
||||
worker.schedule.next_poll = 0
|
||||
self.now += 121
|
||||
self.tick(identifier, worker)
|
||||
paths = [url.rsplit("/", 2)[-2:] for _, url, _ in fixtures.FakeClient.calls[calls:]]
|
||||
self.assertEqual(paths, [["masterService", "masterQueryDetail"]] * 2)
|
||||
self.assertEqual(journal.pending(worker.engine.snapshot)["attempt_id"], pending["attempt_id"])
|
||||
self.assertEqual(journal.pending(worker.engine.snapshot)["phase"], "unknown")
|
||||
self.assertFalse(worker.running)
|
||||
|
||||
def test_business_rejection_keeps_connection_and_idle_does_not_clear_error(self):
|
||||
identifier, worker = self.connected(running=True)
|
||||
client = worker.engine.client
|
||||
response = SimpleNamespace(status_code=200, headers={},
|
||||
content=encode({"resCode": 403}, client.session.key))
|
||||
with patch.object(client.http, "post", return_value=response):
|
||||
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
|
||||
state = self.manager.status(identifier)
|
||||
self.assertEqual(state["connection_state"], "online")
|
||||
self.assertTrue(state["connected"])
|
||||
self.assertFalse(state["running"])
|
||||
self.assertFalse(worker.schedule.game_paused)
|
||||
self.assertTrue(worker.schedule.monitoring)
|
||||
reason = state["error"]
|
||||
self.assertIn("403", reason)
|
||||
worker.engine.last_heartbeat = self.now
|
||||
worker.schedule.next_poll = float("inf")
|
||||
with patch.object(worker.engine, "master_sync_due", return_value=False), patch.object(worker.engine, "growth_sync_due", return_value=False):
|
||||
self.tick(identifier, worker)
|
||||
view = self.manager.view(identifier, "bag")
|
||||
self.job(identifier, {"id": view["job_id"]})
|
||||
self.assertEqual(self.manager.status(identifier)["error"], reason)
|
||||
|
||||
def test_997_is_retrying_without_expiry_and_backoff_is_bounded(self):
|
||||
identifier, worker = self.connected()
|
||||
response = SimpleNamespace(status_code=200, headers={},
|
||||
content=encode({"resCode": 997}, worker.engine.client.session.key))
|
||||
with patch.object(worker.engine.client.http, "post", return_value=response) as post:
|
||||
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
|
||||
for delay in (5, 10, 20, 40, 60, 60):
|
||||
self.assertEqual(worker.schedule.retry_deadline - self.now, delay)
|
||||
state = self.manager.status(identifier)
|
||||
self.assertEqual(state["connection_state"], "retrying")
|
||||
self.assertNotIn("失效", state["connection_message"])
|
||||
calls = post.call_count
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(post.call_count, calls)
|
||||
self.now += delay
|
||||
self.tick(identifier, worker, status="failed")
|
||||
self.assertEqual(post.call_count, calls + 1)
|
||||
|
||||
def test_swallowed_expiry_still_stops_strategy_and_marks_job_failed(self):
|
||||
identifier, worker = self.connected(running=True)
|
||||
|
||||
def swallowed(runtime, operation, payload):
|
||||
try:
|
||||
runtime.engine.client.call("monsterService/monsterLock")
|
||||
except ApiError:
|
||||
return {"message": "Engine caught the error"}
|
||||
|
||||
with patch("ranch_web.actions.execute_action", side_effect=swallowed):
|
||||
with patch.object(worker.engine.client.http, "post", return_value=self.expiry(worker)):
|
||||
result = self.job(identifier, self.manager.submit(identifier, "lock"), status="failed")
|
||||
self.assertIn("失效", result["error"])
|
||||
self.assertEqual(self.manager.status(identifier)["connection_state"], "expired")
|
||||
self.assertFalse(worker.running)
|
||||
|
||||
def test_scheduler_runs_fake_heartbeat_and_poll_without_any_browser_requests(self):
|
||||
identifier, worker = self.connected()
|
||||
heartbeat, poll = threading.Event(), threading.Event()
|
||||
original = worker.engine.client.http.post
|
||||
|
||||
def post(url, *args, **kwargs):
|
||||
result = original(url, *args, **kwargs)
|
||||
if url.endswith("heartBeatService/keepAlive"):
|
||||
heartbeat.set()
|
||||
if url.endswith("masterService/masterQueryDetail"):
|
||||
poll.set()
|
||||
return result
|
||||
|
||||
worker.engine.last_heartbeat = 0
|
||||
worker.schedule.next_poll = 0
|
||||
with patch.object(worker.engine.client.http, "post", side_effect=post):
|
||||
scheduler = Scheduler(self.manager._worker_list, interval=.01)
|
||||
try:
|
||||
self.assertTrue(heartbeat.wait(3), "No heartbeat without browser polling")
|
||||
self.assertTrue(poll.wait(3), "No game poll without browser polling")
|
||||
finally:
|
||||
scheduler.close()
|
||||
self.idle(identifier)
|
||||
self.assertFalse(worker.running)
|
||||
|
||||
def test_idle_local_success_does_not_clear_game_error(self):
|
||||
identifier, worker = self.connected()
|
||||
with patch.object(worker.engine.client.http, "post", side_effect=requests.Timeout("offline")):
|
||||
self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
|
||||
reason = self.manager.status(identifier)["error"]
|
||||
self.assertTrue(reason)
|
||||
self.tick(identifier, worker)
|
||||
view = self.manager.view(identifier, "bag")
|
||||
self.job(identifier, {"id": view["job_id"]})
|
||||
self.job(identifier, self.manager.submit(identifier, "refresh", {"page": "statistics"}))
|
||||
self.assertEqual(self.manager.status(identifier)["error"], reason)
|
||||
self.assertFalse(self.manager.status(identifier)["connected"])
|
||||
|
||||
def test_failed_restore_candidate_does_not_poison_online_client(self):
|
||||
identifier, worker = self.connected()
|
||||
client = worker.engine.client
|
||||
original = deepcopy(self.manager.status(identifier))
|
||||
response = self.expiry(worker)
|
||||
with patch.object(fixtures.FakeClient, "_post", return_value=response):
|
||||
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}), status="failed")
|
||||
self.assertIs(worker.engine.client, client)
|
||||
self.assertFalse(client.closed)
|
||||
state = self.manager.status(identifier)
|
||||
self.assertEqual(state.get("connection_state"), "online")
|
||||
self.assertEqual(state["last_game_success_at"], original["last_game_success_at"])
|
||||
self.assertEqual(state["connection_message"], original["connection_message"])
|
||||
self.assertTrue(state["connected"])
|
||||
|
||||
def test_failed_same_protocol_restore_preserves_counter_high_water_before_wire_and_finally(self):
|
||||
identifier, worker = self.connected()
|
||||
for active_reqid, active_reqsid in ((400, 900), (500, 1200)):
|
||||
with self.subTest(active_reqid=active_reqid):
|
||||
client = worker.engine.client
|
||||
saved = deepcopy(client.session)
|
||||
saved.reqid, saved.reqsid = 400, 900
|
||||
worker.securestore.save(saved, "离线高水位回归")
|
||||
client.session.reqid, client.session.reqsid = active_reqid, active_reqsid
|
||||
wire = []
|
||||
|
||||
def post(url, data, **kwargs):
|
||||
wire.append(fixtures.decode(data, client.session.key))
|
||||
return self.expiry(worker)
|
||||
|
||||
with patch.object(fixtures.FakeClient, "_post", side_effect=post):
|
||||
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}), status="failed")
|
||||
self.assertEqual((wire[0]["reqid"], wire[0]["reqsid"]), (active_reqid, active_reqsid))
|
||||
persisted = worker.securestore.load(worker.active_identity)
|
||||
self.assertEqual((persisted.reqid, persisted.reqsid), (active_reqid + 1, active_reqsid + 1))
|
||||
self.assertEqual((client.session.reqid, client.session.reqsid), (persisted.reqid, persisted.reqsid))
|
||||
client.call("heartBeatService/keepAlive")
|
||||
payload = fixtures.FakeClient.calls[-1][2]
|
||||
self.assertEqual((payload["reqid"], payload["reqsid"]), (active_reqid + 1, active_reqsid + 1))
|
||||
|
||||
def test_counter_merge_requires_exact_protocol_session(self):
|
||||
identifier, worker = self.connected()
|
||||
current = worker.engine.client.session
|
||||
current.reqid, current.reqsid = 500, 900
|
||||
changes = {"token": "different", "key": "different", "server_url": "https://other.invalid/",
|
||||
"account_id": "different", "server_id": 9, "role_id": 999, "channel": "other", "platform": "other"}
|
||||
for field, value in changes.items():
|
||||
candidate = deepcopy(current)
|
||||
candidate.reqid, candidate.reqsid = 800, 1800
|
||||
setattr(candidate, field, value)
|
||||
worker._share_session_counters(candidate)
|
||||
self.assertEqual((current.reqid, current.reqsid), (500, 900), field)
|
||||
self.assertEqual((candidate.reqid, candidate.reqsid), (800, 1800), field)
|
||||
|
||||
def test_first_restore_timeout_without_client_requires_manual_restore(self):
|
||||
identifier, worker = self.connected()
|
||||
identity = worker.active_identity
|
||||
self.manager.close()
|
||||
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
|
||||
self.addCleanup(self.manager.close)
|
||||
self.idle(identifier)
|
||||
worker = self.manager._worker(identifier)
|
||||
with patch.object(fixtures.FakeClient, "_post", side_effect=requests.ReadTimeout("offline")) as post:
|
||||
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": identity}), status="failed")
|
||||
for _ in range(3):
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(post.call_count, 1)
|
||||
state = self.manager.status(identifier)
|
||||
self.assertEqual(state["connection_state"], "offline")
|
||||
self.assertIn("手动", state["connection_message"])
|
||||
self.assertIsNone(state["next_retry_at"])
|
||||
self.assertIsNone(worker.schedule.retry_deadline)
|
||||
self.assertIsNone(worker.engine.client)
|
||||
|
||||
def test_activity_restart_cursor_survives_debug_only_tail_and_records_reason(self):
|
||||
identifier, worker = self.connected()
|
||||
worker.log("可持久保存的运行事件")
|
||||
worker.log("不持久保存的调试内容", debug=True)
|
||||
old_cursor = self.manager.logs(identifier, debug=True)["next"]
|
||||
self.manager.close()
|
||||
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
|
||||
self.addCleanup(self.manager.close)
|
||||
self.idle(identifier)
|
||||
logs = self.manager.logs(identifier, after=old_cursor)
|
||||
self.assertGreater(logs["next"], old_cursor)
|
||||
self.assertTrue(any("重启" in row["message"] and "恢复" in row["message"] for row in logs["entries"]))
|
||||
all_logs = self.manager.logs(identifier, debug=True)["entries"]
|
||||
self.assertTrue(any("可持久保存的运行事件" == row["message"] for row in all_logs))
|
||||
self.assertFalse(any("不持久保存的调试内容" == row["message"] for row in all_logs))
|
||||
|
||||
def test_activity_write_failure_keeps_memory_logs_and_original_game_outcome(self):
|
||||
identifier, worker = self.connected()
|
||||
with patch.object(worker._activity, "append", side_effect=ActivityLogError("Unable to save activity storage")):
|
||||
with patch.object(worker.engine.client.http, "post", return_value=self.expiry(worker)):
|
||||
result = self.job(identifier, self.manager.submit(identifier, "refresh"), status="failed")
|
||||
self.assertIn("失效", result["error"])
|
||||
self.assertEqual(self.manager.status(identifier)["connection_state"], "expired")
|
||||
logs = self.manager.logs(identifier, debug=True)["entries"]
|
||||
self.assertTrue(any("日志无法持久保存" in entry["message"] for entry in logs))
|
||||
self.assertEqual(len({entry["seq"] for entry in logs}), len(logs))
|
||||
|
||||
def test_activity_idle_tick_and_view_do_not_write_log_storage(self):
|
||||
identifier, worker = self.connected()
|
||||
worker.engine.last_heartbeat = self.now
|
||||
worker.schedule.next_poll = float("inf")
|
||||
with patch.object(worker.engine, "master_sync_due", return_value=False), patch.object(worker.engine, "growth_sync_due", return_value=False):
|
||||
with patch.object(worker._activity, "append", wraps=worker._activity.append) as append:
|
||||
self.tick(identifier, worker)
|
||||
view = self.manager.view(identifier, "bag")
|
||||
self.job(identifier, {"id": view["job_id"]})
|
||||
append.assert_not_called()
|
||||
|
||||
def test_activity_failure_does_not_turn_successful_game_read_into_failure(self):
|
||||
identifier, worker = self.connected()
|
||||
with patch.object(worker._activity, "append", side_effect=ActivityLogError("Unable to save activity storage")):
|
||||
self.job(identifier, self.manager.submit(identifier, "refresh"))
|
||||
self.assertEqual(self.manager.status(identifier)["connection_state"], "online")
|
||||
self.assertEqual(self.manager.status(identifier)["error"], "")
|
||||
self.assertTrue(any("日志无法持久保存" in entry["message"] for entry in self.manager.logs(identifier)["entries"]))
|
||||
|
||||
def test_activity_recovers_after_long_memory_fallback_without_reusing_cursor(self):
|
||||
identifier, worker = self.connected()
|
||||
activity = worker._activity
|
||||
with patch.object(activity, "_write", side_effect=ActivityLogError("Unable to save activity storage")):
|
||||
worker.log("临时无法保存的运行事件")
|
||||
for _ in range(300):
|
||||
worker.log("磁盘只读期间的调试事件", debug=True)
|
||||
memory_cursor = self.manager.logs(identifier, debug=True)["next"]
|
||||
for _ in range(300):
|
||||
worker.log("磁盘恢复后的调试事件", debug=True)
|
||||
self.assertIs(worker._activity, activity)
|
||||
logs = self.manager.logs(identifier, debug=True)
|
||||
old_cursor = logs["next"]
|
||||
self.assertGreater(old_cursor, memory_cursor)
|
||||
self.assertEqual(len({entry["seq"] for entry in logs["entries"]}), len(logs["entries"]))
|
||||
self.assertEqual(sum("日志无法持久保存" in entry["message"] for entry in logs["entries"]), 1)
|
||||
self.assertEqual(sum("日志持久化已恢复" in entry["message"] for entry in logs["entries"]), 1)
|
||||
self.manager.close()
|
||||
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
|
||||
self.addCleanup(self.manager.close)
|
||||
self.idle(identifier)
|
||||
after_restart = self.manager.logs(identifier, after=old_cursor)
|
||||
self.assertTrue(after_restart["entries"])
|
||||
self.assertTrue(all(entry["seq"] > old_cursor for entry in after_restart["entries"]))
|
||||
|
||||
def test_activity_initial_load_failure_retries_on_later_events(self):
|
||||
self.manager.close()
|
||||
with patch("ranch_web.service.ActivityLog.load", side_effect=ActivityLogError("Unable to read activity storage")), \
|
||||
patch("ranch_web.activity.ActivityLog._read", side_effect=ActivityLogError("Unable to read activity storage")):
|
||||
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
|
||||
self.addCleanup(self.manager.close)
|
||||
identifier = self.account(False)
|
||||
worker = self.manager._worker(identifier)
|
||||
self.assertTrue(any("首次读取运行日志失败" in entry["message"] for entry in self.manager.logs(identifier)["entries"]))
|
||||
before = self.manager.logs(identifier)["next"]
|
||||
identifier = worker.log("日志存储重新可用")
|
||||
self.assertGreater(identifier, before)
|
||||
self.assertIsNotNone(worker._activity)
|
||||
self.assertTrue(any("日志持久化已恢复" in entry["message"] for entry in worker._activity.load()))
|
||||
|
||||
def test_activity_initial_load_failure_then_immediate_recovery_has_ordered_messages(self):
|
||||
self.manager.close()
|
||||
with patch("ranch_web.service.ActivityLog.load", side_effect=ActivityLogError("Unable to read activity storage")):
|
||||
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
|
||||
self.addCleanup(self.manager.close)
|
||||
identifier = self.account(False)
|
||||
messages = [entry["message"] for entry in self.manager.logs(identifier)["entries"]]
|
||||
self.assertIn("首次读取运行日志失败", messages[0])
|
||||
self.assertIn("日志持久化已恢复", messages[1])
|
||||
self.assertEqual(messages[2], "账号工作区已载入")
|
||||
self.assertFalse(any("无法持久保存" in message for message in messages[1:]))
|
||||
|
||||
def test_restart_and_explicit_logout_require_manual_restore(self):
|
||||
identifier, worker = self.connected()
|
||||
self.manager.close()
|
||||
calls = len(fixtures.FakeClient.calls)
|
||||
self.manager = WebManager(self.root, start_scheduler=False, login_factory=fixtures.FakeLogin)
|
||||
self.addCleanup(self.manager.close)
|
||||
self.idle(identifier)
|
||||
worker = self.manager._worker(identifier)
|
||||
state = self.manager.status(identifier)
|
||||
self.assertEqual(state.get("connection_state"), "offline")
|
||||
self.assertIn("重启", state["connection_message"])
|
||||
self.assertIn("恢复", state["connection_message"])
|
||||
self.assertFalse(state["running"])
|
||||
self.assertEqual(len(fixtures.FakeClient.calls), calls)
|
||||
self.tick(identifier, worker)
|
||||
self.assertEqual(len(fixtures.FakeClient.calls), calls)
|
||||
self.job(identifier, self.manager.submit(identifier, "restore_session", {"session_id": worker.active_identity}))
|
||||
self.job(identifier, self.manager.submit(identifier, "logout"))
|
||||
calls = len(fixtures.FakeClient.calls)
|
||||
self.now += 10000
|
||||
self.tick(identifier, worker)
|
||||
state = self.manager.status(identifier)
|
||||
self.assertEqual(state["connection_state"], "offline")
|
||||
self.assertIn("退出", state["connection_message"])
|
||||
self.assertFalse(state["running"])
|
||||
self.assertEqual(len(fixtures.FakeClient.calls), calls)
|
||||
|
||||
|
||||
class ErrorClassificationTests(unittest.TestCase):
|
||||
def test_failed_heartbeat_does_not_advance_its_success_clock(self):
|
||||
def fail(*args):
|
||||
raise ApiError("offline")
|
||||
|
||||
engine = fixtures.Engine(fixtures.Catalog(), SimpleNamespace(call=fail))
|
||||
engine.last_heartbeat = 123
|
||||
with self.assertRaises(ApiError):
|
||||
engine.keep_alive()
|
||||
self.assertEqual(engine.last_heartbeat, 123)
|
||||
|
||||
def test_wire_expiry_and_transport_timeout_have_distinct_types(self):
|
||||
client = fixtures.FakeClient(fixtures.Session("https://offline.invalid/game/", "synthetic-token"))
|
||||
self.addCleanup(client.close)
|
||||
response = SimpleNamespace(status_code=200, headers={}, content=encode({"resCode": 408}, client.session.key))
|
||||
with patch.object(client.http, "post", return_value=response):
|
||||
with self.assertRaises(ApiError) as expired:
|
||||
client.call("masterService/masterQueryDetail")
|
||||
with patch.object(client.http, "post", side_effect=requests.Timeout("offline")) as post:
|
||||
with self.assertRaises(ApiError) as temporary:
|
||||
client.call("monsterService/monsterLock")
|
||||
self.assertEqual(post.call_count, 1)
|
||||
self.assertEqual(getattr(expired.exception, "code", None), 408)
|
||||
self.assertNotEqual(type(expired.exception), type(temporary.exception))
|
||||
self.assertIsInstance(temporary.exception.__cause__, requests.Timeout)
|
||||
Reference in New Issue
Block a user