fix: preserve strategy intent and recover accounts automatically
This commit is contained in:
@@ -0,0 +1,251 @@
|
||||
"""Recovery deadlines and loop isolation with fake clocks and no transports."""
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from ranch_web.scheduler import AccountSchedule, FAST_KINDS, Scheduler
|
||||
|
||||
|
||||
class SchedulerRecoveryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.now = 1000.0
|
||||
self.wall = 5000.0
|
||||
self.schedule = AccountSchedule(clock=lambda: self.now,
|
||||
wall_clock=lambda: self.wall, uniform=lambda low, high: high)
|
||||
self.config = {"interval": 30, "auto_relogin": {"enabled": False}}
|
||||
self.snapshot = SimpleNamespace(now=1000, raw={"exploreTeams": []})
|
||||
self.engine = SimpleNamespace(demo=False, client=object(), snapshot=self.snapshot,
|
||||
heartbeat_enabled=True, last_heartbeat=self.now,
|
||||
pending_recovery_due=lambda: False, master_sync_due=lambda: False,
|
||||
growth_sync_due=lambda: False, _progression_pending=lambda: None,
|
||||
_breakthrough_pending=None, pending_hatches=[object()],
|
||||
explore_planner=SimpleNamespace(pending=lambda snapshot: None),
|
||||
gift_planner=SimpleNamespace(next_ready_at=None))
|
||||
self.schedule.monitor()
|
||||
|
||||
def due(self, running=False):
|
||||
return self.schedule.due(self.engine, self.config, running)
|
||||
|
||||
def test_stopped_strategy_repeats_password_login_at_configured_or_default_delay(self):
|
||||
for preference, delay in (({"enabled": True, "delay_minutes": 10}, 600),
|
||||
({"enabled": True}, 1800)):
|
||||
with self.subTest(preference=preference):
|
||||
self.setUp()
|
||||
self.config["auto_relogin"] = preference
|
||||
self.schedule.disconnected()
|
||||
self.schedule.game_failed()
|
||||
self.schedule.stop()
|
||||
origin = self.schedule.disconnected_at
|
||||
for attempt in range(3):
|
||||
deadline = self.schedule.auto_relogin_deadline(self.config)
|
||||
self.now = deadline - .001
|
||||
self.assertIsNone(self.due())
|
||||
self.now = deadline
|
||||
self.assertEqual(self.due(), "auto_relogin")
|
||||
self.now += 7 # Authentication consumes time before failing.
|
||||
self.schedule.auto_relogin_failed()
|
||||
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), self.now + delay)
|
||||
self.assertEqual(self.schedule.disconnected_at, origin)
|
||||
self.assertIsNone(self.due())
|
||||
|
||||
def test_retries_and_settings_changes_keep_original_outage(self):
|
||||
self.schedule.disconnected()
|
||||
origin = (self.schedule.disconnected_at, self.schedule.disconnected_wall_at)
|
||||
self.now += 100
|
||||
self.wall += 10000
|
||||
self.schedule.game_failed(retry=True)
|
||||
self.schedule.disconnected()
|
||||
self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10}
|
||||
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), 1600)
|
||||
self.assertEqual((self.schedule.disconnected_at, self.schedule.disconnected_wall_at), origin)
|
||||
self.config["auto_relogin"]["delay_minutes"] = 30
|
||||
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), 2800)
|
||||
|
||||
def test_password_login_due_outranks_read_probe_even_when_stopped(self):
|
||||
self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10}
|
||||
self.schedule.disconnected()
|
||||
self.schedule.game_failed(retry=True)
|
||||
self.now += 600
|
||||
self.assertEqual(self.due(), "auto_relogin")
|
||||
self.config["auto_relogin"]["enabled"] = False
|
||||
self.assertEqual(self.due(), "retry")
|
||||
|
||||
def test_read_probe_backoff_caps_at_sixty_seconds_and_resets_on_recovery(self):
|
||||
for delay in (5, 10, 20, 40, 60, 60, 60):
|
||||
self.assertEqual(self.schedule.game_failed(retry=True), delay)
|
||||
self.assertTrue(self.schedule.monitoring)
|
||||
self.assertIsNone(self.due(running=True))
|
||||
self.now += delay - .001
|
||||
self.assertIsNone(self.due(running=True))
|
||||
self.now += .001
|
||||
self.assertEqual(self.due(running=True), "retry")
|
||||
self.schedule.game_resumed()
|
||||
self.assertEqual(self.schedule.game_failed(retry=True), 5)
|
||||
|
||||
def test_stop_preserves_monitoring_outage_and_pending_probe(self):
|
||||
self.schedule.disconnected()
|
||||
self.schedule.game_failed(retry=True)
|
||||
retry = self.schedule.retry_deadline
|
||||
self.schedule.stop()
|
||||
self.assertTrue(self.schedule.monitoring)
|
||||
self.assertEqual(self.schedule.retry_deadline, retry)
|
||||
self.assertEqual(self.schedule.disconnected_at, 1000)
|
||||
self.now = retry
|
||||
self.assertEqual(self.due(), "retry")
|
||||
|
||||
def test_recovery_rearms_poll_without_enabling_fast_strategies(self):
|
||||
for deadline in (math.inf, self.now + 3600):
|
||||
with self.subTest(deadline=deadline):
|
||||
self.schedule.game_failed(retry=True)
|
||||
self.schedule.next_poll = deadline
|
||||
self.schedule.game_resumed()
|
||||
self.assertFalse(self.schedule.game_paused)
|
||||
self.assertIsNone(self.schedule.retry_deadline)
|
||||
self.assertEqual(self.schedule.next_poll, self.now)
|
||||
self.assertEqual(self.due(), "poll")
|
||||
self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values()))
|
||||
|
||||
def test_ordinary_success_preserves_armed_monitor_deadline(self):
|
||||
for deadline in (self.now - 30, self.now + 30):
|
||||
with self.subTest(deadline=deadline):
|
||||
self.schedule.next_poll = deadline
|
||||
self.schedule.game_resumed()
|
||||
self.assertEqual(self.schedule.next_poll, deadline)
|
||||
|
||||
def test_recovery_keeps_explicitly_disabled_monitor_disabled(self):
|
||||
self.schedule.monitor(False)
|
||||
self.schedule.game_failed(retry=True)
|
||||
self.schedule.game_resumed()
|
||||
self.assertFalse(self.schedule.monitoring)
|
||||
self.assertEqual(self.schedule.next_poll, math.inf)
|
||||
self.assertIsNone(self.due())
|
||||
|
||||
def test_success_can_preserve_failed_auth_origin_until_verified_recovery(self):
|
||||
self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10}
|
||||
self.schedule.disconnected()
|
||||
self.now += 600
|
||||
self.schedule.auto_relogin_failed()
|
||||
deadline = self.schedule.auto_relogin_deadline(self.config)
|
||||
origin = (self.schedule.disconnected_at, self.schedule.disconnected_wall_at)
|
||||
self.schedule.game_resumed(clear_disconnection=False)
|
||||
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), deadline)
|
||||
self.assertEqual((self.schedule.disconnected_at, self.schedule.disconnected_wall_at), origin)
|
||||
self.assertEqual(self.schedule.auto_relogin_failed_at, self.now)
|
||||
self.schedule.game_resumed()
|
||||
self.assertIsNone(self.schedule.auto_relogin_deadline(self.config))
|
||||
self.assertIsNone(self.schedule.auto_relogin_failed_at)
|
||||
|
||||
def test_continuously_due_maintenance_cannot_starve_snapshot_poll(self):
|
||||
for kind in ("heartbeat", "reconcile", "master_sync", "growth_sync"):
|
||||
with self.subTest(kind=kind):
|
||||
self.setUp()
|
||||
if kind == "heartbeat":
|
||||
self.engine.last_heartbeat = 0
|
||||
else:
|
||||
predicate = {"reconcile": "pending_recovery_due", "master_sync": "master_sync_due",
|
||||
"growth_sync": "growth_sync_due"}[kind]
|
||||
setattr(self.engine, predicate, lambda: True)
|
||||
self.assertEqual(self.due(), kind)
|
||||
self.schedule.game_resumed(clear_disconnection=False)
|
||||
self.assertEqual(self.due(), "poll")
|
||||
self.schedule.poll_finished(self.config)
|
||||
self.now += 30
|
||||
self.assertEqual(self.due(), kind)
|
||||
self.assertEqual(self.due(), "poll")
|
||||
|
||||
def test_continuously_due_hatch_cannot_starve_snapshot_poll(self):
|
||||
self.schedule.next_fast["hatch"] = self.now
|
||||
self.assertEqual(self.due(running=True), "hatch")
|
||||
self.assertEqual(self.due(running=True), "poll")
|
||||
|
||||
def test_failed_poll_cools_down_even_when_random_interval_is_zero(self):
|
||||
self.schedule.uniform = lambda low, high: 0
|
||||
self.schedule.maintenance_failed("poll")
|
||||
self.schedule.poll_finished(self.config)
|
||||
self.assertIsNone(self.due())
|
||||
self.now += 29.999
|
||||
self.assertIsNone(self.due())
|
||||
self.now += .001
|
||||
self.assertEqual(self.due(), "poll")
|
||||
|
||||
def test_failed_fast_kind_cools_down_without_blocking_monitor(self):
|
||||
for kind in FAST_KINDS:
|
||||
with self.subTest(kind=kind):
|
||||
self.setUp()
|
||||
self.schedule.next_fast[kind] = self.now
|
||||
self.schedule.maintenance_failed(kind)
|
||||
self.assertEqual(self.due(running=True), "poll")
|
||||
self.schedule.next_poll = math.inf
|
||||
self.now += 29.999
|
||||
self.assertIsNone(self.due(running=True))
|
||||
self.now += .001
|
||||
self.assertEqual(self.due(running=True), kind)
|
||||
|
||||
def test_paused_session_allows_no_maintenance_or_strategy_without_probe(self):
|
||||
self.engine.last_heartbeat = 0
|
||||
self.schedule.next_fast = dict.fromkeys(FAST_KINDS, self.now)
|
||||
self.schedule.game_failed()
|
||||
self.now += 10000
|
||||
self.assertIsNone(self.due(running=True))
|
||||
|
||||
def test_paused_game_does_not_rearm_fast_tasks_from_durable_run_intent(self):
|
||||
self.config["policy"] = {"enabled": True, "fill": True}
|
||||
self.schedule.game_failed(retry=True)
|
||||
self.schedule.arm(self.engine, self.config, True)
|
||||
self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values()))
|
||||
|
||||
def test_recovery_keeps_all_uncertain_write_barriers(self):
|
||||
unknown = {"phase": "unknown", "attempt_id": "offline-attempt"}
|
||||
self.config["policy"] = {"enabled": True, "fill": True}
|
||||
for kind in ("progression", "explore", "breakthrough"):
|
||||
with self.subTest(kind=kind):
|
||||
self.engine._progression_pending = lambda: unknown if kind == "progression" else None
|
||||
self.engine.explore_planner.pending = lambda snapshot: unknown if kind == "explore" else None
|
||||
self.engine._breakthrough_pending = unknown if kind == "breakthrough" else None
|
||||
self.schedule.game_failed(retry=True)
|
||||
self.schedule.game_resumed()
|
||||
self.schedule.arm(self.engine, self.config, True)
|
||||
self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values()))
|
||||
self.assertEqual(unknown, {"phase": "unknown", "attempt_id": "offline-attempt"})
|
||||
|
||||
|
||||
class SchedulerLoopRecoveryTests(unittest.TestCase):
|
||||
def run_passes(self, workers, passes):
|
||||
# Run the real loop synchronously. The fake event ends it after exact passes.
|
||||
with patch("ranch_web.scheduler.threading.Thread"):
|
||||
scheduler = Scheduler(workers)
|
||||
scheduler.closed = Mock()
|
||||
scheduler.closed.wait.side_effect = [False] * passes + [True]
|
||||
with self.assertLogs("ranch_web.scheduler", level="ERROR"):
|
||||
scheduler._run()
|
||||
return scheduler
|
||||
|
||||
def test_one_broken_account_does_not_kill_other_account_or_next_pass(self):
|
||||
first = SimpleNamespace(request_tick=Mock(side_effect=RuntimeError("offline queue error")))
|
||||
second = SimpleNamespace(request_tick=Mock())
|
||||
scheduler = self.run_passes(lambda: [first, second], 3)
|
||||
self.assertEqual(first.request_tick.call_count, 3)
|
||||
self.assertEqual(second.request_tick.call_count, 3)
|
||||
self.assertEqual(scheduler.closed.wait.call_count, 4)
|
||||
|
||||
def test_registry_failure_recovers_on_next_pass(self):
|
||||
first, second = SimpleNamespace(request_tick=Mock()), SimpleNamespace(request_tick=Mock())
|
||||
workers = Mock(side_effect=[RuntimeError("offline registry error"), [first, second]])
|
||||
self.run_passes(workers, 2)
|
||||
first.request_tick.assert_called_once_with()
|
||||
second.request_tick.assert_called_once_with()
|
||||
|
||||
def test_lazy_registry_failure_does_not_stop_future_passes(self):
|
||||
worker = SimpleNamespace(request_tick=Mock())
|
||||
|
||||
def workers():
|
||||
yield worker
|
||||
raise RuntimeError("offline iterator error")
|
||||
|
||||
self.run_passes(workers, 2)
|
||||
self.assertEqual(worker.request_tick.call_count, 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user