252 lines
12 KiB
Python
252 lines
12 KiB
Python
"""Recovery deadlines and loop isolation with fake clocks and no transports."""
|
|
import math
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
|
|
from ranch_web.scheduler import AccountSchedule, FAST_KINDS, Scheduler
|
|
|
|
|
|
class SchedulerRecoveryTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.now = 1000.0
|
|
self.wall = 5000.0
|
|
self.schedule = AccountSchedule(clock=lambda: self.now,
|
|
wall_clock=lambda: self.wall, uniform=lambda low, high: high)
|
|
self.config = {"interval": 30, "auto_relogin": {"enabled": False}}
|
|
self.snapshot = SimpleNamespace(now=1000, raw={"exploreTeams": []})
|
|
self.engine = SimpleNamespace(demo=False, client=object(), snapshot=self.snapshot,
|
|
heartbeat_enabled=True, last_heartbeat=self.now,
|
|
pending_recovery_due=lambda: False, master_sync_due=lambda: False,
|
|
growth_sync_due=lambda: False, _progression_pending=lambda: None,
|
|
_breakthrough_pending=None, pending_hatches=[object()],
|
|
explore_planner=SimpleNamespace(pending=lambda snapshot: None),
|
|
gift_planner=SimpleNamespace(next_ready_at=None))
|
|
self.schedule.monitor()
|
|
|
|
def due(self, running=False):
|
|
return self.schedule.due(self.engine, self.config, running)
|
|
|
|
def test_stopped_strategy_repeats_password_login_at_configured_or_default_delay(self):
|
|
for preference, delay in (({"enabled": True, "delay_minutes": 10}, 600),
|
|
({"enabled": True}, 1800)):
|
|
with self.subTest(preference=preference):
|
|
self.setUp()
|
|
self.config["auto_relogin"] = preference
|
|
self.schedule.disconnected()
|
|
self.schedule.game_failed()
|
|
self.schedule.stop()
|
|
origin = self.schedule.disconnected_at
|
|
for attempt in range(3):
|
|
deadline = self.schedule.auto_relogin_deadline(self.config)
|
|
self.now = deadline - .001
|
|
self.assertIsNone(self.due())
|
|
self.now = deadline
|
|
self.assertEqual(self.due(), "auto_relogin")
|
|
self.now += 7 # Authentication consumes time before failing.
|
|
self.schedule.auto_relogin_failed()
|
|
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), self.now + delay)
|
|
self.assertEqual(self.schedule.disconnected_at, origin)
|
|
self.assertIsNone(self.due())
|
|
|
|
def test_retries_and_settings_changes_keep_original_outage(self):
|
|
self.schedule.disconnected()
|
|
origin = (self.schedule.disconnected_at, self.schedule.disconnected_wall_at)
|
|
self.now += 100
|
|
self.wall += 10000
|
|
self.schedule.game_failed(retry=True)
|
|
self.schedule.disconnected()
|
|
self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10}
|
|
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), 1600)
|
|
self.assertEqual((self.schedule.disconnected_at, self.schedule.disconnected_wall_at), origin)
|
|
self.config["auto_relogin"]["delay_minutes"] = 30
|
|
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), 2800)
|
|
|
|
def test_password_login_due_outranks_read_probe_even_when_stopped(self):
|
|
self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10}
|
|
self.schedule.disconnected()
|
|
self.schedule.game_failed(retry=True)
|
|
self.now += 600
|
|
self.assertEqual(self.due(), "auto_relogin")
|
|
self.config["auto_relogin"]["enabled"] = False
|
|
self.assertEqual(self.due(), "retry")
|
|
|
|
def test_read_probe_backoff_caps_at_sixty_seconds_and_resets_on_recovery(self):
|
|
for delay in (5, 10, 20, 40, 60, 60, 60):
|
|
self.assertEqual(self.schedule.game_failed(retry=True), delay)
|
|
self.assertTrue(self.schedule.monitoring)
|
|
self.assertIsNone(self.due(running=True))
|
|
self.now += delay - .001
|
|
self.assertIsNone(self.due(running=True))
|
|
self.now += .001
|
|
self.assertEqual(self.due(running=True), "retry")
|
|
self.schedule.game_resumed()
|
|
self.assertEqual(self.schedule.game_failed(retry=True), 5)
|
|
|
|
def test_stop_preserves_monitoring_outage_and_pending_probe(self):
|
|
self.schedule.disconnected()
|
|
self.schedule.game_failed(retry=True)
|
|
retry = self.schedule.retry_deadline
|
|
self.schedule.stop()
|
|
self.assertTrue(self.schedule.monitoring)
|
|
self.assertEqual(self.schedule.retry_deadline, retry)
|
|
self.assertEqual(self.schedule.disconnected_at, 1000)
|
|
self.now = retry
|
|
self.assertEqual(self.due(), "retry")
|
|
|
|
def test_recovery_rearms_poll_without_enabling_fast_strategies(self):
|
|
for deadline in (math.inf, self.now + 3600):
|
|
with self.subTest(deadline=deadline):
|
|
self.schedule.game_failed(retry=True)
|
|
self.schedule.next_poll = deadline
|
|
self.schedule.game_resumed()
|
|
self.assertFalse(self.schedule.game_paused)
|
|
self.assertIsNone(self.schedule.retry_deadline)
|
|
self.assertEqual(self.schedule.next_poll, self.now)
|
|
self.assertEqual(self.due(), "poll")
|
|
self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values()))
|
|
|
|
def test_ordinary_success_preserves_armed_monitor_deadline(self):
|
|
for deadline in (self.now - 30, self.now + 30):
|
|
with self.subTest(deadline=deadline):
|
|
self.schedule.next_poll = deadline
|
|
self.schedule.game_resumed()
|
|
self.assertEqual(self.schedule.next_poll, deadline)
|
|
|
|
def test_recovery_keeps_explicitly_disabled_monitor_disabled(self):
|
|
self.schedule.monitor(False)
|
|
self.schedule.game_failed(retry=True)
|
|
self.schedule.game_resumed()
|
|
self.assertFalse(self.schedule.monitoring)
|
|
self.assertEqual(self.schedule.next_poll, math.inf)
|
|
self.assertIsNone(self.due())
|
|
|
|
def test_success_can_preserve_failed_auth_origin_until_verified_recovery(self):
|
|
self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10}
|
|
self.schedule.disconnected()
|
|
self.now += 600
|
|
self.schedule.auto_relogin_failed()
|
|
deadline = self.schedule.auto_relogin_deadline(self.config)
|
|
origin = (self.schedule.disconnected_at, self.schedule.disconnected_wall_at)
|
|
self.schedule.game_resumed(clear_disconnection=False)
|
|
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), deadline)
|
|
self.assertEqual((self.schedule.disconnected_at, self.schedule.disconnected_wall_at), origin)
|
|
self.assertEqual(self.schedule.auto_relogin_failed_at, self.now)
|
|
self.schedule.game_resumed()
|
|
self.assertIsNone(self.schedule.auto_relogin_deadline(self.config))
|
|
self.assertIsNone(self.schedule.auto_relogin_failed_at)
|
|
|
|
def test_continuously_due_maintenance_cannot_starve_snapshot_poll(self):
|
|
for kind in ("heartbeat", "reconcile", "master_sync", "growth_sync"):
|
|
with self.subTest(kind=kind):
|
|
self.setUp()
|
|
if kind == "heartbeat":
|
|
self.engine.last_heartbeat = 0
|
|
else:
|
|
predicate = {"reconcile": "pending_recovery_due", "master_sync": "master_sync_due",
|
|
"growth_sync": "growth_sync_due"}[kind]
|
|
setattr(self.engine, predicate, lambda: True)
|
|
self.assertEqual(self.due(), kind)
|
|
self.schedule.game_resumed(clear_disconnection=False)
|
|
self.assertEqual(self.due(), "poll")
|
|
self.schedule.poll_finished(self.config)
|
|
self.now += 30
|
|
self.assertEqual(self.due(), kind)
|
|
self.assertEqual(self.due(), "poll")
|
|
|
|
def test_continuously_due_hatch_cannot_starve_snapshot_poll(self):
|
|
self.schedule.next_fast["hatch"] = self.now
|
|
self.assertEqual(self.due(running=True), "hatch")
|
|
self.assertEqual(self.due(running=True), "poll")
|
|
|
|
def test_failed_poll_cools_down_even_when_random_interval_is_zero(self):
|
|
self.schedule.uniform = lambda low, high: 0
|
|
self.schedule.maintenance_failed("poll")
|
|
self.schedule.poll_finished(self.config)
|
|
self.assertIsNone(self.due())
|
|
self.now += 29.999
|
|
self.assertIsNone(self.due())
|
|
self.now += .001
|
|
self.assertEqual(self.due(), "poll")
|
|
|
|
def test_failed_fast_kind_cools_down_without_blocking_monitor(self):
|
|
for kind in FAST_KINDS:
|
|
with self.subTest(kind=kind):
|
|
self.setUp()
|
|
self.schedule.next_fast[kind] = self.now
|
|
self.schedule.maintenance_failed(kind)
|
|
self.assertEqual(self.due(running=True), "poll")
|
|
self.schedule.next_poll = math.inf
|
|
self.now += 29.999
|
|
self.assertIsNone(self.due(running=True))
|
|
self.now += .001
|
|
self.assertEqual(self.due(running=True), kind)
|
|
|
|
def test_paused_session_allows_no_maintenance_or_strategy_without_probe(self):
|
|
self.engine.last_heartbeat = 0
|
|
self.schedule.next_fast = dict.fromkeys(FAST_KINDS, self.now)
|
|
self.schedule.game_failed()
|
|
self.now += 10000
|
|
self.assertIsNone(self.due(running=True))
|
|
|
|
def test_paused_game_does_not_rearm_fast_tasks_from_durable_run_intent(self):
|
|
self.config["policy"] = {"enabled": True, "fill": True}
|
|
self.schedule.game_failed(retry=True)
|
|
self.schedule.arm(self.engine, self.config, True)
|
|
self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values()))
|
|
|
|
def test_recovery_keeps_all_uncertain_write_barriers(self):
|
|
unknown = {"phase": "unknown", "attempt_id": "offline-attempt"}
|
|
self.config["policy"] = {"enabled": True, "fill": True}
|
|
for kind in ("progression", "explore", "breakthrough"):
|
|
with self.subTest(kind=kind):
|
|
self.engine._progression_pending = lambda: unknown if kind == "progression" else None
|
|
self.engine.explore_planner.pending = lambda snapshot: unknown if kind == "explore" else None
|
|
self.engine._breakthrough_pending = unknown if kind == "breakthrough" else None
|
|
self.schedule.game_failed(retry=True)
|
|
self.schedule.game_resumed()
|
|
self.schedule.arm(self.engine, self.config, True)
|
|
self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values()))
|
|
self.assertEqual(unknown, {"phase": "unknown", "attempt_id": "offline-attempt"})
|
|
|
|
|
|
class SchedulerLoopRecoveryTests(unittest.TestCase):
|
|
def run_passes(self, workers, passes):
|
|
# Run the real loop synchronously. The fake event ends it after exact passes.
|
|
with patch("ranch_web.scheduler.threading.Thread"):
|
|
scheduler = Scheduler(workers)
|
|
scheduler.closed = Mock()
|
|
scheduler.closed.wait.side_effect = [False] * passes + [True]
|
|
with self.assertLogs("ranch_web.scheduler", level="ERROR"):
|
|
scheduler._run()
|
|
return scheduler
|
|
|
|
def test_one_broken_account_does_not_kill_other_account_or_next_pass(self):
|
|
first = SimpleNamespace(request_tick=Mock(side_effect=RuntimeError("offline queue error")))
|
|
second = SimpleNamespace(request_tick=Mock())
|
|
scheduler = self.run_passes(lambda: [first, second], 3)
|
|
self.assertEqual(first.request_tick.call_count, 3)
|
|
self.assertEqual(second.request_tick.call_count, 3)
|
|
self.assertEqual(scheduler.closed.wait.call_count, 4)
|
|
|
|
def test_registry_failure_recovers_on_next_pass(self):
|
|
first, second = SimpleNamespace(request_tick=Mock()), SimpleNamespace(request_tick=Mock())
|
|
workers = Mock(side_effect=[RuntimeError("offline registry error"), [first, second]])
|
|
self.run_passes(workers, 2)
|
|
first.request_tick.assert_called_once_with()
|
|
second.request_tick.assert_called_once_with()
|
|
|
|
def test_lazy_registry_failure_does_not_stop_future_passes(self):
|
|
worker = SimpleNamespace(request_tick=Mock())
|
|
|
|
def workers():
|
|
yield worker
|
|
raise RuntimeError("offline iterator error")
|
|
|
|
self.run_passes(workers, 2)
|
|
self.assertEqual(worker.request_tick.call_count, 2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|