"""Recovery deadlines and loop isolation with fake clocks and no transports.""" import math from types import SimpleNamespace import unittest from unittest.mock import Mock, patch from ranch_web.scheduler import AccountSchedule, FAST_KINDS, Scheduler class SchedulerRecoveryTests(unittest.TestCase): def setUp(self): self.now = 1000.0 self.wall = 5000.0 self.schedule = AccountSchedule(clock=lambda: self.now, wall_clock=lambda: self.wall, uniform=lambda low, high: high) self.config = {"interval": 30, "auto_relogin": {"enabled": False}} self.snapshot = SimpleNamespace(now=1000, raw={"exploreTeams": []}) self.engine = SimpleNamespace(demo=False, client=object(), snapshot=self.snapshot, heartbeat_enabled=True, last_heartbeat=self.now, pending_recovery_due=lambda: False, master_sync_due=lambda: False, growth_sync_due=lambda: False, _progression_pending=lambda: None, _breakthrough_pending=None, pending_hatches=[object()], explore_planner=SimpleNamespace(pending=lambda snapshot: None), gift_planner=SimpleNamespace(next_ready_at=None)) self.schedule.monitor() def due(self, running=False): return self.schedule.due(self.engine, self.config, running) def test_stopped_strategy_repeats_password_login_at_configured_or_default_delay(self): for preference, delay in (({"enabled": True, "delay_minutes": 10}, 600), ({"enabled": True}, 1800)): with self.subTest(preference=preference): self.setUp() self.config["auto_relogin"] = preference self.schedule.disconnected() self.schedule.game_failed() self.schedule.stop() origin = self.schedule.disconnected_at for attempt in range(3): deadline = self.schedule.auto_relogin_deadline(self.config) self.now = deadline - .001 self.assertIsNone(self.due()) self.now = deadline self.assertEqual(self.due(), "auto_relogin") self.now += 7 # Authentication consumes time before failing. self.schedule.auto_relogin_failed() self.assertEqual(self.schedule.auto_relogin_deadline(self.config), self.now + delay) self.assertEqual(self.schedule.disconnected_at, origin) self.assertIsNone(self.due()) def test_retries_and_settings_changes_keep_original_outage(self): self.schedule.disconnected() origin = (self.schedule.disconnected_at, self.schedule.disconnected_wall_at) self.now += 100 self.wall += 10000 self.schedule.game_failed(retry=True) self.schedule.disconnected() self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10} self.assertEqual(self.schedule.auto_relogin_deadline(self.config), 1600) self.assertEqual((self.schedule.disconnected_at, self.schedule.disconnected_wall_at), origin) self.config["auto_relogin"]["delay_minutes"] = 30 self.assertEqual(self.schedule.auto_relogin_deadline(self.config), 2800) def test_password_login_due_outranks_read_probe_even_when_stopped(self): self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10} self.schedule.disconnected() self.schedule.game_failed(retry=True) self.now += 600 self.assertEqual(self.due(), "auto_relogin") self.config["auto_relogin"]["enabled"] = False self.assertEqual(self.due(), "retry") def test_read_probe_backoff_caps_at_sixty_seconds_and_resets_on_recovery(self): for delay in (5, 10, 20, 40, 60, 60, 60): self.assertEqual(self.schedule.game_failed(retry=True), delay) self.assertTrue(self.schedule.monitoring) self.assertIsNone(self.due(running=True)) self.now += delay - .001 self.assertIsNone(self.due(running=True)) self.now += .001 self.assertEqual(self.due(running=True), "retry") self.schedule.game_resumed() self.assertEqual(self.schedule.game_failed(retry=True), 5) def test_stop_preserves_monitoring_outage_and_pending_probe(self): self.schedule.disconnected() self.schedule.game_failed(retry=True) retry = self.schedule.retry_deadline self.schedule.stop() self.assertTrue(self.schedule.monitoring) self.assertEqual(self.schedule.retry_deadline, retry) self.assertEqual(self.schedule.disconnected_at, 1000) self.now = retry self.assertEqual(self.due(), "retry") def test_recovery_rearms_poll_without_enabling_fast_strategies(self): for deadline in (math.inf, self.now + 3600): with self.subTest(deadline=deadline): self.schedule.game_failed(retry=True) self.schedule.next_poll = deadline self.schedule.game_resumed() self.assertFalse(self.schedule.game_paused) self.assertIsNone(self.schedule.retry_deadline) self.assertEqual(self.schedule.next_poll, self.now) self.assertEqual(self.due(), "poll") self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values())) def test_ordinary_success_preserves_armed_monitor_deadline(self): for deadline in (self.now - 30, self.now + 30): with self.subTest(deadline=deadline): self.schedule.next_poll = deadline self.schedule.game_resumed() self.assertEqual(self.schedule.next_poll, deadline) def test_recovery_keeps_explicitly_disabled_monitor_disabled(self): self.schedule.monitor(False) self.schedule.game_failed(retry=True) self.schedule.game_resumed() self.assertFalse(self.schedule.monitoring) self.assertEqual(self.schedule.next_poll, math.inf) self.assertIsNone(self.due()) def test_success_can_preserve_failed_auth_origin_until_verified_recovery(self): self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10} self.schedule.disconnected() self.now += 600 self.schedule.auto_relogin_failed() deadline = self.schedule.auto_relogin_deadline(self.config) origin = (self.schedule.disconnected_at, self.schedule.disconnected_wall_at) self.schedule.game_resumed(clear_disconnection=False) self.assertEqual(self.schedule.auto_relogin_deadline(self.config), deadline) self.assertEqual((self.schedule.disconnected_at, self.schedule.disconnected_wall_at), origin) self.assertEqual(self.schedule.auto_relogin_failed_at, self.now) self.schedule.game_resumed() self.assertIsNone(self.schedule.auto_relogin_deadline(self.config)) self.assertIsNone(self.schedule.auto_relogin_failed_at) def test_continuously_due_maintenance_cannot_starve_snapshot_poll(self): for kind in ("heartbeat", "reconcile", "master_sync", "growth_sync"): with self.subTest(kind=kind): self.setUp() if kind == "heartbeat": self.engine.last_heartbeat = 0 else: predicate = {"reconcile": "pending_recovery_due", "master_sync": "master_sync_due", "growth_sync": "growth_sync_due"}[kind] setattr(self.engine, predicate, lambda: True) self.assertEqual(self.due(), kind) self.schedule.game_resumed(clear_disconnection=False) self.assertEqual(self.due(), "poll") self.schedule.poll_finished(self.config) self.now += 30 self.assertEqual(self.due(), kind) self.assertEqual(self.due(), "poll") def test_continuously_due_hatch_cannot_starve_snapshot_poll(self): self.schedule.next_fast["hatch"] = self.now self.assertEqual(self.due(running=True), "hatch") self.assertEqual(self.due(running=True), "poll") def test_failed_poll_cools_down_even_when_random_interval_is_zero(self): self.schedule.uniform = lambda low, high: 0 self.schedule.maintenance_failed("poll") self.schedule.poll_finished(self.config) self.assertIsNone(self.due()) self.now += 29.999 self.assertIsNone(self.due()) self.now += .001 self.assertEqual(self.due(), "poll") def test_failed_fast_kind_cools_down_without_blocking_monitor(self): for kind in FAST_KINDS: with self.subTest(kind=kind): self.setUp() self.schedule.next_fast[kind] = self.now self.schedule.maintenance_failed(kind) self.assertEqual(self.due(running=True), "poll") self.schedule.next_poll = math.inf self.now += 29.999 self.assertIsNone(self.due(running=True)) self.now += .001 self.assertEqual(self.due(running=True), kind) def test_paused_session_allows_no_maintenance_or_strategy_without_probe(self): self.engine.last_heartbeat = 0 self.schedule.next_fast = dict.fromkeys(FAST_KINDS, self.now) self.schedule.game_failed() self.now += 10000 self.assertIsNone(self.due(running=True)) def test_paused_game_does_not_rearm_fast_tasks_from_durable_run_intent(self): self.config["policy"] = {"enabled": True, "fill": True} self.schedule.game_failed(retry=True) self.schedule.arm(self.engine, self.config, True) self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values())) def test_recovery_keeps_all_uncertain_write_barriers(self): unknown = {"phase": "unknown", "attempt_id": "offline-attempt"} self.config["policy"] = {"enabled": True, "fill": True} for kind in ("progression", "explore", "breakthrough"): with self.subTest(kind=kind): self.engine._progression_pending = lambda: unknown if kind == "progression" else None self.engine.explore_planner.pending = lambda snapshot: unknown if kind == "explore" else None self.engine._breakthrough_pending = unknown if kind == "breakthrough" else None self.schedule.game_failed(retry=True) self.schedule.game_resumed() self.schedule.arm(self.engine, self.config, True) self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values())) self.assertEqual(unknown, {"phase": "unknown", "attempt_id": "offline-attempt"}) class SchedulerLoopRecoveryTests(unittest.TestCase): def run_passes(self, workers, passes): # Run the real loop synchronously. The fake event ends it after exact passes. with patch("ranch_web.scheduler.threading.Thread"): scheduler = Scheduler(workers) scheduler.closed = Mock() scheduler.closed.wait.side_effect = [False] * passes + [True] with self.assertLogs("ranch_web.scheduler", level="ERROR"): scheduler._run() return scheduler def test_one_broken_account_does_not_kill_other_account_or_next_pass(self): first = SimpleNamespace(request_tick=Mock(side_effect=RuntimeError("offline queue error"))) second = SimpleNamespace(request_tick=Mock()) scheduler = self.run_passes(lambda: [first, second], 3) self.assertEqual(first.request_tick.call_count, 3) self.assertEqual(second.request_tick.call_count, 3) self.assertEqual(scheduler.closed.wait.call_count, 4) def test_registry_failure_recovers_on_next_pass(self): first, second = SimpleNamespace(request_tick=Mock()), SimpleNamespace(request_tick=Mock()) workers = Mock(side_effect=[RuntimeError("offline registry error"), [first, second]]) self.run_passes(workers, 2) first.request_tick.assert_called_once_with() second.request_tick.assert_called_once_with() def test_lazy_registry_failure_does_not_stop_future_passes(self): worker = SimpleNamespace(request_tick=Mock()) def workers(): yield worker raise RuntimeError("offline iterator error") self.run_passes(workers, 2) self.assertEqual(worker.request_tick.call_count, 2) if __name__ == "__main__": unittest.main()