Files
myrzx_bot/ranch_web/tests/test_scheduler_recovery.py

252 lines
12 KiB
Python

"""Recovery deadlines and loop isolation with fake clocks and no transports."""
import math
from types import SimpleNamespace
import unittest
from unittest.mock import Mock, patch
from ranch_web.scheduler import AccountSchedule, FAST_KINDS, Scheduler
class SchedulerRecoveryTests(unittest.TestCase):
def setUp(self):
self.now = 1000.0
self.wall = 5000.0
self.schedule = AccountSchedule(clock=lambda: self.now,
wall_clock=lambda: self.wall, uniform=lambda low, high: high)
self.config = {"interval": 30, "auto_relogin": {"enabled": False}}
self.snapshot = SimpleNamespace(now=1000, raw={"exploreTeams": []})
self.engine = SimpleNamespace(demo=False, client=object(), snapshot=self.snapshot,
heartbeat_enabled=True, last_heartbeat=self.now,
pending_recovery_due=lambda: False, master_sync_due=lambda: False,
growth_sync_due=lambda: False, _progression_pending=lambda: None,
_breakthrough_pending=None, pending_hatches=[object()],
explore_planner=SimpleNamespace(pending=lambda snapshot: None),
gift_planner=SimpleNamespace(next_ready_at=None))
self.schedule.monitor()
def due(self, running=False):
return self.schedule.due(self.engine, self.config, running)
def test_stopped_strategy_repeats_password_login_at_configured_or_default_delay(self):
for preference, delay in (({"enabled": True, "delay_minutes": 10}, 600),
({"enabled": True}, 1800)):
with self.subTest(preference=preference):
self.setUp()
self.config["auto_relogin"] = preference
self.schedule.disconnected()
self.schedule.game_failed()
self.schedule.stop()
origin = self.schedule.disconnected_at
for attempt in range(3):
deadline = self.schedule.auto_relogin_deadline(self.config)
self.now = deadline - .001
self.assertIsNone(self.due())
self.now = deadline
self.assertEqual(self.due(), "auto_relogin")
self.now += 7 # Authentication consumes time before failing.
self.schedule.auto_relogin_failed()
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), self.now + delay)
self.assertEqual(self.schedule.disconnected_at, origin)
self.assertIsNone(self.due())
def test_retries_and_settings_changes_keep_original_outage(self):
self.schedule.disconnected()
origin = (self.schedule.disconnected_at, self.schedule.disconnected_wall_at)
self.now += 100
self.wall += 10000
self.schedule.game_failed(retry=True)
self.schedule.disconnected()
self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10}
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), 1600)
self.assertEqual((self.schedule.disconnected_at, self.schedule.disconnected_wall_at), origin)
self.config["auto_relogin"]["delay_minutes"] = 30
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), 2800)
def test_password_login_due_outranks_read_probe_even_when_stopped(self):
self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10}
self.schedule.disconnected()
self.schedule.game_failed(retry=True)
self.now += 600
self.assertEqual(self.due(), "auto_relogin")
self.config["auto_relogin"]["enabled"] = False
self.assertEqual(self.due(), "retry")
def test_read_probe_backoff_caps_at_sixty_seconds_and_resets_on_recovery(self):
for delay in (5, 10, 20, 40, 60, 60, 60):
self.assertEqual(self.schedule.game_failed(retry=True), delay)
self.assertTrue(self.schedule.monitoring)
self.assertIsNone(self.due(running=True))
self.now += delay - .001
self.assertIsNone(self.due(running=True))
self.now += .001
self.assertEqual(self.due(running=True), "retry")
self.schedule.game_resumed()
self.assertEqual(self.schedule.game_failed(retry=True), 5)
def test_stop_preserves_monitoring_outage_and_pending_probe(self):
self.schedule.disconnected()
self.schedule.game_failed(retry=True)
retry = self.schedule.retry_deadline
self.schedule.stop()
self.assertTrue(self.schedule.monitoring)
self.assertEqual(self.schedule.retry_deadline, retry)
self.assertEqual(self.schedule.disconnected_at, 1000)
self.now = retry
self.assertEqual(self.due(), "retry")
def test_recovery_rearms_poll_without_enabling_fast_strategies(self):
for deadline in (math.inf, self.now + 3600):
with self.subTest(deadline=deadline):
self.schedule.game_failed(retry=True)
self.schedule.next_poll = deadline
self.schedule.game_resumed()
self.assertFalse(self.schedule.game_paused)
self.assertIsNone(self.schedule.retry_deadline)
self.assertEqual(self.schedule.next_poll, self.now)
self.assertEqual(self.due(), "poll")
self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values()))
def test_ordinary_success_preserves_armed_monitor_deadline(self):
for deadline in (self.now - 30, self.now + 30):
with self.subTest(deadline=deadline):
self.schedule.next_poll = deadline
self.schedule.game_resumed()
self.assertEqual(self.schedule.next_poll, deadline)
def test_recovery_keeps_explicitly_disabled_monitor_disabled(self):
self.schedule.monitor(False)
self.schedule.game_failed(retry=True)
self.schedule.game_resumed()
self.assertFalse(self.schedule.monitoring)
self.assertEqual(self.schedule.next_poll, math.inf)
self.assertIsNone(self.due())
def test_success_can_preserve_failed_auth_origin_until_verified_recovery(self):
self.config["auto_relogin"] = {"enabled": True, "delay_minutes": 10}
self.schedule.disconnected()
self.now += 600
self.schedule.auto_relogin_failed()
deadline = self.schedule.auto_relogin_deadline(self.config)
origin = (self.schedule.disconnected_at, self.schedule.disconnected_wall_at)
self.schedule.game_resumed(clear_disconnection=False)
self.assertEqual(self.schedule.auto_relogin_deadline(self.config), deadline)
self.assertEqual((self.schedule.disconnected_at, self.schedule.disconnected_wall_at), origin)
self.assertEqual(self.schedule.auto_relogin_failed_at, self.now)
self.schedule.game_resumed()
self.assertIsNone(self.schedule.auto_relogin_deadline(self.config))
self.assertIsNone(self.schedule.auto_relogin_failed_at)
def test_continuously_due_maintenance_cannot_starve_snapshot_poll(self):
for kind in ("heartbeat", "reconcile", "master_sync", "growth_sync"):
with self.subTest(kind=kind):
self.setUp()
if kind == "heartbeat":
self.engine.last_heartbeat = 0
else:
predicate = {"reconcile": "pending_recovery_due", "master_sync": "master_sync_due",
"growth_sync": "growth_sync_due"}[kind]
setattr(self.engine, predicate, lambda: True)
self.assertEqual(self.due(), kind)
self.schedule.game_resumed(clear_disconnection=False)
self.assertEqual(self.due(), "poll")
self.schedule.poll_finished(self.config)
self.now += 30
self.assertEqual(self.due(), kind)
self.assertEqual(self.due(), "poll")
def test_continuously_due_hatch_cannot_starve_snapshot_poll(self):
self.schedule.next_fast["hatch"] = self.now
self.assertEqual(self.due(running=True), "hatch")
self.assertEqual(self.due(running=True), "poll")
def test_failed_poll_cools_down_even_when_random_interval_is_zero(self):
self.schedule.uniform = lambda low, high: 0
self.schedule.maintenance_failed("poll")
self.schedule.poll_finished(self.config)
self.assertIsNone(self.due())
self.now += 29.999
self.assertIsNone(self.due())
self.now += .001
self.assertEqual(self.due(), "poll")
def test_failed_fast_kind_cools_down_without_blocking_monitor(self):
for kind in FAST_KINDS:
with self.subTest(kind=kind):
self.setUp()
self.schedule.next_fast[kind] = self.now
self.schedule.maintenance_failed(kind)
self.assertEqual(self.due(running=True), "poll")
self.schedule.next_poll = math.inf
self.now += 29.999
self.assertIsNone(self.due(running=True))
self.now += .001
self.assertEqual(self.due(running=True), kind)
def test_paused_session_allows_no_maintenance_or_strategy_without_probe(self):
self.engine.last_heartbeat = 0
self.schedule.next_fast = dict.fromkeys(FAST_KINDS, self.now)
self.schedule.game_failed()
self.now += 10000
self.assertIsNone(self.due(running=True))
def test_paused_game_does_not_rearm_fast_tasks_from_durable_run_intent(self):
self.config["policy"] = {"enabled": True, "fill": True}
self.schedule.game_failed(retry=True)
self.schedule.arm(self.engine, self.config, True)
self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values()))
def test_recovery_keeps_all_uncertain_write_barriers(self):
unknown = {"phase": "unknown", "attempt_id": "offline-attempt"}
self.config["policy"] = {"enabled": True, "fill": True}
for kind in ("progression", "explore", "breakthrough"):
with self.subTest(kind=kind):
self.engine._progression_pending = lambda: unknown if kind == "progression" else None
self.engine.explore_planner.pending = lambda snapshot: unknown if kind == "explore" else None
self.engine._breakthrough_pending = unknown if kind == "breakthrough" else None
self.schedule.game_failed(retry=True)
self.schedule.game_resumed()
self.schedule.arm(self.engine, self.config, True)
self.assertTrue(all(math.isinf(value) for value in self.schedule.next_fast.values()))
self.assertEqual(unknown, {"phase": "unknown", "attempt_id": "offline-attempt"})
class SchedulerLoopRecoveryTests(unittest.TestCase):
def run_passes(self, workers, passes):
# Run the real loop synchronously. The fake event ends it after exact passes.
with patch("ranch_web.scheduler.threading.Thread"):
scheduler = Scheduler(workers)
scheduler.closed = Mock()
scheduler.closed.wait.side_effect = [False] * passes + [True]
with self.assertLogs("ranch_web.scheduler", level="ERROR"):
scheduler._run()
return scheduler
def test_one_broken_account_does_not_kill_other_account_or_next_pass(self):
first = SimpleNamespace(request_tick=Mock(side_effect=RuntimeError("offline queue error")))
second = SimpleNamespace(request_tick=Mock())
scheduler = self.run_passes(lambda: [first, second], 3)
self.assertEqual(first.request_tick.call_count, 3)
self.assertEqual(second.request_tick.call_count, 3)
self.assertEqual(scheduler.closed.wait.call_count, 4)
def test_registry_failure_recovers_on_next_pass(self):
first, second = SimpleNamespace(request_tick=Mock()), SimpleNamespace(request_tick=Mock())
workers = Mock(side_effect=[RuntimeError("offline registry error"), [first, second]])
self.run_passes(workers, 2)
first.request_tick.assert_called_once_with()
second.request_tick.assert_called_once_with()
def test_lazy_registry_failure_does_not_stop_future_passes(self):
worker = SimpleNamespace(request_tick=Mock())
def workers():
yield worker
raise RuntimeError("offline iterator error")
self.run_passes(workers, 2)
self.assertEqual(worker.request_tick.call_count, 2)
if __name__ == "__main__":
unittest.main()