350 lines
17 KiB
Python
350 lines
17 KiB
Python
"""Temporary filesystem and fake Docker/GitHub transaction tests."""
|
|
from contextlib import closing, redirect_stdout
|
|
from dataclasses import replace
|
|
from datetime import timedelta
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from deploy.web import updater
|
|
from ranch_web import updates
|
|
|
|
|
|
def release(tag):
|
|
return {"tag_name": tag, "draft": False, "prerelease": False}
|
|
|
|
|
|
class FakeDocker:
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.events = []
|
|
self.version = "2.0.0.dev1"
|
|
self.old_version = self.version
|
|
self.image = "sha256:" + "a" * 64
|
|
self.old_image = self.image
|
|
self.running = True
|
|
self.fail_pull = self.fail_start = self.fail_rollback = False
|
|
|
|
def inspect_service(self, **kwargs):
|
|
return {"image": self.image, "version": self.version, "running": self.running, "health": "healthy"}
|
|
|
|
def validate(self, image):
|
|
self.events.append(("validate", image))
|
|
|
|
def pull(self, image):
|
|
self.events.append(("pull", image))
|
|
if not self.running:
|
|
raise AssertionError("pull must happen while the old app is running")
|
|
if self.fail_pull:
|
|
raise updater.UpdaterError("fake registry failure")
|
|
|
|
def stop(self):
|
|
self.events.append(("stop", None))
|
|
self.running = False
|
|
|
|
def up(self, image):
|
|
self.events.append(("up", image))
|
|
self.running = True
|
|
if image == self.old_image:
|
|
if self.fail_rollback:
|
|
raise updater.UpdaterError("fake rollback failure")
|
|
self.image = image
|
|
self.version = self.old_version
|
|
else:
|
|
self.image = "sha256:" + "b" * 64
|
|
self.version = image.rsplit(":", 1)[1][1:]
|
|
(self.config.data_dir / "key.bin").write_bytes(b"changed-key")
|
|
with closing(sqlite3.connect(self.config.data_dir / "samples.sqlite3")) as connection, connection:
|
|
connection.execute("insert into samples values (2)")
|
|
if self.fail_start:
|
|
raise updater.UpdaterError("fake health failure")
|
|
|
|
|
|
class HostUpdaterTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temporary = tempfile.TemporaryDirectory(prefix="ranch-host-updater-")
|
|
self.addCleanup(self.temporary.cleanup)
|
|
self.root = Path(self.temporary.name).resolve()
|
|
self.config = updater.Config(
|
|
project_dir=self.root, data_dir=self.root / "data", control_dir=self.root / "control",
|
|
backup_dir=self.root / "backups", failed_data_dir=self.root / "failed-data",
|
|
)
|
|
self.config.data_dir.mkdir()
|
|
self.config.control_dir.mkdir()
|
|
(self.config.data_dir / "key.bin").write_bytes(b"original-encrypted-test-key")
|
|
with closing(sqlite3.connect(self.config.data_dir / "samples.sqlite3")) as connection, connection:
|
|
connection.execute("create table samples (n integer)")
|
|
connection.execute("insert into samples values (1)")
|
|
self.docker = FakeDocker(self.config)
|
|
self.host = updater.HostUpdater(self.config, self.docker)
|
|
self.github = patch.object(updates, "github_releases", return_value=[release("v2.0.0")])
|
|
self.releases = self.github.start()
|
|
self.addCleanup(self.github.stop)
|
|
# Linux CI runs unprivileged; real chown is a host deployment check.
|
|
self.chown = patch.object(updater.os, "chown", create=True)
|
|
self.chown.start()
|
|
self.addCleanup(self.chown.stop)
|
|
self.environment = patch.dict(os.environ, {"RANCH_WEB_UPDATER_DIR": str(self.config.control_dir)})
|
|
self.environment.start()
|
|
self.addCleanup(self.environment.stop)
|
|
|
|
def samples(self, directory=None):
|
|
with closing(sqlite3.connect((directory or self.config.data_dir) / "samples.sqlite3")) as connection:
|
|
return connection.execute("select n from samples order by n").fetchall()
|
|
|
|
def enable(self):
|
|
self.config = replace(self.config, accept_web_requests=True)
|
|
self.host = updater.HostUpdater(self.config, self.docker)
|
|
return self.host.capability()
|
|
|
|
def request(self, capability, **overrides):
|
|
value = {"protocol": 1, "request_id": "c" * 32, "executor_id": capability["executor_id"],
|
|
"version": "v2.0.0", "requested_at": updates.timestamp(), **overrides}
|
|
updates.atomic_json(self.config.data_dir / "update-request.json", value)
|
|
return value
|
|
|
|
def test_pull_failure_does_not_stop_or_backup(self):
|
|
self.docker.fail_pull = True
|
|
with self.assertRaises(updater.UpdaterError):
|
|
self.host.apply()
|
|
self.assertTrue(self.docker.running)
|
|
self.assertNotIn("stop", [event[0] for event in self.docker.events])
|
|
self.assertFalse(self.config.backup_dir.exists())
|
|
self.assertFalse(self.host.journal.exists())
|
|
|
|
def test_update_stops_before_snapshot_and_persists_image_only_after_health(self):
|
|
original_snapshot = updater.snapshot_data
|
|
def snapshot(config, identity):
|
|
self.assertFalse(self.docker.running)
|
|
self.docker.events.append(("snapshot", None))
|
|
return original_snapshot(config, identity)
|
|
with patch.object(updater, "snapshot_data", side_effect=snapshot):
|
|
result = self.host.apply()
|
|
self.assertEqual(result["status"], "succeeded")
|
|
self.assertEqual([event[0] for event in self.docker.events], ["validate", "pull", "stop", "snapshot", "up"])
|
|
self.assertEqual((self.root / ".image.env").read_text(), f"RANCH_WEB_IMAGE={updates.IMAGE}:v2.0.0\n")
|
|
self.assertFalse(self.host.journal.exists())
|
|
self.assertEqual(len(list(self.config.backup_dir.glob("*.tar.gz"))), 1)
|
|
|
|
def test_health_failure_restores_sqlite_key_and_keeps_failed_data(self):
|
|
self.docker.fail_start = True
|
|
result = self.host.apply(request_id="d" * 32)
|
|
self.assertEqual(result["status"], "rolled_back")
|
|
self.assertTrue(self.docker.running)
|
|
self.assertEqual(self.docker.image, self.docker.old_image)
|
|
self.assertEqual(self.samples(), [(1,)])
|
|
self.assertEqual((self.config.data_dir / "key.bin").read_bytes(), b"original-encrypted-test-key")
|
|
failed = self.config.failed_data_dir / ("d" * 32)
|
|
self.assertEqual(self.samples(failed), [(1,), (2,)])
|
|
self.assertEqual((failed / "key.bin").read_bytes(), b"changed-key")
|
|
self.assertIn(self.docker.old_image, (self.root / ".image.env").read_text())
|
|
|
|
def test_restart_and_snapshot_restore_preserve_preferences_but_never_start_strategies(self):
|
|
from ranch_web.service import WebManager
|
|
def complete(manager, account_id, job):
|
|
until = time.monotonic() + 5
|
|
while time.monotonic() < until:
|
|
result = manager.job(account_id, job["id"])
|
|
if result["status"] in {"succeeded", "failed", "cancelled"}:
|
|
self.assertEqual(result["status"], "succeeded", result.get("error"))
|
|
return
|
|
time.sleep(0.01)
|
|
self.fail("offline worker did not finish")
|
|
with patch("socket.socket.connect", side_effect=AssertionError("Network forbidden in deployment restart test")):
|
|
manager = WebManager(self.config.data_dir, start_scheduler=False)
|
|
try:
|
|
account_id = manager.create_account(demo=True)["id"]
|
|
config = manager.get_config(account_id)["config"]
|
|
config["daily_policy"]["enabled"] = True
|
|
pending = manager.set_config(account_id, config)
|
|
complete(manager, account_id, pending["job"])
|
|
complete(manager, account_id, manager.submit(account_id, "start"))
|
|
self.assertTrue(manager.status(account_id)["running"])
|
|
finally:
|
|
manager.close()
|
|
key = (self.config.data_dir / "web.key").read_bytes()
|
|
identity = "f" * 32
|
|
backup = updater.snapshot_data(self.config, identity)
|
|
for restore in (False, True):
|
|
if restore:
|
|
updater.restore_data(self.config, backup, identity)
|
|
restarted = WebManager(self.config.data_dir, start_scheduler=False)
|
|
try:
|
|
self.assertFalse(restarted.status(account_id)["running"])
|
|
self.assertTrue(restarted.get_config(account_id)["config"]["daily_policy"]["enabled"])
|
|
self.assertEqual((self.config.data_dir / "web.key").read_bytes(), key)
|
|
finally:
|
|
restarted.close()
|
|
|
|
def test_backup_failure_resumes_pinned_old_image_without_new_image(self):
|
|
with patch.object(updater, "snapshot_data", side_effect=OSError("fake disk full")):
|
|
result = self.host.apply()
|
|
self.assertEqual(result["status"], "rolled_back")
|
|
starts = [image for event, image in self.docker.events if event == "up"]
|
|
self.assertEqual(starts, [self.docker.old_image])
|
|
self.assertEqual(self.samples(), [(1,)])
|
|
|
|
def test_failed_rollback_retains_journal_and_backup(self):
|
|
self.docker.fail_start = self.docker.fail_rollback = True
|
|
with self.assertRaises(updater.UpdaterError):
|
|
self.host.apply()
|
|
self.assertTrue(self.host.journal.exists())
|
|
self.assertEqual(updates.read_json(self.config.data_dir / "updater-status.json")["status"], "failed")
|
|
self.assertTrue(list(self.config.backup_dir.glob("*.tar.gz")))
|
|
|
|
def test_recovery_can_retry_old_start_after_data_was_already_restored(self):
|
|
self.docker.fail_start = self.docker.fail_rollback = True
|
|
with self.assertRaises(updater.UpdaterError):
|
|
self.host.apply()
|
|
self.docker.fail_rollback = False
|
|
result = self.host.recover()
|
|
self.assertEqual(result["status"], "rolled_back")
|
|
self.assertEqual(self.samples(), [(1,)])
|
|
self.assertFalse(self.host.journal.exists())
|
|
|
|
def test_interrupted_transaction_recovery_resumes_old_service(self):
|
|
identity = "e" * 32
|
|
self.docker.stop()
|
|
updater.snapshot_data(self.config, identity)
|
|
self.docker.up(f"{updates.IMAGE}:v2.0.0")
|
|
updates.atomic_json(self.host.journal, {"request_id": identity, "version": "v2.0.0",
|
|
"old_image": self.docker.old_image, "backup_ready": True, "new_started": True})
|
|
result = self.host.recover()
|
|
self.assertEqual(result["status"], "rolled_back")
|
|
self.assertEqual(self.samples(), [(1,)])
|
|
self.assertFalse(self.host.journal.exists())
|
|
|
|
def test_only_old_desktop_release_leaves_service_untouched(self):
|
|
self.releases.return_value = [release("v1.2.0"), release("v3.0.0")]
|
|
result = self.host.apply()
|
|
self.assertEqual(result["status"], "no_release")
|
|
self.assertEqual(self.docker.events, [])
|
|
|
|
def test_same_stable_version_or_downgrade_never_pulls(self):
|
|
for current in ("2.0.0", "2.1.0"):
|
|
with self.subTest(current=current):
|
|
self.docker.version = current
|
|
with self.assertRaises(updater.UpdaterError):
|
|
self.host.apply("v2.0.0")
|
|
self.assertEqual(self.docker.events, [])
|
|
|
|
def test_host_independently_rejects_not_official_request_tag(self):
|
|
capability = self.enable()
|
|
self.request(capability, version="v2.99.99")
|
|
self.releases.return_value = [release("v2.0.0")]
|
|
result = self.host.run()
|
|
self.assertEqual(result["status"], "failed")
|
|
self.assertTrue(self.docker.running)
|
|
self.assertEqual(self.docker.events, [])
|
|
self.releases.assert_called_once()
|
|
|
|
def test_web_request_requires_operator_opt_in(self):
|
|
capability = self.host.capability()
|
|
self.assertFalse(capability["enabled"])
|
|
self.request(capability)
|
|
self.host.run()
|
|
self.assertTrue(self.docker.running)
|
|
self.assertEqual(self.docker.events, [])
|
|
|
|
def test_web_request_success_and_replay_deduplication(self):
|
|
capability = self.enable()
|
|
request = self.request(capability)
|
|
result = self.host.run()
|
|
self.assertEqual(result["status"], "succeeded")
|
|
self.assertEqual(result["request_id"], request["request_id"])
|
|
events = list(self.docker.events)
|
|
self.host.run()
|
|
self.assertEqual(events, self.docker.events)
|
|
|
|
def test_stale_other_executor_and_injected_command_are_rejected(self):
|
|
capability = self.enable()
|
|
variants = [dict(executor_id="a" * 32), dict(command="rm anything"),
|
|
dict(requested_at=updates.timestamp(updates.utc_now() - timedelta(days=2))),
|
|
dict(requested_at=updates.timestamp(updates.utc_now() + timedelta(hours=1)))]
|
|
for index, variant in enumerate(variants):
|
|
with self.subTest(variant=variant):
|
|
self.request(capability, request_id=f"{index:032x}", **variant)
|
|
self.assertEqual(self.host.run()["status"], "failed")
|
|
self.assertEqual(self.docker.events, [])
|
|
self.releases.assert_not_called()
|
|
|
|
def test_auto_apply_is_explicit_opt_in(self):
|
|
result = self.host.run()
|
|
self.assertTrue(result["available"])
|
|
self.assertEqual(self.docker.events, [])
|
|
config = replace(self.config, auto_apply=True)
|
|
result = updater.HostUpdater(config, self.docker).run()
|
|
self.assertEqual(result["status"], "succeeded")
|
|
|
|
def test_malformed_request_fails_without_network_or_docker(self):
|
|
self.enable()
|
|
updates.atomic_json(self.config.data_dir / "update-request.json", ["invalid"])
|
|
self.assertEqual(self.host.run()["status"], "failed")
|
|
self.assertEqual(self.docker.events, [])
|
|
self.releases.assert_not_called()
|
|
|
|
def test_dry_run_is_read_only_and_network_free(self):
|
|
document = self.root / "updater.toml"
|
|
text = "\n".join(f"{key} = {json.dumps(str(getattr(self.config, key)))}" for key in (
|
|
"project_dir", "data_dir", "control_dir", "backup_dir", "failed_data_dir"))
|
|
document.write_text(text, encoding="utf-8")
|
|
before = sorted(str(item.relative_to(self.root)) for item in self.root.rglob("*"))
|
|
with patch.object(updater.Docker, "run", side_effect=AssertionError("Docker not allowed")), redirect_stdout(io.StringIO()) as output:
|
|
code = updater.main(["--config", str(document), "--apply", "v2.1.0", "--dry-run"])
|
|
self.assertEqual(code, 0)
|
|
self.assertTrue(json.loads(output.getvalue())["dry_run"])
|
|
self.assertEqual(before, sorted(str(item.relative_to(self.root)) for item in self.root.rglob("*")))
|
|
self.releases.assert_not_called()
|
|
|
|
def test_shipped_linux_config_supports_windows_dry_run(self):
|
|
source = Path(updater.__file__).with_name("updater.example.toml")
|
|
with patch.object(updater.Docker, "run", side_effect=AssertionError("Docker not allowed")), redirect_stdout(io.StringIO()) as output:
|
|
code = updater.main(["--config", str(source), "--apply", "v2.0.0", "--dry-run"])
|
|
self.assertEqual(code, 0)
|
|
self.assertTrue(json.loads(output.getvalue())["dry_run"])
|
|
self.releases.assert_not_called()
|
|
|
|
def test_config_rejects_unknown_fields_and_nested_control_paths(self):
|
|
document = self.root / "bad.toml"
|
|
for text in ("download_url = 'https://evil.invalid'", f"control_dir = {json.dumps(str(self.config.data_dir / 'control'))}\ndata_dir = {json.dumps(str(self.config.data_dir))}"):
|
|
document.write_text(text, encoding="utf-8")
|
|
with self.assertRaises(updater.UpdaterError):
|
|
updater.Config.load(document)
|
|
|
|
def test_real_docker_runner_uses_arguments_not_shell_and_never_echoes_output(self):
|
|
docker = updater.Docker(self.config)
|
|
with patch.object(updater.subprocess, "run") as run:
|
|
run.return_value.returncode = 1
|
|
run.return_value.stdout = "password=secret"
|
|
run.return_value.stderr = "token=secret"
|
|
with self.assertRaises(updater.UpdaterError) as error:
|
|
docker.pull(f"{updates.IMAGE}:v2.0.0")
|
|
args, kwargs = run.call_args
|
|
self.assertIsInstance(args[0], list)
|
|
self.assertNotIn("shell", kwargs)
|
|
self.assertIn("pull", args[0])
|
|
self.assertNotIn("secret", str(error.exception))
|
|
|
|
def test_healthy_container_with_dev_label_is_not_a_completed_stable_update(self):
|
|
docker = updater.Docker(self.config)
|
|
target = f"{updates.IMAGE}:v2.0.0"
|
|
with patch.object(docker, "compose"), patch.object(docker, "inspect_service") as inspect:
|
|
inspect.return_value = {"running": True, "health": "healthy", "version": "2.0.0.dev1"}
|
|
with self.assertRaises(updater.UpdaterError):
|
|
docker.up(target)
|
|
inspect.return_value = {"running": True, "health": "unhealthy", "version": "2.0.0"}
|
|
with self.assertRaises(updater.UpdaterError):
|
|
docker.up(target)
|
|
inspect.return_value = {"running": True, "health": "healthy", "version": "2.0.0"}
|
|
docker.up(target)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|