246 lines
11 KiB
Python
246 lines
11 KiB
Python
"""Bounded account-local activity persistence; standard library only.
|
|
|
|
Use one ActivityLog per profile_root (one writer, thread safe)::
|
|
|
|
activity = ActivityLog(profile_root / "activity.json")
|
|
logs.extend(activity.load())
|
|
cursor = activity.sequence
|
|
entry = activity.append({"message": safe_text(message), "debug": False})
|
|
|
|
Only allowlisted short fields are retained. The caller MUST apply safe_text to
|
|
messages before append; this module cannot know account-specific secret values.
|
|
Debug messages are never persisted. A tiny .seq sidecar reserves sequence blocks
|
|
BEFORE publishing any entry, including debug. Restart skips unused reservations,
|
|
so ids strictly increase across restarts but can have gaps. Both load() and
|
|
sequence are idempotent reads; no timer, idle writes, flush or close is needed.
|
|
Storage failures raise ActivityLogError with a path-free message.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections import deque
|
|
from collections.abc import Mapping
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
|
|
from .log_categories import legacy_category, normalize_category
|
|
|
|
|
|
MAX_ENTRIES = 2000
|
|
MAX_MESSAGE_CHARS = 2048
|
|
MAX_BYTES = 2 * 1024 * 1024
|
|
_MAX_SEQUENCE = (1 << 53) - 1 # Browser cursors must remain exact integers.
|
|
_LEVELS = frozenset({"debug", "info", "warning", "error", "critical"})
|
|
|
|
|
|
class ActivityLogError(RuntimeError):
|
|
"""Persistence failed; public error text contains no filesystem paths."""
|
|
|
|
|
|
def _encode(value):
|
|
return json.dumps(value, ensure_ascii=False, allow_nan=False,
|
|
separators=(",", ":")).encode("utf-8")
|
|
|
|
|
|
def _sequence(value):
|
|
return value if type(value) is int and 0 <= value <= _MAX_SEQUENCE else 0
|
|
|
|
|
|
class ActivityLog:
|
|
"""Atomic snapshots capped by both count and serialized bytes.
|
|
|
|
Construction does no IO. load() lazily restores nondebug entries and the
|
|
reserved high-water mark; append() and sequence also load on first use.
|
|
append() returns a NEW canonical dict with id == seq; caller ids are ignored.
|
|
load() returns independent copies of nondebug entries, oldest first.
|
|
|
|
Default file cap is 1 MiB, hard cap 2 MiB, each message <= 2048 characters.
|
|
The .seq file is < 64 bytes and rewritten at most once per 256 appends in a
|
|
running instance. Only real nondebug appends rewrite the bounded snapshot.
|
|
"""
|
|
|
|
def __init__(self, path, max_entries=MAX_ENTRIES, *,
|
|
max_bytes=1024 * 1024, sequence_block_size=256):
|
|
if type(max_entries) is not int or not 1 <= max_entries <= MAX_ENTRIES:
|
|
raise ValueError("max_entries must be between 1 and 2000")
|
|
if type(max_bytes) is not int or not 16384 <= max_bytes <= MAX_BYTES:
|
|
raise ValueError("max_bytes must be between 16384 and 2097152")
|
|
if type(sequence_block_size) is not int or not 1 <= sequence_block_size <= 65536:
|
|
raise ValueError("sequence_block_size must be between 1 and 65536")
|
|
self._path = Path(path)
|
|
self._sequence_path = self._path.with_name(self._path.name + ".seq")
|
|
self._max_entries = max_entries
|
|
self._max_bytes = max_bytes
|
|
self._block_size = sequence_block_size
|
|
self._lock = threading.RLock()
|
|
self._loaded = False
|
|
self._sequence = self._reserved = 0
|
|
self._entries = deque() # (canonical dict, cached serialized bytes)
|
|
|
|
@property
|
|
def sequence(self):
|
|
"""Last emitted seq, or the restored reservation ceiling after restart."""
|
|
with self._lock:
|
|
self._load()
|
|
return self._sequence
|
|
|
|
def load(self):
|
|
"""Restore once, without modifying either file; return nondebug copies."""
|
|
with self._lock:
|
|
self._load()
|
|
return [dict(entry) for entry, _ in self._entries]
|
|
|
|
def append(self, entry, *, after_sequence=0):
|
|
"""Append sanitized fields; return {time,message,category,debug,level,id,seq}.
|
|
|
|
No seq is returned until its high-water mark is durable. If writing
|
|
fails the event is not published and the in-memory cursor is unchanged.
|
|
Unknown fields (tokens, headers, request bodies, paths, etc.) are dropped.
|
|
after_sequence includes ids published by the caller during memory-only
|
|
fallback; reserve above that cursor before publishing a recovered event.
|
|
"""
|
|
if not isinstance(entry, Mapping) or not isinstance(entry.get("message"), str):
|
|
raise ValueError("Activity entry requires a sanitized string message")
|
|
if "debug" in entry and type(entry["debug"]) is not bool:
|
|
raise ValueError("Activity debug flag must be a boolean")
|
|
if type(after_sequence) is not int or not 0 <= after_sequence <= _MAX_SEQUENCE:
|
|
raise ValueError("Activity cursor must be a nonnegative safe integer")
|
|
clean = self._clean(entry)
|
|
with self._lock:
|
|
self._load()
|
|
next_sequence = max(self._sequence, after_sequence) + 1
|
|
if next_sequence > _MAX_SEQUENCE:
|
|
raise ActivityLogError("Activity sequence exhausted")
|
|
if next_sequence > self._reserved:
|
|
reserved = min(_MAX_SEQUENCE, next_sequence + self._block_size - 1)
|
|
self._write(self._sequence_path, _encode({"schema": 1, "sequence": reserved}))
|
|
self._reserved = reserved
|
|
clean["id"] = clean["seq"] = next_sequence
|
|
if not clean["debug"]:
|
|
entries = deque(self._entries)
|
|
entries.append((clean, _encode(clean)))
|
|
self._trim(entries, self._reserved)
|
|
self._write(self._path, self._snapshot(entries, self._reserved))
|
|
self._entries = entries
|
|
self._sequence = next_sequence
|
|
return dict(clean)
|
|
|
|
def log(self, message, *, debug=False, category="general"):
|
|
"""Convenience helper for an already safe_text-sanitized message; int id."""
|
|
return self.append({"message": message, "debug": debug, "category": category})["id"]
|
|
|
|
@staticmethod
|
|
def _clean(entry):
|
|
level = entry.get("level", "info")
|
|
level = level.lower() if isinstance(level, str) else "info"
|
|
level = {"trace": "debug", "warn": "warning"}.get(level, level)
|
|
debug = entry.get("debug", False) is True or level == "debug"
|
|
level = "debug" if debug else level if level in _LEVELS else "info"
|
|
stamp = entry.get("time")
|
|
try:
|
|
if not isinstance(stamp, str) or len(stamp) > 64:
|
|
raise ValueError
|
|
datetime.fromisoformat(stamp)
|
|
except ValueError:
|
|
stamp = datetime.now(timezone.utc).isoformat(timespec="milliseconds")
|
|
message = entry["message"][:MAX_MESSAGE_CHARS].replace("\r", " ").replace("\n", " ")
|
|
message = message.encode("utf-8", errors="replace").decode("utf-8")
|
|
category = normalize_category(entry["category"]) if "category" in entry else legacy_category(message)
|
|
return {"time": stamp, "message": message, "category": category, "debug": debug, "level": level}
|
|
|
|
def _load(self):
|
|
if self._loaded:
|
|
return
|
|
record = self._read(self._path, MAX_BYTES)
|
|
watermark = self._read(self._sequence_path, 1024)
|
|
high = _sequence(watermark.get("sequence")) if watermark.get("schema") == 1 else 0
|
|
entries = {}
|
|
if record.get("schema") == 1:
|
|
high = max(high, _sequence(record.get("sequence")))
|
|
rows = record.get("entries", [])
|
|
if isinstance(rows, list):
|
|
for row in rows:
|
|
if not isinstance(row, dict):
|
|
continue
|
|
seq = _sequence(row.get("seq"))
|
|
high = max(high, seq)
|
|
if not seq or not isinstance(row.get("message"), str):
|
|
continue
|
|
if "debug" in row and type(row["debug"]) is not bool:
|
|
continue
|
|
clean = self._clean(row)
|
|
if clean["debug"]:
|
|
continue
|
|
clean["id"] = clean["seq"] = seq
|
|
entries[seq] = (clean, _encode(clean))
|
|
self._entries = deque(entries[seq] for seq in sorted(entries)[-self._max_entries:])
|
|
self._trim(self._entries, high)
|
|
self._sequence = self._reserved = high
|
|
self._loaded = True
|
|
|
|
@staticmethod
|
|
def _prefix(sequence):
|
|
return b'{"schema":1,"sequence":' + str(sequence).encode("ascii") + b',"entries":['
|
|
|
|
def _trim(self, entries, sequence):
|
|
size = len(self._prefix(sequence)) + 2 + sum(len(raw) for _, raw in entries) + max(0, len(entries) - 1)
|
|
while entries and (len(entries) > self._max_entries or size > self._max_bytes):
|
|
_, raw = entries.popleft()
|
|
size -= len(raw) + (1 if entries else 0)
|
|
|
|
def _snapshot(self, entries, sequence):
|
|
return self._prefix(sequence) + b",".join(raw for _, raw in entries) + b"]}"
|
|
|
|
@staticmethod
|
|
def _check_path(path):
|
|
for candidate in (path, *path.parents):
|
|
if candidate.is_symlink() or getattr(candidate, "is_junction", lambda: False)():
|
|
raise ActivityLogError("Activity storage must not use linked paths")
|
|
|
|
def _read(self, path, limit):
|
|
try:
|
|
self._check_path(path)
|
|
with path.open("rb") as stream:
|
|
data = stream.read(limit + 1)
|
|
if len(data) > limit:
|
|
return {}
|
|
value = json.loads(data)
|
|
return value if isinstance(value, dict) else {}
|
|
except FileNotFoundError:
|
|
return {}
|
|
except (ValueError, UnicodeError, RecursionError):
|
|
return {}
|
|
except OSError:
|
|
raise ActivityLogError("Unable to read activity storage") from None
|
|
|
|
def _write(self, path, data):
|
|
temporary = None
|
|
try:
|
|
self._check_path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
descriptor, name = tempfile.mkstemp(prefix=".activity-", suffix=".tmp", dir=path.parent)
|
|
temporary = Path(name)
|
|
with os.fdopen(descriptor, "wb") as stream:
|
|
os.chmod(temporary, 0o600)
|
|
stream.write(data)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temporary, path)
|
|
if os.name != "nt":
|
|
directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
try:
|
|
os.fsync(directory)
|
|
finally:
|
|
os.close(directory)
|
|
except OSError:
|
|
raise ActivityLogError("Unable to save activity storage") from None
|
|
finally:
|
|
if temporary is not None:
|
|
try:
|
|
temporary.unlink(missing_ok=True)
|
|
except OSError:
|
|
pass
|