317 lines
13 KiB
Python
317 lines
13 KiB
Python
"""Focused tests for RC stack inventory and stop-only retention guard."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import tempfile
|
|
import unittest
|
|
from contextlib import redirect_stdout
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SPEC = importlib.util.spec_from_file_location(
|
|
"rc_stack_retention", ROOT / "scripts" / "rc_stack_retention.py"
|
|
)
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
retention = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(retention)
|
|
|
|
|
|
PROJECT = "modelforge-validation-123271"
|
|
NOW = datetime(2026, 9, 23, 10, 0, tzinfo=UTC)
|
|
CREATED = "2026-09-20T10:00:00Z"
|
|
|
|
|
|
class MockDocker:
|
|
def __init__(self, containers: dict[str, dict[str, object]]) -> None:
|
|
self.containers = containers
|
|
self.calls: list[tuple[str, ...]] = []
|
|
|
|
def run(self, command: tuple[str, ...], **_kwargs: object) -> SimpleNamespace:
|
|
self.assert_docker(command)
|
|
args = tuple(command[1:])
|
|
self.calls.append(args)
|
|
if args[:4] == (
|
|
"container", "ls", "--all", "--quiet"
|
|
):
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout="\n".join(self.containers) + "\n",
|
|
stderr="",
|
|
)
|
|
if args[:2] == ("container", "inspect"):
|
|
identifier = args[2]
|
|
return SimpleNamespace(
|
|
returncode=0,
|
|
stdout=json.dumps([self.containers[identifier]]),
|
|
stderr="",
|
|
)
|
|
if args[:1] == ("stop",):
|
|
identifiers = args[3:]
|
|
for identifier in identifiers:
|
|
state = self.containers[identifier]["State"]
|
|
assert isinstance(state, dict)
|
|
state["Status"] = "exited"
|
|
return SimpleNamespace(returncode=0, stdout="\n".join(identifiers), stderr="")
|
|
raise AssertionError(f"unexpected Docker command: {args}")
|
|
|
|
@staticmethod
|
|
def assert_docker(command: tuple[str, ...]) -> None:
|
|
if command[0] != "docker":
|
|
raise AssertionError(f"unexpected executable: {command[0]}")
|
|
|
|
|
|
def docker_container(
|
|
identifier: str,
|
|
project: str,
|
|
service: str,
|
|
*,
|
|
status: str = "exited",
|
|
source: str | None = None,
|
|
created: str = CREATED,
|
|
oneoff: str = "False",
|
|
) -> dict[str, object]:
|
|
mounts: list[dict[str, str]] = []
|
|
if source is not None:
|
|
mounts.append({"Type": "bind", "Source": source, "Destination": "/app/config"})
|
|
return {
|
|
"Id": identifier,
|
|
"Name": f"/{project}-{service}-1",
|
|
"Created": created,
|
|
"Config": {
|
|
"Labels": {
|
|
retention.PROJECT_LABEL: project,
|
|
retention.SERVICE_LABEL: service,
|
|
retention.ONEOFF_LABEL: oneoff,
|
|
}
|
|
},
|
|
"State": {"Status": status, "Running": status == "running"},
|
|
"Mounts": mounts,
|
|
}
|
|
|
|
|
|
class RcStackRetentionTests(unittest.TestCase):
|
|
def fixture(self, *, api_status: str = "exited", service: str = "postgres", oneoff: str = "False") -> tuple[MockDocker, tempfile.TemporaryDirectory[str]]:
|
|
temporary = tempfile.TemporaryDirectory()
|
|
config = Path(temporary.name) / "config"
|
|
config.mkdir()
|
|
containers = {
|
|
"validation-api-id": docker_container("validation-api-id", PROJECT, "api", source=str(config)),
|
|
"validation-db-id": docker_container(
|
|
"validation-db-id", PROJECT, service, status="running", oneoff=oneoff
|
|
),
|
|
"prod-api-id": docker_container("prod-api-id", "modelforge", "api", status="running"),
|
|
"other-api-id": docker_container(
|
|
"other-api-id", "staging", "api", status="running"
|
|
),
|
|
}
|
|
if api_status != "exited":
|
|
containers["validation-api-id"]["State"] = {"Status": api_status, "Running": True}
|
|
return MockDocker(containers), temporary
|
|
|
|
@staticmethod
|
|
def write_evidence(path: Path, project: str = PROJECT) -> None:
|
|
path.write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": 1,
|
|
"project": project,
|
|
"retained": True,
|
|
"evidence_reference": "acceptance-run-2026-09-20",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
def test_inventory_scopes_validation_projects_and_flags_manifest_and_age(self) -> None:
|
|
docker, temporary = self.fixture()
|
|
self.addCleanup(temporary.cleanup)
|
|
with patch.object(retention.subprocess, "run", docker.run):
|
|
projects = retention.collect_projects(now=NOW, stale_after_hours=24)
|
|
|
|
self.assertEqual([row["project"] for row in projects], [PROJECT])
|
|
self.assertTrue(projects[0]["stale"])
|
|
self.assertEqual(len(projects[0]["missing_manifests"]), 1)
|
|
self.assertEqual(projects[0]["missing_bind_sources"], [])
|
|
self.assertNotIn("volumes", projects[0])
|
|
self.assertNotIn("images", projects[0])
|
|
self.assertTrue(all(call[:1] != ("stop",) for call in docker.calls))
|
|
self.assertTrue(all(call[:1] != ("rm",) for call in docker.calls))
|
|
|
|
def test_inventory_ignores_non_validation_partial_compose_labels(self) -> None:
|
|
docker, temporary = self.fixture()
|
|
self.addCleanup(temporary.cleanup)
|
|
docker.containers["unrelated-id"] = {
|
|
"Id": "unrelated-id",
|
|
"Name": "/projectbrain-worker-1",
|
|
"Config": {"Labels": {retention.PROJECT_LABEL: "projectbrain"}},
|
|
"State": {"Status": "exited"},
|
|
}
|
|
|
|
with patch.object(retention.subprocess, "run", docker.run):
|
|
projects = retention.collect_projects(now=NOW, stale_after_hours=24)
|
|
|
|
self.assertEqual([row["project"] for row in projects], [PROJECT])
|
|
|
|
def test_inventory_fails_closed_for_validation_container_missing_service_label(self) -> None:
|
|
docker, temporary = self.fixture()
|
|
self.addCleanup(temporary.cleanup)
|
|
labels = docker.containers["validation-api-id"]["Config"]["Labels"]
|
|
assert isinstance(labels, dict)
|
|
del labels[retention.SERVICE_LABEL]
|
|
|
|
with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
|
|
retention.RetentionError, "Compose service label is missing"
|
|
):
|
|
retention.collect_projects(now=NOW, stale_after_hours=24)
|
|
|
|
def test_default_cli_is_read_only(self) -> None:
|
|
docker, temporary = self.fixture()
|
|
self.addCleanup(temporary.cleanup)
|
|
output = io.StringIO()
|
|
with patch.object(retention.subprocess, "run", docker.run), redirect_stdout(output):
|
|
status = retention.main([])
|
|
|
|
result = json.loads(output.getvalue())
|
|
self.assertEqual(status, 0)
|
|
self.assertEqual(result["mode"], "read-only")
|
|
self.assertTrue(all(call[:1] != ("stop",) for call in docker.calls))
|
|
|
|
def test_docker_errors_and_ambiguous_inspection_fail_closed(self) -> None:
|
|
failed = SimpleNamespace(returncode=1, stdout="", stderr="daemon unavailable")
|
|
with patch.object(retention.subprocess, "run", return_value=failed), self.assertRaisesRegex(
|
|
retention.RetentionError, "Docker command failed"
|
|
):
|
|
retention.collect_projects(now=NOW)
|
|
|
|
docker, temporary = self.fixture()
|
|
self.addCleanup(temporary.cleanup)
|
|
docker.containers["validation-api-id"]["Id"] = "different-id"
|
|
with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
|
|
retention.RetentionError, "different container"
|
|
):
|
|
retention.collect_projects(now=NOW)
|
|
|
|
def test_project_validation_rejects_production_and_ambiguous_names(self) -> None:
|
|
for project in (
|
|
"modelforge",
|
|
"modelforge-rc-prod-",
|
|
"modelforge-validation-",
|
|
"modelforge-validation-prod-",
|
|
):
|
|
with self.subTest(project=project), self.assertRaises(retention.RetentionError):
|
|
retention.validate_project(project)
|
|
|
|
def test_stop_requires_explicit_exact_project_confirmation_and_evidence(self) -> None:
|
|
docker, temporary = self.fixture()
|
|
self.addCleanup(temporary.cleanup)
|
|
empty_record = Path(temporary.name) / "empty.json"
|
|
empty_record.write_text("", encoding="utf-8")
|
|
with patch.object(retention.subprocess, "run", docker.run):
|
|
with self.assertRaisesRegex(retention.RetentionError, "exact project name"):
|
|
retention.stop_project(
|
|
PROJECT,
|
|
confirm_stop=None,
|
|
confirm_no_active_jobs=True,
|
|
evidence_record=Path("evidence.json"),
|
|
stale_after_hours=24,
|
|
now=NOW,
|
|
)
|
|
with self.assertRaisesRegex(retention.RetentionError, "nonempty regular file"):
|
|
retention.stop_project(
|
|
PROJECT,
|
|
confirm_stop=PROJECT,
|
|
confirm_no_active_jobs=True,
|
|
evidence_record=empty_record,
|
|
stale_after_hours=24,
|
|
now=NOW,
|
|
)
|
|
mismatch = Path(temporary.name) / "mismatch.json"
|
|
self.write_evidence(mismatch, project="modelforge-validation-someone-else")
|
|
with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
|
|
retention.RetentionError, "identify this project"
|
|
):
|
|
retention.stop_project(
|
|
PROJECT,
|
|
confirm_stop=PROJECT,
|
|
confirm_no_active_jobs=True,
|
|
evidence_record=mismatch,
|
|
stale_after_hours=24,
|
|
now=NOW,
|
|
)
|
|
self.assertEqual(docker.calls, [])
|
|
|
|
def test_stop_only_targets_inspected_project_container_ids(self) -> None:
|
|
docker, temporary = self.fixture()
|
|
self.addCleanup(temporary.cleanup)
|
|
evidence = Path(temporary.name) / "evidence.json"
|
|
self.write_evidence(evidence)
|
|
with patch.object(retention.subprocess, "run", docker.run):
|
|
result = retention.stop_project(
|
|
PROJECT,
|
|
confirm_stop=PROJECT,
|
|
confirm_no_active_jobs=True,
|
|
evidence_record=evidence,
|
|
stale_after_hours=24,
|
|
now=NOW,
|
|
)
|
|
|
|
self.assertEqual(result["stopped_container_ids"], ["validation-db-id"])
|
|
self.assertFalse(result["volumes_or_images_removed"])
|
|
stop_calls = [call for call in docker.calls if call[:1] == ("stop",)]
|
|
self.assertEqual(stop_calls, [("stop", "--time", "30", "validation-db-id")])
|
|
self.assertTrue(all("prod-api-id" not in call for call in stop_calls))
|
|
self.assertTrue(all(call[:1] not in {("rm",), ("rmi",), ("volume",)} for call in docker.calls))
|
|
|
|
def test_stop_fails_if_api_or_oneoff_job_is_running(self) -> None:
|
|
for options, message in (
|
|
({"api_status": "running"}, "API is running"),
|
|
({"api_status": "restarting"}, "API is running"),
|
|
({"oneoff": "True"}, "one-off job container"),
|
|
({"service": "migrate"}, "service activity is unknown"),
|
|
):
|
|
with self.subTest(message=message):
|
|
docker, temporary = self.fixture(**options)
|
|
self.addCleanup(temporary.cleanup)
|
|
evidence = Path(temporary.name) / "evidence.json"
|
|
self.write_evidence(evidence)
|
|
with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
|
|
retention.RetentionError, message
|
|
):
|
|
retention.stop_project(
|
|
PROJECT,
|
|
confirm_stop=PROJECT,
|
|
confirm_no_active_jobs=True,
|
|
evidence_record=evidence,
|
|
stale_after_hours=24,
|
|
now=NOW,
|
|
)
|
|
self.assertTrue(all(call[:1] != ("stop",) for call in docker.calls))
|
|
|
|
def test_stop_requires_job_attestation(self) -> None:
|
|
docker, temporary = self.fixture()
|
|
self.addCleanup(temporary.cleanup)
|
|
evidence = Path(temporary.name) / "evidence.json"
|
|
self.write_evidence(evidence)
|
|
with patch.object(retention.subprocess, "run", docker.run), self.assertRaisesRegex(
|
|
retention.RetentionError, "--confirm-no-active-jobs"
|
|
):
|
|
retention.stop_project(
|
|
PROJECT,
|
|
confirm_stop=PROJECT,
|
|
confirm_no_active_jobs=False,
|
|
evidence_record=evidence,
|
|
stale_after_hours=24,
|
|
now=NOW,
|
|
)
|
|
self.assertEqual(docker.calls, [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|