Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d

backend/tests/test_privatemuse_deployment_recovery.py

@@ -0,0 +1,630 @@
+from datetime import UTC, datetime
+from pathlib import Path
+from threading import Barrier, Event, Thread
+from types import SimpleNamespace
+
+from sqlalchemy import event
+from sqlmodel import Session, SQLModel, create_engine, select
+
+import backend.app.services.privatemuse_deployment_recovery as recovery
+from backend.app.db.models import JobRun
+
+
+class FakeResponse:
+    def __init__(self, payload):
+        self.payload = payload
+
+    def json(self):
+        return self.payload
+
+
+class FakePortainer:
+    def __init__(self, exit_code=0):
+        self.exit_code = exit_code
+        self.calls = []
+
+    def get_container_status(self, name):
+        assert name == "gitea-deploy-control"
+        return {
+            "status": "ok",
+            "container": {
+                "id": "controller-id",
+                "name": "gitea-deploy-control",
+                "state": "running",
+            },
+        }
+
+    def _config(self):
+        return SimpleNamespace(endpoint_id="3")
+
+    def _request(self, method, path, **kwargs):
+        self.calls.append((method, path, kwargs))
+        if path.endswith("/containers/controller-id/exec"):
+            return FakeResponse({"Id": "exec-1"})
+        if path.endswith("/exec/exec-1/start"):
+            return FakeResponse({})
+        if path.endswith("/exec/exec-1/json"):
+            return FakeResponse(
+                {
+                    "ID": "exec-1",
+                    "ContainerID": "controller-id",
+                    "Running": False,
+                    "ExitCode": self.exit_code,
+                }
+            )
+        raise AssertionError(path)
+
+
+def recovery_job(job_id):
+    return JobRun(
+        id=job_id,
+        type="deployment.target.register_privatemuse",
+        status="queued",
+        source="chatgpt-mcp",
+        action_id="deployment.target.register_privatemuse",
+        target="jens/privatemuse",
+        risk="high",
+        policy_decision="allowed",
+    )
+
+
+def test_recovery_runs_only_the_exact_observed_privatemuse_incident():
+    fake = FakePortainer()
+
+    result = recovery.PrivateMuseDeploymentRecovery(fake).run()
+
+    assert result == {
+        "status": "reconciled",
+        "repository": "jens/privatemuse",
+        "live_revision": "2103d5795d151885edeeb964466b643b8fd4a0eb",
+        "failed_revision": "6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38",
+        "containers": [
+            "privatemuse-backend-1",
+            "privatemuse-frontend-1",
+            "privatemuse-audio-enhancement-1",
+        ],
+        "compose_project": "privatemuse",
+    }
+    command = fake.calls[0][2]["json"]["Cmd"]
+    assert command[:3] == ["python3", "-c", recovery._REATTEST_SCRIPT]
+    assert command[3:] == [
+        "jens/privatemuse",
+        "2103d5795d151885edeeb964466b643b8fd4a0eb",
+        "6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38",
+        "Existing production containers require a valid healthy live deployment state",
+    ]
+
+
+def test_recovery_script_allows_only_the_two_observed_container_id_changes():
+    script = recovery._REATTEST_SCRIPT
+
+    assert (
+        'allowed_changed_ids = {"privatemuse-backend-1", "privatemuse-frontend-1"}'
+        in script
+    )
+    assert "changed_receipt_ids != allowed_changed_ids" in script
+    assert "changed_artifact_ids != allowed_changed_ids" in script
+    assert 'state.get("revision") == live_revision' in script
+    assert 'state.get("live_revision") == live_revision' in script
+    assert 'state.get("live_status") == "unknown"' in script
+    assert 'attempt.get("revision") == failed_revision' in script
+    assert 'attempt.get("message") == failure_message' in script
+
+
+def test_recovery_reuses_controller_verifiers_and_writes_state_durably():
+    script = recovery._REATTEST_SCRIPT
+
+    assert "controller.validate_recorded_receipt" in script
+    assert "controller.verify_deployment" in script
+    assert "expected_image_ids=expected_image_ids" in script
+    assert "controller.deployment_artifacts" in script
+    assert "controller.verified_current_receipt" in script
+    assert "write_exclusive(backup, original_state, state_metadata)" in script
+    assert "os.replace(temporary, state_path)" in script
+    assert "os.fsync" in script
+
+
+def test_recovery_durably_preserves_the_original_state_before_replacement():
+    script = recovery._REATTEST_SCRIPT
+
+    assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script
+    assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script
+    assert "/mnt/cache/appdata/gitea-deploy-runners/deploy.lock" in script
+    assert "original_state = state_path.read_bytes()" in script
+    assert "write_exclusive(backup, original_state, state_metadata)" in script
+    assert "os.fchmod(handle.fileno(), stat.S_IMODE(metadata.st_mode))" in script
+    assert "os.fchown(handle.fileno(), metadata.st_uid, metadata.st_gid)" in script
+    assert "fsync_directory(state_root)" in script
+
+
+def test_recovery_restores_original_state_when_post_write_verification_fails():
+    script = recovery._REATTEST_SCRIPT
+
+    assert "def restore_original_state():" in script
+    assert (
+        "write_exclusive(restore_temporary, original_state, state_metadata)" in script
+    )
+    assert "os.replace(restore_temporary, state_path)" in script
+    assert "return state_path.read_bytes() == original_state" in script
+    assert "if not restore_original_state():" in script
+    assert "raise SystemExit(45)" in script
+
+
+def test_recovery_restores_original_state_when_post_replace_durability_fails():
+    script = recovery._REATTEST_SCRIPT
+
+    assert "state_replaced = False" in script
+    assert "state_replaced = True" in script
+    assert "if state_replaced and not restore_original_state():" in script
+
+
+def test_recovery_script_has_no_application_runtime_mutation_commands():
+    forbidden = (
+        "docker restart",
+        "docker stop",
+        "docker rm",
+        "docker compose up",
+        "docker compose down",
+        'subprocess.run(["docker", "restart"',
+        'subprocess.run(["docker", "rm"',
+    )
+
+    assert all(value not in recovery._REATTEST_SCRIPT for value in forbidden)
+
+
+def test_recovery_reports_verified_healthy_state_without_enrollment_fallback():
+    fake = FakePortainer(exit_code=35)
+
+    result = recovery.PrivateMuseDeploymentRecovery(fake).run()
+
+    assert result == {
+        "status": "already-enrolled",
+        "repository": "jens/privatemuse",
+    }
+    assert "PrivateMuseDeploymentEnrollment" not in Path(
+        "backend/app/services/privatemuse_deployment_recovery.py"
+    ).read_text(encoding="utf-8")
+
+
+def test_recovery_fails_closed_when_controller_is_unavailable():
+    class UnavailablePortainer(FakePortainer):
+        def get_container_status(self, name):
+            assert name == "gitea-deploy-control"
+            return {"status": "unavailable", "reason": "offline"}
+
+        def _config(self):
+            raise AssertionError("configuration must not be read")
+
+    result = recovery.PrivateMuseDeploymentRecovery(UnavailablePortainer()).run()
+
+    assert result == {"status": "controller_unavailable", "reason": "offline"}
+
+
+def test_recovery_waits_beyond_controller_health_timeout_and_marks_timeout_uncertain(
+    monkeypatch,
+):
+    fake = FakePortainer()
+    fake._request = lambda method, path, **kwargs: (
+        FakeResponse({"Id": "exec-1"})
+        if path.endswith("/containers/controller-id/exec")
+        else FakeResponse({})
+        if path.endswith("/exec/exec-1/start")
+        else FakeResponse({"Running": True, "ExitCode": None})
+    )
+    monotonic_values = iter((0.0, 301.0))
+    monkeypatch.setattr(recovery.time, "monotonic", lambda: next(monotonic_values))
+
+    result = recovery.PrivateMuseDeploymentRecovery(fake).run()
+
+    assert result == {
+        "status": "outcome_unknown",
+        "reason": (
+            "PrivateMuse runtime re-attestation is still running; inspect the "
+            "existing execution and do not retry"
+        ),
+        "execution_id": "exec-1",
+    }
+    source = Path("backend/app/services/privatemuse_deployment_recovery.py").read_text(
+        encoding="utf-8"
+    )
+    assert "deadline = time.monotonic() + 300" in source
+
+
+def test_recovery_marks_connector_failure_after_exec_start_as_uncertain():
+    class InspectFailurePortainer(FakePortainer):
+        def _request(self, method, path, **kwargs):
+            if path.endswith("/containers/controller-id/exec"):
+                return FakeResponse({"Id": "exec-uncertain"})
+            if path.endswith("/exec/exec-uncertain/start"):
+                return FakeResponse({})
+            if path.endswith("/exec/exec-uncertain/json"):
+                raise RuntimeError("connection lost")
+            raise AssertionError(path)
+
+    result = recovery.PrivateMuseDeploymentRecovery(InspectFailurePortainer()).run()
+
+    assert result == {
+        "status": "outcome_unknown",
+        "reason": (
+            "PrivateMuse runtime re-attestation outcome is unknown after execution "
+            "start; inspect the existing execution and do not retry"
+        ),
+        "execution_id": "exec-uncertain",
+    }
+
+
+def test_recovery_reconciles_durable_unknown_exec_without_starting_another(
+    tmp_path,
+):
+    engine = create_engine(f"sqlite:///{tmp_path / 'recovery.db'}")
+    SQLModel.metadata.create_all(engine)
+
+    class FirstAttemptPortainer(FakePortainer):
+        def _request(self, method, path, **kwargs):
+            self.calls.append((method, path, kwargs))
+            if path.endswith("/containers/controller-id/exec"):
+                return FakeResponse({"Id": "exec-durable"})
+            if path.endswith("/exec/exec-durable/start"):
+                return FakeResponse({})
+            if path.endswith("/exec/exec-durable/json"):
+                raise RuntimeError("connection lost")
+            raise AssertionError(path)
+
+    first = FirstAttemptPortainer()
+    with Session(engine) as session:
+        _tracked, admitted = recovery.PrivateMuseDeploymentRecovery.admit_job(
+            session, recovery_job("job-original")
+        )
+        assert admitted is True
+        uncertain = recovery.PrivateMuseDeploymentRecovery(first).run(
+            session=session,
+            job_id="job-original",
+        )
+
+    assert uncertain["status"] == "outcome_unknown"
+    assert uncertain["execution_id"] == "exec-durable"
+    assert uncertain["original_job_id"] == "job-original"
+
+    class ReconcilePortainer(FakePortainer):
+        def _request(self, method, path, **kwargs):
+            self.calls.append((method, path, kwargs))
+            assert method == "GET"
+            assert path.endswith("/exec/exec-durable/json")
+            return FakeResponse(
+                {
+                    "ID": "exec-durable",
+                    "ContainerID": "controller-id",
+                    "Running": False,
+                    "ExitCode": 0,
+                }
+            )
+
+    reconciler = ReconcilePortainer()
+    with Session(engine) as session:
+        reconciled = recovery.PrivateMuseDeploymentRecovery(reconciler).run(
+            session=session,
+            job_id="job-fresh-key",
+        )
+
+    assert reconciled["status"] == "reconciled"
+    assert reconciled["execution_id"] == "exec-durable"
+    assert reconciled["original_job_id"] == "job-original"
+    assert [method for method, _path, _kwargs in reconciler.calls] == ["GET"]
+
+
+def test_recovery_keeps_unknown_exec_guard_when_controller_identity_changed(tmp_path):
+    engine = create_engine(f"sqlite:///{tmp_path / 'identity.db'}")
+    SQLModel.metadata.create_all(engine)
+    with Session(engine) as session:
+        session.add(
+            recovery.AppSetting(
+                key=recovery._RECOVERY_STATE_SETTING,
+                value_json={
+                    "state": "outcome_unknown",
+                    "executionId": "exec-existing",
+                    "controllerId": "old-controller",
+                    "jobId": "job-original",
+                },
+                is_secret=False,
+            )
+        )
+        session.commit()
+
+    portainer = FakePortainer()
+    with Session(engine) as session:
+        result = recovery.PrivateMuseDeploymentRecovery(portainer).run(
+            session=session,
+            job_id="job-fresh-key",
+        )
+
+    assert result["status"] == "outcome_unknown"
+    assert result["execution_id"] == "exec-existing"
+    assert result["original_job_id"] == "job-original"
+    assert "controller identity changed" in result["reason"]
+    assert portainer.calls == []
+
+
+def test_recovery_admission_creates_only_the_winning_job(tmp_path):
+    engine = create_engine(f"sqlite:///{tmp_path / 'admission.db'}")
+    SQLModel.metadata.create_all(engine)
+
+    with Session(engine) as first_session:
+        first_state, first_admitted = recovery.PrivateMuseDeploymentRecovery.admit_job(
+            first_session, recovery_job("job-winner")
+        )
+    with Session(engine) as second_session:
+        second_state, second_admitted = (
+            recovery.PrivateMuseDeploymentRecovery.admit_job(
+                second_session, recovery_job("job-loser")
+            )
+        )
+        jobs = second_session.exec(select(JobRun)).all()
+
+    assert first_admitted is True
+    assert second_admitted is False
+    assert first_state["jobId"] == "job-winner"
+    assert second_state["jobId"] == "job-winner"
+    assert [job.id for job in jobs] == ["job-winner"]
+
+
+def test_recovery_admission_cas_uses_generation_when_timestamps_collide(
+    tmp_path, monkeypatch
+):
+    fixed = datetime(2026, 9, 20, 12, 0, 0, tzinfo=UTC).replace(tzinfo=None)
+    monkeypatch.setattr(recovery, "utc_now", lambda: fixed)
+    engine = create_engine(
+        f"sqlite:///{tmp_path / 'admission-generation.db'}",
+        connect_args={"check_same_thread": False, "timeout": 10},
+    )
+    SQLModel.metadata.create_all(engine)
+    with Session(engine) as session:
+        session.add(
+            recovery.AppSetting(
+                key=recovery._RECOVERY_STATE_SETTING,
+                value_json={"state": "idle", "version": 0},
+                is_secret=False,
+                updated_at=fixed,
+            )
+        )
+        session.commit()
+
+    update_barrier = Barrier(2)
+
+    def align_cas_updates(_conn, _cursor, statement, _parameters, _context, _many):
+        if statement.startswith("UPDATE app_settings SET"):
+            update_barrier.wait(timeout=10)
+
+    event.listen(engine, "before_cursor_execute", align_cas_updates)
+    admissions = []
+
+    def admit(job_id):
+        with Session(engine) as session:
+            _state, admitted = recovery.PrivateMuseDeploymentRecovery.admit_job(
+                session, recovery_job(job_id)
+            )
+            admissions.append((job_id, admitted))
+
+    workers = [Thread(target=admit, args=(f"job-{index}",)) for index in range(2)]
+    for worker in workers:
+        worker.start()
+    for worker in workers:
+        worker.join(timeout=10)
+        assert not worker.is_alive()
+    event.remove(engine, "before_cursor_execute", align_cas_updates)
+
+    with Session(engine) as session:
+        jobs = session.exec(select(JobRun)).all()
+        state = session.get(recovery.AppSetting, recovery._RECOVERY_STATE_SETTING)
+
+    assert sorted(admitted for _job_id, admitted in admissions) == [False, True]
+    assert len(jobs) == 1
+    assert state is not None
+    assert state.value_json["jobId"] == jobs[0].id
+    assert state.value_json["version"] == 1
+
+
+def test_reserved_recovery_can_only_be_started_by_its_admitted_job(tmp_path):
+    engine = create_engine(f"sqlite:///{tmp_path / 'owner.db'}")
+    SQLModel.metadata.create_all(engine)
+
+    class NoPreflightPortainer(FakePortainer):
+        def get_container_status(self, name):
+            raise AssertionError("a losing caller must not reach Portainer preflight")
+
+    with Session(engine) as session:
+        recovery.PrivateMuseDeploymentRecovery.admit_job(
+            session, recovery_job("job-owner")
+        )
+        result = recovery.PrivateMuseDeploymentRecovery(NoPreflightPortainer()).run(
+            session=session,
+            job_id="job-loser",
+        )
+
+    assert result["status"] == "in_progress"
+    assert result["original_job_id"] == "job-owner"
+    assert "no second execution was started" in result["reason"]
+
+
+def test_terminal_reconciliation_cannot_regress_to_late_unknown(tmp_path):
+    engine = create_engine(
+        f"sqlite:///{tmp_path / 'monotonic.db'}",
+        connect_args={"check_same_thread": False},
+    )
+    SQLModel.metadata.create_all(engine)
+    with Session(engine) as session:
+        recovery.PrivateMuseDeploymentRecovery.admit_job(
+            session, recovery_job("job-race")
+        )
+
+    inspect_started = Event()
+    reconciliation_finished = Event()
+    worker_result = {}
+
+    class DelayedWorkerPortainer(FakePortainer):
+        def _request(self, method, path, **kwargs):
+            if path.endswith("/containers/controller-id/exec"):
+                return FakeResponse({"Id": "exec-race"})
+            if path.endswith("/exec/exec-race/start"):
+                return FakeResponse({})
+            if path.endswith("/exec/exec-race/json"):
+                inspect_started.set()
+                assert reconciliation_finished.wait(timeout=10)
+                raise RuntimeError("late connection loss")
+            raise AssertionError(path)
+
+    def run_delayed_worker():
+        with Session(engine) as worker_session:
+            worker_result.update(
+                recovery.PrivateMuseDeploymentRecovery(
+                    DelayedWorkerPortainer()
+                ).run(session=worker_session, job_id="job-race")
+            )
+
+    worker = Thread(target=run_delayed_worker)
+    worker.start()
+    assert inspect_started.wait(timeout=10)
+
+    class TerminalInspectPortainer(FakePortainer):
+        def _request(self, method, path, **kwargs):
+            assert method == "GET"
+            assert path.endswith("/exec/exec-race/json")
+            return FakeResponse(
+                {
+                    "ID": "exec-race",
+                    "ContainerID": "controller-id",
+                    "Running": False,
+                    "ExitCode": 0,
+                }
+            )
+
+    with Session(engine) as reconcile_session:
+        reconciled = recovery.PrivateMuseDeploymentRecovery(
+            TerminalInspectPortainer()
+        ).run(session=reconcile_session, job_id="job-fresh-key")
+    reconciliation_finished.set()
+    worker.join(timeout=10)
+    assert not worker.is_alive()
+
+    with Session(engine) as verification_session:
+        row = verification_session.get(
+            recovery.AppSetting, recovery._RECOVERY_STATE_SETTING
+        )
+
+    assert reconciled["status"] == "reconciled"
+    assert worker_result["status"] == "reconciled"
+    assert row is not None
+    assert row.value_json["state"] == "completed"
+    assert row.value_json["result"]["status"] == "reconciled"
+
+
+def test_recovery_state_cas_prevents_same_timestamp_terminal_regression(
+    tmp_path, monkeypatch
+):
+    fixed = datetime(2026, 9, 20, 12, 0, 0, tzinfo=UTC).replace(tzinfo=None)
+    monkeypatch.setattr(recovery, "utc_now", lambda: fixed)
+    engine = create_engine(
+        f"sqlite:///{tmp_path / 'terminal-generation.db'}",
+        connect_args={"check_same_thread": False, "timeout": 10},
+    )
+    SQLModel.metadata.create_all(engine)
+    tracked = {
+        "state": "started",
+        "claimId": "claim-race",
+        "jobId": "job-race",
+        "controllerId": "controller-id",
+        "executionId": "exec-race",
+        "version": 7,
+    }
+    with Session(engine) as session:
+        session.add(
+            recovery.AppSetting(
+                key=recovery._RECOVERY_STATE_SETTING,
+                value_json=tracked,
+                is_secret=False,
+                updated_at=fixed,
+            )
+        )
+        session.commit()
+
+    completed_at_update = Event()
+    unknown_at_update = Event()
+    completed_finished = Event()
+
+    def order_cas_updates(_conn, _cursor, statement, parameters, _context, _many):
+        if not statement.startswith("UPDATE app_settings SET"):
+            return
+        payload = str(parameters[0])
+        if '"state": "completed"' in payload:
+            completed_at_update.set()
+            assert unknown_at_update.wait(timeout=10)
+        elif '"state": "outcome_unknown"' in payload:
+            unknown_at_update.set()
+            assert completed_at_update.wait(timeout=10)
+            assert completed_finished.wait(timeout=10)
+
+    event.listen(engine, "before_cursor_execute", order_cas_updates)
+    results = {}
+
+    def persist_completed():
+        with Session(engine) as session:
+            results["completed"] = recovery.PrivateMuseDeploymentRecovery._persist_state(
+                session,
+                dict(tracked),
+                state="completed",
+                execution_id="exec-race",
+                result={"status": "reconciled"},
+            )
+        completed_finished.set()
+
+    def persist_unknown():
+        with Session(engine) as session:
+            results["unknown"] = recovery.PrivateMuseDeploymentRecovery._persist_state(
+                session,
+                dict(tracked),
+                state="outcome_unknown",
+                execution_id="exec-race",
+                result={"status": "outcome_unknown"},
+            )
+
+    completed_worker = Thread(target=persist_completed)
+    unknown_worker = Thread(target=persist_unknown)
+    completed_worker.start()
+    unknown_worker.start()
+    completed_worker.join(timeout=10)
+    unknown_worker.join(timeout=10)
+    assert not completed_worker.is_alive()
+    assert not unknown_worker.is_alive()
+    event.remove(engine, "before_cursor_execute", order_cas_updates)
+
+    with Session(engine) as session:
+        state = session.get(recovery.AppSetting, recovery._RECOVERY_STATE_SETTING)
+
+    assert results["completed"]["state"] == "completed"
+    assert results["unknown"]["state"] == "completed"
+    assert state is not None
+    assert state.value_json["state"] == "completed"
+    assert state.value_json["result"]["status"] == "reconciled"
+    assert state.value_json["version"] == 8
+
+
+def test_recovery_fails_closed_when_runtime_provenance_differs():
+    result = recovery.PrivateMuseDeploymentRecovery(FakePortainer(exit_code=38)).run()
+
+    assert result == {
+        "status": "validation_failed",
+        "reason": "PrivateMuse immutable container/image provenance differs from the recorded receipt",
+    }
+
+
+def test_privatemuse_action_execution_accepts_reconciled_status():
+    source = Path("backend/app/services/action_execution.py").read_text(
+        encoding="utf-8"
+    )
+
+    assert (
+        "from .privatemuse_deployment_recovery import PrivateMuseDeploymentRecovery"
+        in source
+    )
+    assert "tracked, admitted = recovery.admit_job(session, candidate_job)" in source
+    assert "job_id=job.id if admitted else candidate_job.id" in source
+    assert 'status in {"enrolled", "already-enrolled", "reconciled"}' in source