from datetime import UTC, datetime from pathlib import Path from threading import Barrier, Event, Thread from types import SimpleNamespace from sqlalchemy import event from sqlmodel import Session, SQLModel, create_engine, select import backend.app.services.privatemuse_deployment_recovery as recovery from backend.app.db.models import JobRun class FakeResponse: def __init__(self, payload): self.payload = payload def json(self): return self.payload class FakePortainer: def __init__(self, exit_code=0): self.exit_code = exit_code self.calls = [] def get_container_status(self, name): assert name == "gitea-deploy-control" return { "status": "ok", "container": { "id": "controller-id", "name": "gitea-deploy-control", "state": "running", }, } def _config(self): return SimpleNamespace(endpoint_id="3") def _request(self, method, path, **kwargs): self.calls.append((method, path, kwargs)) if path.endswith("/containers/controller-id/exec"): return FakeResponse({"Id": "exec-1"}) if path.endswith("/exec/exec-1/start"): return FakeResponse({}) if path.endswith("/exec/exec-1/json"): return FakeResponse( { "ID": "exec-1", "ContainerID": "controller-id", "Running": False, "ExitCode": self.exit_code, } ) raise AssertionError(path) def recovery_job(job_id): return JobRun( id=job_id, type="deployment.target.register_privatemuse", status="queued", source="chatgpt-mcp", action_id="deployment.target.register_privatemuse", target="jens/privatemuse", risk="high", policy_decision="allowed", ) def test_recovery_runs_only_the_exact_observed_privatemuse_incident(): fake = FakePortainer() result = recovery.PrivateMuseDeploymentRecovery(fake).run() assert result == { "status": "reconciled", "repository": "jens/privatemuse", "live_revision": "2103d5795d151885edeeb964466b643b8fd4a0eb", "failed_revision": "6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38", "containers": [ "privatemuse-backend-1", "privatemuse-frontend-1", "privatemuse-audio-enhancement-1", ], "compose_project": "privatemuse", } command = fake.calls[0][2]["json"]["Cmd"] assert command[:3] == ["python3", "-c", recovery._REATTEST_SCRIPT] assert command[3:] == [ "jens/privatemuse", "2103d5795d151885edeeb964466b643b8fd4a0eb", "6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38", "Existing production containers require a valid healthy live deployment state", ] def test_recovery_script_allows_only_the_two_observed_container_id_changes(): script = recovery._REATTEST_SCRIPT assert ( 'allowed_changed_ids = {"privatemuse-backend-1", "privatemuse-frontend-1"}' in script ) assert "changed_receipt_ids != allowed_changed_ids" in script assert "changed_artifact_ids != allowed_changed_ids" in script assert 'state.get("revision") == live_revision' in script assert 'state.get("live_revision") == live_revision' in script assert 'state.get("live_status") == "unknown"' in script assert 'attempt.get("revision") == failed_revision' in script assert 'attempt.get("message") == failure_message' in script def test_recovery_reuses_controller_verifiers_and_writes_state_durably(): script = recovery._REATTEST_SCRIPT assert "controller.validate_recorded_receipt" in script assert "controller.verify_deployment" in script assert "expected_image_ids=expected_image_ids" in script assert "controller.deployment_artifacts" in script assert "controller.verified_current_receipt" in script assert "write_exclusive(backup, original_state, state_metadata)" in script assert "os.replace(temporary, state_path)" in script assert "os.fsync" in script def test_recovery_durably_preserves_the_original_state_before_replacement(): script = recovery._REATTEST_SCRIPT assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script assert "/mnt/cache/appdata/gitea-deploy-runners/deploy.lock" in script assert "original_state = state_path.read_bytes()" in script assert "write_exclusive(backup, original_state, state_metadata)" in script assert "os.fchmod(handle.fileno(), stat.S_IMODE(metadata.st_mode))" in script assert "os.fchown(handle.fileno(), metadata.st_uid, metadata.st_gid)" in script assert "fsync_directory(state_root)" in script def test_recovery_restores_original_state_when_post_write_verification_fails(): script = recovery._REATTEST_SCRIPT assert "def restore_original_state():" in script assert ( "write_exclusive(restore_temporary, original_state, state_metadata)" in script ) assert "os.replace(restore_temporary, state_path)" in script assert "return state_path.read_bytes() == original_state" in script assert "if not restore_original_state():" in script assert "raise SystemExit(45)" in script def test_recovery_restores_original_state_when_post_replace_durability_fails(): script = recovery._REATTEST_SCRIPT assert "state_replaced = False" in script assert "state_replaced = True" in script assert "if state_replaced and not restore_original_state():" in script def test_recovery_script_has_no_application_runtime_mutation_commands(): forbidden = ( "docker restart", "docker stop", "docker rm", "docker compose up", "docker compose down", 'subprocess.run(["docker", "restart"', 'subprocess.run(["docker", "rm"', ) assert all(value not in recovery._REATTEST_SCRIPT for value in forbidden) def test_recovery_reports_verified_healthy_state_without_enrollment_fallback(): fake = FakePortainer(exit_code=35) result = recovery.PrivateMuseDeploymentRecovery(fake).run() assert result == { "status": "already-enrolled", "repository": "jens/privatemuse", } assert "PrivateMuseDeploymentEnrollment" not in Path( "backend/app/services/privatemuse_deployment_recovery.py" ).read_text(encoding="utf-8") def test_recovery_fails_closed_when_controller_is_unavailable(): class UnavailablePortainer(FakePortainer): def get_container_status(self, name): assert name == "gitea-deploy-control" return {"status": "unavailable", "reason": "offline"} def _config(self): raise AssertionError("configuration must not be read") result = recovery.PrivateMuseDeploymentRecovery(UnavailablePortainer()).run() assert result == {"status": "controller_unavailable", "reason": "offline"} def test_recovery_waits_beyond_controller_health_timeout_and_marks_timeout_uncertain( monkeypatch, ): fake = FakePortainer() fake._request = lambda method, path, **kwargs: ( FakeResponse({"Id": "exec-1"}) if path.endswith("/containers/controller-id/exec") else FakeResponse({}) if path.endswith("/exec/exec-1/start") else FakeResponse({"Running": True, "ExitCode": None}) ) monotonic_values = iter((0.0, 301.0)) monkeypatch.setattr(recovery.time, "monotonic", lambda: next(monotonic_values)) result = recovery.PrivateMuseDeploymentRecovery(fake).run() assert result == { "status": "outcome_unknown", "reason": ( "PrivateMuse runtime re-attestation is still running; inspect the " "existing execution and do not retry" ), "execution_id": "exec-1", } source = Path("backend/app/services/privatemuse_deployment_recovery.py").read_text( encoding="utf-8" ) assert "deadline = time.monotonic() + 300" in source def test_recovery_marks_connector_failure_after_exec_start_as_uncertain(): class InspectFailurePortainer(FakePortainer): def _request(self, method, path, **kwargs): if path.endswith("/containers/controller-id/exec"): return FakeResponse({"Id": "exec-uncertain"}) if path.endswith("/exec/exec-uncertain/start"): return FakeResponse({}) if path.endswith("/exec/exec-uncertain/json"): raise RuntimeError("connection lost") raise AssertionError(path) result = recovery.PrivateMuseDeploymentRecovery(InspectFailurePortainer()).run() assert result == { "status": "outcome_unknown", "reason": ( "PrivateMuse runtime re-attestation outcome is unknown after execution " "start; inspect the existing execution and do not retry" ), "execution_id": "exec-uncertain", } def test_recovery_reconciles_durable_unknown_exec_without_starting_another( tmp_path, ): engine = create_engine(f"sqlite:///{tmp_path / 'recovery.db'}") SQLModel.metadata.create_all(engine) class FirstAttemptPortainer(FakePortainer): def _request(self, method, path, **kwargs): self.calls.append((method, path, kwargs)) if path.endswith("/containers/controller-id/exec"): return FakeResponse({"Id": "exec-durable"}) if path.endswith("/exec/exec-durable/start"): return FakeResponse({}) if path.endswith("/exec/exec-durable/json"): raise RuntimeError("connection lost") raise AssertionError(path) first = FirstAttemptPortainer() with Session(engine) as session: _tracked, admitted = recovery.PrivateMuseDeploymentRecovery.admit_job( session, recovery_job("job-original") ) assert admitted is True uncertain = recovery.PrivateMuseDeploymentRecovery(first).run( session=session, job_id="job-original", ) assert uncertain["status"] == "outcome_unknown" assert uncertain["execution_id"] == "exec-durable" assert uncertain["original_job_id"] == "job-original" class ReconcilePortainer(FakePortainer): def _request(self, method, path, **kwargs): self.calls.append((method, path, kwargs)) assert method == "GET" assert path.endswith("/exec/exec-durable/json") return FakeResponse( { "ID": "exec-durable", "ContainerID": "controller-id", "Running": False, "ExitCode": 0, } ) reconciler = ReconcilePortainer() with Session(engine) as session: reconciled = recovery.PrivateMuseDeploymentRecovery(reconciler).run( session=session, job_id="job-fresh-key", ) assert reconciled["status"] == "reconciled" assert reconciled["execution_id"] == "exec-durable" assert reconciled["original_job_id"] == "job-original" assert [method for method, _path, _kwargs in reconciler.calls] == ["GET"] def test_recovery_keeps_unknown_exec_guard_when_controller_identity_changed(tmp_path): engine = create_engine(f"sqlite:///{tmp_path / 'identity.db'}") SQLModel.metadata.create_all(engine) with Session(engine) as session: session.add( recovery.AppSetting( key=recovery._RECOVERY_STATE_SETTING, value_json={ "state": "outcome_unknown", "executionId": "exec-existing", "controllerId": "old-controller", "jobId": "job-original", }, is_secret=False, ) ) session.commit() portainer = FakePortainer() with Session(engine) as session: result = recovery.PrivateMuseDeploymentRecovery(portainer).run( session=session, job_id="job-fresh-key", ) assert result["status"] == "outcome_unknown" assert result["execution_id"] == "exec-existing" assert result["original_job_id"] == "job-original" assert "controller identity changed" in result["reason"] assert portainer.calls == [] def test_recovery_admission_creates_only_the_winning_job(tmp_path): engine = create_engine(f"sqlite:///{tmp_path / 'admission.db'}") SQLModel.metadata.create_all(engine) with Session(engine) as first_session: first_state, first_admitted = recovery.PrivateMuseDeploymentRecovery.admit_job( first_session, recovery_job("job-winner") ) with Session(engine) as second_session: second_state, second_admitted = ( recovery.PrivateMuseDeploymentRecovery.admit_job( second_session, recovery_job("job-loser") ) ) jobs = second_session.exec(select(JobRun)).all() assert first_admitted is True assert second_admitted is False assert first_state["jobId"] == "job-winner" assert second_state["jobId"] == "job-winner" assert [job.id for job in jobs] == ["job-winner"] def test_recovery_admission_cas_uses_generation_when_timestamps_collide( tmp_path, monkeypatch ): fixed = datetime(2026, 9, 20, 12, 0, 0, tzinfo=UTC).replace(tzinfo=None) monkeypatch.setattr(recovery, "utc_now", lambda: fixed) engine = create_engine( f"sqlite:///{tmp_path / 'admission-generation.db'}", connect_args={"check_same_thread": False, "timeout": 10}, ) SQLModel.metadata.create_all(engine) with Session(engine) as session: session.add( recovery.AppSetting( key=recovery._RECOVERY_STATE_SETTING, value_json={"state": "idle", "version": 0}, is_secret=False, updated_at=fixed, ) ) session.commit() update_barrier = Barrier(2) def align_cas_updates(_conn, _cursor, statement, _parameters, _context, _many): if statement.startswith("UPDATE app_settings SET"): update_barrier.wait(timeout=10) event.listen(engine, "before_cursor_execute", align_cas_updates) admissions = [] def admit(job_id): with Session(engine) as session: _state, admitted = recovery.PrivateMuseDeploymentRecovery.admit_job( session, recovery_job(job_id) ) admissions.append((job_id, admitted)) workers = [Thread(target=admit, args=(f"job-{index}",)) for index in range(2)] for worker in workers: worker.start() for worker in workers: worker.join(timeout=10) assert not worker.is_alive() event.remove(engine, "before_cursor_execute", align_cas_updates) with Session(engine) as session: jobs = session.exec(select(JobRun)).all() state = session.get(recovery.AppSetting, recovery._RECOVERY_STATE_SETTING) assert sorted(admitted for _job_id, admitted in admissions) == [False, True] assert len(jobs) == 1 assert state is not None assert state.value_json["jobId"] == jobs[0].id assert state.value_json["version"] == 1 def test_reserved_recovery_can_only_be_started_by_its_admitted_job(tmp_path): engine = create_engine(f"sqlite:///{tmp_path / 'owner.db'}") SQLModel.metadata.create_all(engine) class NoPreflightPortainer(FakePortainer): def get_container_status(self, name): raise AssertionError("a losing caller must not reach Portainer preflight") with Session(engine) as session: recovery.PrivateMuseDeploymentRecovery.admit_job( session, recovery_job("job-owner") ) result = recovery.PrivateMuseDeploymentRecovery(NoPreflightPortainer()).run( session=session, job_id="job-loser", ) assert result["status"] == "in_progress" assert result["original_job_id"] == "job-owner" assert "no second execution was started" in result["reason"] def test_terminal_reconciliation_cannot_regress_to_late_unknown(tmp_path): engine = create_engine( f"sqlite:///{tmp_path / 'monotonic.db'}", connect_args={"check_same_thread": False}, ) SQLModel.metadata.create_all(engine) with Session(engine) as session: recovery.PrivateMuseDeploymentRecovery.admit_job( session, recovery_job("job-race") ) inspect_started = Event() reconciliation_finished = Event() worker_result = {} class DelayedWorkerPortainer(FakePortainer): def _request(self, method, path, **kwargs): if path.endswith("/containers/controller-id/exec"): return FakeResponse({"Id": "exec-race"}) if path.endswith("/exec/exec-race/start"): return FakeResponse({}) if path.endswith("/exec/exec-race/json"): inspect_started.set() assert reconciliation_finished.wait(timeout=10) raise RuntimeError("late connection loss") raise AssertionError(path) def run_delayed_worker(): with Session(engine) as worker_session: worker_result.update( recovery.PrivateMuseDeploymentRecovery( DelayedWorkerPortainer() ).run(session=worker_session, job_id="job-race") ) worker = Thread(target=run_delayed_worker) worker.start() assert inspect_started.wait(timeout=10) class TerminalInspectPortainer(FakePortainer): def _request(self, method, path, **kwargs): assert method == "GET" assert path.endswith("/exec/exec-race/json") return FakeResponse( { "ID": "exec-race", "ContainerID": "controller-id", "Running": False, "ExitCode": 0, } ) with Session(engine) as reconcile_session: reconciled = recovery.PrivateMuseDeploymentRecovery( TerminalInspectPortainer() ).run(session=reconcile_session, job_id="job-fresh-key") reconciliation_finished.set() worker.join(timeout=10) assert not worker.is_alive() with Session(engine) as verification_session: row = verification_session.get( recovery.AppSetting, recovery._RECOVERY_STATE_SETTING ) assert reconciled["status"] == "reconciled" assert worker_result["status"] == "reconciled" assert row is not None assert row.value_json["state"] == "completed" assert row.value_json["result"]["status"] == "reconciled" def test_recovery_state_cas_prevents_same_timestamp_terminal_regression( tmp_path, monkeypatch ): fixed = datetime(2026, 9, 20, 12, 0, 0, tzinfo=UTC).replace(tzinfo=None) monkeypatch.setattr(recovery, "utc_now", lambda: fixed) engine = create_engine( f"sqlite:///{tmp_path / 'terminal-generation.db'}", connect_args={"check_same_thread": False, "timeout": 10}, ) SQLModel.metadata.create_all(engine) tracked = { "state": "started", "claimId": "claim-race", "jobId": "job-race", "controllerId": "controller-id", "executionId": "exec-race", "version": 7, } with Session(engine) as session: session.add( recovery.AppSetting( key=recovery._RECOVERY_STATE_SETTING, value_json=tracked, is_secret=False, updated_at=fixed, ) ) session.commit() completed_at_update = Event() unknown_at_update = Event() completed_finished = Event() def order_cas_updates(_conn, _cursor, statement, parameters, _context, _many): if not statement.startswith("UPDATE app_settings SET"): return payload = str(parameters[0]) if '"state": "completed"' in payload: completed_at_update.set() assert unknown_at_update.wait(timeout=10) elif '"state": "outcome_unknown"' in payload: unknown_at_update.set() assert completed_at_update.wait(timeout=10) assert completed_finished.wait(timeout=10) event.listen(engine, "before_cursor_execute", order_cas_updates) results = {} def persist_completed(): with Session(engine) as session: results["completed"] = recovery.PrivateMuseDeploymentRecovery._persist_state( session, dict(tracked), state="completed", execution_id="exec-race", result={"status": "reconciled"}, ) completed_finished.set() def persist_unknown(): with Session(engine) as session: results["unknown"] = recovery.PrivateMuseDeploymentRecovery._persist_state( session, dict(tracked), state="outcome_unknown", execution_id="exec-race", result={"status": "outcome_unknown"}, ) completed_worker = Thread(target=persist_completed) unknown_worker = Thread(target=persist_unknown) completed_worker.start() unknown_worker.start() completed_worker.join(timeout=10) unknown_worker.join(timeout=10) assert not completed_worker.is_alive() assert not unknown_worker.is_alive() event.remove(engine, "before_cursor_execute", order_cas_updates) with Session(engine) as session: state = session.get(recovery.AppSetting, recovery._RECOVERY_STATE_SETTING) assert results["completed"]["state"] == "completed" assert results["unknown"]["state"] == "completed" assert state is not None assert state.value_json["state"] == "completed" assert state.value_json["result"]["status"] == "reconciled" assert state.value_json["version"] == 8 def test_recovery_fails_closed_when_runtime_provenance_differs(): result = recovery.PrivateMuseDeploymentRecovery(FakePortainer(exit_code=38)).run() assert result == { "status": "validation_failed", "reason": "PrivateMuse immutable container/image provenance differs from the recorded receipt", } def test_privatemuse_action_execution_accepts_reconciled_status(): source = Path("backend/app/services/action_execution.py").read_text( encoding="utf-8" ) assert ( "from .privatemuse_deployment_recovery import PrivateMuseDeploymentRecovery" in source ) assert "tracked, admitted = recovery.admit_job(session, candidate_job)" in source assert "job_id=job.id if admitted else candidate_job.id" in source assert 'status in {"enrolled", "already-enrolled", "reconciled"}' in source