631 lines
22 KiB
Python
631 lines
22 KiB
Python
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from threading import Barrier, Event, Thread
|
|
from types import SimpleNamespace
|
|
|
|
from sqlalchemy import event
|
|
from sqlmodel import Session, SQLModel, create_engine, select
|
|
|
|
import backend.app.services.privatemuse_deployment_recovery as recovery
|
|
from backend.app.db.models import JobRun
|
|
|
|
|
|
class FakeResponse:
|
|
def __init__(self, payload):
|
|
self.payload = payload
|
|
|
|
def json(self):
|
|
return self.payload
|
|
|
|
|
|
class FakePortainer:
|
|
def __init__(self, exit_code=0):
|
|
self.exit_code = exit_code
|
|
self.calls = []
|
|
|
|
def get_container_status(self, name):
|
|
assert name == "gitea-deploy-control"
|
|
return {
|
|
"status": "ok",
|
|
"container": {
|
|
"id": "controller-id",
|
|
"name": "gitea-deploy-control",
|
|
"state": "running",
|
|
},
|
|
}
|
|
|
|
def _config(self):
|
|
return SimpleNamespace(endpoint_id="3")
|
|
|
|
def _request(self, method, path, **kwargs):
|
|
self.calls.append((method, path, kwargs))
|
|
if path.endswith("/containers/controller-id/exec"):
|
|
return FakeResponse({"Id": "exec-1"})
|
|
if path.endswith("/exec/exec-1/start"):
|
|
return FakeResponse({})
|
|
if path.endswith("/exec/exec-1/json"):
|
|
return FakeResponse(
|
|
{
|
|
"ID": "exec-1",
|
|
"ContainerID": "controller-id",
|
|
"Running": False,
|
|
"ExitCode": self.exit_code,
|
|
}
|
|
)
|
|
raise AssertionError(path)
|
|
|
|
|
|
def recovery_job(job_id):
|
|
return JobRun(
|
|
id=job_id,
|
|
type="deployment.target.register_privatemuse",
|
|
status="queued",
|
|
source="chatgpt-mcp",
|
|
action_id="deployment.target.register_privatemuse",
|
|
target="jens/privatemuse",
|
|
risk="high",
|
|
policy_decision="allowed",
|
|
)
|
|
|
|
|
|
def test_recovery_runs_only_the_exact_observed_privatemuse_incident():
|
|
fake = FakePortainer()
|
|
|
|
result = recovery.PrivateMuseDeploymentRecovery(fake).run()
|
|
|
|
assert result == {
|
|
"status": "reconciled",
|
|
"repository": "jens/privatemuse",
|
|
"live_revision": "2103d5795d151885edeeb964466b643b8fd4a0eb",
|
|
"failed_revision": "6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38",
|
|
"containers": [
|
|
"privatemuse-backend-1",
|
|
"privatemuse-frontend-1",
|
|
"privatemuse-audio-enhancement-1",
|
|
],
|
|
"compose_project": "privatemuse",
|
|
}
|
|
command = fake.calls[0][2]["json"]["Cmd"]
|
|
assert command[:3] == ["python3", "-c", recovery._REATTEST_SCRIPT]
|
|
assert command[3:] == [
|
|
"jens/privatemuse",
|
|
"2103d5795d151885edeeb964466b643b8fd4a0eb",
|
|
"6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38",
|
|
"Existing production containers require a valid healthy live deployment state",
|
|
]
|
|
|
|
|
|
def test_recovery_script_allows_only_the_two_observed_container_id_changes():
|
|
script = recovery._REATTEST_SCRIPT
|
|
|
|
assert (
|
|
'allowed_changed_ids = {"privatemuse-backend-1", "privatemuse-frontend-1"}'
|
|
in script
|
|
)
|
|
assert "changed_receipt_ids != allowed_changed_ids" in script
|
|
assert "changed_artifact_ids != allowed_changed_ids" in script
|
|
assert 'state.get("revision") == live_revision' in script
|
|
assert 'state.get("live_revision") == live_revision' in script
|
|
assert 'state.get("live_status") == "unknown"' in script
|
|
assert 'attempt.get("revision") == failed_revision' in script
|
|
assert 'attempt.get("message") == failure_message' in script
|
|
|
|
|
|
def test_recovery_reuses_controller_verifiers_and_writes_state_durably():
|
|
script = recovery._REATTEST_SCRIPT
|
|
|
|
assert "controller.validate_recorded_receipt" in script
|
|
assert "controller.verify_deployment" in script
|
|
assert "expected_image_ids=expected_image_ids" in script
|
|
assert "controller.deployment_artifacts" in script
|
|
assert "controller.verified_current_receipt" in script
|
|
assert "write_exclusive(backup, original_state, state_metadata)" in script
|
|
assert "os.replace(temporary, state_path)" in script
|
|
assert "os.fsync" in script
|
|
|
|
|
|
def test_recovery_durably_preserves_the_original_state_before_replacement():
|
|
script = recovery._REATTEST_SCRIPT
|
|
|
|
assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script
|
|
assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script
|
|
assert "/mnt/cache/appdata/gitea-deploy-runners/deploy.lock" in script
|
|
assert "original_state = state_path.read_bytes()" in script
|
|
assert "write_exclusive(backup, original_state, state_metadata)" in script
|
|
assert "os.fchmod(handle.fileno(), stat.S_IMODE(metadata.st_mode))" in script
|
|
assert "os.fchown(handle.fileno(), metadata.st_uid, metadata.st_gid)" in script
|
|
assert "fsync_directory(state_root)" in script
|
|
|
|
|
|
def test_recovery_restores_original_state_when_post_write_verification_fails():
|
|
script = recovery._REATTEST_SCRIPT
|
|
|
|
assert "def restore_original_state():" in script
|
|
assert (
|
|
"write_exclusive(restore_temporary, original_state, state_metadata)" in script
|
|
)
|
|
assert "os.replace(restore_temporary, state_path)" in script
|
|
assert "return state_path.read_bytes() == original_state" in script
|
|
assert "if not restore_original_state():" in script
|
|
assert "raise SystemExit(45)" in script
|
|
|
|
|
|
def test_recovery_restores_original_state_when_post_replace_durability_fails():
|
|
script = recovery._REATTEST_SCRIPT
|
|
|
|
assert "state_replaced = False" in script
|
|
assert "state_replaced = True" in script
|
|
assert "if state_replaced and not restore_original_state():" in script
|
|
|
|
|
|
def test_recovery_script_has_no_application_runtime_mutation_commands():
|
|
forbidden = (
|
|
"docker restart",
|
|
"docker stop",
|
|
"docker rm",
|
|
"docker compose up",
|
|
"docker compose down",
|
|
'subprocess.run(["docker", "restart"',
|
|
'subprocess.run(["docker", "rm"',
|
|
)
|
|
|
|
assert all(value not in recovery._REATTEST_SCRIPT for value in forbidden)
|
|
|
|
|
|
def test_recovery_reports_verified_healthy_state_without_enrollment_fallback():
|
|
fake = FakePortainer(exit_code=35)
|
|
|
|
result = recovery.PrivateMuseDeploymentRecovery(fake).run()
|
|
|
|
assert result == {
|
|
"status": "already-enrolled",
|
|
"repository": "jens/privatemuse",
|
|
}
|
|
assert "PrivateMuseDeploymentEnrollment" not in Path(
|
|
"backend/app/services/privatemuse_deployment_recovery.py"
|
|
).read_text(encoding="utf-8")
|
|
|
|
|
|
def test_recovery_fails_closed_when_controller_is_unavailable():
|
|
class UnavailablePortainer(FakePortainer):
|
|
def get_container_status(self, name):
|
|
assert name == "gitea-deploy-control"
|
|
return {"status": "unavailable", "reason": "offline"}
|
|
|
|
def _config(self):
|
|
raise AssertionError("configuration must not be read")
|
|
|
|
result = recovery.PrivateMuseDeploymentRecovery(UnavailablePortainer()).run()
|
|
|
|
assert result == {"status": "controller_unavailable", "reason": "offline"}
|
|
|
|
|
|
def test_recovery_waits_beyond_controller_health_timeout_and_marks_timeout_uncertain(
|
|
monkeypatch,
|
|
):
|
|
fake = FakePortainer()
|
|
fake._request = lambda method, path, **kwargs: (
|
|
FakeResponse({"Id": "exec-1"})
|
|
if path.endswith("/containers/controller-id/exec")
|
|
else FakeResponse({})
|
|
if path.endswith("/exec/exec-1/start")
|
|
else FakeResponse({"Running": True, "ExitCode": None})
|
|
)
|
|
monotonic_values = iter((0.0, 301.0))
|
|
monkeypatch.setattr(recovery.time, "monotonic", lambda: next(monotonic_values))
|
|
|
|
result = recovery.PrivateMuseDeploymentRecovery(fake).run()
|
|
|
|
assert result == {
|
|
"status": "outcome_unknown",
|
|
"reason": (
|
|
"PrivateMuse runtime re-attestation is still running; inspect the "
|
|
"existing execution and do not retry"
|
|
),
|
|
"execution_id": "exec-1",
|
|
}
|
|
source = Path("backend/app/services/privatemuse_deployment_recovery.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
assert "deadline = time.monotonic() + 300" in source
|
|
|
|
|
|
def test_recovery_marks_connector_failure_after_exec_start_as_uncertain():
|
|
class InspectFailurePortainer(FakePortainer):
|
|
def _request(self, method, path, **kwargs):
|
|
if path.endswith("/containers/controller-id/exec"):
|
|
return FakeResponse({"Id": "exec-uncertain"})
|
|
if path.endswith("/exec/exec-uncertain/start"):
|
|
return FakeResponse({})
|
|
if path.endswith("/exec/exec-uncertain/json"):
|
|
raise RuntimeError("connection lost")
|
|
raise AssertionError(path)
|
|
|
|
result = recovery.PrivateMuseDeploymentRecovery(InspectFailurePortainer()).run()
|
|
|
|
assert result == {
|
|
"status": "outcome_unknown",
|
|
"reason": (
|
|
"PrivateMuse runtime re-attestation outcome is unknown after execution "
|
|
"start; inspect the existing execution and do not retry"
|
|
),
|
|
"execution_id": "exec-uncertain",
|
|
}
|
|
|
|
|
|
def test_recovery_reconciles_durable_unknown_exec_without_starting_another(
|
|
tmp_path,
|
|
):
|
|
engine = create_engine(f"sqlite:///{tmp_path / 'recovery.db'}")
|
|
SQLModel.metadata.create_all(engine)
|
|
|
|
class FirstAttemptPortainer(FakePortainer):
|
|
def _request(self, method, path, **kwargs):
|
|
self.calls.append((method, path, kwargs))
|
|
if path.endswith("/containers/controller-id/exec"):
|
|
return FakeResponse({"Id": "exec-durable"})
|
|
if path.endswith("/exec/exec-durable/start"):
|
|
return FakeResponse({})
|
|
if path.endswith("/exec/exec-durable/json"):
|
|
raise RuntimeError("connection lost")
|
|
raise AssertionError(path)
|
|
|
|
first = FirstAttemptPortainer()
|
|
with Session(engine) as session:
|
|
_tracked, admitted = recovery.PrivateMuseDeploymentRecovery.admit_job(
|
|
session, recovery_job("job-original")
|
|
)
|
|
assert admitted is True
|
|
uncertain = recovery.PrivateMuseDeploymentRecovery(first).run(
|
|
session=session,
|
|
job_id="job-original",
|
|
)
|
|
|
|
assert uncertain["status"] == "outcome_unknown"
|
|
assert uncertain["execution_id"] == "exec-durable"
|
|
assert uncertain["original_job_id"] == "job-original"
|
|
|
|
class ReconcilePortainer(FakePortainer):
|
|
def _request(self, method, path, **kwargs):
|
|
self.calls.append((method, path, kwargs))
|
|
assert method == "GET"
|
|
assert path.endswith("/exec/exec-durable/json")
|
|
return FakeResponse(
|
|
{
|
|
"ID": "exec-durable",
|
|
"ContainerID": "controller-id",
|
|
"Running": False,
|
|
"ExitCode": 0,
|
|
}
|
|
)
|
|
|
|
reconciler = ReconcilePortainer()
|
|
with Session(engine) as session:
|
|
reconciled = recovery.PrivateMuseDeploymentRecovery(reconciler).run(
|
|
session=session,
|
|
job_id="job-fresh-key",
|
|
)
|
|
|
|
assert reconciled["status"] == "reconciled"
|
|
assert reconciled["execution_id"] == "exec-durable"
|
|
assert reconciled["original_job_id"] == "job-original"
|
|
assert [method for method, _path, _kwargs in reconciler.calls] == ["GET"]
|
|
|
|
|
|
def test_recovery_keeps_unknown_exec_guard_when_controller_identity_changed(tmp_path):
|
|
engine = create_engine(f"sqlite:///{tmp_path / 'identity.db'}")
|
|
SQLModel.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
session.add(
|
|
recovery.AppSetting(
|
|
key=recovery._RECOVERY_STATE_SETTING,
|
|
value_json={
|
|
"state": "outcome_unknown",
|
|
"executionId": "exec-existing",
|
|
"controllerId": "old-controller",
|
|
"jobId": "job-original",
|
|
},
|
|
is_secret=False,
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
portainer = FakePortainer()
|
|
with Session(engine) as session:
|
|
result = recovery.PrivateMuseDeploymentRecovery(portainer).run(
|
|
session=session,
|
|
job_id="job-fresh-key",
|
|
)
|
|
|
|
assert result["status"] == "outcome_unknown"
|
|
assert result["execution_id"] == "exec-existing"
|
|
assert result["original_job_id"] == "job-original"
|
|
assert "controller identity changed" in result["reason"]
|
|
assert portainer.calls == []
|
|
|
|
|
|
def test_recovery_admission_creates_only_the_winning_job(tmp_path):
|
|
engine = create_engine(f"sqlite:///{tmp_path / 'admission.db'}")
|
|
SQLModel.metadata.create_all(engine)
|
|
|
|
with Session(engine) as first_session:
|
|
first_state, first_admitted = recovery.PrivateMuseDeploymentRecovery.admit_job(
|
|
first_session, recovery_job("job-winner")
|
|
)
|
|
with Session(engine) as second_session:
|
|
second_state, second_admitted = (
|
|
recovery.PrivateMuseDeploymentRecovery.admit_job(
|
|
second_session, recovery_job("job-loser")
|
|
)
|
|
)
|
|
jobs = second_session.exec(select(JobRun)).all()
|
|
|
|
assert first_admitted is True
|
|
assert second_admitted is False
|
|
assert first_state["jobId"] == "job-winner"
|
|
assert second_state["jobId"] == "job-winner"
|
|
assert [job.id for job in jobs] == ["job-winner"]
|
|
|
|
|
|
def test_recovery_admission_cas_uses_generation_when_timestamps_collide(
|
|
tmp_path, monkeypatch
|
|
):
|
|
fixed = datetime(2026, 9, 20, 12, 0, 0, tzinfo=UTC).replace(tzinfo=None)
|
|
monkeypatch.setattr(recovery, "utc_now", lambda: fixed)
|
|
engine = create_engine(
|
|
f"sqlite:///{tmp_path / 'admission-generation.db'}",
|
|
connect_args={"check_same_thread": False, "timeout": 10},
|
|
)
|
|
SQLModel.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
session.add(
|
|
recovery.AppSetting(
|
|
key=recovery._RECOVERY_STATE_SETTING,
|
|
value_json={"state": "idle", "version": 0},
|
|
is_secret=False,
|
|
updated_at=fixed,
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
update_barrier = Barrier(2)
|
|
|
|
def align_cas_updates(_conn, _cursor, statement, _parameters, _context, _many):
|
|
if statement.startswith("UPDATE app_settings SET"):
|
|
update_barrier.wait(timeout=10)
|
|
|
|
event.listen(engine, "before_cursor_execute", align_cas_updates)
|
|
admissions = []
|
|
|
|
def admit(job_id):
|
|
with Session(engine) as session:
|
|
_state, admitted = recovery.PrivateMuseDeploymentRecovery.admit_job(
|
|
session, recovery_job(job_id)
|
|
)
|
|
admissions.append((job_id, admitted))
|
|
|
|
workers = [Thread(target=admit, args=(f"job-{index}",)) for index in range(2)]
|
|
for worker in workers:
|
|
worker.start()
|
|
for worker in workers:
|
|
worker.join(timeout=10)
|
|
assert not worker.is_alive()
|
|
event.remove(engine, "before_cursor_execute", align_cas_updates)
|
|
|
|
with Session(engine) as session:
|
|
jobs = session.exec(select(JobRun)).all()
|
|
state = session.get(recovery.AppSetting, recovery._RECOVERY_STATE_SETTING)
|
|
|
|
assert sorted(admitted for _job_id, admitted in admissions) == [False, True]
|
|
assert len(jobs) == 1
|
|
assert state is not None
|
|
assert state.value_json["jobId"] == jobs[0].id
|
|
assert state.value_json["version"] == 1
|
|
|
|
|
|
def test_reserved_recovery_can_only_be_started_by_its_admitted_job(tmp_path):
|
|
engine = create_engine(f"sqlite:///{tmp_path / 'owner.db'}")
|
|
SQLModel.metadata.create_all(engine)
|
|
|
|
class NoPreflightPortainer(FakePortainer):
|
|
def get_container_status(self, name):
|
|
raise AssertionError("a losing caller must not reach Portainer preflight")
|
|
|
|
with Session(engine) as session:
|
|
recovery.PrivateMuseDeploymentRecovery.admit_job(
|
|
session, recovery_job("job-owner")
|
|
)
|
|
result = recovery.PrivateMuseDeploymentRecovery(NoPreflightPortainer()).run(
|
|
session=session,
|
|
job_id="job-loser",
|
|
)
|
|
|
|
assert result["status"] == "in_progress"
|
|
assert result["original_job_id"] == "job-owner"
|
|
assert "no second execution was started" in result["reason"]
|
|
|
|
|
|
def test_terminal_reconciliation_cannot_regress_to_late_unknown(tmp_path):
|
|
engine = create_engine(
|
|
f"sqlite:///{tmp_path / 'monotonic.db'}",
|
|
connect_args={"check_same_thread": False},
|
|
)
|
|
SQLModel.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
recovery.PrivateMuseDeploymentRecovery.admit_job(
|
|
session, recovery_job("job-race")
|
|
)
|
|
|
|
inspect_started = Event()
|
|
reconciliation_finished = Event()
|
|
worker_result = {}
|
|
|
|
class DelayedWorkerPortainer(FakePortainer):
|
|
def _request(self, method, path, **kwargs):
|
|
if path.endswith("/containers/controller-id/exec"):
|
|
return FakeResponse({"Id": "exec-race"})
|
|
if path.endswith("/exec/exec-race/start"):
|
|
return FakeResponse({})
|
|
if path.endswith("/exec/exec-race/json"):
|
|
inspect_started.set()
|
|
assert reconciliation_finished.wait(timeout=10)
|
|
raise RuntimeError("late connection loss")
|
|
raise AssertionError(path)
|
|
|
|
def run_delayed_worker():
|
|
with Session(engine) as worker_session:
|
|
worker_result.update(
|
|
recovery.PrivateMuseDeploymentRecovery(
|
|
DelayedWorkerPortainer()
|
|
).run(session=worker_session, job_id="job-race")
|
|
)
|
|
|
|
worker = Thread(target=run_delayed_worker)
|
|
worker.start()
|
|
assert inspect_started.wait(timeout=10)
|
|
|
|
class TerminalInspectPortainer(FakePortainer):
|
|
def _request(self, method, path, **kwargs):
|
|
assert method == "GET"
|
|
assert path.endswith("/exec/exec-race/json")
|
|
return FakeResponse(
|
|
{
|
|
"ID": "exec-race",
|
|
"ContainerID": "controller-id",
|
|
"Running": False,
|
|
"ExitCode": 0,
|
|
}
|
|
)
|
|
|
|
with Session(engine) as reconcile_session:
|
|
reconciled = recovery.PrivateMuseDeploymentRecovery(
|
|
TerminalInspectPortainer()
|
|
).run(session=reconcile_session, job_id="job-fresh-key")
|
|
reconciliation_finished.set()
|
|
worker.join(timeout=10)
|
|
assert not worker.is_alive()
|
|
|
|
with Session(engine) as verification_session:
|
|
row = verification_session.get(
|
|
recovery.AppSetting, recovery._RECOVERY_STATE_SETTING
|
|
)
|
|
|
|
assert reconciled["status"] == "reconciled"
|
|
assert worker_result["status"] == "reconciled"
|
|
assert row is not None
|
|
assert row.value_json["state"] == "completed"
|
|
assert row.value_json["result"]["status"] == "reconciled"
|
|
|
|
|
|
def test_recovery_state_cas_prevents_same_timestamp_terminal_regression(
|
|
tmp_path, monkeypatch
|
|
):
|
|
fixed = datetime(2026, 9, 20, 12, 0, 0, tzinfo=UTC).replace(tzinfo=None)
|
|
monkeypatch.setattr(recovery, "utc_now", lambda: fixed)
|
|
engine = create_engine(
|
|
f"sqlite:///{tmp_path / 'terminal-generation.db'}",
|
|
connect_args={"check_same_thread": False, "timeout": 10},
|
|
)
|
|
SQLModel.metadata.create_all(engine)
|
|
tracked = {
|
|
"state": "started",
|
|
"claimId": "claim-race",
|
|
"jobId": "job-race",
|
|
"controllerId": "controller-id",
|
|
"executionId": "exec-race",
|
|
"version": 7,
|
|
}
|
|
with Session(engine) as session:
|
|
session.add(
|
|
recovery.AppSetting(
|
|
key=recovery._RECOVERY_STATE_SETTING,
|
|
value_json=tracked,
|
|
is_secret=False,
|
|
updated_at=fixed,
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
completed_at_update = Event()
|
|
unknown_at_update = Event()
|
|
completed_finished = Event()
|
|
|
|
def order_cas_updates(_conn, _cursor, statement, parameters, _context, _many):
|
|
if not statement.startswith("UPDATE app_settings SET"):
|
|
return
|
|
payload = str(parameters[0])
|
|
if '"state": "completed"' in payload:
|
|
completed_at_update.set()
|
|
assert unknown_at_update.wait(timeout=10)
|
|
elif '"state": "outcome_unknown"' in payload:
|
|
unknown_at_update.set()
|
|
assert completed_at_update.wait(timeout=10)
|
|
assert completed_finished.wait(timeout=10)
|
|
|
|
event.listen(engine, "before_cursor_execute", order_cas_updates)
|
|
results = {}
|
|
|
|
def persist_completed():
|
|
with Session(engine) as session:
|
|
results["completed"] = recovery.PrivateMuseDeploymentRecovery._persist_state(
|
|
session,
|
|
dict(tracked),
|
|
state="completed",
|
|
execution_id="exec-race",
|
|
result={"status": "reconciled"},
|
|
)
|
|
completed_finished.set()
|
|
|
|
def persist_unknown():
|
|
with Session(engine) as session:
|
|
results["unknown"] = recovery.PrivateMuseDeploymentRecovery._persist_state(
|
|
session,
|
|
dict(tracked),
|
|
state="outcome_unknown",
|
|
execution_id="exec-race",
|
|
result={"status": "outcome_unknown"},
|
|
)
|
|
|
|
completed_worker = Thread(target=persist_completed)
|
|
unknown_worker = Thread(target=persist_unknown)
|
|
completed_worker.start()
|
|
unknown_worker.start()
|
|
completed_worker.join(timeout=10)
|
|
unknown_worker.join(timeout=10)
|
|
assert not completed_worker.is_alive()
|
|
assert not unknown_worker.is_alive()
|
|
event.remove(engine, "before_cursor_execute", order_cas_updates)
|
|
|
|
with Session(engine) as session:
|
|
state = session.get(recovery.AppSetting, recovery._RECOVERY_STATE_SETTING)
|
|
|
|
assert results["completed"]["state"] == "completed"
|
|
assert results["unknown"]["state"] == "completed"
|
|
assert state is not None
|
|
assert state.value_json["state"] == "completed"
|
|
assert state.value_json["result"]["status"] == "reconciled"
|
|
assert state.value_json["version"] == 8
|
|
|
|
|
|
def test_recovery_fails_closed_when_runtime_provenance_differs():
|
|
result = recovery.PrivateMuseDeploymentRecovery(FakePortainer(exit_code=38)).run()
|
|
|
|
assert result == {
|
|
"status": "validation_failed",
|
|
"reason": "PrivateMuse immutable container/image provenance differs from the recorded receipt",
|
|
}
|
|
|
|
|
|
def test_privatemuse_action_execution_accepts_reconciled_status():
|
|
source = Path("backend/app/services/action_execution.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
assert (
|
|
"from .privatemuse_deployment_recovery import PrivateMuseDeploymentRecovery"
|
|
in source
|
|
)
|
|
assert "tracked, admitted = recovery.admit_job(session, candidate_job)" in source
|
|
assert "job_id=job.id if admitted else candidate_job.id" in source
|
|
assert 'status in {"enrolled", "already-enrolled", "reconciled"}' in source
|