from __future__ import annotations import time from typing import Any from urllib.parse import quote from uuid import uuid4 from sqlalchemy import func, update from sqlalchemy.exc import IntegrityError from sqlmodel import Session from ..connectors.portainer import PortainerConnector from ..core.redaction import redact_message from ..core.time import utc_now from ..db.models import AppSetting, JobRun _CONTROLLER = "gitea-deploy-control" _REPOSITORY = "jens/privatemuse" _LIVE_REVISION = "2103d5795d151885edeeb964466b643b8fd4a0eb" _FAILED_REVISION = "6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38" _FAILURE_MESSAGE = ( "Existing production containers require a valid healthy live deployment state" ) _CONTAINERS = [ "privatemuse-backend-1", "privatemuse-frontend-1", "privatemuse-audio-enhancement-1", ] _RECOVERY_STATE_SETTING = "deployment:privatemuse:receipt-recovery:v1" _ACTIVE_RECOVERY_STATES = {"reserved", "created", "started", "outcome_unknown"} _STATE_TRANSITIONS = { "reserved": {"reserved", "created", "outcome_unknown", "completed"}, "created": {"started", "outcome_unknown", "completed"}, "started": {"outcome_unknown", "completed"}, "outcome_unknown": {"outcome_unknown", "completed"}, } # One-time, fail-closed re-attestation for the exact observed PrivateMuse state. # It never builds, restarts, recreates or stops application containers. The # controller's own receipt/runtime verifiers prove that only the two container # IDs recreated by the prior rollback differ; all immutable evidence must match. _REATTEST_SCRIPT = r""" import fcntl import importlib.util import json import os from pathlib import Path import stat import subprocess import sys import time repository, live_revision, failed_revision, failure_message = sys.argv[1:5] expected_repository = "jens/privatemuse" expected_live_revision = "2103d5795d151885edeeb964466b643b8fd4a0eb" expected_failed_revision = "6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38" expected_failure_message = "Existing production containers require a valid healthy live deployment state" if ( repository != expected_repository or live_revision != expected_live_revision or failed_revision != expected_failed_revision or failure_message != expected_failure_message ): raise SystemExit(40) controller_path = Path("/opt/gitea-deploy/deploy.py") registry_path = Path("/opt/gitea-deploy/projects.json") state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state") state_path = state_root / "privatemuse.json" project_lock_path = state_root / "locks" / "privatemuse.lock" global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock") def fsync_directory(directory): flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) directory_fd = os.open(str(directory), flags) try: os.fsync(directory_fd) finally: os.close(directory_fd) def write_exclusive(path, payload, metadata): try: with path.open("xb") as handle: handle.write(payload) handle.flush() os.fchmod(handle.fileno(), stat.S_IMODE(metadata.st_mode)) os.fchown(handle.fileno(), metadata.st_uid, metadata.st_gid) os.fsync(handle.fileno()) except Exception: try: path.unlink(missing_ok=True) except Exception: pass raise if controller_path.is_symlink() or not controller_path.is_file(): raise SystemExit(20) if registry_path.is_symlink() or not registry_path.is_file(): raise SystemExit(21) try: projects = json.loads(registry_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(22) project = projects.get(repository) if isinstance(projects, dict) else None if not isinstance(project, dict): raise SystemExit(23) expected_target = { "branch": "main", "slug": "privatemuse", "source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/privatemuse", "exact_source": True, "env_source": "/srv/example/appdata/privatemuse/.env", "compose_files": ["docker-compose.yml", "docker-compose.unraid.yml"], "compose_project": "privatemuse", "containers": [ "privatemuse-backend-1", "privatemuse-frontend-1", "privatemuse-audio-enhancement-1", ], "verify_oci_revision": False, "health_timeout": 180, "environment": {"PRIVATEMUSE_RELEASE": "{short_revision}"}, } for key, expected in expected_target.items(): if project.get(key) != expected: raise SystemExit(24) allowed_metadata_keys = { "gitea_repository_id", "gitea_webhook_secret_file", "validation", } expected_validation = { "validation_gate_version": 1, "required": True, "contexts": ["Managed validation / full (push)"], "timeout_seconds": 1800, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30, } repository_id = project.get("gitea_repository_id") if ( set(project) - set(expected_target) != allowed_metadata_keys or not isinstance(repository_id, int) or isinstance(repository_id, bool) or repository_id <= 0 or project.get("gitea_webhook_secret_file") != "jens__privatemuse" or project.get("validation") != expected_validation ): raise SystemExit(24) owner, name = repository.split("/", 1) bare_repo = Path("/gitea-repositories") / owner / f"{name}.git" if not bare_repo.is_dir(): raise SystemExit(25) def git_run(*args): return subprocess.run( ["git", f"--git-dir={bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False, ) for revision in (live_revision, failed_revision): resolved = git_run("rev-parse", f"{revision}^{{commit}}") if resolved.returncode != 0 or resolved.stdout.strip() != revision: raise SystemExit(26) main = git_run("rev-parse", "refs/heads/main") if main.returncode != 0 or main.stdout.strip() != failed_revision: raise SystemExit(27) if git_run("merge-base", "--is-ancestor", live_revision, failed_revision).returncode != 0: raise SystemExit(28) try: spec = importlib.util.spec_from_file_location("itworx_deploy_controller", controller_path) controller = importlib.util.module_from_spec(spec) if spec and spec.loader else None if controller is None or spec is None or spec.loader is None: raise RuntimeError("controller import unavailable") spec.loader.exec_module(controller) except Exception: raise SystemExit(29) state_root.mkdir(parents=True, exist_ok=True) project_lock_path.parent.mkdir(parents=True, exist_ok=True) try: project_lock = project_lock_path.open("a+", encoding="utf-8") global_lock = global_lock_path.open("a+", encoding="utf-8") except Exception: raise SystemExit(30) with project_lock: try: fcntl.flock(project_lock, fcntl.LOCK_EX) except Exception: raise SystemExit(31) with global_lock: try: fcntl.flock(global_lock, fcntl.LOCK_EX) except Exception: raise SystemExit(31) try: current_projects = json.loads(registry_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(32) if current_projects.get(repository) != project: raise SystemExit(33) if state_path.is_symlink() or not state_path.is_file(): raise SystemExit(34) try: state_metadata = state_path.stat() original_state = state_path.read_bytes() state = json.loads(original_state.decode("utf-8")) except Exception: raise SystemExit(34) if not isinstance(state, dict): raise SystemExit(34) project = dict(project) project["_repository_name"] = repository project["_repository_path"] = str(bare_repo) if ( state.get("schema_version") == 3 and state.get("repository") == repository and state.get("live_status") == "healthy" ): healthy_revision = str(state.get("live_revision") or "") healthy_receipt = state.get("live_receipt") healthy_tree = ( str(healthy_receipt.get("tree") or "") if isinstance(healthy_receipt, dict) else "" ) verified_healthy = None if ( healthy_revision in {live_revision, failed_revision} and state.get("revision") == healthy_revision and state.get("status") == "success" and state.get("live_legacy_provenance") is False and isinstance(healthy_receipt, dict) and healthy_receipt.get("revision") == healthy_revision and healthy_tree ): try: verified_healthy = controller.verified_current_receipt( repository, project, healthy_revision, healthy_tree, ) except Exception: verified_healthy = None if verified_healthy == healthy_receipt: raise SystemExit(35) raise SystemExit(44) attempt = state.get("last_attempt") receipt = state.get("live_receipt") artifacts = state.get("live_artifacts") expected_attempt_metadata = { "id": "6e4c959eea2e", "repository": repository, "revision": failed_revision, "tree": "daa0e757f4b3edac2c2830572a05bfdf271e8bba", "treeDigest": "e763339aca7139a29b3d4fc8a8f856557881ef177f15115d0c7fd52b093b3654", "builtAt": "2026-09-19T23:07:14.894Z", } expected_receipt_keys = { "build", "containers", "live", "repository", "revision", "schema", "tree", "treeDigest", } expected_artifact_keys = {"compose_sha256", "containers"} if not ( state.get("schema_version") == 3 and state.get("repository") == repository and state.get("revision") == live_revision and state.get("live_revision") == live_revision and state.get("live_status") == "unknown" and state.get("live_legacy_provenance") is False and state.get("status") == "failed" and state.get("message") == failure_message and isinstance(attempt, dict) and attempt.get("revision") == failed_revision and attempt.get("status") == "failed" and attempt.get("message") == failure_message and attempt.get("metadata") == expected_attempt_metadata and isinstance(receipt, dict) and set(receipt) == expected_receipt_keys and isinstance(artifacts, dict) and set(artifacts) == expected_artifact_keys ): raise SystemExit(44) expected_containers = { "privatemuse-audio-enhancement-1", "privatemuse-backend-1", "privatemuse-frontend-1", } expected_container_keys = { "container_id", "container_revision", "image_created", "image_id", "image_ref", "image_revision", "image_source", "image_tree", "image_version", } try: build = controller.validate_recorded_receipt(project, receipt, repository, live_revision) recorded_receipt_containers = receipt.get("containers") if ( not isinstance(recorded_receipt_containers, dict) or set(recorded_receipt_containers) != expected_containers ): raise RuntimeError("receipt containers differ") expected_image_ids = { str(container_name): str(dict(raw).get("image_id") or "") for container_name, raw in recorded_receipt_containers.items() if isinstance(raw, dict) and set(raw) == expected_container_keys } if set(expected_image_ids) != expected_containers or not all(expected_image_ids.values()): raise RuntimeError("receipt image IDs missing") observed_receipt = controller.verify_deployment( project, build, expected_image_ids=expected_image_ids, ) observed_artifacts = controller.deployment_artifacts(project) except Exception: raise SystemExit(36) immutable_receipt_keys = ( "schema", "repository", "revision", "tree", "treeDigest", "build", "live", ) if any(observed_receipt.get(key) != receipt.get(key) for key in immutable_receipt_keys): raise SystemExit(37) observed_receipt_containers = observed_receipt.get("containers") if ( not isinstance(observed_receipt_containers, dict) or set(observed_receipt_containers) != expected_containers ): raise SystemExit(37) allowed_changed_ids = {"privatemuse-backend-1", "privatemuse-frontend-1"} changed_receipt_ids = set() for container_name in sorted(expected_containers): old = dict(recorded_receipt_containers[container_name]) new = dict(observed_receipt_containers[container_name]) if set(old) != expected_container_keys or set(new) != expected_container_keys: raise SystemExit(38) if old.get("container_id") != new.get("container_id"): changed_receipt_ids.add(container_name) old_without_id = {key: value for key, value in old.items() if key != "container_id"} new_without_id = {key: value for key, value in new.items() if key != "container_id"} if old_without_id != new_without_id: raise SystemExit(38) if changed_receipt_ids != allowed_changed_ids: raise SystemExit(39) recorded_artifact_containers = artifacts.get("containers") observed_artifact_containers = observed_artifacts.get("containers") if ( set(observed_artifacts) != expected_artifact_keys or not isinstance(recorded_artifact_containers, dict) or not isinstance(observed_artifact_containers, dict) or set(recorded_artifact_containers) != expected_containers or set(observed_artifact_containers) != expected_containers or artifacts.get("compose_sha256") != observed_artifacts.get("compose_sha256") ): raise SystemExit(41) changed_artifact_ids = set() for container_name in sorted(expected_containers): old = dict(recorded_artifact_containers[container_name]) new = dict(observed_artifact_containers[container_name]) if set(old) != expected_container_keys or set(new) != expected_container_keys: raise SystemExit(41) if old.get("container_id") != new.get("container_id"): changed_artifact_ids.add(container_name) old_without_id = {key: value for key, value in old.items() if key != "container_id"} new_without_id = {key: value for key, value in new.items() if key != "container_id"} if ( old_without_id != new_without_id or new.get("container_id") != dict(observed_receipt_containers[container_name]).get("container_id") ): raise SystemExit(41) if changed_artifact_ids != allowed_changed_ids: raise SystemExit(41) now = int(time.time()) repaired = dict(state) repaired["live_status"] = "healthy" repaired["live_artifacts"] = observed_artifacts repaired["artifacts"] = observed_artifacts repaired["live_receipt"] = observed_receipt repaired["status"] = "success" repaired["message"] = "Verified PrivateMuse runtime re-attested after rollback recreation" repaired["updated_at_epoch"] = now repaired["last_attempt"] = { "revision": live_revision, "status": "success", "message": "Verified live PrivateMuse runtime receipt re-attestation", "updated_at_epoch": now, "metadata": { "reconciled_failed_revision": failed_revision, "container_identity_only": True, "changed_containers": sorted(allowed_changed_ids), }, } backup = state_path.with_name(f"privatemuse.json.reattest-backup-{now}-{os.getpid()}") temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.reattest.tmp") restore_temporary = state_path.with_name( f".{state_path.name}.{os.getpid()}.reattest-restore.tmp" ) def restore_original_state(): try: write_exclusive(restore_temporary, original_state, state_metadata) os.replace(restore_temporary, state_path) fsync_directory(state_root) return state_path.read_bytes() == original_state except Exception: try: restore_temporary.unlink(missing_ok=True) except Exception: pass return False state_replaced = False try: write_exclusive(backup, original_state, state_metadata) fsync_directory(state_root) repaired_state = ( json.dumps(repaired, indent=2, sort_keys=True) + "\n" ).encode("utf-8") write_exclusive(temporary, repaired_state, state_metadata) os.replace(temporary, state_path) state_replaced = True fsync_directory(state_root) except Exception: try: temporary.unlink(missing_ok=True) except Exception: pass if state_replaced and not restore_original_state(): raise SystemExit(45) raise SystemExit(42) try: written = json.loads(state_path.read_text(encoding="utf-8")) verified = controller.verified_current_receipt( repository, project, live_revision, str(receipt.get("tree") or ""), ) final_artifacts = controller.deployment_artifacts(project) except Exception: written = None verified = None final_artifacts = None if ( written != repaired or not isinstance(verified, dict) or verified != observed_receipt or final_artifacts != observed_artifacts ): if not restore_original_state(): raise SystemExit(45) raise SystemExit(43) print("re-attested", flush=True) """ class PrivateMuseDeploymentRecovery: """Recover only the exact known PrivateMuse receipt drift, otherwise fail closed.""" def __init__(self, portainer: PortainerConnector | None = None): self.portainer = portainer or PortainerConnector() @staticmethod def tracked_job_id(session: Session) -> str | None: row = session.get(AppSetting, _RECOVERY_STATE_SETTING) if not row: return None state = dict(row.value_json or {}) job_id = str(state.get("jobId") or "") return job_id or None @staticmethod def guarded_unknown_result( session: Session, *, reason: str ) -> dict[str, Any] | None: row = session.get(AppSetting, _RECOVERY_STATE_SETTING) state = dict(row.value_json or {}) if row else {} if state.get("state") not in _ACTIVE_RECOVERY_STATES: return None execution_id = str(state.get("executionId") or "") return { "status": "outcome_unknown", "reason": reason, **({"execution_id": execution_id} if execution_id else {}), "original_job_id": str(state.get("jobId") or ""), } @staticmethod def _ensure_state_row(session: Session) -> AppSetting: row = session.get(AppSetting, _RECOVERY_STATE_SETTING) if row: return row try: row = AppSetting( key=_RECOVERY_STATE_SETTING, value_json={"state": "idle", "version": 0}, is_secret=False, ) session.add(row) session.commit() except IntegrityError: session.rollback() row = session.get(AppSetting, _RECOVERY_STATE_SETTING) if not row: raise RuntimeError("PrivateMuse recovery state could not be initialized") return row @classmethod def admit_job( cls, session: Session, job: JobRun, ) -> tuple[dict[str, Any], bool]: """Atomically reserve recovery ownership and create its sole JobRun.""" cls._ensure_state_row(session) for _ in range(5): session.expire_all() row = session.get(AppSetting, _RECOVERY_STATE_SETTING) if not row: raise RuntimeError("PrivateMuse recovery state disappeared") current = dict(row.value_json or {}) if current.get("state") in _ACTIVE_RECOVERY_STATES or current.get( "state" ) == "completed": return current, False now = utc_now() current_version = int(current.get("version") or 0) claimed = { "state": "reserved", "claimId": uuid4().hex, "jobId": job.id, "version": current_version + 1, "reservedAt": now.isoformat(), "updatedAt": now.isoformat(), } statement = ( update(AppSetting) .where( AppSetting.key == _RECOVERY_STATE_SETTING, AppSetting.updated_at == row.updated_at, func.coalesce( AppSetting.value_json["version"].as_integer(), 0 ) == current_version, ) .values(value_json=claimed, updated_at=now) ) result = session.exec(statement) if result.rowcount == 1: session.add(job) session.commit() session.refresh(job) return claimed, True session.rollback() raise RuntimeError("PrivateMuse recovery state is busy") @classmethod def _persist_state( cls, session: Session, tracked: dict[str, Any], *, state: str, execution_id: str = "", controller_id: str = "", result: dict[str, Any] | None = None, ) -> dict[str, Any]: for _ in range(5): session.expire_all() row = session.get(AppSetting, _RECOVERY_STATE_SETTING) if not row: raise RuntimeError("PrivateMuse recovery state is unavailable") current = dict(row.value_json or {}) if ( current.get("claimId") != tracked.get("claimId") or current.get("jobId") != tracked.get("jobId") ): raise RuntimeError("PrivateMuse recovery ownership changed") if current.get("state") == "completed": tracked.clear() tracked.update(current) return current current_state = str(current.get("state") or "") if state not in _STATE_TRANSITIONS.get(current_state, set()): raise RuntimeError( f"PrivateMuse recovery cannot transition from {current_state} to {state}" ) current_execution_id = str(current.get("executionId") or "") if ( current_execution_id and execution_id and current_execution_id != execution_id ): raise RuntimeError("PrivateMuse recovery execution identity changed") current_controller_id = str(current.get("controllerId") or "") if ( current_controller_id and controller_id and current_controller_id != controller_id ): raise RuntimeError("PrivateMuse recovery controller identity changed") now = utc_now() current_version = int(current.get("version") or 0) updated = { **current, "state": state, "version": current_version + 1, "updatedAt": now.isoformat(), } if execution_id: updated["executionId"] = execution_id if controller_id: updated["controllerId"] = controller_id if result is not None: updated["result"] = result statement = ( update(AppSetting) .where( AppSetting.key == _RECOVERY_STATE_SETTING, AppSetting.updated_at == row.updated_at, func.coalesce( AppSetting.value_json["version"].as_integer(), 0 ) == current_version, ) .values(value_json=updated, updated_at=now) ) update_result = session.exec(statement) if update_result.rowcount == 1: session.commit() tracked.clear() tracked.update(updated) return updated session.rollback() raise RuntimeError("PrivateMuse recovery state changed concurrently") @staticmethod def _require_owner_state( session: Session, tracked: dict[str, Any], *, expected_state: str, execution_id: str = "", ) -> dict[str, Any]: session.expire_all() row = session.get(AppSetting, _RECOVERY_STATE_SETTING) current = dict(row.value_json or {}) if row else {} if ( current.get("claimId") != tracked.get("claimId") or current.get("jobId") != tracked.get("jobId") or current.get("state") != expected_state or ( execution_id and str(current.get("executionId") or "") != execution_id ) ): raise RuntimeError("PrivateMuse recovery ownership or phase changed") return current @staticmethod def _with_tracking( result: dict[str, Any], tracked: dict[str, Any] | None, execution_id: str, ) -> dict[str, Any]: if tracked is None: return result return { **result, **({"execution_id": execution_id} if execution_id else {}), "original_job_id": str(tracked.get("jobId") or ""), } def _record_unknown( self, *, reason: str, execution_id: str, session: Session | None, tracked: dict[str, Any] | None, ) -> dict[str, Any]: result = self._with_tracking( { "status": "outcome_unknown", "reason": reason, **({"execution_id": execution_id} if execution_id else {}), }, tracked, execution_id, ) if session is not None and tracked is not None: persisted = self._persist_state( session, tracked, state="outcome_unknown", execution_id=execution_id, result=result, ) if persisted.get("state") == "completed" and isinstance( persisted.get("result"), dict ): return dict(persisted["result"]) return result def _record_terminal( self, result: dict[str, Any], *, execution_id: str, session: Session | None, tracked: dict[str, Any] | None, ) -> dict[str, Any]: result = self._with_tracking(result, tracked, execution_id) if session is not None and tracked is not None: persisted = self._persist_state( session, tracked, state="completed", execution_id=execution_id, result=result, ) if isinstance(persisted.get("result"), dict): return dict(persisted["result"]) return result @staticmethod def _exit_result(exit_code: int) -> dict[str, Any]: if exit_code == 0: return { "status": "reconciled", "repository": _REPOSITORY, "live_revision": _LIVE_REVISION, "failed_revision": _FAILED_REVISION, "containers": list(_CONTAINERS), "compose_project": "privatemuse", } if exit_code == 35: return {"status": "already-enrolled", "repository": _REPOSITORY} reasons = { 20: "deployment controller executable is unavailable or unsafe", 21: "deployment projects registry is unavailable or unsafe", 22: "deployment projects registry cannot be read", 23: "PrivateMuse deployment target is missing", 24: "PrivateMuse deployment target differs from the bounded contract", 25: "mounted PrivateMuse bare repository is unavailable", 26: "known PrivateMuse recovery revisions cannot be resolved", 27: "PrivateMuse main changed after the bounded recovery contract was recorded", 28: "known PrivateMuse live revision is not an ancestor of the failed revision", 29: "deployment controller verifier cannot be loaded", 30: "deployment lock files cannot be opened", 31: "deployment locks cannot be acquired", 32: "deployment projects registry changed or became unreadable while waiting", 33: "PrivateMuse deployment target changed while recovery was waiting", 34: "PrivateMuse deployment state is unavailable or invalid", 36: ( "controller could not re-attest the current PrivateMuse runtime " "against recorded immutable image IDs" ), 37: "PrivateMuse receipt identity changed during re-attestation", 38: ( "PrivateMuse immutable container/image provenance differs from " "the recorded receipt" ), 39: "PrivateMuse container identity drift differs from the known rollback incident", 40: "fixed PrivateMuse re-attestation contract was altered", 41: "PrivateMuse recorded live artifacts differ from the known rollback incident", 42: "re-attested PrivateMuse deployment state could not be written durably", 43: "controller rejected the repaired PrivateMuse receipt after write", 44: "PrivateMuse deployment state differs from the exact recoverable incident", 45: ( "controller rejected the repaired PrivateMuse receipt and the " "original state could not be restored" ), } return { "status": "validation_failed", "reason": reasons.get( exit_code, "PrivateMuse runtime re-attestation failed closed" ), } def _inspect_tracked_execution( self, *, config: Any, current_controller_id: str, session: Session, tracked: dict[str, Any], ) -> dict[str, Any]: execution_id = str(tracked.get("executionId") or "") original_controller_id = str(tracked.get("controllerId") or "") if not execution_id: return self._record_unknown( reason=( "PrivateMuse recovery crossed its durable reservation boundary " "without a recorded execution ID; do not retry" ), execution_id="", session=session, tracked=tracked, ) if original_controller_id != current_controller_id: return self._record_unknown( reason=( "PrivateMuse recovery controller identity changed while the " "existing execution outcome remained unknown; do not retry" ), execution_id=execution_id, session=session, tracked=tracked, ) endpoint = quote(str(config.endpoint_id), safe="") safe_exec = quote(execution_id, safe="") try: inspected = self.portainer._request( "GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json" ).json() except (RuntimeError, ValueError, TypeError) as exc: return self._record_unknown( reason=( "PrivateMuse runtime re-attestation outcome remains unknown: " f"{redact_message(str(exc))}; inspect the existing execution and do not retry" ), execution_id=execution_id, session=session, tracked=tracked, ) if ( str(inspected.get("ID") or "") != execution_id or str(inspected.get("ContainerID") or "") != original_controller_id ): return self._record_unknown( reason=( "PrivateMuse recovery execution identity could not be verified; " "do not retry" ), execution_id=execution_id, session=session, tracked=tracked, ) if bool(inspected.get("Running")): return self._record_unknown( reason=( "PrivateMuse runtime re-attestation is still running; inspect the " "existing execution and do not retry" ), execution_id=execution_id, session=session, tracked=tracked, ) if tracked.get("state") == "created": return self._record_unknown( reason=( "PrivateMuse recovery execution was created but its start boundary " "was not durably confirmed; do not retry" ), execution_id=execution_id, session=session, tracked=tracked, ) raw_exit = inspected.get("ExitCode") exit_code = int(raw_exit) if raw_exit is not None else -1 return self._record_terminal( self._exit_result(exit_code), execution_id=execution_id, session=session, tracked=tracked, ) def run( self, *, session: Session | None = None, job_id: str | None = None, ) -> dict[str, Any]: preexisting: dict[str, Any] | None = None tracked: dict[str, Any] | None = None owns_reservation = False if session is not None: if not job_id: raise ValueError("job_id is required for durable recovery execution") session.expire_all() row = session.get(AppSetting, _RECOVERY_STATE_SETTING) state = dict(row.value_json or {}) if row else {} if state.get("state") == "completed": stored_result = state.get("result") if isinstance(stored_result, dict): return dict(stored_result) raise RuntimeError( "PrivateMuse recovery has a terminal guard without a durable result" ) if state.get("state") not in _ACTIVE_RECOVERY_STATES: raise RuntimeError( "PrivateMuse recovery requires durable job admission before execution" ) tracked = state owns_reservation = ( state.get("state") == "reserved" and state.get("jobId") == job_id ) if state.get("state") == "reserved" and not owns_reservation: return self._with_tracking( { "status": "in_progress", "reason": ( "PrivateMuse recovery is durably admitted and awaiting its " "owning worker; no second execution was started" ), }, tracked, "", ) if not owns_reservation: preexisting = state status = self.portainer.get_container_status(_CONTROLLER) if status.get("status") != "ok": if session is not None and preexisting is not None: return self._record_unknown( reason=( "PrivateMuse recovery outcome remains unknown because the " "deployment controller is unavailable; do not retry" ), execution_id=str(preexisting.get("executionId") or ""), session=session, tracked=preexisting, ) unavailable = { "status": "controller_unavailable", "reason": redact_message( str(status.get("reason") or status.get("status") or "unavailable") ), } return ( self._record_terminal( unavailable, execution_id="", session=session, tracked=tracked, ) if tracked is not None else unavailable ) controller = dict(status.get("container") or {}) if ( controller.get("name") != _CONTROLLER or controller.get("state") != "running" ): if session is not None and preexisting is not None: return self._record_unknown( reason=( "PrivateMuse recovery outcome remains unknown because the " "deployment controller is not running; do not retry" ), execution_id=str(preexisting.get("executionId") or ""), session=session, tracked=preexisting, ) unavailable = { "status": "controller_unavailable", "reason": "deployment controller is not running", } return ( self._record_terminal( unavailable, execution_id="", session=session, tracked=tracked, ) if tracked is not None else unavailable ) container_id = str(controller.get("id") or "") if not container_id: if session is not None and preexisting is not None: return self._record_unknown( reason=( "PrivateMuse recovery outcome remains unknown because the " "deployment controller identity is unavailable; do not retry" ), execution_id=str(preexisting.get("executionId") or ""), session=session, tracked=preexisting, ) unavailable = { "status": "controller_unavailable", "reason": "deployment controller identity is unavailable", } return ( self._record_terminal( unavailable, execution_id="", session=session, tracked=tracked, ) if tracked is not None else unavailable ) config = self.portainer._config() if not config: if session is not None and preexisting is not None: return self._record_unknown( reason=( "PrivateMuse recovery outcome remains unknown because " "Portainer deployment control is unavailable; do not retry" ), execution_id=str(preexisting.get("executionId") or ""), session=session, tracked=preexisting, ) unavailable = { "status": "connector_unavailable", "reason": "Portainer deployment control is unavailable", } return ( self._record_terminal( unavailable, execution_id="", session=session, tracked=tracked, ) if tracked is not None else unavailable ) if session is not None: if preexisting is not None: return self._inspect_tracked_execution( config=config, current_controller_id=container_id, session=session, tracked=tracked, ) if tracked is None: raise RuntimeError("PrivateMuse recovery admission state is unavailable") tracked = self._persist_state( session, tracked, state="reserved", controller_id=container_id, ) command = [ "python3", "-c", _REATTEST_SCRIPT, _REPOSITORY, _LIVE_REVISION, _FAILED_REVISION, _FAILURE_MESSAGE, ] endpoint = quote(str(config.endpoint_id), safe="") safe_container = quote(container_id, safe="") exec_id = "" start_requested = False try: if session is not None and tracked is not None: self._require_owner_state( session, tracked, expected_state="reserved", ) created = self.portainer._request( "POST", f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec", json={ "AttachStdout": False, "AttachStderr": False, "Tty": False, "Cmd": command, }, ).json() exec_id = str(created.get("Id") or "") if not exec_id: return self._record_terminal( { "status": "error", "reason": "PrivateMuse runtime re-attestation could not start", }, execution_id="", session=session, tracked=tracked, ) if session is not None and tracked is not None: self._persist_state( session, tracked, state="created", execution_id=exec_id, ) safe_exec = quote(exec_id, safe="") if session is not None and tracked is not None: self._require_owner_state( session, tracked, expected_state="created", execution_id=exec_id, ) start_requested = True self.portainer._request( "POST", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start", json={"Detach": True, "Tty": False}, ) if session is not None and tracked is not None: self._persist_state( session, tracked, state="started", execution_id=exec_id, ) deadline = time.monotonic() + 300 exit_code: int | None = None while time.monotonic() < deadline: inspected = self.portainer._request( "GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json" ).json() if ( str(inspected.get("ID") or "") != exec_id or str(inspected.get("ContainerID") or "") != container_id ): return self._record_unknown( reason=( "PrivateMuse recovery execution identity could not be " "verified; do not retry" ), execution_id=exec_id, session=session, tracked=tracked, ) if not bool(inspected.get("Running")): raw_exit = inspected.get("ExitCode") exit_code = int(raw_exit) if raw_exit is not None else -1 break time.sleep(1.0) except (RuntimeError, ValueError, TypeError) as exc: if exec_id and start_requested: return self._record_unknown( reason=( "PrivateMuse runtime re-attestation outcome is unknown after " "execution start; inspect the existing execution and do not retry" ), execution_id=exec_id, session=session, tracked=tracked, ) return self._record_terminal( {"status": "error", "reason": redact_message(str(exc))}, execution_id=exec_id, session=session, tracked=tracked, ) if exit_code is None: return self._record_unknown( reason=( "PrivateMuse runtime re-attestation is still running; inspect the " "existing execution and do not retry" ), execution_id=exec_id, session=session, tracked=tracked, ) return self._record_terminal( self._exit_result(exit_code), execution_id=exec_id, session=session, tracked=tracked, ) def privatemuse_recovery_job_cancellation_reason( session: Session, job: JobRun ) -> str | None: if job.action_id != "deployment.target.register_privatemuse": return None row = session.get(AppSetting, _RECOVERY_STATE_SETTING) state = dict(row.value_json or {}) if row else {} if ( state.get("state") not in _ACTIVE_RECOVERY_STATES or state.get("jobId") != job.id ): return None return ( "This job owns an unresolved detached PrivateMuse recovery execution; " "reconcile that execution before changing the job record." )