from __future__ import annotations import json import time from typing import Any from urllib.parse import quote from ..connectors.portainer import PortainerConnector from ..core.redaction import redact_message from .portfolio_deployment_enrollment import PortfolioDeploymentEnrollment _CONTROLLER = "gitea-deploy-control" _REPOSITORY = "jens/portfolio" _LIVE_REVISION = "52c6f2fc74a1e39ad288f1c5e987a9cd5b072d3e" _FAILED_REVISION = "b7955d90b63860ffff8e0bd84be123aaba632043" _FAILURE_MESSAGE = "Existing production containers require a valid healthy live deployment state" # One-time, fail-closed re-attestation for the exact observed Portfolio state. # The recovery never builds, restarts, recreates or stops application containers. # It uses the controller's own receipt/runtime verifier and accepts only a change # in ephemeral container IDs while immutable image/provenance evidence remains # exactly equal to the previously recorded verified receipt. _REATTEST_SCRIPT = r''' import fcntl import importlib.util import json import os from pathlib import Path import shutil import stat import subprocess import sys import time repository, live_revision, failed_revision, failure_message = sys.argv[1:5] expected_repository = "jens/portfolio" expected_live_revision = "52c6f2fc74a1e39ad288f1c5e987a9cd5b072d3e" expected_failed_revision = "b7955d90b63860ffff8e0bd84be123aaba632043" expected_failure_message = "Existing production containers require a valid healthy live deployment state" if ( repository != expected_repository or live_revision != expected_live_revision or failed_revision != expected_failed_revision or failure_message != expected_failure_message ): raise SystemExit(40) controller_path = Path("/opt/gitea-deploy/deploy.py") registry_path = Path("/controller-config/projects.json") state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state") state_path = state_root / "portfolio.json" project_lock_path = state_root / "locks" / "portfolio.lock" global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock") if controller_path.is_symlink() or not controller_path.is_file(): raise SystemExit(20) if registry_path.is_symlink() or not registry_path.is_file(): raise SystemExit(21) try: projects = json.loads(registry_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(22) project = projects.get(repository) if isinstance(projects, dict) else None if not isinstance(project, dict): raise SystemExit(23) if ( project.get("branch") != "main" or project.get("slug") != "portfolio" or project.get("compose_project") != "portfolio" or project.get("containers") != ["Portfolio", "Portfolio-api"] or project.get("compose_files") != ["docker-compose.yml"] or project.get("exact_source") is not True or project.get("verify_oci_revision") is not False ): raise SystemExit(24) owner, name = repository.split("/", 1) bare_repo = Path("/gitea-repositories") / owner / f"{name}.git" if not bare_repo.is_dir(): raise SystemExit(25) def git_run(*args): return subprocess.run( ["git", f"--git-dir={bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False, ) for revision in (live_revision, failed_revision): resolved = git_run("rev-parse", f"{revision}^{{commit}}") if resolved.returncode != 0 or resolved.stdout.strip() != revision: raise SystemExit(26) main = git_run("rev-parse", "refs/heads/main") if main.returncode != 0 or main.stdout.strip() != failed_revision: raise SystemExit(27) if git_run("merge-base", "--is-ancestor", live_revision, failed_revision).returncode != 0: raise SystemExit(28) try: spec = importlib.util.spec_from_file_location("itworx_deploy_controller", controller_path) controller = importlib.util.module_from_spec(spec) if spec and spec.loader else None if controller is None or spec is None or spec.loader is None: raise RuntimeError("controller import unavailable") spec.loader.exec_module(controller) except Exception: raise SystemExit(29) state_root.mkdir(parents=True, exist_ok=True) project_lock_path.parent.mkdir(parents=True, exist_ok=True) try: project_lock = project_lock_path.open("a+", encoding="utf-8") global_lock = global_lock_path.open("a+", encoding="utf-8") except Exception: raise SystemExit(30) with project_lock: try: fcntl.flock(project_lock, fcntl.LOCK_EX) except Exception: raise SystemExit(31) with global_lock: try: fcntl.flock(global_lock, fcntl.LOCK_EX) except Exception: raise SystemExit(31) try: current_projects = json.loads(registry_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(32) if current_projects.get(repository) != project: raise SystemExit(33) if state_path.is_symlink() or not state_path.is_file(): raise SystemExit(34) try: state = json.loads(state_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(34) if not isinstance(state, dict): raise SystemExit(34) attempt = state.get("last_attempt") receipt = state.get("live_receipt") artifacts = state.get("live_artifacts") if not ( state.get("schema_version") == 3 and state.get("repository") == repository and state.get("revision") == live_revision and state.get("live_revision") == live_revision and state.get("live_status") == "unknown" and state.get("live_legacy_provenance") is False and state.get("status") == "failed" and isinstance(attempt, dict) and attempt.get("revision") == failed_revision and attempt.get("status") == "failed" and attempt.get("message") == failure_message and isinstance(receipt, dict) and bool(receipt) and isinstance(artifacts, dict) ): raise SystemExit(35) project = dict(project) project["_repository_name"] = repository project["_repository_path"] = str(bare_repo) try: build = controller.validate_recorded_receipt(project, receipt, repository, live_revision) recorded_receipt_containers = receipt.get("containers") if not isinstance(recorded_receipt_containers, dict): raise RuntimeError("receipt containers missing") expected_image_ids = { str(container_name): str(dict(raw).get("image_id") or "") for container_name, raw in recorded_receipt_containers.items() if isinstance(raw, dict) } if set(expected_image_ids) != {"Portfolio", "Portfolio-api"} or not all(expected_image_ids.values()): raise RuntimeError("receipt image IDs missing") observed_receipt = controller.verify_deployment( project, build, expected_image_ids=expected_image_ids, ) observed_artifacts = controller.deployment_artifacts(project) except Exception: raise SystemExit(36) immutable_receipt_keys = ("schema", "repository", "revision", "tree", "treeDigest", "build", "live") if any(observed_receipt.get(key) != receipt.get(key) for key in immutable_receipt_keys): raise SystemExit(37) observed_receipt_containers = observed_receipt.get("containers") if not isinstance(observed_receipt_containers, dict) or set(observed_receipt_containers) != set(recorded_receipt_containers): raise SystemExit(37) container_id_changed = False for container_name in sorted(recorded_receipt_containers): old = dict(recorded_receipt_containers[container_name]) new = dict(observed_receipt_containers[container_name]) if old.get("container_id") != new.get("container_id"): container_id_changed = True old_without_id = {key: value for key, value in old.items() if key != "container_id"} new_without_id = {key: value for key, value in new.items() if key != "container_id"} if old_without_id != new_without_id: raise SystemExit(38) if not container_id_changed: raise SystemExit(39) recorded_artifact_containers = artifacts.get("containers") observed_artifact_containers = observed_artifacts.get("containers") if ( not isinstance(recorded_artifact_containers, dict) or not isinstance(observed_artifact_containers, dict) or set(recorded_artifact_containers) != set(observed_artifact_containers) or artifacts.get("compose_sha256") != observed_artifacts.get("compose_sha256") ): raise SystemExit(41) for container_name in sorted(recorded_artifact_containers): old = dict(recorded_artifact_containers[container_name]) new = dict(observed_artifact_containers[container_name]) old_without_id = {key: value for key, value in old.items() if key != "container_id"} new_without_id = {key: value for key, value in new.items() if key != "container_id"} if old_without_id != new_without_id: raise SystemExit(41) now = int(time.time()) repaired = dict(state) repaired["live_status"] = "healthy" repaired["live_artifacts"] = observed_artifacts repaired["artifacts"] = observed_artifacts repaired["live_receipt"] = observed_receipt repaired["status"] = "success" repaired["message"] = "Verified Portfolio runtime re-attested after container recreation" repaired["updated_at_epoch"] = now repaired["last_attempt"] = { "revision": live_revision, "status": "success", "message": "Verified live Portfolio runtime receipt re-attestation", "updated_at_epoch": now, "metadata": { "reconciled_failed_revision": failed_revision, "container_identity_only": True, }, } backup = state_path.with_name(f"portfolio.json.reattest-backup-{now}-{os.getpid()}") temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.reattest.tmp") try: mode = stat.S_IMODE(state_path.stat().st_mode) shutil.copy2(state_path, backup) with temporary.open("x", encoding="utf-8") as handle: handle.write(json.dumps(repaired, indent=2, sort_keys=True) + "\n") handle.flush() os.fsync(handle.fileno()) os.chmod(temporary, mode) os.replace(temporary, state_path) directory_fd = os.open(str(state_root), os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) except Exception: try: temporary.unlink(missing_ok=True) except Exception: pass raise SystemExit(42) try: verified = controller.verified_current_receipt( repository, project, live_revision, str(receipt.get("tree") or ""), ) except Exception: verified = None if not isinstance(verified, dict) or verified != observed_receipt: raise SystemExit(43) print("re-attested", flush=True) ''' class PortfolioDeploymentRecovery: """Recover the exact known Portfolio state drift, otherwise use normal enrollment.""" def __init__(self, portainer: PortainerConnector | None = None): self.portainer = portainer or PortainerConnector() def run(self) -> dict[str, Any]: status = self.portainer.get_container_status(_CONTROLLER) controller = dict(status.get("container") or {}) if status.get("status") == "ok" else {} container_id = str(controller.get("id") or "") config = self.portainer._config() if controller.get("name") != _CONTROLLER or controller.get("state") != "running" or not container_id or not config: return PortfolioDeploymentEnrollment(portainer=self.portainer).run() command = [ "python3", "-c", _REATTEST_SCRIPT, _REPOSITORY, _LIVE_REVISION, _FAILED_REVISION, _FAILURE_MESSAGE, ] endpoint = quote(str(config.endpoint_id), safe="") safe_container = quote(container_id, safe="") try: created = self.portainer._request( "POST", f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec", json={"AttachStdout": False, "AttachStderr": False, "Tty": False, "Cmd": command}, ).json() exec_id = str(created.get("Id") or "") if not exec_id: return {"status": "error", "reason": "Portfolio runtime re-attestation could not start"} safe_exec = quote(exec_id, safe="") self.portainer._request( "POST", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start", json={"Detach": True, "Tty": False}, ) deadline = time.monotonic() + 45 exit_code: int | None = None while time.monotonic() < deadline: inspected = self.portainer._request( "GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json" ).json() if not bool(inspected.get("Running")): raw_exit = inspected.get("ExitCode") exit_code = int(raw_exit) if raw_exit is not None else -1 break time.sleep(0.25) except (RuntimeError, ValueError, TypeError) as exc: return {"status": "error", "reason": redact_message(str(exc))} if exit_code is None: return {"status": "error", "reason": "Portfolio runtime re-attestation timed out"} if exit_code == 0: return { "status": "reconciled", "repository": _REPOSITORY, "live_revision": _LIVE_REVISION, "failed_revision": _FAILED_REVISION, "containers": ["Portfolio", "Portfolio-api"], "compose_project": "portfolio", } # Exit 35 means the exact recoverable state is no longer present. In # normal/healthy states the established enrollment routine remains the # authoritative path. Every other re-attestation failure is fail-closed. if exit_code == 35: return PortfolioDeploymentEnrollment(portainer=self.portainer).run() reasons = { 20: "deployment controller executable is unavailable or unsafe", 21: "deployment projects registry is unavailable or unsafe", 22: "deployment projects registry cannot be read", 23: "Portfolio deployment target is missing", 24: "Portfolio deployment target differs from the bounded paired contract", 25: "mounted Portfolio bare repository is unavailable", 26: "known Portfolio recovery revisions cannot be resolved", 27: "Portfolio main changed after the bounded recovery contract was recorded", 28: "known Portfolio live revision is not an ancestor of the failed revision", 29: "deployment controller verifier cannot be loaded", 30: "deployment lock files cannot be opened", 31: "deployment locks cannot be acquired", 32: "deployment projects registry changed or became unreadable while waiting", 33: "Portfolio deployment target changed while recovery was waiting", 34: "Portfolio deployment state is unavailable or invalid", 36: "controller could not re-attest the current Portfolio runtime against recorded immutable image IDs", 37: "Portfolio receipt identity changed during re-attestation", 38: "Portfolio immutable container/image provenance differs from the recorded receipt", 39: "Portfolio container identities did not change; the known recovery cause is absent", 40: "fixed Portfolio re-attestation contract was altered", 41: "Portfolio recorded live artifacts differ beyond ephemeral container IDs", 42: "re-attested Portfolio deployment state could not be written durably", 43: "controller rejected the repaired Portfolio receipt after write", } return { "status": "validation_failed", "reason": reasons.get(exit_code, "Portfolio runtime re-attestation failed closed"), }