from __future__ import annotations import time from typing import Any from urllib.parse import quote from ..connectors.portainer import PortainerConnector from ..core.redaction import redact_message _CONTROLLER = "gitea-deploy-control" _REPOSITORY = "jens/media-codec-dashboard" _CONTAINER = "Media-Codec-Dashboard" # Fail-closed reconciliation for the verified production runtime. # It does not deploy, restart, recreate or stop any container. It only repairs # controller state after proving the exact live container/image/mount topology # and immutable runtime revision from independent OCI labels plus Gitea ancestry. _RECONCILE_SCRIPT = r''' import fcntl import importlib.util import json import os from pathlib import Path import re import shutil import stat import subprocess import sys import time repository, expected_container = sys.argv[1:3] registry_path = Path("/controller-config/projects.json") host_registry_path = Path("/controller-hub-releases/controller-config/projects.json") state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state") state_path = state_root / "media-codec-dashboard.json" project_lock_path = state_root / "locks" / "media-codec-dashboard.lock" global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock") controller_path = Path("/opt/gitea-deploy/deploy.py") if repository != "jens/media-codec-dashboard": raise SystemExit(40) if expected_container != "Media-Codec-Dashboard": raise SystemExit(40) if registry_path.is_symlink() or not registry_path.is_file(): raise SystemExit(20) try: if host_registry_path.is_symlink() or not host_registry_path.is_file() or not os.path.samefile(registry_path, host_registry_path): raise SystemExit(36) except OSError: raise SystemExit(36) try: projects = json.loads(registry_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(31) project = projects.get(repository) if isinstance(projects, dict) else None if not isinstance(project, dict): raise SystemExit(22) if project.get("branch") != "main" or project.get("slug") != "media-codec-dashboard": raise SystemExit(23) if project.get("compose_project") != "media-codec-dashboard" or project.get("containers") != [expected_container]: raise SystemExit(23) compose_files = project.get("compose_files") if not isinstance(compose_files, list) or "docker-compose.yml" not in compose_files: raise SystemExit(23) source_dir = Path(str(project.get("source_dir") or "")) allowed_roots = (Path("/mnt/cache/appdata/gitea-deploy-runners/sources"), Path("/srv/example/appdata")) if not source_dir.is_absolute() or not any(root == source_dir or root in source_dir.parents for root in allowed_roots): raise SystemExit(23) owner, name = repository.split("/", 1) bare_repo = Path("/gitea-repositories") / owner / f"{name}.git" canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories") canonical_bare_repo = canonical_repository_root / owner / f"{name}.git" if not bare_repo.is_dir(): raise SystemExit(24) def git_run(*args): return subprocess.run(["git", f"--git-dir={bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False) def canonical_git_run(*args): return subprocess.run(["git", f"--git-dir={canonical_bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False) def _is_current_verified_receipt_snapshot_baseline(state, repository, expected_container, live_revision, attempt_revision): attempt = state.get("last_attempt") artifacts = state.get("live_artifacts") receipt = state.get("live_receipt") if not isinstance(attempt, dict) or not isinstance(artifacts, dict) or not isinstance(receipt, dict): return False containers = artifacts.get("containers") receipt_containers = receipt.get("containers") return ( state.get("schema_version") == 3 and state.get("repository") == repository and state.get("revision") == live_revision and state.get("live_revision") == live_revision and state.get("status") == "failed" and state.get("live_status") == "unknown" and state.get("live_legacy_provenance") is False and attempt.get("revision") == attempt_revision and attempt.get("status") == "failed" and attempt.get("message") == "Existing production containers require a valid healthy live deployment state" and attempt.get("updated_at_epoch") == state.get("updated_at_epoch") and re.fullmatch(r"[0-9a-f]{64}", str(artifacts.get("compose_sha256") or "")) is not None and isinstance(containers, dict) and set(containers) == {expected_container} and receipt.get("schema") == "gitea-deploy-receipt.v1" and receipt.get("repository") == repository and receipt.get("revision") == live_revision and receipt.get("live") is None and isinstance(receipt_containers, dict) and set(receipt_containers) == {expected_container} and receipt_containers == containers ) def _is_exact_legacy_state_ahead_baseline( state, repository, expected_container, recorded_live, live_revision, container_artifact, git_facts, ): attempt = state.get("last_attempt") artifacts = state.get("live_artifacts") receipt = state.get("live_receipt") if not isinstance(attempt, dict) or not isinstance(artifacts, dict) or not isinstance(receipt, dict): return False metadata = attempt.get("metadata") build = receipt.get("build") if not isinstance(metadata, dict) or not isinstance(build, dict): return False artifact_containers = artifacts.get("containers") receipt_containers = receipt.get("containers") if not isinstance(artifact_containers, dict) or not isinstance(receipt_containers, dict): return False stored_artifact = artifact_containers.get(expected_container) if not isinstance(stored_artifact, dict) or receipt_containers != artifact_containers: return False def _same_immutable_artifact_after_container_recreate(stored, current): expected_fields = { "container_id", "image_id", "image_ref", "container_revision", "image_revision", "image_source", "image_tree", "image_version", "image_created", } immutable_fields = expected_fields - {"container_id"} return ( isinstance(current, dict) and set(stored) == expected_fields and set(current) == expected_fields and re.fullmatch(r"[0-9a-f]{64}", str(stored.get("container_id") or "")) is not None and re.fullmatch(r"[0-9a-f]{64}", str(current.get("container_id") or "")) is not None and all(stored.get(field) == current.get(field) for field in immutable_fields) ) attempt_revision = str(attempt.get("revision") or "") expected_facts = { "recorded_to_live": 1, "canonical_recorded_to_live": 1, "live_to_recorded": 0, "canonical_live_to_recorded": 0, "mounted_recorded_main": 0, "canonical_recorded_main": 0, "canonical_live_main": 0, "recorded_to_attempt": 0, "canonical_recorded_to_attempt": 0, "attempt_to_main": 0, "canonical_attempt_to_main": 0, } return ( state.get("schema_version") == 3 and state.get("repository") == repository and state.get("revision") == recorded_live and state.get("live_revision") == recorded_live and state.get("status") == "failed" and state.get("live_status") == "unknown" and state.get("live_legacy_provenance") is False and re.fullmatch(r"[0-9a-f]{40}", recorded_live) is not None and re.fullmatch(r"[0-9a-f]{40}", live_revision) is not None and recorded_live != live_revision and re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is not None and attempt_revision != recorded_live and attempt.get("status") == "failed" and attempt.get("message") == "Existing production containers require a valid healthy live deployment state" and attempt.get("updated_at_epoch") == state.get("updated_at_epoch") and set(metadata) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"} and metadata.get("id") == attempt_revision[:12] and metadata.get("repository") == repository and metadata.get("revision") == attempt_revision and re.fullmatch(r"[0-9a-f]{40}", str(metadata.get("tree") or "")) is not None and re.fullmatch(r"[0-9a-f]{64}", str(metadata.get("treeDigest") or "")) is not None and isinstance(metadata.get("builtAt"), str) and bool(metadata.get("builtAt")) and set(artifacts) == {"compose_sha256", "containers"} and re.fullmatch(r"[0-9a-f]{64}", str(artifacts.get("compose_sha256") or "")) is not None and set(artifact_containers) == {expected_container} and _same_immutable_artifact_after_container_recreate(stored_artifact, container_artifact) and set(receipt) == {"schema", "repository", "revision", "tree", "treeDigest", "build", "containers", "live"} and receipt.get("schema") == "gitea-deploy-receipt.v1" and receipt.get("repository") == repository and receipt.get("revision") == recorded_live and receipt.get("live") is None and re.fullmatch(r"[0-9a-f]{40}", str(receipt.get("tree") or "")) is not None and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("treeDigest") or "")) is not None and set(receipt_containers) == {expected_container} and set(build) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"} and build.get("id") == recorded_live[:12] and build.get("repository") == repository and build.get("revision") == recorded_live and build.get("tree") == receipt.get("tree") and build.get("treeDigest") == receipt.get("treeDigest") and isinstance(build.get("builtAt"), str) and bool(build.get("builtAt")) and container_artifact.get("container_revision") == live_revision and container_artifact.get("image_revision") == live_revision and re.fullmatch(r"[0-9a-f]{64}", str(container_artifact.get("container_id") or "")) is not None and re.fullmatch(r"sha256:[0-9a-f]{64}", str(container_artifact.get("image_id") or "")) is not None and git_facts == expected_facts ) main = git_run("rev-parse", "refs/heads/main") main_revision = main.stdout.strip() if main.returncode != 0 or re.fullmatch(r"[0-9a-f]{40}", main_revision) is None: raise SystemExit(26) state_root.mkdir(parents=True, exist_ok=True) project_lock_path.parent.mkdir(parents=True, exist_ok=True) try: project_lock = project_lock_path.open("a+", encoding="utf-8") global_lock = global_lock_path.open("a+", encoding="utf-8") except Exception: raise SystemExit(29) with project_lock: try: fcntl.flock(project_lock, fcntl.LOCK_EX) except Exception: raise SystemExit(30) with global_lock: try: fcntl.flock(global_lock, fcntl.LOCK_EX) except Exception: raise SystemExit(30) try: current_projects = json.loads(registry_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(31) if current_projects.get(repository) != project: raise SystemExit(37) if state_path.is_symlink() or not state_path.is_file(): raise SystemExit(42) try: state = json.loads(state_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(42) if not isinstance(state, dict) or state.get("schema_version") != 3 or state.get("repository") != repository: raise SystemExit(43) inspected = subprocess.run(["docker", "inspect", expected_container], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False) if inspected.returncode != 0: raise SystemExit(44) try: container = json.loads(inspected.stdout)[0] except Exception: raise SystemExit(44) runtime_state = dict(container.get("State") or {}) labels = dict(dict(container.get("Config") or {}).get("Labels") or {}) image_ref = str(dict(container.get("Config") or {}).get("Image") or "") image_id = str(container.get("Image") or "") container_revision = str(labels.get("org.opencontainers.image.revision") or "") if not runtime_state.get("Running") or str(dict(runtime_state.get("Health") or {}).get("Status") or "") != "healthy": raise SystemExit(45) if labels.get("com.docker.compose.project") != "media-codec-dashboard": raise SystemExit(45) if not image_id.startswith("sha256:") or re.fullmatch(r"[0-9a-f]{40}", container_revision) is None: raise SystemExit(46) project_names = subprocess.run(["docker", "ps", "--all", "--filter", "label=com.docker.compose.project=media-codec-dashboard", "--format", "{{.Names}}"], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False) if project_names.returncode != 0 or {line.strip() for line in project_names.stdout.splitlines() if line.strip()} != {expected_container}: raise SystemExit(47) mounts = {str(item.get("Destination") or ""): str(item.get("Source") or "") for item in list(container.get("Mounts") or []) if isinstance(item, dict)} required_mounts = { "/app/data": "/srv/example/appdata/media-codec-dashboard/data", "/app/data/quality-runtime": "/srv/example/appdata/tdarr/logs/media-codec-dashboard-quality-audit", "/run/secrets": "/srv/example/appdata/media-codec-dashboard/secrets", "/tdarr-output": "/srv/example/Media", } if any(mounts.get(destination) != source for destination, source in required_mounts.items()): raise SystemExit(48) bindings = dict(dict(container.get("HostConfig") or {}).get("PortBindings") or {}) host_ports = {str(item.get("HostPort") or "") for item in list(bindings.get("1211/tcp") or []) if isinstance(item, dict)} if "1211" not in host_ports: raise SystemExit(49) image_result = subprocess.run(["docker", "image", "inspect", image_id], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False) if image_result.returncode != 0: raise SystemExit(50) try: image = json.loads(image_result.stdout)[0] except Exception: raise SystemExit(50) image_labels = dict(dict(image.get("Config") or {}).get("Labels") or {}) image_revision = str(image_labels.get("org.opencontainers.image.revision") or "") live_revision = image_revision if re.fullmatch(r"[0-9a-f]{40}", live_revision) is None or container_revision != image_revision: raise SystemExit(51) container_artifact = { "container_id": str(container.get("Id") or ""), "image_id": image_id, "image_ref": image_ref, "container_revision": container_revision, "image_revision": image_revision, "image_source": str(image_labels.get("org.opencontainers.image.source") or ""), "image_tree": str(image_labels.get("org.opencontainers.image.source-tree") or ""), "image_version": str(image_labels.get("org.opencontainers.image.version") or ""), "image_created": str(image_labels.get("org.opencontainers.image.created") or ""), } resolved = git_run("rev-parse", f"{live_revision}^{{commit}}") if resolved.returncode != 0 or resolved.stdout.strip() != live_revision: raise SystemExit(25) if git_run("merge-base", "--is-ancestor", live_revision, main_revision).returncode != 0: raise SystemExit(27) recorded_live = str(state.get("live_revision") or state.get("revision") or "") recorded_status = str(state.get("status") or "") recorded_live_status = str(state.get("live_status") or "") last_attempt = dict(state.get("last_attempt") or {}) attempt_revision = str(last_attempt.get("revision") or "") attempt_status = str(last_attempt.get("status") or "") reconciled_from = recorded_live if recorded_live == live_revision: if recorded_status == "success" and state.get("live_legacy_provenance") is True: print("already-reconciled", flush=True) raise SystemExit(0) current_artifacts = state.get("live_artifacts") current_receipt = state.get("live_receipt") current_artifact_containers = ( current_artifacts.get("containers") if isinstance(current_artifacts, dict) else None ) current_stored_artifact = ( current_artifact_containers.get(expected_container) if isinstance(current_artifact_containers, dict) else None ) current_attempt_metadata = ( last_attempt.get("metadata") if isinstance(last_attempt, dict) else None ) current_artifact_fields = { "container_id", "image_id", "image_ref", "container_revision", "image_revision", "image_source", "image_tree", "image_version", "image_created", } current_immutable_fields = current_artifact_fields - {"container_id"} current_live_unknown_snapshot_baseline = ( recorded_status == "failed" and recorded_live_status == "unknown" and state.get("live_legacy_provenance") is True and isinstance(current_artifacts, dict) and set(current_artifacts) == {"compose_sha256", "containers"} and current_artifacts.get("compose_sha256") == "" and isinstance(current_artifact_containers, dict) and set(current_artifact_containers) == {expected_container} and isinstance(current_stored_artifact, dict) and set(current_stored_artifact) == current_artifact_fields and set(container_artifact) == current_artifact_fields and re.fullmatch(r"[0-9a-f]{64}", str(current_stored_artifact.get("container_id") or "")) is not None and re.fullmatch(r"[0-9a-f]{64}", str(container_artifact.get("container_id") or "")) is not None and all(current_stored_artifact.get(field) == container_artifact.get(field) for field in current_immutable_fields) and isinstance(current_receipt, dict) and current_receipt == {} and attempt_status == "failed" # Do not bind recovery to one historical error string. The proof # is the preserved immutable artifact plus the currently healthy # runtime, exact candidate metadata/timestamp, and Git ancestry. and attempt.get("updated_at_epoch") == state.get("updated_at_epoch") and isinstance(current_attempt_metadata, dict) and set(current_attempt_metadata) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"} and current_attempt_metadata.get("id") == attempt_revision[:12] and current_attempt_metadata.get("repository") == repository and current_attempt_metadata.get("revision") == attempt_revision and re.fullmatch(r"[0-9a-f]{40}", str(current_attempt_metadata.get("tree") or "")) is not None and re.fullmatch(r"[0-9a-f]{64}", str(current_attempt_metadata.get("treeDigest") or "")) is not None and isinstance(current_attempt_metadata.get("builtAt"), str) and bool(current_attempt_metadata.get("builtAt")) ) current_verified_receipt_snapshot_baseline = _is_current_verified_receipt_snapshot_baseline( state, repository, expected_container, live_revision, attempt_revision ) current_live_failed_baseline = ( recorded_status == "failed" and (recorded_live_status == "healthy" or current_live_unknown_snapshot_baseline or current_verified_receipt_snapshot_baseline) and attempt_status == "failed" and re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is not None and attempt_revision != live_revision ) if not current_live_failed_baseline: raise SystemExit(55) resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}") if resolved_attempt.returncode != 0 or resolved_attempt.stdout.strip() != attempt_revision: raise SystemExit(25) if git_run("merge-base", "--is-ancestor", live_revision, attempt_revision).returncode != 0: raise SystemExit(56) if git_run("merge-base", "--is-ancestor", attempt_revision, main_revision).returncode != 0: raise SystemExit(57) if current_verified_receipt_snapshot_baseline: if controller_path.is_symlink() or not controller_path.is_file(): raise SystemExit(62) try: spec = importlib.util.spec_from_file_location("dashboard_deploy_controller", controller_path) controller = importlib.util.module_from_spec(spec) if spec and spec.loader else None if controller is None or spec is None or spec.loader is None: raise RuntimeError("controller import unavailable") spec.loader.exec_module(controller) verified_project = dict(project) verified_project["_repository_name"] = repository verified_project["_repository_path"] = str(bare_repo) build = controller.validate_recorded_receipt( verified_project, current_receipt, repository, live_revision ) observed_receipt = controller.verify_deployment( verified_project, build, expected_image_ids={ expected_container: current_receipt["containers"][expected_container]["image_id"] }, ) observed_artifacts = controller.deployment_artifacts(verified_project) except Exception: raise SystemExit(63) if observed_receipt != current_receipt or observed_artifacts != current_artifacts: raise SystemExit(64) reconciled_from = attempt_revision else: if re.fullmatch(r"[0-9a-f]{40}", recorded_live) is None: raise SystemExit(58) stale_success_baseline = ( recorded_status == "success" and recorded_live_status == "healthy" and attempt_status == "success" and attempt_revision == recorded_live ) stale_failed_attempt = recorded_status == "failed" if not (stale_success_baseline or stale_failed_attempt): raise SystemExit(59) resolved_recorded = git_run("rev-parse", f"{recorded_live}^{{commit}}") if resolved_recorded.returncode != 0 or resolved_recorded.stdout.strip() != recorded_live: raise SystemExit(25) recorded_to_live = git_run("merge-base", "--is-ancestor", recorded_live, live_revision) if recorded_to_live.returncode not in {0, 1}: raise SystemExit(61) if recorded_to_live.returncode != 0: canonical_recorded = canonical_git_run("rev-parse", f"{recorded_live}^{{commit}}") canonical_live = canonical_git_run("rev-parse", f"{live_revision}^{{commit}}") canonical_main = canonical_git_run("rev-parse", "refs/heads/main") canonical_recorded_to_live = canonical_git_run("merge-base", "--is-ancestor", recorded_live, live_revision) canonical_recorded_main = canonical_git_run("merge-base", "--is-ancestor", recorded_live, main_revision) canonical_live_main = canonical_git_run("merge-base", "--is-ancestor", live_revision, main_revision) canonical_revisions_available = ( canonical_bare_repo.is_dir() and not canonical_bare_repo.is_symlink() and canonical_recorded.returncode == 0 and canonical_recorded.stdout.strip() == recorded_live and canonical_live.returncode == 0 and canonical_live.stdout.strip() == live_revision and canonical_main.returncode == 0 and canonical_main.stdout.strip() == main_revision ) canonical_ancestry_fallback = ( canonical_revisions_available and canonical_recorded_to_live.returncode == 0 and canonical_recorded_main.returncode == 0 and canonical_live_main.returncode == 0 ) if not canonical_ancestry_fallback: if not canonical_revisions_available: raise SystemExit(61) if canonical_recorded_to_live.returncode not in {0, 1}: raise SystemExit(61) if canonical_recorded_main.returncode not in {0, 1}: raise SystemExit(61) if canonical_live_main.returncode not in {0, 1}: raise SystemExit(61) live_to_recorded = git_run("merge-base", "--is-ancestor", live_revision, recorded_live) canonical_live_to_recorded = canonical_git_run("merge-base", "--is-ancestor", live_revision, recorded_live) mounted_recorded_main = git_run("merge-base", "--is-ancestor", recorded_live, main_revision) mounted_merge_base = git_run("merge-base", recorded_live, live_revision) canonical_merge_base = canonical_git_run("merge-base", recorded_live, live_revision) mounted_recorded_time = git_run("show", "-s", "--format=%ct", recorded_live) mounted_live_time = git_run("show", "-s", "--format=%ct", live_revision) canonical_recorded_time = canonical_git_run("show", "-s", "--format=%ct", recorded_live) canonical_live_time = canonical_git_run("show", "-s", "--format=%ct", live_revision) mounted_recorded_distance = git_run("rev-list", "--count", f"{recorded_live}..{main_revision}") mounted_live_distance = git_run("rev-list", "--count", f"{live_revision}..{main_revision}") canonical_recorded_distance = canonical_git_run("rev-list", "--count", f"{recorded_live}..{main_revision}") canonical_live_distance = canonical_git_run("rev-list", "--count", f"{live_revision}..{main_revision}") resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}") canonical_resolved_attempt = canonical_git_run("rev-parse", f"{attempt_revision}^{{commit}}") recorded_to_attempt = git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision) canonical_recorded_to_attempt = canonical_git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision) attempt_to_main = git_run("merge-base", "--is-ancestor", attempt_revision, main_revision) canonical_attempt_to_main = canonical_git_run("merge-base", "--is-ancestor", attempt_revision, main_revision) def git_nonnegative_int(result): if result.returncode != 0: return None value = result.stdout.strip() if not value.isdigit(): return None return int(value) recorded_commit_epoch = git_nonnegative_int(mounted_recorded_time) live_commit_epoch = git_nonnegative_int(mounted_live_time) canonical_recorded_commit_epoch = git_nonnegative_int(canonical_recorded_time) canonical_live_commit_epoch = git_nonnegative_int(canonical_live_time) recorded_distance = git_nonnegative_int(mounted_recorded_distance) live_distance = git_nonnegative_int(mounted_live_distance) canonical_recorded_distance_value = git_nonnegative_int(canonical_recorded_distance) canonical_live_distance_value = git_nonnegative_int(canonical_live_distance) try: recorded_updated_at = int(state.get("updated_at_epoch") or 0) except (TypeError, ValueError): recorded_updated_at = 0 mounted_merge_base_value = mounted_merge_base.stdout.strip() canonical_merge_base_value = canonical_merge_base.stdout.strip() history_transition_baseline = ( stale_success_baseline and recorded_to_live.returncode == 1 and canonical_recorded_to_live.returncode == 1 and live_to_recorded.returncode == 1 and canonical_live_to_recorded.returncode == 1 and mounted_recorded_main.returncode == 0 and canonical_recorded_main.returncode == 0 and canonical_live_main.returncode == 0 and mounted_merge_base.returncode == 0 and canonical_merge_base.returncode == 0 and re.fullmatch(r"[0-9a-f]{40}", mounted_merge_base_value) is not None and canonical_merge_base_value == mounted_merge_base_value and mounted_merge_base_value not in {recorded_live, live_revision} and recorded_commit_epoch is not None and live_commit_epoch is not None and canonical_recorded_commit_epoch == recorded_commit_epoch and canonical_live_commit_epoch == live_commit_epoch and 0 < recorded_updated_at and recorded_commit_epoch <= recorded_updated_at and recorded_updated_at < live_commit_epoch and recorded_commit_epoch < live_commit_epoch and recorded_distance is not None and live_distance is not None and canonical_recorded_distance_value == recorded_distance and canonical_live_distance_value == live_distance and 0 <= live_distance < recorded_distance ) exact_legacy_state_ahead_baseline = ( resolved_attempt.returncode == 0 and resolved_attempt.stdout.strip() == attempt_revision and canonical_resolved_attempt.returncode == 0 and canonical_resolved_attempt.stdout.strip() == attempt_revision and _is_exact_legacy_state_ahead_baseline( state, repository, expected_container, recorded_live, live_revision, container_artifact, { "recorded_to_live": recorded_to_live.returncode, "canonical_recorded_to_live": canonical_recorded_to_live.returncode, "live_to_recorded": live_to_recorded.returncode, "canonical_live_to_recorded": canonical_live_to_recorded.returncode, "mounted_recorded_main": mounted_recorded_main.returncode, "canonical_recorded_main": canonical_recorded_main.returncode, "canonical_live_main": canonical_live_main.returncode, "recorded_to_attempt": recorded_to_attempt.returncode, "canonical_recorded_to_attempt": canonical_recorded_to_attempt.returncode, "attempt_to_main": attempt_to_main.returncode, "canonical_attempt_to_main": canonical_attempt_to_main.returncode, }, ) ) if not (history_transition_baseline or exact_legacy_state_ahead_baseline): if canonical_recorded_to_live.returncode == 1: raise SystemExit(60) raise SystemExit(61) artifacts = {"containers": {expected_container: container_artifact}, "compose_sha256": ""} now = int(time.time()) payload = { "schema_version": 3, "repository": repository, "slug": "media-codec-dashboard", "compose_project": "media-codec-dashboard", "revision": live_revision, "live_revision": live_revision, "live_status": "healthy", "live_legacy_provenance": True, "live_artifacts": artifacts, "live_receipt": {}, "status": "success", "message": "Verified existing Media Codec Dashboard runtime reconciled to actual immutable live image", "updated_at_epoch": now, "artifacts": artifacts, "last_attempt": { "revision": live_revision, "status": "success", "message": "Verified runtime state reconciliation", "updated_at_epoch": now, "metadata": {"reconciled_from": reconciled_from}, }, "previous_revision": recorded_live, "rollback_restored_revision": "", } if recorded_live == live_revision and current_verified_receipt_snapshot_baseline: payload = dict(state) payload.update({ "live_status": "healthy", "status": "success", "message": "Verified Media Codec Dashboard receipt re-attested after failed forward deployment", "updated_at_epoch": now, "artifacts": current_artifacts, "last_attempt": { "revision": live_revision, "status": "success", "message": "Verified existing runtime receipt re-attestation", "updated_at_epoch": now, "metadata": {"reconciled_failed_revision": reconciled_from}, }, }) backup = state_path.with_name(f"{state_path.name}.reconcile-backup-{now}-{os.getpid()}") try: shutil.copy2(state_path, backup) temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.tmp") with temporary.open("x", encoding="utf-8") as handle: handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n") handle.flush() os.fsync(handle.fileno()) os.chmod(temporary, stat.S_IMODE(state_path.stat().st_mode)) os.replace(temporary, state_path) directory_fd = os.open(str(state_root), os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) except Exception: raise SystemExit(53) verify = json.loads(state_path.read_text(encoding="utf-8")) expected_legacy = not (recorded_live == live_revision and current_verified_receipt_snapshot_baseline) if not (verify.get("live_revision") == live_revision and verify.get("live_status") == "healthy" and verify.get("live_legacy_provenance") is expected_legacy and dict(dict(verify.get("live_artifacts") or {}).get("containers") or {}).get(expected_container, {}).get("image_id") == image_id): raise SystemExit(54) if not expected_legacy and (verify.get("live_receipt") != current_receipt or verify.get("live_artifacts") != current_artifacts): raise SystemExit(64) print("reconciled", flush=True) ''' class MediaCodecDashboardDeploymentReconciliation: """Repair only stale controller state proven against the live immutable dashboard runtime.""" def __init__(self, portainer: PortainerConnector | None = None): self.portainer = portainer or PortainerConnector() def run(self) -> dict[str, Any]: status = self.portainer.get_container_status(_CONTROLLER) if status.get("status") != "ok": return {"status": "controller_unavailable", "reason": redact_message(str(status.get("reason") or status.get("status") or "unavailable"))} controller = dict(status.get("container") or {}) if controller.get("name") != _CONTROLLER or controller.get("state") != "running": return {"status": "controller_unavailable", "reason": "deployment controller is not running"} container_id = str(controller.get("id") or "") config = self.portainer._config() if not config or not container_id: return {"status": "connector_unavailable", "reason": "Portainer deployment control is unavailable"} command = ["python3", "-c", _RECONCILE_SCRIPT, _REPOSITORY, _CONTAINER] endpoint = quote(str(config.endpoint_id), safe="") safe_container = quote(container_id, safe="") try: created = self.portainer._request("POST", f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec", json={"AttachStdout": False, "AttachStderr": False, "Tty": False, "Cmd": command}).json() exec_id = str(created.get("Id") or "") if not exec_id: return {"status": "error", "reason": "dashboard state reconciliation could not start"} safe_exec = quote(exec_id, safe="") self.portainer._request("POST", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start", json={"Detach": True, "Tty": False}) deadline = time.monotonic() + 45 exit_code: int | None = None while time.monotonic() < deadline: inspected = self.portainer._request("GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json").json() if not bool(inspected.get("Running")): raw_exit = inspected.get("ExitCode") exit_code = int(raw_exit) if raw_exit is not None else -1 break time.sleep(0.25) except (RuntimeError, ValueError, TypeError) as exc: return {"status": "error", "reason": redact_message(str(exc))} if exit_code is None: return {"status": "error", "reason": "dashboard state reconciliation timed out"} reasons = { 20: "deployment projects registry is unavailable or unsafe", 22: "Media Codec Dashboard deployment target is missing", 23: "Media Codec Dashboard deployment target topology differs from the bounded contract", 24: "mounted Media Codec Dashboard bare repository is unavailable", 25: "deployment revisions cannot be resolved", 26: "Media Codec Dashboard main cannot be resolved", 27: "live Media Codec Dashboard revision is not an ancestor of main", 29: "deployment lock files cannot be opened", 30: "deployment locks cannot be acquired", 31: "deployment projects registry cannot be read", 36: "deployment projects registry host alias is unavailable or mismatched", 37: "deployment target changed while reconciliation was waiting", 40: "fixed Media Codec Dashboard reconciliation contract was altered", 42: "deployment state file is unavailable or invalid", 43: "deployment state is not a modern Media Codec Dashboard state", 44: "live Media Codec Dashboard container cannot be inspected", 45: "live Media Codec Dashboard container is not healthy in the expected Compose project", 46: "live Media Codec Dashboard immutable image identity is invalid", 47: "Media Codec Dashboard Compose project contains unexpected containers", 48: "live Media Codec Dashboard mount topology differs from the verified runtime", 49: "live Media Codec Dashboard port mapping differs from the verified runtime", 50: "live Media Codec Dashboard image cannot be inspected", 51: "live Media Codec Dashboard image revision label does not match the verified runtime", 52: "controller state drift is not the exact known recoverable state", 53: "reconciled deployment state could not be written durably", 54: "reconciled deployment state did not verify after write", 55: "controller state for the current live revision is internally inconsistent", 56: "failed deployment attempt is not a descendant of the live revision", 57: "failed deployment attempt is not an ancestor of main", 58: "recorded live deployment revision is invalid", 59: "stale controller state metadata is not a supported recoverable baseline", 60: "recorded live deployment revision is not an ancestor of the verified runtime", 61: "mounted deployment repository ancestry differs from canonical Gitea repository", 62: "deployment controller verifier is unavailable or unsafe", 63: "recorded dashboard receipt cannot be reverified against the live runtime", 64: "verified dashboard receipt or artifacts differ from the recorded state", } if exit_code != 0: return {"status": "validation_failed", "reason": reasons.get(exit_code, "Media Codec Dashboard deployment reconciliation failed closed")} return { "status": "reconciled", "repository": _REPOSITORY, "container": _CONTAINER, }