769 lines
42 KiB
Python
769 lines
42 KiB
Python
from __future__ import annotations
|
|
|
|
import time
|
|
from typing import Any
|
|
from urllib.parse import quote
|
|
|
|
from ..connectors.portainer import PortainerConnector
|
|
from ..core.redaction import redact_message
|
|
|
|
_CONTROLLER = "gitea-deploy-control"
|
|
_REPOSITORY = "jens/media-codec-dashboard"
|
|
_CONTAINER = "Media-Codec-Dashboard"
|
|
|
|
# Fail-closed reconciliation for the verified production runtime.
|
|
# It does not deploy, restart, recreate or stop any container. It only repairs
|
|
# controller state after proving the exact live container/image/mount topology
|
|
# and immutable runtime revision from independent OCI labels plus Gitea ancestry.
|
|
_RECONCILE_SCRIPT = r'''
|
|
import fcntl
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
repository, expected_container = sys.argv[1:3]
|
|
registry_path = Path("/controller-config/projects.json")
|
|
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
|
|
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
|
state_path = state_root / "media-codec-dashboard.json"
|
|
project_lock_path = state_root / "locks" / "media-codec-dashboard.lock"
|
|
global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
|
controller_path = Path("/opt/gitea-deploy/deploy.py")
|
|
|
|
if repository != "jens/media-codec-dashboard":
|
|
raise SystemExit(40)
|
|
if expected_container != "Media-Codec-Dashboard":
|
|
raise SystemExit(40)
|
|
|
|
if registry_path.is_symlink() or not registry_path.is_file():
|
|
raise SystemExit(20)
|
|
try:
|
|
if host_registry_path.is_symlink() or not host_registry_path.is_file() or not os.path.samefile(registry_path, host_registry_path):
|
|
raise SystemExit(36)
|
|
except OSError:
|
|
raise SystemExit(36)
|
|
|
|
try:
|
|
projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(31)
|
|
project = projects.get(repository) if isinstance(projects, dict) else None
|
|
if not isinstance(project, dict):
|
|
raise SystemExit(22)
|
|
if project.get("branch") != "main" or project.get("slug") != "media-codec-dashboard":
|
|
raise SystemExit(23)
|
|
if project.get("compose_project") != "media-codec-dashboard" or project.get("containers") != [expected_container]:
|
|
raise SystemExit(23)
|
|
compose_files = project.get("compose_files")
|
|
if not isinstance(compose_files, list) or "docker-compose.yml" not in compose_files:
|
|
raise SystemExit(23)
|
|
source_dir = Path(str(project.get("source_dir") or ""))
|
|
allowed_roots = (Path("/mnt/cache/appdata/gitea-deploy-runners/sources"), Path("/srv/example/appdata"))
|
|
if not source_dir.is_absolute() or not any(root == source_dir or root in source_dir.parents for root in allowed_roots):
|
|
raise SystemExit(23)
|
|
|
|
owner, name = repository.split("/", 1)
|
|
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
|
canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories")
|
|
canonical_bare_repo = canonical_repository_root / owner / f"{name}.git"
|
|
if not bare_repo.is_dir():
|
|
raise SystemExit(24)
|
|
|
|
def git_run(*args):
|
|
return subprocess.run(["git", f"--git-dir={bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
|
|
|
def canonical_git_run(*args):
|
|
return subprocess.run(["git", f"--git-dir={canonical_bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
|
|
|
def _is_current_verified_receipt_snapshot_baseline(state, repository, expected_container, live_revision, attempt_revision):
|
|
attempt = state.get("last_attempt")
|
|
artifacts = state.get("live_artifacts")
|
|
receipt = state.get("live_receipt")
|
|
if not isinstance(attempt, dict) or not isinstance(artifacts, dict) or not isinstance(receipt, dict):
|
|
return False
|
|
containers = artifacts.get("containers")
|
|
receipt_containers = receipt.get("containers")
|
|
return (
|
|
state.get("schema_version") == 3
|
|
and state.get("repository") == repository
|
|
and state.get("revision") == live_revision
|
|
and state.get("live_revision") == live_revision
|
|
and state.get("status") == "failed"
|
|
and state.get("live_status") == "unknown"
|
|
and state.get("live_legacy_provenance") is False
|
|
and attempt.get("revision") == attempt_revision
|
|
and attempt.get("status") == "failed"
|
|
and attempt.get("message") == "Existing production containers require a valid healthy live deployment state"
|
|
and attempt.get("updated_at_epoch") == state.get("updated_at_epoch")
|
|
and re.fullmatch(r"[0-9a-f]{64}", str(artifacts.get("compose_sha256") or "")) is not None
|
|
and isinstance(containers, dict)
|
|
and set(containers) == {expected_container}
|
|
and receipt.get("schema") == "gitea-deploy-receipt.v1"
|
|
and receipt.get("repository") == repository
|
|
and receipt.get("revision") == live_revision
|
|
and receipt.get("live") is None
|
|
and isinstance(receipt_containers, dict)
|
|
and set(receipt_containers) == {expected_container}
|
|
and receipt_containers == containers
|
|
)
|
|
|
|
def _is_exact_legacy_state_ahead_baseline(
|
|
state,
|
|
repository,
|
|
expected_container,
|
|
recorded_live,
|
|
live_revision,
|
|
container_artifact,
|
|
git_facts,
|
|
):
|
|
attempt = state.get("last_attempt")
|
|
artifacts = state.get("live_artifacts")
|
|
receipt = state.get("live_receipt")
|
|
if not isinstance(attempt, dict) or not isinstance(artifacts, dict) or not isinstance(receipt, dict):
|
|
return False
|
|
metadata = attempt.get("metadata")
|
|
build = receipt.get("build")
|
|
if not isinstance(metadata, dict) or not isinstance(build, dict):
|
|
return False
|
|
artifact_containers = artifacts.get("containers")
|
|
receipt_containers = receipt.get("containers")
|
|
if not isinstance(artifact_containers, dict) or not isinstance(receipt_containers, dict):
|
|
return False
|
|
stored_artifact = artifact_containers.get(expected_container)
|
|
if not isinstance(stored_artifact, dict) or receipt_containers != artifact_containers:
|
|
return False
|
|
|
|
def _same_immutable_artifact_after_container_recreate(stored, current):
|
|
expected_fields = {
|
|
"container_id",
|
|
"image_id",
|
|
"image_ref",
|
|
"container_revision",
|
|
"image_revision",
|
|
"image_source",
|
|
"image_tree",
|
|
"image_version",
|
|
"image_created",
|
|
}
|
|
immutable_fields = expected_fields - {"container_id"}
|
|
return (
|
|
isinstance(current, dict)
|
|
and set(stored) == expected_fields
|
|
and set(current) == expected_fields
|
|
and re.fullmatch(r"[0-9a-f]{64}", str(stored.get("container_id") or "")) is not None
|
|
and re.fullmatch(r"[0-9a-f]{64}", str(current.get("container_id") or "")) is not None
|
|
and all(stored.get(field) == current.get(field) for field in immutable_fields)
|
|
)
|
|
|
|
attempt_revision = str(attempt.get("revision") or "")
|
|
expected_facts = {
|
|
"recorded_to_live": 1,
|
|
"canonical_recorded_to_live": 1,
|
|
"live_to_recorded": 0,
|
|
"canonical_live_to_recorded": 0,
|
|
"mounted_recorded_main": 0,
|
|
"canonical_recorded_main": 0,
|
|
"canonical_live_main": 0,
|
|
"recorded_to_attempt": 0,
|
|
"canonical_recorded_to_attempt": 0,
|
|
"attempt_to_main": 0,
|
|
"canonical_attempt_to_main": 0,
|
|
}
|
|
return (
|
|
state.get("schema_version") == 3
|
|
and state.get("repository") == repository
|
|
and state.get("revision") == recorded_live
|
|
and state.get("live_revision") == recorded_live
|
|
and state.get("status") == "failed"
|
|
and state.get("live_status") == "unknown"
|
|
and state.get("live_legacy_provenance") is False
|
|
and re.fullmatch(r"[0-9a-f]{40}", recorded_live) is not None
|
|
and re.fullmatch(r"[0-9a-f]{40}", live_revision) is not None
|
|
and recorded_live != live_revision
|
|
and re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is not None
|
|
and attempt_revision != recorded_live
|
|
and attempt.get("status") == "failed"
|
|
and attempt.get("message") == "Existing production containers require a valid healthy live deployment state"
|
|
and attempt.get("updated_at_epoch") == state.get("updated_at_epoch")
|
|
and set(metadata) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}
|
|
and metadata.get("id") == attempt_revision[:12]
|
|
and metadata.get("repository") == repository
|
|
and metadata.get("revision") == attempt_revision
|
|
and re.fullmatch(r"[0-9a-f]{40}", str(metadata.get("tree") or "")) is not None
|
|
and re.fullmatch(r"[0-9a-f]{64}", str(metadata.get("treeDigest") or "")) is not None
|
|
and isinstance(metadata.get("builtAt"), str)
|
|
and bool(metadata.get("builtAt"))
|
|
and set(artifacts) == {"compose_sha256", "containers"}
|
|
and re.fullmatch(r"[0-9a-f]{64}", str(artifacts.get("compose_sha256") or "")) is not None
|
|
and set(artifact_containers) == {expected_container}
|
|
and _same_immutable_artifact_after_container_recreate(stored_artifact, container_artifact)
|
|
and set(receipt) == {"schema", "repository", "revision", "tree", "treeDigest", "build", "containers", "live"}
|
|
and receipt.get("schema") == "gitea-deploy-receipt.v1"
|
|
and receipt.get("repository") == repository
|
|
and receipt.get("revision") == recorded_live
|
|
and receipt.get("live") is None
|
|
and re.fullmatch(r"[0-9a-f]{40}", str(receipt.get("tree") or "")) is not None
|
|
and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("treeDigest") or "")) is not None
|
|
and set(receipt_containers) == {expected_container}
|
|
and set(build) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}
|
|
and build.get("id") == recorded_live[:12]
|
|
and build.get("repository") == repository
|
|
and build.get("revision") == recorded_live
|
|
and build.get("tree") == receipt.get("tree")
|
|
and build.get("treeDigest") == receipt.get("treeDigest")
|
|
and isinstance(build.get("builtAt"), str)
|
|
and bool(build.get("builtAt"))
|
|
and container_artifact.get("container_revision") == live_revision
|
|
and container_artifact.get("image_revision") == live_revision
|
|
and re.fullmatch(r"[0-9a-f]{64}", str(container_artifact.get("container_id") or "")) is not None
|
|
and re.fullmatch(r"sha256:[0-9a-f]{64}", str(container_artifact.get("image_id") or "")) is not None
|
|
and git_facts == expected_facts
|
|
)
|
|
|
|
main = git_run("rev-parse", "refs/heads/main")
|
|
main_revision = main.stdout.strip()
|
|
if main.returncode != 0 or re.fullmatch(r"[0-9a-f]{40}", main_revision) is None:
|
|
raise SystemExit(26)
|
|
|
|
state_root.mkdir(parents=True, exist_ok=True)
|
|
project_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
project_lock = project_lock_path.open("a+", encoding="utf-8")
|
|
global_lock = global_lock_path.open("a+", encoding="utf-8")
|
|
except Exception:
|
|
raise SystemExit(29)
|
|
|
|
with project_lock:
|
|
try:
|
|
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
|
except Exception:
|
|
raise SystemExit(30)
|
|
with global_lock:
|
|
try:
|
|
fcntl.flock(global_lock, fcntl.LOCK_EX)
|
|
except Exception:
|
|
raise SystemExit(30)
|
|
|
|
try:
|
|
current_projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(31)
|
|
if current_projects.get(repository) != project:
|
|
raise SystemExit(37)
|
|
|
|
if state_path.is_symlink() or not state_path.is_file():
|
|
raise SystemExit(42)
|
|
try:
|
|
state = json.loads(state_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(42)
|
|
if not isinstance(state, dict) or state.get("schema_version") != 3 or state.get("repository") != repository:
|
|
raise SystemExit(43)
|
|
|
|
inspected = subprocess.run(["docker", "inspect", expected_container], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
|
if inspected.returncode != 0:
|
|
raise SystemExit(44)
|
|
try:
|
|
container = json.loads(inspected.stdout)[0]
|
|
except Exception:
|
|
raise SystemExit(44)
|
|
runtime_state = dict(container.get("State") or {})
|
|
labels = dict(dict(container.get("Config") or {}).get("Labels") or {})
|
|
image_ref = str(dict(container.get("Config") or {}).get("Image") or "")
|
|
image_id = str(container.get("Image") or "")
|
|
container_revision = str(labels.get("org.opencontainers.image.revision") or "")
|
|
if not runtime_state.get("Running") or str(dict(runtime_state.get("Health") or {}).get("Status") or "") != "healthy":
|
|
raise SystemExit(45)
|
|
if labels.get("com.docker.compose.project") != "media-codec-dashboard":
|
|
raise SystemExit(45)
|
|
if not image_id.startswith("sha256:") or re.fullmatch(r"[0-9a-f]{40}", container_revision) is None:
|
|
raise SystemExit(46)
|
|
|
|
project_names = subprocess.run(["docker", "ps", "--all", "--filter", "label=com.docker.compose.project=media-codec-dashboard", "--format", "{{.Names}}"], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
|
if project_names.returncode != 0 or {line.strip() for line in project_names.stdout.splitlines() if line.strip()} != {expected_container}:
|
|
raise SystemExit(47)
|
|
|
|
mounts = {str(item.get("Destination") or ""): str(item.get("Source") or "") for item in list(container.get("Mounts") or []) if isinstance(item, dict)}
|
|
required_mounts = {
|
|
"/app/data": "/srv/example/appdata/media-codec-dashboard/data",
|
|
"/app/data/quality-runtime": "/srv/example/appdata/tdarr/logs/media-codec-dashboard-quality-audit",
|
|
"/run/secrets": "/srv/example/appdata/media-codec-dashboard/secrets",
|
|
"/tdarr-output": "/srv/example/Media",
|
|
}
|
|
if any(mounts.get(destination) != source for destination, source in required_mounts.items()):
|
|
raise SystemExit(48)
|
|
|
|
bindings = dict(dict(container.get("HostConfig") or {}).get("PortBindings") or {})
|
|
host_ports = {str(item.get("HostPort") or "") for item in list(bindings.get("1211/tcp") or []) if isinstance(item, dict)}
|
|
if "1211" not in host_ports:
|
|
raise SystemExit(49)
|
|
|
|
image_result = subprocess.run(["docker", "image", "inspect", image_id], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
|
if image_result.returncode != 0:
|
|
raise SystemExit(50)
|
|
try:
|
|
image = json.loads(image_result.stdout)[0]
|
|
except Exception:
|
|
raise SystemExit(50)
|
|
image_labels = dict(dict(image.get("Config") or {}).get("Labels") or {})
|
|
image_revision = str(image_labels.get("org.opencontainers.image.revision") or "")
|
|
live_revision = image_revision
|
|
if re.fullmatch(r"[0-9a-f]{40}", live_revision) is None or container_revision != image_revision:
|
|
raise SystemExit(51)
|
|
container_artifact = {
|
|
"container_id": str(container.get("Id") or ""),
|
|
"image_id": image_id,
|
|
"image_ref": image_ref,
|
|
"container_revision": container_revision,
|
|
"image_revision": image_revision,
|
|
"image_source": str(image_labels.get("org.opencontainers.image.source") or ""),
|
|
"image_tree": str(image_labels.get("org.opencontainers.image.source-tree") or ""),
|
|
"image_version": str(image_labels.get("org.opencontainers.image.version") or ""),
|
|
"image_created": str(image_labels.get("org.opencontainers.image.created") or ""),
|
|
}
|
|
resolved = git_run("rev-parse", f"{live_revision}^{{commit}}")
|
|
if resolved.returncode != 0 or resolved.stdout.strip() != live_revision:
|
|
raise SystemExit(25)
|
|
if git_run("merge-base", "--is-ancestor", live_revision, main_revision).returncode != 0:
|
|
raise SystemExit(27)
|
|
|
|
recorded_live = str(state.get("live_revision") or state.get("revision") or "")
|
|
recorded_status = str(state.get("status") or "")
|
|
recorded_live_status = str(state.get("live_status") or "")
|
|
last_attempt = dict(state.get("last_attempt") or {})
|
|
attempt_revision = str(last_attempt.get("revision") or "")
|
|
attempt_status = str(last_attempt.get("status") or "")
|
|
reconciled_from = recorded_live
|
|
if recorded_live == live_revision:
|
|
if recorded_status == "success" and state.get("live_legacy_provenance") is True:
|
|
print("already-reconciled", flush=True)
|
|
raise SystemExit(0)
|
|
|
|
current_artifacts = state.get("live_artifacts")
|
|
current_receipt = state.get("live_receipt")
|
|
current_artifact_containers = (
|
|
current_artifacts.get("containers")
|
|
if isinstance(current_artifacts, dict)
|
|
else None
|
|
)
|
|
current_stored_artifact = (
|
|
current_artifact_containers.get(expected_container)
|
|
if isinstance(current_artifact_containers, dict)
|
|
else None
|
|
)
|
|
current_attempt_metadata = (
|
|
last_attempt.get("metadata")
|
|
if isinstance(last_attempt, dict)
|
|
else None
|
|
)
|
|
current_artifact_fields = {
|
|
"container_id",
|
|
"image_id",
|
|
"image_ref",
|
|
"container_revision",
|
|
"image_revision",
|
|
"image_source",
|
|
"image_tree",
|
|
"image_version",
|
|
"image_created",
|
|
}
|
|
current_immutable_fields = current_artifact_fields - {"container_id"}
|
|
current_live_unknown_snapshot_baseline = (
|
|
recorded_status == "failed"
|
|
and recorded_live_status == "unknown"
|
|
and state.get("live_legacy_provenance") is True
|
|
and isinstance(current_artifacts, dict)
|
|
and set(current_artifacts) == {"compose_sha256", "containers"}
|
|
and current_artifacts.get("compose_sha256") == ""
|
|
and isinstance(current_artifact_containers, dict)
|
|
and set(current_artifact_containers) == {expected_container}
|
|
and isinstance(current_stored_artifact, dict)
|
|
and set(current_stored_artifact) == current_artifact_fields
|
|
and set(container_artifact) == current_artifact_fields
|
|
and re.fullmatch(r"[0-9a-f]{64}", str(current_stored_artifact.get("container_id") or "")) is not None
|
|
and re.fullmatch(r"[0-9a-f]{64}", str(container_artifact.get("container_id") or "")) is not None
|
|
and all(current_stored_artifact.get(field) == container_artifact.get(field) for field in current_immutable_fields)
|
|
and isinstance(current_receipt, dict)
|
|
and current_receipt == {}
|
|
and attempt_status == "failed"
|
|
# Do not bind recovery to one historical error string. The proof
|
|
# is the preserved immutable artifact plus the currently healthy
|
|
# runtime, exact candidate metadata/timestamp, and Git ancestry.
|
|
and attempt.get("updated_at_epoch") == state.get("updated_at_epoch")
|
|
and isinstance(current_attempt_metadata, dict)
|
|
and set(current_attempt_metadata) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}
|
|
and current_attempt_metadata.get("id") == attempt_revision[:12]
|
|
and current_attempt_metadata.get("repository") == repository
|
|
and current_attempt_metadata.get("revision") == attempt_revision
|
|
and re.fullmatch(r"[0-9a-f]{40}", str(current_attempt_metadata.get("tree") or "")) is not None
|
|
and re.fullmatch(r"[0-9a-f]{64}", str(current_attempt_metadata.get("treeDigest") or "")) is not None
|
|
and isinstance(current_attempt_metadata.get("builtAt"), str)
|
|
and bool(current_attempt_metadata.get("builtAt"))
|
|
)
|
|
current_verified_receipt_snapshot_baseline = _is_current_verified_receipt_snapshot_baseline(
|
|
state, repository, expected_container, live_revision, attempt_revision
|
|
)
|
|
current_live_failed_baseline = (
|
|
recorded_status == "failed"
|
|
and (recorded_live_status == "healthy" or current_live_unknown_snapshot_baseline or current_verified_receipt_snapshot_baseline)
|
|
and attempt_status == "failed"
|
|
and re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is not None
|
|
and attempt_revision != live_revision
|
|
)
|
|
if not current_live_failed_baseline:
|
|
raise SystemExit(55)
|
|
resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}")
|
|
if resolved_attempt.returncode != 0 or resolved_attempt.stdout.strip() != attempt_revision:
|
|
raise SystemExit(25)
|
|
if git_run("merge-base", "--is-ancestor", live_revision, attempt_revision).returncode != 0:
|
|
raise SystemExit(56)
|
|
if git_run("merge-base", "--is-ancestor", attempt_revision, main_revision).returncode != 0:
|
|
raise SystemExit(57)
|
|
if current_verified_receipt_snapshot_baseline:
|
|
if controller_path.is_symlink() or not controller_path.is_file():
|
|
raise SystemExit(62)
|
|
try:
|
|
spec = importlib.util.spec_from_file_location("dashboard_deploy_controller", controller_path)
|
|
controller = importlib.util.module_from_spec(spec) if spec and spec.loader else None
|
|
if controller is None or spec is None or spec.loader is None:
|
|
raise RuntimeError("controller import unavailable")
|
|
spec.loader.exec_module(controller)
|
|
verified_project = dict(project)
|
|
verified_project["_repository_name"] = repository
|
|
verified_project["_repository_path"] = str(bare_repo)
|
|
build = controller.validate_recorded_receipt(
|
|
verified_project, current_receipt, repository, live_revision
|
|
)
|
|
observed_receipt = controller.verify_deployment(
|
|
verified_project,
|
|
build,
|
|
expected_image_ids={
|
|
expected_container: current_receipt["containers"][expected_container]["image_id"]
|
|
},
|
|
)
|
|
observed_artifacts = controller.deployment_artifacts(verified_project)
|
|
except Exception:
|
|
raise SystemExit(63)
|
|
if observed_receipt != current_receipt or observed_artifacts != current_artifacts:
|
|
raise SystemExit(64)
|
|
reconciled_from = attempt_revision
|
|
else:
|
|
if re.fullmatch(r"[0-9a-f]{40}", recorded_live) is None:
|
|
raise SystemExit(58)
|
|
stale_success_baseline = (
|
|
recorded_status == "success" and recorded_live_status == "healthy"
|
|
and attempt_status == "success" and attempt_revision == recorded_live
|
|
)
|
|
stale_failed_attempt = recorded_status == "failed"
|
|
if not (stale_success_baseline or stale_failed_attempt):
|
|
raise SystemExit(59)
|
|
resolved_recorded = git_run("rev-parse", f"{recorded_live}^{{commit}}")
|
|
if resolved_recorded.returncode != 0 or resolved_recorded.stdout.strip() != recorded_live:
|
|
raise SystemExit(25)
|
|
recorded_to_live = git_run("merge-base", "--is-ancestor", recorded_live, live_revision)
|
|
if recorded_to_live.returncode not in {0, 1}:
|
|
raise SystemExit(61)
|
|
if recorded_to_live.returncode != 0:
|
|
canonical_recorded = canonical_git_run("rev-parse", f"{recorded_live}^{{commit}}")
|
|
canonical_live = canonical_git_run("rev-parse", f"{live_revision}^{{commit}}")
|
|
canonical_main = canonical_git_run("rev-parse", "refs/heads/main")
|
|
canonical_recorded_to_live = canonical_git_run("merge-base", "--is-ancestor", recorded_live, live_revision)
|
|
canonical_recorded_main = canonical_git_run("merge-base", "--is-ancestor", recorded_live, main_revision)
|
|
canonical_live_main = canonical_git_run("merge-base", "--is-ancestor", live_revision, main_revision)
|
|
canonical_revisions_available = (
|
|
canonical_bare_repo.is_dir()
|
|
and not canonical_bare_repo.is_symlink()
|
|
and canonical_recorded.returncode == 0
|
|
and canonical_recorded.stdout.strip() == recorded_live
|
|
and canonical_live.returncode == 0
|
|
and canonical_live.stdout.strip() == live_revision
|
|
and canonical_main.returncode == 0
|
|
and canonical_main.stdout.strip() == main_revision
|
|
)
|
|
canonical_ancestry_fallback = (
|
|
canonical_revisions_available
|
|
and canonical_recorded_to_live.returncode == 0
|
|
and canonical_recorded_main.returncode == 0
|
|
and canonical_live_main.returncode == 0
|
|
)
|
|
if not canonical_ancestry_fallback:
|
|
if not canonical_revisions_available:
|
|
raise SystemExit(61)
|
|
if canonical_recorded_to_live.returncode not in {0, 1}:
|
|
raise SystemExit(61)
|
|
if canonical_recorded_main.returncode not in {0, 1}:
|
|
raise SystemExit(61)
|
|
if canonical_live_main.returncode not in {0, 1}:
|
|
raise SystemExit(61)
|
|
|
|
live_to_recorded = git_run("merge-base", "--is-ancestor", live_revision, recorded_live)
|
|
canonical_live_to_recorded = canonical_git_run("merge-base", "--is-ancestor", live_revision, recorded_live)
|
|
mounted_recorded_main = git_run("merge-base", "--is-ancestor", recorded_live, main_revision)
|
|
mounted_merge_base = git_run("merge-base", recorded_live, live_revision)
|
|
canonical_merge_base = canonical_git_run("merge-base", recorded_live, live_revision)
|
|
mounted_recorded_time = git_run("show", "-s", "--format=%ct", recorded_live)
|
|
mounted_live_time = git_run("show", "-s", "--format=%ct", live_revision)
|
|
canonical_recorded_time = canonical_git_run("show", "-s", "--format=%ct", recorded_live)
|
|
canonical_live_time = canonical_git_run("show", "-s", "--format=%ct", live_revision)
|
|
mounted_recorded_distance = git_run("rev-list", "--count", f"{recorded_live}..{main_revision}")
|
|
mounted_live_distance = git_run("rev-list", "--count", f"{live_revision}..{main_revision}")
|
|
canonical_recorded_distance = canonical_git_run("rev-list", "--count", f"{recorded_live}..{main_revision}")
|
|
canonical_live_distance = canonical_git_run("rev-list", "--count", f"{live_revision}..{main_revision}")
|
|
resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}")
|
|
canonical_resolved_attempt = canonical_git_run("rev-parse", f"{attempt_revision}^{{commit}}")
|
|
recorded_to_attempt = git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision)
|
|
canonical_recorded_to_attempt = canonical_git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision)
|
|
attempt_to_main = git_run("merge-base", "--is-ancestor", attempt_revision, main_revision)
|
|
canonical_attempt_to_main = canonical_git_run("merge-base", "--is-ancestor", attempt_revision, main_revision)
|
|
|
|
def git_nonnegative_int(result):
|
|
if result.returncode != 0:
|
|
return None
|
|
value = result.stdout.strip()
|
|
if not value.isdigit():
|
|
return None
|
|
return int(value)
|
|
|
|
recorded_commit_epoch = git_nonnegative_int(mounted_recorded_time)
|
|
live_commit_epoch = git_nonnegative_int(mounted_live_time)
|
|
canonical_recorded_commit_epoch = git_nonnegative_int(canonical_recorded_time)
|
|
canonical_live_commit_epoch = git_nonnegative_int(canonical_live_time)
|
|
recorded_distance = git_nonnegative_int(mounted_recorded_distance)
|
|
live_distance = git_nonnegative_int(mounted_live_distance)
|
|
canonical_recorded_distance_value = git_nonnegative_int(canonical_recorded_distance)
|
|
canonical_live_distance_value = git_nonnegative_int(canonical_live_distance)
|
|
try:
|
|
recorded_updated_at = int(state.get("updated_at_epoch") or 0)
|
|
except (TypeError, ValueError):
|
|
recorded_updated_at = 0
|
|
|
|
mounted_merge_base_value = mounted_merge_base.stdout.strip()
|
|
canonical_merge_base_value = canonical_merge_base.stdout.strip()
|
|
history_transition_baseline = (
|
|
stale_success_baseline
|
|
and recorded_to_live.returncode == 1
|
|
and canonical_recorded_to_live.returncode == 1
|
|
and live_to_recorded.returncode == 1
|
|
and canonical_live_to_recorded.returncode == 1
|
|
and mounted_recorded_main.returncode == 0
|
|
and canonical_recorded_main.returncode == 0
|
|
and canonical_live_main.returncode == 0
|
|
and mounted_merge_base.returncode == 0
|
|
and canonical_merge_base.returncode == 0
|
|
and re.fullmatch(r"[0-9a-f]{40}", mounted_merge_base_value) is not None
|
|
and canonical_merge_base_value == mounted_merge_base_value
|
|
and mounted_merge_base_value not in {recorded_live, live_revision}
|
|
and recorded_commit_epoch is not None
|
|
and live_commit_epoch is not None
|
|
and canonical_recorded_commit_epoch == recorded_commit_epoch
|
|
and canonical_live_commit_epoch == live_commit_epoch
|
|
and 0 < recorded_updated_at
|
|
and recorded_commit_epoch <= recorded_updated_at
|
|
and recorded_updated_at < live_commit_epoch
|
|
and recorded_commit_epoch < live_commit_epoch
|
|
and recorded_distance is not None
|
|
and live_distance is not None
|
|
and canonical_recorded_distance_value == recorded_distance
|
|
and canonical_live_distance_value == live_distance
|
|
and 0 <= live_distance < recorded_distance
|
|
)
|
|
exact_legacy_state_ahead_baseline = (
|
|
resolved_attempt.returncode == 0
|
|
and resolved_attempt.stdout.strip() == attempt_revision
|
|
and canonical_resolved_attempt.returncode == 0
|
|
and canonical_resolved_attempt.stdout.strip() == attempt_revision
|
|
and _is_exact_legacy_state_ahead_baseline(
|
|
state,
|
|
repository,
|
|
expected_container,
|
|
recorded_live,
|
|
live_revision,
|
|
container_artifact,
|
|
{
|
|
"recorded_to_live": recorded_to_live.returncode,
|
|
"canonical_recorded_to_live": canonical_recorded_to_live.returncode,
|
|
"live_to_recorded": live_to_recorded.returncode,
|
|
"canonical_live_to_recorded": canonical_live_to_recorded.returncode,
|
|
"mounted_recorded_main": mounted_recorded_main.returncode,
|
|
"canonical_recorded_main": canonical_recorded_main.returncode,
|
|
"canonical_live_main": canonical_live_main.returncode,
|
|
"recorded_to_attempt": recorded_to_attempt.returncode,
|
|
"canonical_recorded_to_attempt": canonical_recorded_to_attempt.returncode,
|
|
"attempt_to_main": attempt_to_main.returncode,
|
|
"canonical_attempt_to_main": canonical_attempt_to_main.returncode,
|
|
},
|
|
)
|
|
)
|
|
if not (history_transition_baseline or exact_legacy_state_ahead_baseline):
|
|
if canonical_recorded_to_live.returncode == 1:
|
|
raise SystemExit(60)
|
|
raise SystemExit(61)
|
|
|
|
artifacts = {"containers": {expected_container: container_artifact}, "compose_sha256": ""}
|
|
now = int(time.time())
|
|
payload = {
|
|
"schema_version": 3,
|
|
"repository": repository,
|
|
"slug": "media-codec-dashboard",
|
|
"compose_project": "media-codec-dashboard",
|
|
"revision": live_revision,
|
|
"live_revision": live_revision,
|
|
"live_status": "healthy",
|
|
"live_legacy_provenance": True,
|
|
"live_artifacts": artifacts,
|
|
"live_receipt": {},
|
|
"status": "success",
|
|
"message": "Verified existing Media Codec Dashboard runtime reconciled to actual immutable live image",
|
|
"updated_at_epoch": now,
|
|
"artifacts": artifacts,
|
|
"last_attempt": {
|
|
"revision": live_revision,
|
|
"status": "success",
|
|
"message": "Verified runtime state reconciliation",
|
|
"updated_at_epoch": now,
|
|
"metadata": {"reconciled_from": reconciled_from},
|
|
},
|
|
"previous_revision": recorded_live,
|
|
"rollback_restored_revision": "",
|
|
}
|
|
if recorded_live == live_revision and current_verified_receipt_snapshot_baseline:
|
|
payload = dict(state)
|
|
payload.update({
|
|
"live_status": "healthy",
|
|
"status": "success",
|
|
"message": "Verified Media Codec Dashboard receipt re-attested after failed forward deployment",
|
|
"updated_at_epoch": now,
|
|
"artifacts": current_artifacts,
|
|
"last_attempt": {
|
|
"revision": live_revision,
|
|
"status": "success",
|
|
"message": "Verified existing runtime receipt re-attestation",
|
|
"updated_at_epoch": now,
|
|
"metadata": {"reconciled_failed_revision": reconciled_from},
|
|
},
|
|
})
|
|
|
|
backup = state_path.with_name(f"{state_path.name}.reconcile-backup-{now}-{os.getpid()}")
|
|
try:
|
|
shutil.copy2(state_path, backup)
|
|
temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.tmp")
|
|
with temporary.open("x", encoding="utf-8") as handle:
|
|
handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.chmod(temporary, stat.S_IMODE(state_path.stat().st_mode))
|
|
os.replace(temporary, state_path)
|
|
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
except Exception:
|
|
raise SystemExit(53)
|
|
|
|
verify = json.loads(state_path.read_text(encoding="utf-8"))
|
|
expected_legacy = not (recorded_live == live_revision and current_verified_receipt_snapshot_baseline)
|
|
if not (verify.get("live_revision") == live_revision and verify.get("live_status") == "healthy" and verify.get("live_legacy_provenance") is expected_legacy and dict(dict(verify.get("live_artifacts") or {}).get("containers") or {}).get(expected_container, {}).get("image_id") == image_id):
|
|
raise SystemExit(54)
|
|
if not expected_legacy and (verify.get("live_receipt") != current_receipt or verify.get("live_artifacts") != current_artifacts):
|
|
raise SystemExit(64)
|
|
|
|
print("reconciled", flush=True)
|
|
'''
|
|
|
|
|
|
class MediaCodecDashboardDeploymentReconciliation:
|
|
"""Repair only stale controller state proven against the live immutable dashboard runtime."""
|
|
|
|
def __init__(self, portainer: PortainerConnector | None = None):
|
|
self.portainer = portainer or PortainerConnector()
|
|
|
|
def run(self) -> dict[str, Any]:
|
|
status = self.portainer.get_container_status(_CONTROLLER)
|
|
if status.get("status") != "ok":
|
|
return {"status": "controller_unavailable", "reason": redact_message(str(status.get("reason") or status.get("status") or "unavailable"))}
|
|
controller = dict(status.get("container") or {})
|
|
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
|
return {"status": "controller_unavailable", "reason": "deployment controller is not running"}
|
|
container_id = str(controller.get("id") or "")
|
|
config = self.portainer._config()
|
|
if not config or not container_id:
|
|
return {"status": "connector_unavailable", "reason": "Portainer deployment control is unavailable"}
|
|
|
|
command = ["python3", "-c", _RECONCILE_SCRIPT, _REPOSITORY, _CONTAINER]
|
|
endpoint = quote(str(config.endpoint_id), safe="")
|
|
safe_container = quote(container_id, safe="")
|
|
try:
|
|
created = self.portainer._request("POST", f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec", json={"AttachStdout": False, "AttachStderr": False, "Tty": False, "Cmd": command}).json()
|
|
exec_id = str(created.get("Id") or "")
|
|
if not exec_id:
|
|
return {"status": "error", "reason": "dashboard state reconciliation could not start"}
|
|
safe_exec = quote(exec_id, safe="")
|
|
self.portainer._request("POST", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start", json={"Detach": True, "Tty": False})
|
|
deadline = time.monotonic() + 45
|
|
exit_code: int | None = None
|
|
while time.monotonic() < deadline:
|
|
inspected = self.portainer._request("GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json").json()
|
|
if not bool(inspected.get("Running")):
|
|
raw_exit = inspected.get("ExitCode")
|
|
exit_code = int(raw_exit) if raw_exit is not None else -1
|
|
break
|
|
time.sleep(0.25)
|
|
except (RuntimeError, ValueError, TypeError) as exc:
|
|
return {"status": "error", "reason": redact_message(str(exc))}
|
|
|
|
if exit_code is None:
|
|
return {"status": "error", "reason": "dashboard state reconciliation timed out"}
|
|
reasons = {
|
|
20: "deployment projects registry is unavailable or unsafe",
|
|
22: "Media Codec Dashboard deployment target is missing",
|
|
23: "Media Codec Dashboard deployment target topology differs from the bounded contract",
|
|
24: "mounted Media Codec Dashboard bare repository is unavailable",
|
|
25: "deployment revisions cannot be resolved",
|
|
26: "Media Codec Dashboard main cannot be resolved",
|
|
27: "live Media Codec Dashboard revision is not an ancestor of main",
|
|
29: "deployment lock files cannot be opened",
|
|
30: "deployment locks cannot be acquired",
|
|
31: "deployment projects registry cannot be read",
|
|
36: "deployment projects registry host alias is unavailable or mismatched",
|
|
37: "deployment target changed while reconciliation was waiting",
|
|
40: "fixed Media Codec Dashboard reconciliation contract was altered",
|
|
42: "deployment state file is unavailable or invalid",
|
|
43: "deployment state is not a modern Media Codec Dashboard state",
|
|
44: "live Media Codec Dashboard container cannot be inspected",
|
|
45: "live Media Codec Dashboard container is not healthy in the expected Compose project",
|
|
46: "live Media Codec Dashboard immutable image identity is invalid",
|
|
47: "Media Codec Dashboard Compose project contains unexpected containers",
|
|
48: "live Media Codec Dashboard mount topology differs from the verified runtime",
|
|
49: "live Media Codec Dashboard port mapping differs from the verified runtime",
|
|
50: "live Media Codec Dashboard image cannot be inspected",
|
|
51: "live Media Codec Dashboard image revision label does not match the verified runtime",
|
|
52: "controller state drift is not the exact known recoverable state",
|
|
53: "reconciled deployment state could not be written durably",
|
|
54: "reconciled deployment state did not verify after write",
|
|
55: "controller state for the current live revision is internally inconsistent",
|
|
56: "failed deployment attempt is not a descendant of the live revision",
|
|
57: "failed deployment attempt is not an ancestor of main",
|
|
58: "recorded live deployment revision is invalid",
|
|
59: "stale controller state metadata is not a supported recoverable baseline",
|
|
60: "recorded live deployment revision is not an ancestor of the verified runtime",
|
|
61: "mounted deployment repository ancestry differs from canonical Gitea repository",
|
|
62: "deployment controller verifier is unavailable or unsafe",
|
|
63: "recorded dashboard receipt cannot be reverified against the live runtime",
|
|
64: "verified dashboard receipt or artifacts differ from the recorded state",
|
|
}
|
|
if exit_code != 0:
|
|
return {"status": "validation_failed", "reason": reasons.get(exit_code, "Media Codec Dashboard deployment reconciliation failed closed")}
|
|
return {
|
|
"status": "reconciled",
|
|
"repository": _REPOSITORY,
|
|
"container": _CONTAINER,
|
|
}
|