Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d
This commit is contained in:
Public Source Publisher committed
commit
1dfbca08a1
528 files changed
+131054
No files matched your search
@@ -0,0 +1,768 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = "jens/media-codec-dashboard"
|
||||
_CONTAINER = "Media-Codec-Dashboard"
|
||||
|
||||
# Fail-closed reconciliation for the verified production runtime.
|
||||
# It does not deploy, restart, recreate or stop any container. It only repairs
|
||||
# controller state after proving the exact live container/image/mount topology
|
||||
# and immutable runtime revision from independent OCI labels plus Gitea ancestry.
|
||||
_RECONCILE_SCRIPT = r'''
|
||||
import fcntl
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
repository, expected_container = sys.argv[1:3]
|
||||
registry_path = Path("/opt/gitea-deploy/projects.json")
|
||||
host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
|
||||
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
||||
state_path = state_root / "media-codec-dashboard.json"
|
||||
project_lock_path = state_root / "locks" / "media-codec-dashboard.lock"
|
||||
global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
||||
controller_path = Path("/opt/gitea-deploy/deploy.py")
|
||||
|
||||
if repository != "jens/media-codec-dashboard":
|
||||
raise SystemExit(40)
|
||||
if expected_container != "Media-Codec-Dashboard":
|
||||
raise SystemExit(40)
|
||||
|
||||
if registry_path.is_symlink() or not registry_path.is_file():
|
||||
raise SystemExit(20)
|
||||
try:
|
||||
if host_registry_path.is_symlink() or not host_registry_path.is_file() or not os.path.samefile(registry_path, host_registry_path):
|
||||
raise SystemExit(36)
|
||||
except OSError:
|
||||
raise SystemExit(36)
|
||||
|
||||
try:
|
||||
projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
project = projects.get(repository) if isinstance(projects, dict) else None
|
||||
if not isinstance(project, dict):
|
||||
raise SystemExit(22)
|
||||
if project.get("branch") != "main" or project.get("slug") != "media-codec-dashboard":
|
||||
raise SystemExit(23)
|
||||
if project.get("compose_project") != "media-codec-dashboard" or project.get("containers") != [expected_container]:
|
||||
raise SystemExit(23)
|
||||
compose_files = project.get("compose_files")
|
||||
if not isinstance(compose_files, list) or "docker-compose.yml" not in compose_files:
|
||||
raise SystemExit(23)
|
||||
source_dir = Path(str(project.get("source_dir") or ""))
|
||||
allowed_roots = (Path("/mnt/cache/appdata/gitea-deploy-runners/sources"), Path("/srv/example/appdata"))
|
||||
if not source_dir.is_absolute() or not any(root == source_dir or root in source_dir.parents for root in allowed_roots):
|
||||
raise SystemExit(23)
|
||||
|
||||
owner, name = repository.split("/", 1)
|
||||
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
||||
canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories")
|
||||
canonical_bare_repo = canonical_repository_root / owner / f"{name}.git"
|
||||
if not bare_repo.is_dir():
|
||||
raise SystemExit(24)
|
||||
|
||||
def git_run(*args):
|
||||
return subprocess.run(["git", f"--git-dir={bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
|
||||
def canonical_git_run(*args):
|
||||
return subprocess.run(["git", f"--git-dir={canonical_bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
|
||||
def _is_current_verified_receipt_snapshot_baseline(state, repository, expected_container, live_revision, attempt_revision):
|
||||
attempt = state.get("last_attempt")
|
||||
artifacts = state.get("live_artifacts")
|
||||
receipt = state.get("live_receipt")
|
||||
if not isinstance(attempt, dict) or not isinstance(artifacts, dict) or not isinstance(receipt, dict):
|
||||
return False
|
||||
containers = artifacts.get("containers")
|
||||
receipt_containers = receipt.get("containers")
|
||||
return (
|
||||
state.get("schema_version") == 3
|
||||
and state.get("repository") == repository
|
||||
and state.get("revision") == live_revision
|
||||
and state.get("live_revision") == live_revision
|
||||
and state.get("status") == "failed"
|
||||
and state.get("live_status") == "unknown"
|
||||
and state.get("live_legacy_provenance") is False
|
||||
and attempt.get("revision") == attempt_revision
|
||||
and attempt.get("status") == "failed"
|
||||
and attempt.get("message") == "Existing production containers require a valid healthy live deployment state"
|
||||
and attempt.get("updated_at_epoch") == state.get("updated_at_epoch")
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(artifacts.get("compose_sha256") or "")) is not None
|
||||
and isinstance(containers, dict)
|
||||
and set(containers) == {expected_container}
|
||||
and receipt.get("schema") == "gitea-deploy-receipt.v1"
|
||||
and receipt.get("repository") == repository
|
||||
and receipt.get("revision") == live_revision
|
||||
and receipt.get("live") is None
|
||||
and isinstance(receipt_containers, dict)
|
||||
and set(receipt_containers) == {expected_container}
|
||||
and receipt_containers == containers
|
||||
)
|
||||
|
||||
def _is_exact_legacy_state_ahead_baseline(
|
||||
state,
|
||||
repository,
|
||||
expected_container,
|
||||
recorded_live,
|
||||
live_revision,
|
||||
container_artifact,
|
||||
git_facts,
|
||||
):
|
||||
attempt = state.get("last_attempt")
|
||||
artifacts = state.get("live_artifacts")
|
||||
receipt = state.get("live_receipt")
|
||||
if not isinstance(attempt, dict) or not isinstance(artifacts, dict) or not isinstance(receipt, dict):
|
||||
return False
|
||||
metadata = attempt.get("metadata")
|
||||
build = receipt.get("build")
|
||||
if not isinstance(metadata, dict) or not isinstance(build, dict):
|
||||
return False
|
||||
artifact_containers = artifacts.get("containers")
|
||||
receipt_containers = receipt.get("containers")
|
||||
if not isinstance(artifact_containers, dict) or not isinstance(receipt_containers, dict):
|
||||
return False
|
||||
stored_artifact = artifact_containers.get(expected_container)
|
||||
if not isinstance(stored_artifact, dict) or receipt_containers != artifact_containers:
|
||||
return False
|
||||
|
||||
def _same_immutable_artifact_after_container_recreate(stored, current):
|
||||
expected_fields = {
|
||||
"container_id",
|
||||
"image_id",
|
||||
"image_ref",
|
||||
"container_revision",
|
||||
"image_revision",
|
||||
"image_source",
|
||||
"image_tree",
|
||||
"image_version",
|
||||
"image_created",
|
||||
}
|
||||
immutable_fields = expected_fields - {"container_id"}
|
||||
return (
|
||||
isinstance(current, dict)
|
||||
and set(stored) == expected_fields
|
||||
and set(current) == expected_fields
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(stored.get("container_id") or "")) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(current.get("container_id") or "")) is not None
|
||||
and all(stored.get(field) == current.get(field) for field in immutable_fields)
|
||||
)
|
||||
|
||||
attempt_revision = str(attempt.get("revision") or "")
|
||||
expected_facts = {
|
||||
"recorded_to_live": 1,
|
||||
"canonical_recorded_to_live": 1,
|
||||
"live_to_recorded": 0,
|
||||
"canonical_live_to_recorded": 0,
|
||||
"mounted_recorded_main": 0,
|
||||
"canonical_recorded_main": 0,
|
||||
"canonical_live_main": 0,
|
||||
"recorded_to_attempt": 0,
|
||||
"canonical_recorded_to_attempt": 0,
|
||||
"attempt_to_main": 0,
|
||||
"canonical_attempt_to_main": 0,
|
||||
}
|
||||
return (
|
||||
state.get("schema_version") == 3
|
||||
and state.get("repository") == repository
|
||||
and state.get("revision") == recorded_live
|
||||
and state.get("live_revision") == recorded_live
|
||||
and state.get("status") == "failed"
|
||||
and state.get("live_status") == "unknown"
|
||||
and state.get("live_legacy_provenance") is False
|
||||
and re.fullmatch(r"[0-9a-f]{40}", recorded_live) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{40}", live_revision) is not None
|
||||
and recorded_live != live_revision
|
||||
and re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is not None
|
||||
and attempt_revision != recorded_live
|
||||
and attempt.get("status") == "failed"
|
||||
and attempt.get("message") == "Existing production containers require a valid healthy live deployment state"
|
||||
and attempt.get("updated_at_epoch") == state.get("updated_at_epoch")
|
||||
and set(metadata) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}
|
||||
and metadata.get("id") == attempt_revision[:12]
|
||||
and metadata.get("repository") == repository
|
||||
and metadata.get("revision") == attempt_revision
|
||||
and re.fullmatch(r"[0-9a-f]{40}", str(metadata.get("tree") or "")) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(metadata.get("treeDigest") or "")) is not None
|
||||
and isinstance(metadata.get("builtAt"), str)
|
||||
and bool(metadata.get("builtAt"))
|
||||
and set(artifacts) == {"compose_sha256", "containers"}
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(artifacts.get("compose_sha256") or "")) is not None
|
||||
and set(artifact_containers) == {expected_container}
|
||||
and _same_immutable_artifact_after_container_recreate(stored_artifact, container_artifact)
|
||||
and set(receipt) == {"schema", "repository", "revision", "tree", "treeDigest", "build", "containers", "live"}
|
||||
and receipt.get("schema") == "gitea-deploy-receipt.v1"
|
||||
and receipt.get("repository") == repository
|
||||
and receipt.get("revision") == recorded_live
|
||||
and receipt.get("live") is None
|
||||
and re.fullmatch(r"[0-9a-f]{40}", str(receipt.get("tree") or "")) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(receipt.get("treeDigest") or "")) is not None
|
||||
and set(receipt_containers) == {expected_container}
|
||||
and set(build) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}
|
||||
and build.get("id") == recorded_live[:12]
|
||||
and build.get("repository") == repository
|
||||
and build.get("revision") == recorded_live
|
||||
and build.get("tree") == receipt.get("tree")
|
||||
and build.get("treeDigest") == receipt.get("treeDigest")
|
||||
and isinstance(build.get("builtAt"), str)
|
||||
and bool(build.get("builtAt"))
|
||||
and container_artifact.get("container_revision") == live_revision
|
||||
and container_artifact.get("image_revision") == live_revision
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(container_artifact.get("container_id") or "")) is not None
|
||||
and re.fullmatch(r"sha256:[0-9a-f]{64}", str(container_artifact.get("image_id") or "")) is not None
|
||||
and git_facts == expected_facts
|
||||
)
|
||||
|
||||
main = git_run("rev-parse", "refs/heads/main")
|
||||
main_revision = main.stdout.strip()
|
||||
if main.returncode != 0 or re.fullmatch(r"[0-9a-f]{40}", main_revision) is None:
|
||||
raise SystemExit(26)
|
||||
|
||||
state_root.mkdir(parents=True, exist_ok=True)
|
||||
project_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
project_lock = project_lock_path.open("a+", encoding="utf-8")
|
||||
global_lock = global_lock_path.open("a+", encoding="utf-8")
|
||||
except Exception:
|
||||
raise SystemExit(29)
|
||||
|
||||
with project_lock:
|
||||
try:
|
||||
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
with global_lock:
|
||||
try:
|
||||
fcntl.flock(global_lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
|
||||
try:
|
||||
current_projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
if current_projects.get(repository) != project:
|
||||
raise SystemExit(37)
|
||||
|
||||
if state_path.is_symlink() or not state_path.is_file():
|
||||
raise SystemExit(42)
|
||||
try:
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(42)
|
||||
if not isinstance(state, dict) or state.get("schema_version") != 3 or state.get("repository") != repository:
|
||||
raise SystemExit(43)
|
||||
|
||||
inspected = subprocess.run(["docker", "inspect", expected_container], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
if inspected.returncode != 0:
|
||||
raise SystemExit(44)
|
||||
try:
|
||||
container = json.loads(inspected.stdout)[0]
|
||||
except Exception:
|
||||
raise SystemExit(44)
|
||||
runtime_state = dict(container.get("State") or {})
|
||||
labels = dict(dict(container.get("Config") or {}).get("Labels") or {})
|
||||
image_ref = str(dict(container.get("Config") or {}).get("Image") or "")
|
||||
image_id = str(container.get("Image") or "")
|
||||
container_revision = str(labels.get("org.opencontainers.image.revision") or "")
|
||||
if not runtime_state.get("Running") or str(dict(runtime_state.get("Health") or {}).get("Status") or "") != "healthy":
|
||||
raise SystemExit(45)
|
||||
if labels.get("com.docker.compose.project") != "media-codec-dashboard":
|
||||
raise SystemExit(45)
|
||||
if not image_id.startswith("sha256:") or re.fullmatch(r"[0-9a-f]{40}", container_revision) is None:
|
||||
raise SystemExit(46)
|
||||
|
||||
project_names = subprocess.run(["docker", "ps", "--all", "--filter", "label=com.docker.compose.project=media-codec-dashboard", "--format", "{{.Names}}"], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
if project_names.returncode != 0 or {line.strip() for line in project_names.stdout.splitlines() if line.strip()} != {expected_container}:
|
||||
raise SystemExit(47)
|
||||
|
||||
mounts = {str(item.get("Destination") or ""): str(item.get("Source") or "") for item in list(container.get("Mounts") or []) if isinstance(item, dict)}
|
||||
required_mounts = {
|
||||
"/app/data": "/srv/example/appdata/media-codec-dashboard/data",
|
||||
"/app/data/quality-runtime": "/srv/example/appdata/tdarr/logs/media-codec-dashboard-quality-audit",
|
||||
"/run/secrets": "/srv/example/appdata/media-codec-dashboard/secrets",
|
||||
"/tdarr-output": "/srv/example/Media",
|
||||
}
|
||||
if any(mounts.get(destination) != source for destination, source in required_mounts.items()):
|
||||
raise SystemExit(48)
|
||||
|
||||
bindings = dict(dict(container.get("HostConfig") or {}).get("PortBindings") or {})
|
||||
host_ports = {str(item.get("HostPort") or "") for item in list(bindings.get("1211/tcp") or []) if isinstance(item, dict)}
|
||||
if "1211" not in host_ports:
|
||||
raise SystemExit(49)
|
||||
|
||||
image_result = subprocess.run(["docker", "image", "inspect", image_id], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False)
|
||||
if image_result.returncode != 0:
|
||||
raise SystemExit(50)
|
||||
try:
|
||||
image = json.loads(image_result.stdout)[0]
|
||||
except Exception:
|
||||
raise SystemExit(50)
|
||||
image_labels = dict(dict(image.get("Config") or {}).get("Labels") or {})
|
||||
image_revision = str(image_labels.get("org.opencontainers.image.revision") or "")
|
||||
live_revision = image_revision
|
||||
if re.fullmatch(r"[0-9a-f]{40}", live_revision) is None or container_revision != image_revision:
|
||||
raise SystemExit(51)
|
||||
container_artifact = {
|
||||
"container_id": str(container.get("Id") or ""),
|
||||
"image_id": image_id,
|
||||
"image_ref": image_ref,
|
||||
"container_revision": container_revision,
|
||||
"image_revision": image_revision,
|
||||
"image_source": str(image_labels.get("org.opencontainers.image.source") or ""),
|
||||
"image_tree": str(image_labels.get("org.opencontainers.image.source-tree") or ""),
|
||||
"image_version": str(image_labels.get("org.opencontainers.image.version") or ""),
|
||||
"image_created": str(image_labels.get("org.opencontainers.image.created") or ""),
|
||||
}
|
||||
resolved = git_run("rev-parse", f"{live_revision}^{{commit}}")
|
||||
if resolved.returncode != 0 or resolved.stdout.strip() != live_revision:
|
||||
raise SystemExit(25)
|
||||
if git_run("merge-base", "--is-ancestor", live_revision, main_revision).returncode != 0:
|
||||
raise SystemExit(27)
|
||||
|
||||
recorded_live = str(state.get("live_revision") or state.get("revision") or "")
|
||||
recorded_status = str(state.get("status") or "")
|
||||
recorded_live_status = str(state.get("live_status") or "")
|
||||
last_attempt = dict(state.get("last_attempt") or {})
|
||||
attempt_revision = str(last_attempt.get("revision") or "")
|
||||
attempt_status = str(last_attempt.get("status") or "")
|
||||
reconciled_from = recorded_live
|
||||
if recorded_live == live_revision:
|
||||
if recorded_status == "success" and state.get("live_legacy_provenance") is True:
|
||||
print("already-reconciled", flush=True)
|
||||
raise SystemExit(0)
|
||||
|
||||
current_artifacts = state.get("live_artifacts")
|
||||
current_receipt = state.get("live_receipt")
|
||||
current_artifact_containers = (
|
||||
current_artifacts.get("containers")
|
||||
if isinstance(current_artifacts, dict)
|
||||
else None
|
||||
)
|
||||
current_stored_artifact = (
|
||||
current_artifact_containers.get(expected_container)
|
||||
if isinstance(current_artifact_containers, dict)
|
||||
else None
|
||||
)
|
||||
current_attempt_metadata = (
|
||||
last_attempt.get("metadata")
|
||||
if isinstance(last_attempt, dict)
|
||||
else None
|
||||
)
|
||||
current_artifact_fields = {
|
||||
"container_id",
|
||||
"image_id",
|
||||
"image_ref",
|
||||
"container_revision",
|
||||
"image_revision",
|
||||
"image_source",
|
||||
"image_tree",
|
||||
"image_version",
|
||||
"image_created",
|
||||
}
|
||||
current_immutable_fields = current_artifact_fields - {"container_id"}
|
||||
current_live_unknown_snapshot_baseline = (
|
||||
recorded_status == "failed"
|
||||
and recorded_live_status == "unknown"
|
||||
and state.get("live_legacy_provenance") is True
|
||||
and isinstance(current_artifacts, dict)
|
||||
and set(current_artifacts) == {"compose_sha256", "containers"}
|
||||
and current_artifacts.get("compose_sha256") == ""
|
||||
and isinstance(current_artifact_containers, dict)
|
||||
and set(current_artifact_containers) == {expected_container}
|
||||
and isinstance(current_stored_artifact, dict)
|
||||
and set(current_stored_artifact) == current_artifact_fields
|
||||
and set(container_artifact) == current_artifact_fields
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(current_stored_artifact.get("container_id") or "")) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(container_artifact.get("container_id") or "")) is not None
|
||||
and all(current_stored_artifact.get(field) == container_artifact.get(field) for field in current_immutable_fields)
|
||||
and isinstance(current_receipt, dict)
|
||||
and current_receipt == {}
|
||||
and attempt_status == "failed"
|
||||
# Do not bind recovery to one historical error string. The proof
|
||||
# is the preserved immutable artifact plus the currently healthy
|
||||
# runtime, exact candidate metadata/timestamp, and Git ancestry.
|
||||
and attempt.get("updated_at_epoch") == state.get("updated_at_epoch")
|
||||
and isinstance(current_attempt_metadata, dict)
|
||||
and set(current_attempt_metadata) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}
|
||||
and current_attempt_metadata.get("id") == attempt_revision[:12]
|
||||
and current_attempt_metadata.get("repository") == repository
|
||||
and current_attempt_metadata.get("revision") == attempt_revision
|
||||
and re.fullmatch(r"[0-9a-f]{40}", str(current_attempt_metadata.get("tree") or "")) is not None
|
||||
and re.fullmatch(r"[0-9a-f]{64}", str(current_attempt_metadata.get("treeDigest") or "")) is not None
|
||||
and isinstance(current_attempt_metadata.get("builtAt"), str)
|
||||
and bool(current_attempt_metadata.get("builtAt"))
|
||||
)
|
||||
current_verified_receipt_snapshot_baseline = _is_current_verified_receipt_snapshot_baseline(
|
||||
state, repository, expected_container, live_revision, attempt_revision
|
||||
)
|
||||
current_live_failed_baseline = (
|
||||
recorded_status == "failed"
|
||||
and (recorded_live_status == "healthy" or current_live_unknown_snapshot_baseline or current_verified_receipt_snapshot_baseline)
|
||||
and attempt_status == "failed"
|
||||
and re.fullmatch(r"[0-9a-f]{40}", attempt_revision) is not None
|
||||
and attempt_revision != live_revision
|
||||
)
|
||||
if not current_live_failed_baseline:
|
||||
raise SystemExit(55)
|
||||
resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}")
|
||||
if resolved_attempt.returncode != 0 or resolved_attempt.stdout.strip() != attempt_revision:
|
||||
raise SystemExit(25)
|
||||
if git_run("merge-base", "--is-ancestor", live_revision, attempt_revision).returncode != 0:
|
||||
raise SystemExit(56)
|
||||
if git_run("merge-base", "--is-ancestor", attempt_revision, main_revision).returncode != 0:
|
||||
raise SystemExit(57)
|
||||
if current_verified_receipt_snapshot_baseline:
|
||||
if controller_path.is_symlink() or not controller_path.is_file():
|
||||
raise SystemExit(62)
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location("dashboard_deploy_controller", controller_path)
|
||||
controller = importlib.util.module_from_spec(spec) if spec and spec.loader else None
|
||||
if controller is None or spec is None or spec.loader is None:
|
||||
raise RuntimeError("controller import unavailable")
|
||||
spec.loader.exec_module(controller)
|
||||
verified_project = dict(project)
|
||||
verified_project["_repository_name"] = repository
|
||||
verified_project["_repository_path"] = str(bare_repo)
|
||||
build = controller.validate_recorded_receipt(
|
||||
verified_project, current_receipt, repository, live_revision
|
||||
)
|
||||
observed_receipt = controller.verify_deployment(
|
||||
verified_project,
|
||||
build,
|
||||
expected_image_ids={
|
||||
expected_container: current_receipt["containers"][expected_container]["image_id"]
|
||||
},
|
||||
)
|
||||
observed_artifacts = controller.deployment_artifacts(verified_project)
|
||||
except Exception:
|
||||
raise SystemExit(63)
|
||||
if observed_receipt != current_receipt or observed_artifacts != current_artifacts:
|
||||
raise SystemExit(64)
|
||||
reconciled_from = attempt_revision
|
||||
else:
|
||||
if re.fullmatch(r"[0-9a-f]{40}", recorded_live) is None:
|
||||
raise SystemExit(58)
|
||||
stale_success_baseline = (
|
||||
recorded_status == "success" and recorded_live_status == "healthy"
|
||||
and attempt_status == "success" and attempt_revision == recorded_live
|
||||
)
|
||||
stale_failed_attempt = recorded_status == "failed"
|
||||
if not (stale_success_baseline or stale_failed_attempt):
|
||||
raise SystemExit(59)
|
||||
resolved_recorded = git_run("rev-parse", f"{recorded_live}^{{commit}}")
|
||||
if resolved_recorded.returncode != 0 or resolved_recorded.stdout.strip() != recorded_live:
|
||||
raise SystemExit(25)
|
||||
recorded_to_live = git_run("merge-base", "--is-ancestor", recorded_live, live_revision)
|
||||
if recorded_to_live.returncode not in {0, 1}:
|
||||
raise SystemExit(61)
|
||||
if recorded_to_live.returncode != 0:
|
||||
canonical_recorded = canonical_git_run("rev-parse", f"{recorded_live}^{{commit}}")
|
||||
canonical_live = canonical_git_run("rev-parse", f"{live_revision}^{{commit}}")
|
||||
canonical_main = canonical_git_run("rev-parse", "refs/heads/main")
|
||||
canonical_recorded_to_live = canonical_git_run("merge-base", "--is-ancestor", recorded_live, live_revision)
|
||||
canonical_recorded_main = canonical_git_run("merge-base", "--is-ancestor", recorded_live, main_revision)
|
||||
canonical_live_main = canonical_git_run("merge-base", "--is-ancestor", live_revision, main_revision)
|
||||
canonical_revisions_available = (
|
||||
canonical_bare_repo.is_dir()
|
||||
and not canonical_bare_repo.is_symlink()
|
||||
and canonical_recorded.returncode == 0
|
||||
and canonical_recorded.stdout.strip() == recorded_live
|
||||
and canonical_live.returncode == 0
|
||||
and canonical_live.stdout.strip() == live_revision
|
||||
and canonical_main.returncode == 0
|
||||
and canonical_main.stdout.strip() == main_revision
|
||||
)
|
||||
canonical_ancestry_fallback = (
|
||||
canonical_revisions_available
|
||||
and canonical_recorded_to_live.returncode == 0
|
||||
and canonical_recorded_main.returncode == 0
|
||||
and canonical_live_main.returncode == 0
|
||||
)
|
||||
if not canonical_ancestry_fallback:
|
||||
if not canonical_revisions_available:
|
||||
raise SystemExit(61)
|
||||
if canonical_recorded_to_live.returncode not in {0, 1}:
|
||||
raise SystemExit(61)
|
||||
if canonical_recorded_main.returncode not in {0, 1}:
|
||||
raise SystemExit(61)
|
||||
if canonical_live_main.returncode not in {0, 1}:
|
||||
raise SystemExit(61)
|
||||
|
||||
live_to_recorded = git_run("merge-base", "--is-ancestor", live_revision, recorded_live)
|
||||
canonical_live_to_recorded = canonical_git_run("merge-base", "--is-ancestor", live_revision, recorded_live)
|
||||
mounted_recorded_main = git_run("merge-base", "--is-ancestor", recorded_live, main_revision)
|
||||
mounted_merge_base = git_run("merge-base", recorded_live, live_revision)
|
||||
canonical_merge_base = canonical_git_run("merge-base", recorded_live, live_revision)
|
||||
mounted_recorded_time = git_run("show", "-s", "--format=%ct", recorded_live)
|
||||
mounted_live_time = git_run("show", "-s", "--format=%ct", live_revision)
|
||||
canonical_recorded_time = canonical_git_run("show", "-s", "--format=%ct", recorded_live)
|
||||
canonical_live_time = canonical_git_run("show", "-s", "--format=%ct", live_revision)
|
||||
mounted_recorded_distance = git_run("rev-list", "--count", f"{recorded_live}..{main_revision}")
|
||||
mounted_live_distance = git_run("rev-list", "--count", f"{live_revision}..{main_revision}")
|
||||
canonical_recorded_distance = canonical_git_run("rev-list", "--count", f"{recorded_live}..{main_revision}")
|
||||
canonical_live_distance = canonical_git_run("rev-list", "--count", f"{live_revision}..{main_revision}")
|
||||
resolved_attempt = git_run("rev-parse", f"{attempt_revision}^{{commit}}")
|
||||
canonical_resolved_attempt = canonical_git_run("rev-parse", f"{attempt_revision}^{{commit}}")
|
||||
recorded_to_attempt = git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision)
|
||||
canonical_recorded_to_attempt = canonical_git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision)
|
||||
attempt_to_main = git_run("merge-base", "--is-ancestor", attempt_revision, main_revision)
|
||||
canonical_attempt_to_main = canonical_git_run("merge-base", "--is-ancestor", attempt_revision, main_revision)
|
||||
|
||||
def git_nonnegative_int(result):
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
value = result.stdout.strip()
|
||||
if not value.isdigit():
|
||||
return None
|
||||
return int(value)
|
||||
|
||||
recorded_commit_epoch = git_nonnegative_int(mounted_recorded_time)
|
||||
live_commit_epoch = git_nonnegative_int(mounted_live_time)
|
||||
canonical_recorded_commit_epoch = git_nonnegative_int(canonical_recorded_time)
|
||||
canonical_live_commit_epoch = git_nonnegative_int(canonical_live_time)
|
||||
recorded_distance = git_nonnegative_int(mounted_recorded_distance)
|
||||
live_distance = git_nonnegative_int(mounted_live_distance)
|
||||
canonical_recorded_distance_value = git_nonnegative_int(canonical_recorded_distance)
|
||||
canonical_live_distance_value = git_nonnegative_int(canonical_live_distance)
|
||||
try:
|
||||
recorded_updated_at = int(state.get("updated_at_epoch") or 0)
|
||||
except (TypeError, ValueError):
|
||||
recorded_updated_at = 0
|
||||
|
||||
mounted_merge_base_value = mounted_merge_base.stdout.strip()
|
||||
canonical_merge_base_value = canonical_merge_base.stdout.strip()
|
||||
history_transition_baseline = (
|
||||
stale_success_baseline
|
||||
and recorded_to_live.returncode == 1
|
||||
and canonical_recorded_to_live.returncode == 1
|
||||
and live_to_recorded.returncode == 1
|
||||
and canonical_live_to_recorded.returncode == 1
|
||||
and mounted_recorded_main.returncode == 0
|
||||
and canonical_recorded_main.returncode == 0
|
||||
and canonical_live_main.returncode == 0
|
||||
and mounted_merge_base.returncode == 0
|
||||
and canonical_merge_base.returncode == 0
|
||||
and re.fullmatch(r"[0-9a-f]{40}", mounted_merge_base_value) is not None
|
||||
and canonical_merge_base_value == mounted_merge_base_value
|
||||
and mounted_merge_base_value not in {recorded_live, live_revision}
|
||||
and recorded_commit_epoch is not None
|
||||
and live_commit_epoch is not None
|
||||
and canonical_recorded_commit_epoch == recorded_commit_epoch
|
||||
and canonical_live_commit_epoch == live_commit_epoch
|
||||
and 0 < recorded_updated_at
|
||||
and recorded_commit_epoch <= recorded_updated_at
|
||||
and recorded_updated_at < live_commit_epoch
|
||||
and recorded_commit_epoch < live_commit_epoch
|
||||
and recorded_distance is not None
|
||||
and live_distance is not None
|
||||
and canonical_recorded_distance_value == recorded_distance
|
||||
and canonical_live_distance_value == live_distance
|
||||
and 0 <= live_distance < recorded_distance
|
||||
)
|
||||
exact_legacy_state_ahead_baseline = (
|
||||
resolved_attempt.returncode == 0
|
||||
and resolved_attempt.stdout.strip() == attempt_revision
|
||||
and canonical_resolved_attempt.returncode == 0
|
||||
and canonical_resolved_attempt.stdout.strip() == attempt_revision
|
||||
and _is_exact_legacy_state_ahead_baseline(
|
||||
state,
|
||||
repository,
|
||||
expected_container,
|
||||
recorded_live,
|
||||
live_revision,
|
||||
container_artifact,
|
||||
{
|
||||
"recorded_to_live": recorded_to_live.returncode,
|
||||
"canonical_recorded_to_live": canonical_recorded_to_live.returncode,
|
||||
"live_to_recorded": live_to_recorded.returncode,
|
||||
"canonical_live_to_recorded": canonical_live_to_recorded.returncode,
|
||||
"mounted_recorded_main": mounted_recorded_main.returncode,
|
||||
"canonical_recorded_main": canonical_recorded_main.returncode,
|
||||
"canonical_live_main": canonical_live_main.returncode,
|
||||
"recorded_to_attempt": recorded_to_attempt.returncode,
|
||||
"canonical_recorded_to_attempt": canonical_recorded_to_attempt.returncode,
|
||||
"attempt_to_main": attempt_to_main.returncode,
|
||||
"canonical_attempt_to_main": canonical_attempt_to_main.returncode,
|
||||
},
|
||||
)
|
||||
)
|
||||
if not (history_transition_baseline or exact_legacy_state_ahead_baseline):
|
||||
if canonical_recorded_to_live.returncode == 1:
|
||||
raise SystemExit(60)
|
||||
raise SystemExit(61)
|
||||
|
||||
artifacts = {"containers": {expected_container: container_artifact}, "compose_sha256": ""}
|
||||
now = int(time.time())
|
||||
payload = {
|
||||
"schema_version": 3,
|
||||
"repository": repository,
|
||||
"slug": "media-codec-dashboard",
|
||||
"compose_project": "media-codec-dashboard",
|
||||
"revision": live_revision,
|
||||
"live_revision": live_revision,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": True,
|
||||
"live_artifacts": artifacts,
|
||||
"live_receipt": {},
|
||||
"status": "success",
|
||||
"message": "Verified existing Media Codec Dashboard runtime reconciled to actual immutable live image",
|
||||
"updated_at_epoch": now,
|
||||
"artifacts": artifacts,
|
||||
"last_attempt": {
|
||||
"revision": live_revision,
|
||||
"status": "success",
|
||||
"message": "Verified runtime state reconciliation",
|
||||
"updated_at_epoch": now,
|
||||
"metadata": {"reconciled_from": reconciled_from},
|
||||
},
|
||||
"previous_revision": recorded_live,
|
||||
"rollback_restored_revision": "",
|
||||
}
|
||||
if recorded_live == live_revision and current_verified_receipt_snapshot_baseline:
|
||||
payload = dict(state)
|
||||
payload.update({
|
||||
"live_status": "healthy",
|
||||
"status": "success",
|
||||
"message": "Verified Media Codec Dashboard receipt re-attested after failed forward deployment",
|
||||
"updated_at_epoch": now,
|
||||
"artifacts": current_artifacts,
|
||||
"last_attempt": {
|
||||
"revision": live_revision,
|
||||
"status": "success",
|
||||
"message": "Verified existing runtime receipt re-attestation",
|
||||
"updated_at_epoch": now,
|
||||
"metadata": {"reconciled_failed_revision": reconciled_from},
|
||||
},
|
||||
})
|
||||
|
||||
backup = state_path.with_name(f"{state_path.name}.reconcile-backup-{now}-{os.getpid()}")
|
||||
try:
|
||||
shutil.copy2(state_path, backup)
|
||||
temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.tmp")
|
||||
with temporary.open("x", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, stat.S_IMODE(state_path.stat().st_mode))
|
||||
os.replace(temporary, state_path)
|
||||
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except Exception:
|
||||
raise SystemExit(53)
|
||||
|
||||
verify = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
expected_legacy = not (recorded_live == live_revision and current_verified_receipt_snapshot_baseline)
|
||||
if not (verify.get("live_revision") == live_revision and verify.get("live_status") == "healthy" and verify.get("live_legacy_provenance") is expected_legacy and dict(dict(verify.get("live_artifacts") or {}).get("containers") or {}).get(expected_container, {}).get("image_id") == image_id):
|
||||
raise SystemExit(54)
|
||||
if not expected_legacy and (verify.get("live_receipt") != current_receipt or verify.get("live_artifacts") != current_artifacts):
|
||||
raise SystemExit(64)
|
||||
|
||||
print("reconciled", flush=True)
|
||||
'''
|
||||
|
||||
|
||||
class MediaCodecDashboardDeploymentReconciliation:
|
||||
"""Repair only stale controller state proven against the live immutable dashboard runtime."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
if status.get("status") != "ok":
|
||||
return {"status": "controller_unavailable", "reason": redact_message(str(status.get("reason") or status.get("status") or "unavailable"))}
|
||||
controller = dict(status.get("container") or {})
|
||||
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
||||
return {"status": "controller_unavailable", "reason": "deployment controller is not running"}
|
||||
container_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if not config or not container_id:
|
||||
return {"status": "connector_unavailable", "reason": "Portainer deployment control is unavailable"}
|
||||
|
||||
command = ["python3", "-c", _RECONCILE_SCRIPT, _REPOSITORY, _CONTAINER]
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
safe_container = quote(container_id, safe="")
|
||||
try:
|
||||
created = self.portainer._request("POST", f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec", json={"AttachStdout": False, "AttachStderr": False, "Tty": False, "Cmd": command}).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return {"status": "error", "reason": "dashboard state reconciliation could not start"}
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
self.portainer._request("POST", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start", json={"Detach": True, "Tty": False})
|
||||
deadline = time.monotonic() + 45
|
||||
exit_code: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = self.portainer._request("GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json").json()
|
||||
if not bool(inspected.get("Running")):
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
break
|
||||
time.sleep(0.25)
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if exit_code is None:
|
||||
return {"status": "error", "reason": "dashboard state reconciliation timed out"}
|
||||
reasons = {
|
||||
20: "deployment projects registry is unavailable or unsafe",
|
||||
22: "Media Codec Dashboard deployment target is missing",
|
||||
23: "Media Codec Dashboard deployment target topology differs from the bounded contract",
|
||||
24: "mounted Media Codec Dashboard bare repository is unavailable",
|
||||
25: "deployment revisions cannot be resolved",
|
||||
26: "Media Codec Dashboard main cannot be resolved",
|
||||
27: "live Media Codec Dashboard revision is not an ancestor of main",
|
||||
29: "deployment lock files cannot be opened",
|
||||
30: "deployment locks cannot be acquired",
|
||||
31: "deployment projects registry cannot be read",
|
||||
36: "deployment projects registry host alias is unavailable or mismatched",
|
||||
37: "deployment target changed while reconciliation was waiting",
|
||||
40: "fixed Media Codec Dashboard reconciliation contract was altered",
|
||||
42: "deployment state file is unavailable or invalid",
|
||||
43: "deployment state is not a modern Media Codec Dashboard state",
|
||||
44: "live Media Codec Dashboard container cannot be inspected",
|
||||
45: "live Media Codec Dashboard container is not healthy in the expected Compose project",
|
||||
46: "live Media Codec Dashboard immutable image identity is invalid",
|
||||
47: "Media Codec Dashboard Compose project contains unexpected containers",
|
||||
48: "live Media Codec Dashboard mount topology differs from the verified runtime",
|
||||
49: "live Media Codec Dashboard port mapping differs from the verified runtime",
|
||||
50: "live Media Codec Dashboard image cannot be inspected",
|
||||
51: "live Media Codec Dashboard image revision label does not match the verified runtime",
|
||||
52: "controller state drift is not the exact known recoverable state",
|
||||
53: "reconciled deployment state could not be written durably",
|
||||
54: "reconciled deployment state did not verify after write",
|
||||
55: "controller state for the current live revision is internally inconsistent",
|
||||
56: "failed deployment attempt is not a descendant of the live revision",
|
||||
57: "failed deployment attempt is not an ancestor of main",
|
||||
58: "recorded live deployment revision is invalid",
|
||||
59: "stale controller state metadata is not a supported recoverable baseline",
|
||||
60: "recorded live deployment revision is not an ancestor of the verified runtime",
|
||||
61: "mounted deployment repository ancestry differs from canonical Gitea repository",
|
||||
62: "deployment controller verifier is unavailable or unsafe",
|
||||
63: "recorded dashboard receipt cannot be reverified against the live runtime",
|
||||
64: "verified dashboard receipt or artifacts differ from the recorded state",
|
||||
}
|
||||
if exit_code != 0:
|
||||
return {"status": "validation_failed", "reason": reasons.get(exit_code, "Media Codec Dashboard deployment reconciliation failed closed")}
|
||||
return {
|
||||
"status": "reconciled",
|
||||
"repository": _REPOSITORY,
|
||||
"container": _CONTAINER,
|
||||
}
|
||||
Reference in new issue
Block a user