Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d
This commit is contained in:
Public Source Publisher committed
commit
1dfbca08a1
528 files changed
+131054
No files matched your search
@@ -0,0 +1,387 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from ..connectors.portainer import PortainerConnector
|
||||
from ..core.redaction import redact_message
|
||||
from .portfolio_deployment_enrollment import PortfolioDeploymentEnrollment
|
||||
|
||||
_CONTROLLER = "gitea-deploy-control"
|
||||
_REPOSITORY = "jens/portfolio"
|
||||
_LIVE_REVISION = "52c6f2fc74a1e39ad288f1c5e987a9cd5b072d3e"
|
||||
_FAILED_REVISION = "b7955d90b63860ffff8e0bd84be123aaba632043"
|
||||
_FAILURE_MESSAGE = "Existing production containers require a valid healthy live deployment state"
|
||||
|
||||
# One-time, fail-closed re-attestation for the exact observed Portfolio state.
|
||||
# The recovery never builds, restarts, recreates or stops application containers.
|
||||
# It uses the controller's own receipt/runtime verifier and accepts only a change
|
||||
# in ephemeral container IDs while immutable image/provenance evidence remains
|
||||
# exactly equal to the previously recorded verified receipt.
|
||||
_REATTEST_SCRIPT = r'''
|
||||
import fcntl
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
repository, live_revision, failed_revision, failure_message = sys.argv[1:5]
|
||||
expected_repository = "jens/portfolio"
|
||||
expected_live_revision = "52c6f2fc74a1e39ad288f1c5e987a9cd5b072d3e"
|
||||
expected_failed_revision = "b7955d90b63860ffff8e0bd84be123aaba632043"
|
||||
expected_failure_message = "Existing production containers require a valid healthy live deployment state"
|
||||
if (
|
||||
repository != expected_repository
|
||||
or live_revision != expected_live_revision
|
||||
or failed_revision != expected_failed_revision
|
||||
or failure_message != expected_failure_message
|
||||
):
|
||||
raise SystemExit(40)
|
||||
|
||||
controller_path = Path("/opt/gitea-deploy/deploy.py")
|
||||
registry_path = Path("/opt/gitea-deploy/projects.json")
|
||||
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
||||
state_path = state_root / "portfolio.json"
|
||||
project_lock_path = state_root / "locks" / "portfolio.lock"
|
||||
global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
||||
|
||||
if controller_path.is_symlink() or not controller_path.is_file():
|
||||
raise SystemExit(20)
|
||||
if registry_path.is_symlink() or not registry_path.is_file():
|
||||
raise SystemExit(21)
|
||||
try:
|
||||
projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(22)
|
||||
project = projects.get(repository) if isinstance(projects, dict) else None
|
||||
if not isinstance(project, dict):
|
||||
raise SystemExit(23)
|
||||
if (
|
||||
project.get("branch") != "main"
|
||||
or project.get("slug") != "portfolio"
|
||||
or project.get("compose_project") != "portfolio"
|
||||
or project.get("containers") != ["Portfolio", "Portfolio-api"]
|
||||
or project.get("compose_files") != ["docker-compose.yml"]
|
||||
or project.get("exact_source") is not True
|
||||
or project.get("verify_oci_revision") is not False
|
||||
):
|
||||
raise SystemExit(24)
|
||||
|
||||
owner, name = repository.split("/", 1)
|
||||
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
||||
if not bare_repo.is_dir():
|
||||
raise SystemExit(25)
|
||||
|
||||
def git_run(*args):
|
||||
return subprocess.run(
|
||||
["git", f"--git-dir={bare_repo}", *args],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
for revision in (live_revision, failed_revision):
|
||||
resolved = git_run("rev-parse", f"{revision}^{{commit}}")
|
||||
if resolved.returncode != 0 or resolved.stdout.strip() != revision:
|
||||
raise SystemExit(26)
|
||||
main = git_run("rev-parse", "refs/heads/main")
|
||||
if main.returncode != 0 or main.stdout.strip() != failed_revision:
|
||||
raise SystemExit(27)
|
||||
if git_run("merge-base", "--is-ancestor", live_revision, failed_revision).returncode != 0:
|
||||
raise SystemExit(28)
|
||||
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location("itworx_deploy_controller", controller_path)
|
||||
controller = importlib.util.module_from_spec(spec) if spec and spec.loader else None
|
||||
if controller is None or spec is None or spec.loader is None:
|
||||
raise RuntimeError("controller import unavailable")
|
||||
spec.loader.exec_module(controller)
|
||||
except Exception:
|
||||
raise SystemExit(29)
|
||||
|
||||
state_root.mkdir(parents=True, exist_ok=True)
|
||||
project_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
project_lock = project_lock_path.open("a+", encoding="utf-8")
|
||||
global_lock = global_lock_path.open("a+", encoding="utf-8")
|
||||
except Exception:
|
||||
raise SystemExit(30)
|
||||
|
||||
with project_lock:
|
||||
try:
|
||||
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
with global_lock:
|
||||
try:
|
||||
fcntl.flock(global_lock, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
raise SystemExit(31)
|
||||
|
||||
try:
|
||||
current_projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(32)
|
||||
if current_projects.get(repository) != project:
|
||||
raise SystemExit(33)
|
||||
|
||||
if state_path.is_symlink() or not state_path.is_file():
|
||||
raise SystemExit(34)
|
||||
try:
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
raise SystemExit(34)
|
||||
if not isinstance(state, dict):
|
||||
raise SystemExit(34)
|
||||
attempt = state.get("last_attempt")
|
||||
receipt = state.get("live_receipt")
|
||||
artifacts = state.get("live_artifacts")
|
||||
if not (
|
||||
state.get("schema_version") == 3
|
||||
and state.get("repository") == repository
|
||||
and state.get("revision") == live_revision
|
||||
and state.get("live_revision") == live_revision
|
||||
and state.get("live_status") == "unknown"
|
||||
and state.get("live_legacy_provenance") is False
|
||||
and state.get("status") == "failed"
|
||||
and isinstance(attempt, dict)
|
||||
and attempt.get("revision") == failed_revision
|
||||
and attempt.get("status") == "failed"
|
||||
and attempt.get("message") == failure_message
|
||||
and isinstance(receipt, dict)
|
||||
and bool(receipt)
|
||||
and isinstance(artifacts, dict)
|
||||
):
|
||||
raise SystemExit(35)
|
||||
|
||||
project = dict(project)
|
||||
project["_repository_name"] = repository
|
||||
project["_repository_path"] = str(bare_repo)
|
||||
|
||||
try:
|
||||
build = controller.validate_recorded_receipt(project, receipt, repository, live_revision)
|
||||
recorded_receipt_containers = receipt.get("containers")
|
||||
if not isinstance(recorded_receipt_containers, dict):
|
||||
raise RuntimeError("receipt containers missing")
|
||||
expected_image_ids = {
|
||||
str(container_name): str(dict(raw).get("image_id") or "")
|
||||
for container_name, raw in recorded_receipt_containers.items()
|
||||
if isinstance(raw, dict)
|
||||
}
|
||||
if set(expected_image_ids) != {"Portfolio", "Portfolio-api"} or not all(expected_image_ids.values()):
|
||||
raise RuntimeError("receipt image IDs missing")
|
||||
observed_receipt = controller.verify_deployment(
|
||||
project,
|
||||
build,
|
||||
expected_image_ids=expected_image_ids,
|
||||
)
|
||||
observed_artifacts = controller.deployment_artifacts(project)
|
||||
except Exception:
|
||||
raise SystemExit(36)
|
||||
|
||||
immutable_receipt_keys = ("schema", "repository", "revision", "tree", "treeDigest", "build", "live")
|
||||
if any(observed_receipt.get(key) != receipt.get(key) for key in immutable_receipt_keys):
|
||||
raise SystemExit(37)
|
||||
observed_receipt_containers = observed_receipt.get("containers")
|
||||
if not isinstance(observed_receipt_containers, dict) or set(observed_receipt_containers) != set(recorded_receipt_containers):
|
||||
raise SystemExit(37)
|
||||
|
||||
container_id_changed = False
|
||||
for container_name in sorted(recorded_receipt_containers):
|
||||
old = dict(recorded_receipt_containers[container_name])
|
||||
new = dict(observed_receipt_containers[container_name])
|
||||
if old.get("container_id") != new.get("container_id"):
|
||||
container_id_changed = True
|
||||
old_without_id = {key: value for key, value in old.items() if key != "container_id"}
|
||||
new_without_id = {key: value for key, value in new.items() if key != "container_id"}
|
||||
if old_without_id != new_without_id:
|
||||
raise SystemExit(38)
|
||||
if not container_id_changed:
|
||||
raise SystemExit(39)
|
||||
|
||||
recorded_artifact_containers = artifacts.get("containers")
|
||||
observed_artifact_containers = observed_artifacts.get("containers")
|
||||
if (
|
||||
not isinstance(recorded_artifact_containers, dict)
|
||||
or not isinstance(observed_artifact_containers, dict)
|
||||
or set(recorded_artifact_containers) != set(observed_artifact_containers)
|
||||
or artifacts.get("compose_sha256") != observed_artifacts.get("compose_sha256")
|
||||
):
|
||||
raise SystemExit(41)
|
||||
for container_name in sorted(recorded_artifact_containers):
|
||||
old = dict(recorded_artifact_containers[container_name])
|
||||
new = dict(observed_artifact_containers[container_name])
|
||||
old_without_id = {key: value for key, value in old.items() if key != "container_id"}
|
||||
new_without_id = {key: value for key, value in new.items() if key != "container_id"}
|
||||
if old_without_id != new_without_id:
|
||||
raise SystemExit(41)
|
||||
|
||||
now = int(time.time())
|
||||
repaired = dict(state)
|
||||
repaired["live_status"] = "healthy"
|
||||
repaired["live_artifacts"] = observed_artifacts
|
||||
repaired["artifacts"] = observed_artifacts
|
||||
repaired["live_receipt"] = observed_receipt
|
||||
repaired["status"] = "success"
|
||||
repaired["message"] = "Verified Portfolio runtime re-attested after container recreation"
|
||||
repaired["updated_at_epoch"] = now
|
||||
repaired["last_attempt"] = {
|
||||
"revision": live_revision,
|
||||
"status": "success",
|
||||
"message": "Verified live Portfolio runtime receipt re-attestation",
|
||||
"updated_at_epoch": now,
|
||||
"metadata": {
|
||||
"reconciled_failed_revision": failed_revision,
|
||||
"container_identity_only": True,
|
||||
},
|
||||
}
|
||||
|
||||
backup = state_path.with_name(f"portfolio.json.reattest-backup-{now}-{os.getpid()}")
|
||||
temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.reattest.tmp")
|
||||
try:
|
||||
mode = stat.S_IMODE(state_path.stat().st_mode)
|
||||
shutil.copy2(state_path, backup)
|
||||
with temporary.open("x", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(repaired, indent=2, sort_keys=True) + "\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, mode)
|
||||
os.replace(temporary, state_path)
|
||||
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
except Exception:
|
||||
try:
|
||||
temporary.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(42)
|
||||
|
||||
try:
|
||||
verified = controller.verified_current_receipt(
|
||||
repository,
|
||||
project,
|
||||
live_revision,
|
||||
str(receipt.get("tree") or ""),
|
||||
)
|
||||
except Exception:
|
||||
verified = None
|
||||
if not isinstance(verified, dict) or verified != observed_receipt:
|
||||
raise SystemExit(43)
|
||||
|
||||
print("re-attested", flush=True)
|
||||
'''
|
||||
|
||||
|
||||
class PortfolioDeploymentRecovery:
|
||||
"""Recover the exact known Portfolio state drift, otherwise use normal enrollment."""
|
||||
|
||||
def __init__(self, portainer: PortainerConnector | None = None):
|
||||
self.portainer = portainer or PortainerConnector()
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
status = self.portainer.get_container_status(_CONTROLLER)
|
||||
controller = dict(status.get("container") or {}) if status.get("status") == "ok" else {}
|
||||
container_id = str(controller.get("id") or "")
|
||||
config = self.portainer._config()
|
||||
if controller.get("name") != _CONTROLLER or controller.get("state") != "running" or not container_id or not config:
|
||||
return PortfolioDeploymentEnrollment(portainer=self.portainer).run()
|
||||
|
||||
command = [
|
||||
"python3",
|
||||
"-c",
|
||||
_REATTEST_SCRIPT,
|
||||
_REPOSITORY,
|
||||
_LIVE_REVISION,
|
||||
_FAILED_REVISION,
|
||||
_FAILURE_MESSAGE,
|
||||
]
|
||||
endpoint = quote(str(config.endpoint_id), safe="")
|
||||
safe_container = quote(container_id, safe="")
|
||||
try:
|
||||
created = self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
||||
json={"AttachStdout": False, "AttachStderr": False, "Tty": False, "Cmd": command},
|
||||
).json()
|
||||
exec_id = str(created.get("Id") or "")
|
||||
if not exec_id:
|
||||
return {"status": "error", "reason": "Portfolio runtime re-attestation could not start"}
|
||||
safe_exec = quote(exec_id, safe="")
|
||||
self.portainer._request(
|
||||
"POST",
|
||||
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
||||
json={"Detach": True, "Tty": False},
|
||||
)
|
||||
deadline = time.monotonic() + 45
|
||||
exit_code: int | None = None
|
||||
while time.monotonic() < deadline:
|
||||
inspected = self.portainer._request(
|
||||
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
|
||||
).json()
|
||||
if not bool(inspected.get("Running")):
|
||||
raw_exit = inspected.get("ExitCode")
|
||||
exit_code = int(raw_exit) if raw_exit is not None else -1
|
||||
break
|
||||
time.sleep(0.25)
|
||||
except (RuntimeError, ValueError, TypeError) as exc:
|
||||
return {"status": "error", "reason": redact_message(str(exc))}
|
||||
|
||||
if exit_code is None:
|
||||
return {"status": "error", "reason": "Portfolio runtime re-attestation timed out"}
|
||||
if exit_code == 0:
|
||||
return {
|
||||
"status": "reconciled",
|
||||
"repository": _REPOSITORY,
|
||||
"live_revision": _LIVE_REVISION,
|
||||
"failed_revision": _FAILED_REVISION,
|
||||
"containers": ["Portfolio", "Portfolio-api"],
|
||||
"compose_project": "portfolio",
|
||||
}
|
||||
|
||||
# Exit 35 means the exact recoverable state is no longer present. In
|
||||
# normal/healthy states the established enrollment routine remains the
|
||||
# authoritative path. Every other re-attestation failure is fail-closed.
|
||||
if exit_code == 35:
|
||||
return PortfolioDeploymentEnrollment(portainer=self.portainer).run()
|
||||
|
||||
reasons = {
|
||||
20: "deployment controller executable is unavailable or unsafe",
|
||||
21: "deployment projects registry is unavailable or unsafe",
|
||||
22: "deployment projects registry cannot be read",
|
||||
23: "Portfolio deployment target is missing",
|
||||
24: "Portfolio deployment target differs from the bounded paired contract",
|
||||
25: "mounted Portfolio bare repository is unavailable",
|
||||
26: "known Portfolio recovery revisions cannot be resolved",
|
||||
27: "Portfolio main changed after the bounded recovery contract was recorded",
|
||||
28: "known Portfolio live revision is not an ancestor of the failed revision",
|
||||
29: "deployment controller verifier cannot be loaded",
|
||||
30: "deployment lock files cannot be opened",
|
||||
31: "deployment locks cannot be acquired",
|
||||
32: "deployment projects registry changed or became unreadable while waiting",
|
||||
33: "Portfolio deployment target changed while recovery was waiting",
|
||||
34: "Portfolio deployment state is unavailable or invalid",
|
||||
36: "controller could not re-attest the current Portfolio runtime against recorded immutable image IDs",
|
||||
37: "Portfolio receipt identity changed during re-attestation",
|
||||
38: "Portfolio immutable container/image provenance differs from the recorded receipt",
|
||||
39: "Portfolio container identities did not change; the known recovery cause is absent",
|
||||
40: "fixed Portfolio re-attestation contract was altered",
|
||||
41: "Portfolio recorded live artifacts differ beyond ephemeral container IDs",
|
||||
42: "re-attested Portfolio deployment state could not be written durably",
|
||||
43: "controller rejected the repaired Portfolio receipt after write",
|
||||
}
|
||||
return {
|
||||
"status": "validation_failed",
|
||||
"reason": reasons.get(exit_code, "Portfolio runtime re-attestation failed closed"),
|
||||
}
|
||||
Reference in new issue
Block a user