1229 lines
47 KiB
Python
1229 lines
47 KiB
Python
from __future__ import annotations
|
|
|
|
import time
|
|
from typing import Any
|
|
from urllib.parse import quote
|
|
from uuid import uuid4
|
|
|
|
from sqlalchemy import func, update
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlmodel import Session
|
|
|
|
from ..connectors.portainer import PortainerConnector
|
|
from ..core.redaction import redact_message
|
|
from ..core.time import utc_now
|
|
from ..db.models import AppSetting, JobRun
|
|
|
|
_CONTROLLER = "gitea-deploy-control"
|
|
_REPOSITORY = "jens/privatemuse"
|
|
_LIVE_REVISION = "2103d5795d151885edeeb964466b643b8fd4a0eb"
|
|
_FAILED_REVISION = "6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38"
|
|
_FAILURE_MESSAGE = (
|
|
"Existing production containers require a valid healthy live deployment state"
|
|
)
|
|
_CONTAINERS = [
|
|
"privatemuse-backend-1",
|
|
"privatemuse-frontend-1",
|
|
"privatemuse-audio-enhancement-1",
|
|
]
|
|
_RECOVERY_STATE_SETTING = "deployment:privatemuse:receipt-recovery:v1"
|
|
_ACTIVE_RECOVERY_STATES = {"reserved", "created", "started", "outcome_unknown"}
|
|
_STATE_TRANSITIONS = {
|
|
"reserved": {"reserved", "created", "outcome_unknown", "completed"},
|
|
"created": {"started", "outcome_unknown", "completed"},
|
|
"started": {"outcome_unknown", "completed"},
|
|
"outcome_unknown": {"outcome_unknown", "completed"},
|
|
}
|
|
|
|
# One-time, fail-closed re-attestation for the exact observed PrivateMuse state.
|
|
# It never builds, restarts, recreates or stops application containers. The
|
|
# controller's own receipt/runtime verifiers prove that only the two container
|
|
# IDs recreated by the prior rollback differ; all immutable evidence must match.
|
|
_REATTEST_SCRIPT = r"""
|
|
import fcntl
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
repository, live_revision, failed_revision, failure_message = sys.argv[1:5]
|
|
expected_repository = "jens/privatemuse"
|
|
expected_live_revision = "2103d5795d151885edeeb964466b643b8fd4a0eb"
|
|
expected_failed_revision = "6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38"
|
|
expected_failure_message = "Existing production containers require a valid healthy live deployment state"
|
|
if (
|
|
repository != expected_repository
|
|
or live_revision != expected_live_revision
|
|
or failed_revision != expected_failed_revision
|
|
or failure_message != expected_failure_message
|
|
):
|
|
raise SystemExit(40)
|
|
|
|
controller_path = Path("/opt/gitea-deploy/deploy.py")
|
|
registry_path = Path("/opt/gitea-deploy/projects.json")
|
|
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
|
state_path = state_root / "privatemuse.json"
|
|
project_lock_path = state_root / "locks" / "privatemuse.lock"
|
|
global_lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
|
|
|
def fsync_directory(directory):
|
|
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
|
|
directory_fd = os.open(str(directory), flags)
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
|
|
def write_exclusive(path, payload, metadata):
|
|
try:
|
|
with path.open("xb") as handle:
|
|
handle.write(payload)
|
|
handle.flush()
|
|
os.fchmod(handle.fileno(), stat.S_IMODE(metadata.st_mode))
|
|
os.fchown(handle.fileno(), metadata.st_uid, metadata.st_gid)
|
|
os.fsync(handle.fileno())
|
|
except Exception:
|
|
try:
|
|
path.unlink(missing_ok=True)
|
|
except Exception:
|
|
pass
|
|
raise
|
|
|
|
if controller_path.is_symlink() or not controller_path.is_file():
|
|
raise SystemExit(20)
|
|
if registry_path.is_symlink() or not registry_path.is_file():
|
|
raise SystemExit(21)
|
|
try:
|
|
projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(22)
|
|
project = projects.get(repository) if isinstance(projects, dict) else None
|
|
if not isinstance(project, dict):
|
|
raise SystemExit(23)
|
|
|
|
expected_target = {
|
|
"branch": "main",
|
|
"slug": "privatemuse",
|
|
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/privatemuse",
|
|
"exact_source": True,
|
|
"env_source": "/srv/example/appdata/privatemuse/.env",
|
|
"compose_files": ["docker-compose.yml", "docker-compose.unraid.yml"],
|
|
"compose_project": "privatemuse",
|
|
"containers": [
|
|
"privatemuse-backend-1",
|
|
"privatemuse-frontend-1",
|
|
"privatemuse-audio-enhancement-1",
|
|
],
|
|
"verify_oci_revision": False,
|
|
"health_timeout": 180,
|
|
"environment": {"PRIVATEMUSE_RELEASE": "{short_revision}"},
|
|
}
|
|
for key, expected in expected_target.items():
|
|
if project.get(key) != expected:
|
|
raise SystemExit(24)
|
|
allowed_metadata_keys = {
|
|
"gitea_repository_id",
|
|
"gitea_webhook_secret_file",
|
|
"validation",
|
|
}
|
|
expected_validation = {
|
|
"validation_gate_version": 1,
|
|
"required": True,
|
|
"contexts": ["Managed validation / full (push)"],
|
|
"timeout_seconds": 1800,
|
|
"poll_interval_seconds": 5,
|
|
"max_poll_interval_seconds": 30,
|
|
}
|
|
repository_id = project.get("gitea_repository_id")
|
|
if (
|
|
set(project) - set(expected_target) != allowed_metadata_keys
|
|
or not isinstance(repository_id, int)
|
|
or isinstance(repository_id, bool)
|
|
or repository_id <= 0
|
|
or project.get("gitea_webhook_secret_file") != "jens__privatemuse"
|
|
or project.get("validation") != expected_validation
|
|
):
|
|
raise SystemExit(24)
|
|
|
|
owner, name = repository.split("/", 1)
|
|
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
|
if not bare_repo.is_dir():
|
|
raise SystemExit(25)
|
|
|
|
def git_run(*args):
|
|
return subprocess.run(
|
|
["git", f"--git-dir={bare_repo}", *args],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
|
|
for revision in (live_revision, failed_revision):
|
|
resolved = git_run("rev-parse", f"{revision}^{{commit}}")
|
|
if resolved.returncode != 0 or resolved.stdout.strip() != revision:
|
|
raise SystemExit(26)
|
|
main = git_run("rev-parse", "refs/heads/main")
|
|
if main.returncode != 0 or main.stdout.strip() != failed_revision:
|
|
raise SystemExit(27)
|
|
if git_run("merge-base", "--is-ancestor", live_revision, failed_revision).returncode != 0:
|
|
raise SystemExit(28)
|
|
|
|
try:
|
|
spec = importlib.util.spec_from_file_location("itworx_deploy_controller", controller_path)
|
|
controller = importlib.util.module_from_spec(spec) if spec and spec.loader else None
|
|
if controller is None or spec is None or spec.loader is None:
|
|
raise RuntimeError("controller import unavailable")
|
|
spec.loader.exec_module(controller)
|
|
except Exception:
|
|
raise SystemExit(29)
|
|
|
|
state_root.mkdir(parents=True, exist_ok=True)
|
|
project_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
project_lock = project_lock_path.open("a+", encoding="utf-8")
|
|
global_lock = global_lock_path.open("a+", encoding="utf-8")
|
|
except Exception:
|
|
raise SystemExit(30)
|
|
|
|
with project_lock:
|
|
try:
|
|
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
|
except Exception:
|
|
raise SystemExit(31)
|
|
with global_lock:
|
|
try:
|
|
fcntl.flock(global_lock, fcntl.LOCK_EX)
|
|
except Exception:
|
|
raise SystemExit(31)
|
|
|
|
try:
|
|
current_projects = json.loads(registry_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(32)
|
|
if current_projects.get(repository) != project:
|
|
raise SystemExit(33)
|
|
|
|
if state_path.is_symlink() or not state_path.is_file():
|
|
raise SystemExit(34)
|
|
try:
|
|
state_metadata = state_path.stat()
|
|
original_state = state_path.read_bytes()
|
|
state = json.loads(original_state.decode("utf-8"))
|
|
except Exception:
|
|
raise SystemExit(34)
|
|
if not isinstance(state, dict):
|
|
raise SystemExit(34)
|
|
|
|
project = dict(project)
|
|
project["_repository_name"] = repository
|
|
project["_repository_path"] = str(bare_repo)
|
|
|
|
if (
|
|
state.get("schema_version") == 3
|
|
and state.get("repository") == repository
|
|
and state.get("live_status") == "healthy"
|
|
):
|
|
healthy_revision = str(state.get("live_revision") or "")
|
|
healthy_receipt = state.get("live_receipt")
|
|
healthy_tree = (
|
|
str(healthy_receipt.get("tree") or "")
|
|
if isinstance(healthy_receipt, dict)
|
|
else ""
|
|
)
|
|
verified_healthy = None
|
|
if (
|
|
healthy_revision in {live_revision, failed_revision}
|
|
and state.get("revision") == healthy_revision
|
|
and state.get("status") == "success"
|
|
and state.get("live_legacy_provenance") is False
|
|
and isinstance(healthy_receipt, dict)
|
|
and healthy_receipt.get("revision") == healthy_revision
|
|
and healthy_tree
|
|
):
|
|
try:
|
|
verified_healthy = controller.verified_current_receipt(
|
|
repository,
|
|
project,
|
|
healthy_revision,
|
|
healthy_tree,
|
|
)
|
|
except Exception:
|
|
verified_healthy = None
|
|
if verified_healthy == healthy_receipt:
|
|
raise SystemExit(35)
|
|
raise SystemExit(44)
|
|
|
|
attempt = state.get("last_attempt")
|
|
receipt = state.get("live_receipt")
|
|
artifacts = state.get("live_artifacts")
|
|
expected_attempt_metadata = {
|
|
"id": "6e4c959eea2e",
|
|
"repository": repository,
|
|
"revision": failed_revision,
|
|
"tree": "daa0e757f4b3edac2c2830572a05bfdf271e8bba",
|
|
"treeDigest": "e763339aca7139a29b3d4fc8a8f856557881ef177f15115d0c7fd52b093b3654",
|
|
"builtAt": "2026-09-19T23:07:14.894Z",
|
|
}
|
|
expected_receipt_keys = {
|
|
"build",
|
|
"containers",
|
|
"live",
|
|
"repository",
|
|
"revision",
|
|
"schema",
|
|
"tree",
|
|
"treeDigest",
|
|
}
|
|
expected_artifact_keys = {"compose_sha256", "containers"}
|
|
if not (
|
|
state.get("schema_version") == 3
|
|
and state.get("repository") == repository
|
|
and state.get("revision") == live_revision
|
|
and state.get("live_revision") == live_revision
|
|
and state.get("live_status") == "unknown"
|
|
and state.get("live_legacy_provenance") is False
|
|
and state.get("status") == "failed"
|
|
and state.get("message") == failure_message
|
|
and isinstance(attempt, dict)
|
|
and attempt.get("revision") == failed_revision
|
|
and attempt.get("status") == "failed"
|
|
and attempt.get("message") == failure_message
|
|
and attempt.get("metadata") == expected_attempt_metadata
|
|
and isinstance(receipt, dict)
|
|
and set(receipt) == expected_receipt_keys
|
|
and isinstance(artifacts, dict)
|
|
and set(artifacts) == expected_artifact_keys
|
|
):
|
|
raise SystemExit(44)
|
|
|
|
expected_containers = {
|
|
"privatemuse-audio-enhancement-1",
|
|
"privatemuse-backend-1",
|
|
"privatemuse-frontend-1",
|
|
}
|
|
expected_container_keys = {
|
|
"container_id",
|
|
"container_revision",
|
|
"image_created",
|
|
"image_id",
|
|
"image_ref",
|
|
"image_revision",
|
|
"image_source",
|
|
"image_tree",
|
|
"image_version",
|
|
}
|
|
try:
|
|
build = controller.validate_recorded_receipt(project, receipt, repository, live_revision)
|
|
recorded_receipt_containers = receipt.get("containers")
|
|
if (
|
|
not isinstance(recorded_receipt_containers, dict)
|
|
or set(recorded_receipt_containers) != expected_containers
|
|
):
|
|
raise RuntimeError("receipt containers differ")
|
|
expected_image_ids = {
|
|
str(container_name): str(dict(raw).get("image_id") or "")
|
|
for container_name, raw in recorded_receipt_containers.items()
|
|
if isinstance(raw, dict) and set(raw) == expected_container_keys
|
|
}
|
|
if set(expected_image_ids) != expected_containers or not all(expected_image_ids.values()):
|
|
raise RuntimeError("receipt image IDs missing")
|
|
observed_receipt = controller.verify_deployment(
|
|
project,
|
|
build,
|
|
expected_image_ids=expected_image_ids,
|
|
)
|
|
observed_artifacts = controller.deployment_artifacts(project)
|
|
except Exception:
|
|
raise SystemExit(36)
|
|
|
|
immutable_receipt_keys = (
|
|
"schema",
|
|
"repository",
|
|
"revision",
|
|
"tree",
|
|
"treeDigest",
|
|
"build",
|
|
"live",
|
|
)
|
|
if any(observed_receipt.get(key) != receipt.get(key) for key in immutable_receipt_keys):
|
|
raise SystemExit(37)
|
|
observed_receipt_containers = observed_receipt.get("containers")
|
|
if (
|
|
not isinstance(observed_receipt_containers, dict)
|
|
or set(observed_receipt_containers) != expected_containers
|
|
):
|
|
raise SystemExit(37)
|
|
|
|
allowed_changed_ids = {"privatemuse-backend-1", "privatemuse-frontend-1"}
|
|
changed_receipt_ids = set()
|
|
for container_name in sorted(expected_containers):
|
|
old = dict(recorded_receipt_containers[container_name])
|
|
new = dict(observed_receipt_containers[container_name])
|
|
if set(old) != expected_container_keys or set(new) != expected_container_keys:
|
|
raise SystemExit(38)
|
|
if old.get("container_id") != new.get("container_id"):
|
|
changed_receipt_ids.add(container_name)
|
|
old_without_id = {key: value for key, value in old.items() if key != "container_id"}
|
|
new_without_id = {key: value for key, value in new.items() if key != "container_id"}
|
|
if old_without_id != new_without_id:
|
|
raise SystemExit(38)
|
|
if changed_receipt_ids != allowed_changed_ids:
|
|
raise SystemExit(39)
|
|
|
|
recorded_artifact_containers = artifacts.get("containers")
|
|
observed_artifact_containers = observed_artifacts.get("containers")
|
|
if (
|
|
set(observed_artifacts) != expected_artifact_keys
|
|
or not isinstance(recorded_artifact_containers, dict)
|
|
or not isinstance(observed_artifact_containers, dict)
|
|
or set(recorded_artifact_containers) != expected_containers
|
|
or set(observed_artifact_containers) != expected_containers
|
|
or artifacts.get("compose_sha256") != observed_artifacts.get("compose_sha256")
|
|
):
|
|
raise SystemExit(41)
|
|
changed_artifact_ids = set()
|
|
for container_name in sorted(expected_containers):
|
|
old = dict(recorded_artifact_containers[container_name])
|
|
new = dict(observed_artifact_containers[container_name])
|
|
if set(old) != expected_container_keys or set(new) != expected_container_keys:
|
|
raise SystemExit(41)
|
|
if old.get("container_id") != new.get("container_id"):
|
|
changed_artifact_ids.add(container_name)
|
|
old_without_id = {key: value for key, value in old.items() if key != "container_id"}
|
|
new_without_id = {key: value for key, value in new.items() if key != "container_id"}
|
|
if (
|
|
old_without_id != new_without_id
|
|
or new.get("container_id")
|
|
!= dict(observed_receipt_containers[container_name]).get("container_id")
|
|
):
|
|
raise SystemExit(41)
|
|
if changed_artifact_ids != allowed_changed_ids:
|
|
raise SystemExit(41)
|
|
|
|
now = int(time.time())
|
|
repaired = dict(state)
|
|
repaired["live_status"] = "healthy"
|
|
repaired["live_artifacts"] = observed_artifacts
|
|
repaired["artifacts"] = observed_artifacts
|
|
repaired["live_receipt"] = observed_receipt
|
|
repaired["status"] = "success"
|
|
repaired["message"] = "Verified PrivateMuse runtime re-attested after rollback recreation"
|
|
repaired["updated_at_epoch"] = now
|
|
repaired["last_attempt"] = {
|
|
"revision": live_revision,
|
|
"status": "success",
|
|
"message": "Verified live PrivateMuse runtime receipt re-attestation",
|
|
"updated_at_epoch": now,
|
|
"metadata": {
|
|
"reconciled_failed_revision": failed_revision,
|
|
"container_identity_only": True,
|
|
"changed_containers": sorted(allowed_changed_ids),
|
|
},
|
|
}
|
|
|
|
backup = state_path.with_name(f"privatemuse.json.reattest-backup-{now}-{os.getpid()}")
|
|
temporary = state_path.with_name(f".{state_path.name}.{os.getpid()}.reattest.tmp")
|
|
restore_temporary = state_path.with_name(
|
|
f".{state_path.name}.{os.getpid()}.reattest-restore.tmp"
|
|
)
|
|
|
|
def restore_original_state():
|
|
try:
|
|
write_exclusive(restore_temporary, original_state, state_metadata)
|
|
os.replace(restore_temporary, state_path)
|
|
fsync_directory(state_root)
|
|
return state_path.read_bytes() == original_state
|
|
except Exception:
|
|
try:
|
|
restore_temporary.unlink(missing_ok=True)
|
|
except Exception:
|
|
pass
|
|
return False
|
|
|
|
state_replaced = False
|
|
try:
|
|
write_exclusive(backup, original_state, state_metadata)
|
|
fsync_directory(state_root)
|
|
repaired_state = (
|
|
json.dumps(repaired, indent=2, sort_keys=True) + "\n"
|
|
).encode("utf-8")
|
|
write_exclusive(temporary, repaired_state, state_metadata)
|
|
os.replace(temporary, state_path)
|
|
state_replaced = True
|
|
fsync_directory(state_root)
|
|
except Exception:
|
|
try:
|
|
temporary.unlink(missing_ok=True)
|
|
except Exception:
|
|
pass
|
|
if state_replaced and not restore_original_state():
|
|
raise SystemExit(45)
|
|
raise SystemExit(42)
|
|
|
|
try:
|
|
written = json.loads(state_path.read_text(encoding="utf-8"))
|
|
verified = controller.verified_current_receipt(
|
|
repository,
|
|
project,
|
|
live_revision,
|
|
str(receipt.get("tree") or ""),
|
|
)
|
|
final_artifacts = controller.deployment_artifacts(project)
|
|
except Exception:
|
|
written = None
|
|
verified = None
|
|
final_artifacts = None
|
|
if (
|
|
written != repaired
|
|
or not isinstance(verified, dict)
|
|
or verified != observed_receipt
|
|
or final_artifacts != observed_artifacts
|
|
):
|
|
if not restore_original_state():
|
|
raise SystemExit(45)
|
|
raise SystemExit(43)
|
|
|
|
print("re-attested", flush=True)
|
|
"""
|
|
|
|
|
|
class PrivateMuseDeploymentRecovery:
|
|
"""Recover only the exact known PrivateMuse receipt drift, otherwise fail closed."""
|
|
|
|
def __init__(self, portainer: PortainerConnector | None = None):
|
|
self.portainer = portainer or PortainerConnector()
|
|
|
|
@staticmethod
|
|
def tracked_job_id(session: Session) -> str | None:
|
|
row = session.get(AppSetting, _RECOVERY_STATE_SETTING)
|
|
if not row:
|
|
return None
|
|
state = dict(row.value_json or {})
|
|
job_id = str(state.get("jobId") or "")
|
|
return job_id or None
|
|
|
|
@staticmethod
|
|
def guarded_unknown_result(
|
|
session: Session, *, reason: str
|
|
) -> dict[str, Any] | None:
|
|
row = session.get(AppSetting, _RECOVERY_STATE_SETTING)
|
|
state = dict(row.value_json or {}) if row else {}
|
|
if state.get("state") not in _ACTIVE_RECOVERY_STATES:
|
|
return None
|
|
execution_id = str(state.get("executionId") or "")
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"reason": reason,
|
|
**({"execution_id": execution_id} if execution_id else {}),
|
|
"original_job_id": str(state.get("jobId") or ""),
|
|
}
|
|
|
|
@staticmethod
|
|
def _ensure_state_row(session: Session) -> AppSetting:
|
|
row = session.get(AppSetting, _RECOVERY_STATE_SETTING)
|
|
if row:
|
|
return row
|
|
try:
|
|
row = AppSetting(
|
|
key=_RECOVERY_STATE_SETTING,
|
|
value_json={"state": "idle", "version": 0},
|
|
is_secret=False,
|
|
)
|
|
session.add(row)
|
|
session.commit()
|
|
except IntegrityError:
|
|
session.rollback()
|
|
row = session.get(AppSetting, _RECOVERY_STATE_SETTING)
|
|
if not row:
|
|
raise RuntimeError("PrivateMuse recovery state could not be initialized")
|
|
return row
|
|
|
|
@classmethod
|
|
def admit_job(
|
|
cls,
|
|
session: Session,
|
|
job: JobRun,
|
|
) -> tuple[dict[str, Any], bool]:
|
|
"""Atomically reserve recovery ownership and create its sole JobRun."""
|
|
cls._ensure_state_row(session)
|
|
for _ in range(5):
|
|
session.expire_all()
|
|
row = session.get(AppSetting, _RECOVERY_STATE_SETTING)
|
|
if not row:
|
|
raise RuntimeError("PrivateMuse recovery state disappeared")
|
|
current = dict(row.value_json or {})
|
|
if current.get("state") in _ACTIVE_RECOVERY_STATES or current.get(
|
|
"state"
|
|
) == "completed":
|
|
return current, False
|
|
|
|
now = utc_now()
|
|
current_version = int(current.get("version") or 0)
|
|
claimed = {
|
|
"state": "reserved",
|
|
"claimId": uuid4().hex,
|
|
"jobId": job.id,
|
|
"version": current_version + 1,
|
|
"reservedAt": now.isoformat(),
|
|
"updatedAt": now.isoformat(),
|
|
}
|
|
statement = (
|
|
update(AppSetting)
|
|
.where(
|
|
AppSetting.key == _RECOVERY_STATE_SETTING,
|
|
AppSetting.updated_at == row.updated_at,
|
|
func.coalesce(
|
|
AppSetting.value_json["version"].as_integer(), 0
|
|
)
|
|
== current_version,
|
|
)
|
|
.values(value_json=claimed, updated_at=now)
|
|
)
|
|
result = session.exec(statement)
|
|
if result.rowcount == 1:
|
|
session.add(job)
|
|
session.commit()
|
|
session.refresh(job)
|
|
return claimed, True
|
|
session.rollback()
|
|
raise RuntimeError("PrivateMuse recovery state is busy")
|
|
|
|
@classmethod
|
|
def _persist_state(
|
|
cls,
|
|
session: Session,
|
|
tracked: dict[str, Any],
|
|
*,
|
|
state: str,
|
|
execution_id: str = "",
|
|
controller_id: str = "",
|
|
result: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
for _ in range(5):
|
|
session.expire_all()
|
|
row = session.get(AppSetting, _RECOVERY_STATE_SETTING)
|
|
if not row:
|
|
raise RuntimeError("PrivateMuse recovery state is unavailable")
|
|
current = dict(row.value_json or {})
|
|
if (
|
|
current.get("claimId") != tracked.get("claimId")
|
|
or current.get("jobId") != tracked.get("jobId")
|
|
):
|
|
raise RuntimeError("PrivateMuse recovery ownership changed")
|
|
if current.get("state") == "completed":
|
|
tracked.clear()
|
|
tracked.update(current)
|
|
return current
|
|
current_state = str(current.get("state") or "")
|
|
if state not in _STATE_TRANSITIONS.get(current_state, set()):
|
|
raise RuntimeError(
|
|
f"PrivateMuse recovery cannot transition from {current_state} to {state}"
|
|
)
|
|
current_execution_id = str(current.get("executionId") or "")
|
|
if (
|
|
current_execution_id
|
|
and execution_id
|
|
and current_execution_id != execution_id
|
|
):
|
|
raise RuntimeError("PrivateMuse recovery execution identity changed")
|
|
current_controller_id = str(current.get("controllerId") or "")
|
|
if (
|
|
current_controller_id
|
|
and controller_id
|
|
and current_controller_id != controller_id
|
|
):
|
|
raise RuntimeError("PrivateMuse recovery controller identity changed")
|
|
|
|
now = utc_now()
|
|
current_version = int(current.get("version") or 0)
|
|
updated = {
|
|
**current,
|
|
"state": state,
|
|
"version": current_version + 1,
|
|
"updatedAt": now.isoformat(),
|
|
}
|
|
if execution_id:
|
|
updated["executionId"] = execution_id
|
|
if controller_id:
|
|
updated["controllerId"] = controller_id
|
|
if result is not None:
|
|
updated["result"] = result
|
|
statement = (
|
|
update(AppSetting)
|
|
.where(
|
|
AppSetting.key == _RECOVERY_STATE_SETTING,
|
|
AppSetting.updated_at == row.updated_at,
|
|
func.coalesce(
|
|
AppSetting.value_json["version"].as_integer(), 0
|
|
)
|
|
== current_version,
|
|
)
|
|
.values(value_json=updated, updated_at=now)
|
|
)
|
|
update_result = session.exec(statement)
|
|
if update_result.rowcount == 1:
|
|
session.commit()
|
|
tracked.clear()
|
|
tracked.update(updated)
|
|
return updated
|
|
session.rollback()
|
|
raise RuntimeError("PrivateMuse recovery state changed concurrently")
|
|
|
|
@staticmethod
|
|
def _require_owner_state(
|
|
session: Session,
|
|
tracked: dict[str, Any],
|
|
*,
|
|
expected_state: str,
|
|
execution_id: str = "",
|
|
) -> dict[str, Any]:
|
|
session.expire_all()
|
|
row = session.get(AppSetting, _RECOVERY_STATE_SETTING)
|
|
current = dict(row.value_json or {}) if row else {}
|
|
if (
|
|
current.get("claimId") != tracked.get("claimId")
|
|
or current.get("jobId") != tracked.get("jobId")
|
|
or current.get("state") != expected_state
|
|
or (
|
|
execution_id
|
|
and str(current.get("executionId") or "") != execution_id
|
|
)
|
|
):
|
|
raise RuntimeError("PrivateMuse recovery ownership or phase changed")
|
|
return current
|
|
|
|
@staticmethod
|
|
def _with_tracking(
|
|
result: dict[str, Any],
|
|
tracked: dict[str, Any] | None,
|
|
execution_id: str,
|
|
) -> dict[str, Any]:
|
|
if tracked is None:
|
|
return result
|
|
return {
|
|
**result,
|
|
**({"execution_id": execution_id} if execution_id else {}),
|
|
"original_job_id": str(tracked.get("jobId") or ""),
|
|
}
|
|
|
|
def _record_unknown(
|
|
self,
|
|
*,
|
|
reason: str,
|
|
execution_id: str,
|
|
session: Session | None,
|
|
tracked: dict[str, Any] | None,
|
|
) -> dict[str, Any]:
|
|
result = self._with_tracking(
|
|
{
|
|
"status": "outcome_unknown",
|
|
"reason": reason,
|
|
**({"execution_id": execution_id} if execution_id else {}),
|
|
},
|
|
tracked,
|
|
execution_id,
|
|
)
|
|
if session is not None and tracked is not None:
|
|
persisted = self._persist_state(
|
|
session,
|
|
tracked,
|
|
state="outcome_unknown",
|
|
execution_id=execution_id,
|
|
result=result,
|
|
)
|
|
if persisted.get("state") == "completed" and isinstance(
|
|
persisted.get("result"), dict
|
|
):
|
|
return dict(persisted["result"])
|
|
return result
|
|
|
|
def _record_terminal(
|
|
self,
|
|
result: dict[str, Any],
|
|
*,
|
|
execution_id: str,
|
|
session: Session | None,
|
|
tracked: dict[str, Any] | None,
|
|
) -> dict[str, Any]:
|
|
result = self._with_tracking(result, tracked, execution_id)
|
|
if session is not None and tracked is not None:
|
|
persisted = self._persist_state(
|
|
session,
|
|
tracked,
|
|
state="completed",
|
|
execution_id=execution_id,
|
|
result=result,
|
|
)
|
|
if isinstance(persisted.get("result"), dict):
|
|
return dict(persisted["result"])
|
|
return result
|
|
|
|
@staticmethod
|
|
def _exit_result(exit_code: int) -> dict[str, Any]:
|
|
if exit_code == 0:
|
|
return {
|
|
"status": "reconciled",
|
|
"repository": _REPOSITORY,
|
|
"live_revision": _LIVE_REVISION,
|
|
"failed_revision": _FAILED_REVISION,
|
|
"containers": list(_CONTAINERS),
|
|
"compose_project": "privatemuse",
|
|
}
|
|
if exit_code == 35:
|
|
return {"status": "already-enrolled", "repository": _REPOSITORY}
|
|
|
|
reasons = {
|
|
20: "deployment controller executable is unavailable or unsafe",
|
|
21: "deployment projects registry is unavailable or unsafe",
|
|
22: "deployment projects registry cannot be read",
|
|
23: "PrivateMuse deployment target is missing",
|
|
24: "PrivateMuse deployment target differs from the bounded contract",
|
|
25: "mounted PrivateMuse bare repository is unavailable",
|
|
26: "known PrivateMuse recovery revisions cannot be resolved",
|
|
27: "PrivateMuse main changed after the bounded recovery contract was recorded",
|
|
28: "known PrivateMuse live revision is not an ancestor of the failed revision",
|
|
29: "deployment controller verifier cannot be loaded",
|
|
30: "deployment lock files cannot be opened",
|
|
31: "deployment locks cannot be acquired",
|
|
32: "deployment projects registry changed or became unreadable while waiting",
|
|
33: "PrivateMuse deployment target changed while recovery was waiting",
|
|
34: "PrivateMuse deployment state is unavailable or invalid",
|
|
36: (
|
|
"controller could not re-attest the current PrivateMuse runtime "
|
|
"against recorded immutable image IDs"
|
|
),
|
|
37: "PrivateMuse receipt identity changed during re-attestation",
|
|
38: (
|
|
"PrivateMuse immutable container/image provenance differs from "
|
|
"the recorded receipt"
|
|
),
|
|
39: "PrivateMuse container identity drift differs from the known rollback incident",
|
|
40: "fixed PrivateMuse re-attestation contract was altered",
|
|
41: "PrivateMuse recorded live artifacts differ from the known rollback incident",
|
|
42: "re-attested PrivateMuse deployment state could not be written durably",
|
|
43: "controller rejected the repaired PrivateMuse receipt after write",
|
|
44: "PrivateMuse deployment state differs from the exact recoverable incident",
|
|
45: (
|
|
"controller rejected the repaired PrivateMuse receipt and the "
|
|
"original state could not be restored"
|
|
),
|
|
}
|
|
return {
|
|
"status": "validation_failed",
|
|
"reason": reasons.get(
|
|
exit_code, "PrivateMuse runtime re-attestation failed closed"
|
|
),
|
|
}
|
|
|
|
def _inspect_tracked_execution(
|
|
self,
|
|
*,
|
|
config: Any,
|
|
current_controller_id: str,
|
|
session: Session,
|
|
tracked: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
execution_id = str(tracked.get("executionId") or "")
|
|
original_controller_id = str(tracked.get("controllerId") or "")
|
|
if not execution_id:
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse recovery crossed its durable reservation boundary "
|
|
"without a recorded execution ID; do not retry"
|
|
),
|
|
execution_id="",
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if original_controller_id != current_controller_id:
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse recovery controller identity changed while the "
|
|
"existing execution outcome remained unknown; do not retry"
|
|
),
|
|
execution_id=execution_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
endpoint = quote(str(config.endpoint_id), safe="")
|
|
safe_exec = quote(execution_id, safe="")
|
|
try:
|
|
inspected = self.portainer._request(
|
|
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
|
|
).json()
|
|
except (RuntimeError, ValueError, TypeError) as exc:
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse runtime re-attestation outcome remains unknown: "
|
|
f"{redact_message(str(exc))}; inspect the existing execution and do not retry"
|
|
),
|
|
execution_id=execution_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if (
|
|
str(inspected.get("ID") or "") != execution_id
|
|
or str(inspected.get("ContainerID") or "") != original_controller_id
|
|
):
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse recovery execution identity could not be verified; "
|
|
"do not retry"
|
|
),
|
|
execution_id=execution_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if bool(inspected.get("Running")):
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse runtime re-attestation is still running; inspect the "
|
|
"existing execution and do not retry"
|
|
),
|
|
execution_id=execution_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if tracked.get("state") == "created":
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse recovery execution was created but its start boundary "
|
|
"was not durably confirmed; do not retry"
|
|
),
|
|
execution_id=execution_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
raw_exit = inspected.get("ExitCode")
|
|
exit_code = int(raw_exit) if raw_exit is not None else -1
|
|
return self._record_terminal(
|
|
self._exit_result(exit_code),
|
|
execution_id=execution_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
|
|
def run(
|
|
self,
|
|
*,
|
|
session: Session | None = None,
|
|
job_id: str | None = None,
|
|
) -> dict[str, Any]:
|
|
preexisting: dict[str, Any] | None = None
|
|
tracked: dict[str, Any] | None = None
|
|
owns_reservation = False
|
|
if session is not None:
|
|
if not job_id:
|
|
raise ValueError("job_id is required for durable recovery execution")
|
|
session.expire_all()
|
|
row = session.get(AppSetting, _RECOVERY_STATE_SETTING)
|
|
state = dict(row.value_json or {}) if row else {}
|
|
if state.get("state") == "completed":
|
|
stored_result = state.get("result")
|
|
if isinstance(stored_result, dict):
|
|
return dict(stored_result)
|
|
raise RuntimeError(
|
|
"PrivateMuse recovery has a terminal guard without a durable result"
|
|
)
|
|
if state.get("state") not in _ACTIVE_RECOVERY_STATES:
|
|
raise RuntimeError(
|
|
"PrivateMuse recovery requires durable job admission before execution"
|
|
)
|
|
tracked = state
|
|
owns_reservation = (
|
|
state.get("state") == "reserved" and state.get("jobId") == job_id
|
|
)
|
|
if state.get("state") == "reserved" and not owns_reservation:
|
|
return self._with_tracking(
|
|
{
|
|
"status": "in_progress",
|
|
"reason": (
|
|
"PrivateMuse recovery is durably admitted and awaiting its "
|
|
"owning worker; no second execution was started"
|
|
),
|
|
},
|
|
tracked,
|
|
"",
|
|
)
|
|
if not owns_reservation:
|
|
preexisting = state
|
|
|
|
status = self.portainer.get_container_status(_CONTROLLER)
|
|
if status.get("status") != "ok":
|
|
if session is not None and preexisting is not None:
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse recovery outcome remains unknown because the "
|
|
"deployment controller is unavailable; do not retry"
|
|
),
|
|
execution_id=str(preexisting.get("executionId") or ""),
|
|
session=session,
|
|
tracked=preexisting,
|
|
)
|
|
unavailable = {
|
|
"status": "controller_unavailable",
|
|
"reason": redact_message(
|
|
str(status.get("reason") or status.get("status") or "unavailable")
|
|
),
|
|
}
|
|
return (
|
|
self._record_terminal(
|
|
unavailable,
|
|
execution_id="",
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if tracked is not None
|
|
else unavailable
|
|
)
|
|
controller = dict(status.get("container") or {})
|
|
if (
|
|
controller.get("name") != _CONTROLLER
|
|
or controller.get("state") != "running"
|
|
):
|
|
if session is not None and preexisting is not None:
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse recovery outcome remains unknown because the "
|
|
"deployment controller is not running; do not retry"
|
|
),
|
|
execution_id=str(preexisting.get("executionId") or ""),
|
|
session=session,
|
|
tracked=preexisting,
|
|
)
|
|
unavailable = {
|
|
"status": "controller_unavailable",
|
|
"reason": "deployment controller is not running",
|
|
}
|
|
return (
|
|
self._record_terminal(
|
|
unavailable,
|
|
execution_id="",
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if tracked is not None
|
|
else unavailable
|
|
)
|
|
container_id = str(controller.get("id") or "")
|
|
if not container_id:
|
|
if session is not None and preexisting is not None:
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse recovery outcome remains unknown because the "
|
|
"deployment controller identity is unavailable; do not retry"
|
|
),
|
|
execution_id=str(preexisting.get("executionId") or ""),
|
|
session=session,
|
|
tracked=preexisting,
|
|
)
|
|
unavailable = {
|
|
"status": "controller_unavailable",
|
|
"reason": "deployment controller identity is unavailable",
|
|
}
|
|
return (
|
|
self._record_terminal(
|
|
unavailable,
|
|
execution_id="",
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if tracked is not None
|
|
else unavailable
|
|
)
|
|
config = self.portainer._config()
|
|
if not config:
|
|
if session is not None and preexisting is not None:
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse recovery outcome remains unknown because "
|
|
"Portainer deployment control is unavailable; do not retry"
|
|
),
|
|
execution_id=str(preexisting.get("executionId") or ""),
|
|
session=session,
|
|
tracked=preexisting,
|
|
)
|
|
unavailable = {
|
|
"status": "connector_unavailable",
|
|
"reason": "Portainer deployment control is unavailable",
|
|
}
|
|
return (
|
|
self._record_terminal(
|
|
unavailable,
|
|
execution_id="",
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if tracked is not None
|
|
else unavailable
|
|
)
|
|
|
|
if session is not None:
|
|
if preexisting is not None:
|
|
return self._inspect_tracked_execution(
|
|
config=config,
|
|
current_controller_id=container_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if tracked is None:
|
|
raise RuntimeError("PrivateMuse recovery admission state is unavailable")
|
|
tracked = self._persist_state(
|
|
session,
|
|
tracked,
|
|
state="reserved",
|
|
controller_id=container_id,
|
|
)
|
|
|
|
command = [
|
|
"python3",
|
|
"-c",
|
|
_REATTEST_SCRIPT,
|
|
_REPOSITORY,
|
|
_LIVE_REVISION,
|
|
_FAILED_REVISION,
|
|
_FAILURE_MESSAGE,
|
|
]
|
|
endpoint = quote(str(config.endpoint_id), safe="")
|
|
safe_container = quote(container_id, safe="")
|
|
exec_id = ""
|
|
start_requested = False
|
|
try:
|
|
if session is not None and tracked is not None:
|
|
self._require_owner_state(
|
|
session,
|
|
tracked,
|
|
expected_state="reserved",
|
|
)
|
|
created = self.portainer._request(
|
|
"POST",
|
|
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
|
json={
|
|
"AttachStdout": False,
|
|
"AttachStderr": False,
|
|
"Tty": False,
|
|
"Cmd": command,
|
|
},
|
|
).json()
|
|
exec_id = str(created.get("Id") or "")
|
|
if not exec_id:
|
|
return self._record_terminal(
|
|
{
|
|
"status": "error",
|
|
"reason": "PrivateMuse runtime re-attestation could not start",
|
|
},
|
|
execution_id="",
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if session is not None and tracked is not None:
|
|
self._persist_state(
|
|
session,
|
|
tracked,
|
|
state="created",
|
|
execution_id=exec_id,
|
|
)
|
|
safe_exec = quote(exec_id, safe="")
|
|
if session is not None and tracked is not None:
|
|
self._require_owner_state(
|
|
session,
|
|
tracked,
|
|
expected_state="created",
|
|
execution_id=exec_id,
|
|
)
|
|
start_requested = True
|
|
self.portainer._request(
|
|
"POST",
|
|
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
|
json={"Detach": True, "Tty": False},
|
|
)
|
|
if session is not None and tracked is not None:
|
|
self._persist_state(
|
|
session,
|
|
tracked,
|
|
state="started",
|
|
execution_id=exec_id,
|
|
)
|
|
deadline = time.monotonic() + 300
|
|
exit_code: int | None = None
|
|
while time.monotonic() < deadline:
|
|
inspected = self.portainer._request(
|
|
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
|
|
).json()
|
|
if (
|
|
str(inspected.get("ID") or "") != exec_id
|
|
or str(inspected.get("ContainerID") or "") != container_id
|
|
):
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse recovery execution identity could not be "
|
|
"verified; do not retry"
|
|
),
|
|
execution_id=exec_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
if not bool(inspected.get("Running")):
|
|
raw_exit = inspected.get("ExitCode")
|
|
exit_code = int(raw_exit) if raw_exit is not None else -1
|
|
break
|
|
time.sleep(1.0)
|
|
except (RuntimeError, ValueError, TypeError) as exc:
|
|
if exec_id and start_requested:
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse runtime re-attestation outcome is unknown after "
|
|
"execution start; inspect the existing execution and do not retry"
|
|
),
|
|
execution_id=exec_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
return self._record_terminal(
|
|
{"status": "error", "reason": redact_message(str(exc))},
|
|
execution_id=exec_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
|
|
if exit_code is None:
|
|
return self._record_unknown(
|
|
reason=(
|
|
"PrivateMuse runtime re-attestation is still running; inspect the "
|
|
"existing execution and do not retry"
|
|
),
|
|
execution_id=exec_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
return self._record_terminal(
|
|
self._exit_result(exit_code),
|
|
execution_id=exec_id,
|
|
session=session,
|
|
tracked=tracked,
|
|
)
|
|
|
|
|
|
def privatemuse_recovery_job_cancellation_reason(
|
|
session: Session, job: JobRun
|
|
) -> str | None:
|
|
if job.action_id != "deployment.target.register_privatemuse":
|
|
return None
|
|
row = session.get(AppSetting, _RECOVERY_STATE_SETTING)
|
|
state = dict(row.value_json or {}) if row else {}
|
|
if (
|
|
state.get("state") not in _ACTIVE_RECOVERY_STATES
|
|
or state.get("jobId") != job.id
|
|
):
|
|
return None
|
|
return (
|
|
"This job owns an unresolved detached PrivateMuse recovery execution; "
|
|
"reconcile that execution before changing the job record."
|
|
)
|