Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d

backend/app/services/privatemuse_deployment_enrollment.py

@@ -0,0 +1,546 @@
+from __future__ import annotations
+
+import json
+import time
+from typing import Any
+from urllib.parse import quote
+
+from ..connectors.portainer import PortainerConnector
+from ..core.redaction import redact_message
+
+_CONTROLLER = "gitea-deploy-control"
+_REPOSITORY = "jens/privatemuse"
+_LEGACY_REVISION = "715f7d152d35745521a2e014e334a1509bdce6a2"
+
+_PROJECT: dict[str, object] = {
+    "branch": "main",
+    "slug": "privatemuse",
+    "source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/privatemuse",
+    "exact_source": True,
+    "env_source": "/srv/example/appdata/privatemuse/.env",
+    "compose_files": ["docker-compose.yml", "docker-compose.unraid.yml"],
+    "compose_project": "privatemuse",
+    "containers": [
+        "privatemuse-backend-1",
+        "privatemuse-frontend-1",
+        "privatemuse-audio-enhancement-1",
+    ],
+    "verify_oci_revision": False,
+    "health_timeout": 180,
+    "environment": {"PRIVATEMUSE_RELEASE": "{short_revision}"},
+}
+
+# This is a fixed, application-owned one-time migration. Callers cannot supply
+# commands, paths, container names, Compose files or deployment JSON. It uses
+# the same controller-owned registry, global production lock and official
+# deploy.py bootstrap-legacy path as the deployment broker itself.
+_ENROLL_SCRIPT = r'''
+import fcntl
+import importlib.util
+import json
+import os
+from pathlib import Path
+import shutil
+import stat
+import subprocess
+import sys
+import time
+
+registry_path = Path("/opt/gitea-deploy/projects.json")
+host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
+state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
+state_path = state_root / "privatemuse.json"
+project_lock_path = state_root / "locks" / "privatemuse.lock"
+lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
+repository, legacy_revision, raw_project = sys.argv[1:4]
+project = json.loads(raw_project)
+
+expected_project = {
+    "branch": "main",
+    "slug": "privatemuse",
+    "source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/privatemuse",
+    "exact_source": True,
+    "env_source": "/srv/example/appdata/privatemuse/.env",
+    "compose_files": ["docker-compose.yml", "docker-compose.unraid.yml"],
+    "compose_project": "privatemuse",
+    "containers": [
+        "privatemuse-backend-1",
+        "privatemuse-frontend-1",
+        "privatemuse-audio-enhancement-1",
+    ],
+    "verify_oci_revision": False,
+    "health_timeout": 180,
+    "environment": {"PRIVATEMUSE_RELEASE": "{short_revision}"},
+}
+expected_legacy = "715f7d152d35745521a2e014e334a1509bdce6a2"
+stale_audio = {
+    "privatemuse-audio-runtime-base-1",
+    "privatemuse-audio-speech-1",
+    "privatemuse-audio-music-1",
+    "privatemuse-audio-sound-1",
+    "privatemuse-audio-transcription-1",
+    "privatemuse-audio-lipsync-1",
+}
+
+if repository != "jens/privatemuse" or legacy_revision != expected_legacy or project != expected_project:
+    raise SystemExit(40)
+if registry_path.is_symlink() or not registry_path.is_file():
+    raise SystemExit(20)
+registry_stat = registry_path.stat()
+if not stat.S_ISREG(registry_stat.st_mode):
+    raise SystemExit(21)
+try:
+    if (
+        host_registry_path.is_symlink()
+        or not host_registry_path.is_file()
+        or not os.path.samefile(registry_path, host_registry_path)
+    ):
+        raise SystemExit(36)
+except OSError:
+    raise SystemExit(36)
+
+env_source = Path(str(project["env_source"]))
+if env_source.is_symlink() or not env_source.is_file():
+    raise SystemExit(41)
+
+owner, name = repository.split("/", 1)
+bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
+if not bare_repo.is_dir():
+    raise SystemExit(24)
+
+def git_run(*args):
+    return subprocess.run(
+        ["git", f"--git-dir={bare_repo}", *args],
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        timeout=10,
+        check=False,
+    )
+
+legacy = git_run("rev-parse", f"{legacy_revision}^{{commit}}")
+if legacy.returncode != 0 or legacy.stdout.strip() != legacy_revision:
+    raise SystemExit(25)
+main = git_run("rev-parse", "refs/heads/main")
+main_revision = main.stdout.strip()
+if main.returncode != 0 or len(main_revision) != 40:
+    raise SystemExit(26)
+if git_run("merge-base", "--is-ancestor", legacy_revision, main_revision).returncode != 0:
+    raise SystemExit(27)
+for compose_file in project["compose_files"]:
+    if git_run("cat-file", "-e", f"{main_revision}:{compose_file}").returncode != 0:
+        raise SystemExit(28)
+
+try:
+    if state_root.is_symlink() or not state_root.is_dir():
+        raise SystemExit(50)
+    project_lock_path.parent.mkdir(parents=True, exist_ok=True)
+    if project_lock_path.parent.is_symlink() or project_lock_path.is_symlink():
+        raise SystemExit(29)
+    project_lock_handle = project_lock_path.open("a+", encoding="utf-8")
+    lock_handle = lock_path.open("a+", encoding="utf-8")
+except Exception:
+    raise SystemExit(29)
+
+with project_lock_handle as project_lock, lock_handle as lock:
+    try:
+        fcntl.flock(project_lock, fcntl.LOCK_EX)
+        fcntl.flock(lock, fcntl.LOCK_EX)
+    except Exception:
+        raise SystemExit(30)
+
+    try:
+        projects = json.loads(host_registry_path.read_text(encoding="utf-8"))
+    except Exception:
+        raise SystemExit(31)
+    if not isinstance(projects, dict):
+        raise SystemExit(22)
+
+    current = projects.get(repository)
+    canonical_target_match = current == project
+    if current is not None and current != project:
+        if not isinstance(current, dict):
+            raise SystemExit(54)
+        if current.get("appops_managed_target") is True:
+            raise SystemExit(55)
+        target_fields = [
+            "branch",
+            "slug",
+            "source_dir",
+            "exact_source",
+            "env_source",
+            "compose_files",
+            "compose_project",
+            "containers",
+            "verify_oci_revision",
+            "health_timeout",
+            "environment",
+        ]
+        for index, key in enumerate(target_fields, start=60):
+            if key not in current or current.get(key) != project[key]:
+                raise SystemExit(index)
+        extra_keys = set(current) - set(project)
+        allowed_metadata_keys = {
+            "gitea_repository_id",
+            "gitea_webhook_secret_file",
+            "validation",
+        }
+        expected_validation = {
+            "validation_gate_version": 1,
+            "required": True,
+            "contexts": ["Managed validation / full (push)"],
+            "timeout_seconds": 1800,
+            "poll_interval_seconds": 5,
+            "max_poll_interval_seconds": 30,
+        }
+        repository_id = current.get("gitea_repository_id")
+        if (
+            extra_keys != allowed_metadata_keys
+            or not isinstance(repository_id, int)
+            or isinstance(repository_id, bool)
+            or repository_id <= 0
+            or current.get("gitea_webhook_secret_file") != "jens__privatemuse"
+            or current.get("validation") != expected_validation
+        ):
+            raise SystemExit(71)
+        canonical_target_match = True
+
+    existing_state = None
+    if state_path.exists():
+        if state_path.is_symlink() or not state_path.is_file():
+            raise SystemExit(42)
+        try:
+            existing_state = json.loads(state_path.read_text(encoding="utf-8"))
+        except Exception:
+            raise SystemExit(42)
+        if not isinstance(existing_state, dict):
+            raise SystemExit(42)
+        if existing_state.get("schema_version") == 3:
+            if existing_state.get("repository") != repository:
+                raise SystemExit(43)
+            if not canonical_target_match:
+                raise SystemExit(23)
+            print("already-enrolled", flush=True)
+            raise SystemExit(0)
+        if not (
+            existing_state.get("repository") == repository
+            and existing_state.get("status") == "success"
+            and existing_state.get("revision") == legacy_revision
+            and "schema_version" not in existing_state
+            and "live_receipt" not in existing_state
+            and "live_legacy_provenance" not in existing_state
+        ):
+            raise SystemExit(43)
+
+    expected_names = set(project["containers"])
+    observed_result = subprocess.run(
+        [
+            "docker",
+            "ps",
+            "--all",
+            "--filter",
+            f"label=com.docker.compose.project={project['compose_project']}",
+            "--format",
+            "{{.Names}}",
+        ],
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        timeout=15,
+        check=False,
+    )
+    if observed_result.returncode != 0:
+        raise SystemExit(44)
+    observed_names = {line.strip() for line in observed_result.stdout.splitlines() if line.strip()}
+    unexpected = observed_names - expected_names - stale_audio
+    if unexpected:
+        raise SystemExit(45)
+
+    for required in sorted(expected_names):
+        inspected = subprocess.run(
+            [
+                "docker",
+                "inspect",
+                "--format",
+                "{{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{index .Config.Labels \"com.docker.compose.project\"}}",
+                required,
+            ],
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            timeout=10,
+            check=False,
+        )
+        if inspected.returncode != 0:
+            raise SystemExit(46)
+        try:
+            running, health, compose_project = inspected.stdout.strip().split("|", 2)
+        except ValueError:
+            raise SystemExit(46)
+        if running != "true" or health in {"starting", "unhealthy"} or compose_project != "privatemuse":
+            raise SystemExit(46)
+
+    # Remove only known, stopped provider containers. Images, named volumes,
+    # model directories and persistent media are deliberately left untouched.
+    for stale in sorted(observed_names & stale_audio):
+        inspected = subprocess.run(
+            [
+                "docker",
+                "inspect",
+                "--format",
+                "{{.State.Running}}|{{index .Config.Labels \"com.docker.compose.project\"}}",
+                stale,
+            ],
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            timeout=10,
+            check=False,
+        )
+        if inspected.returncode != 0:
+            raise SystemExit(47)
+        try:
+            running, compose_project = inspected.stdout.strip().split("|", 1)
+        except ValueError:
+            raise SystemExit(47)
+        if running == "true" or compose_project != "privatemuse":
+            raise SystemExit(48)
+        removed = subprocess.run(
+            ["docker", "rm", stale],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            timeout=20,
+            check=False,
+        )
+        if removed.returncode != 0:
+            raise SystemExit(49)
+
+    if current is None:
+        backup = host_registry_path.with_name(
+            f"projects.json.privatemuse-backup-{int(time.time())}-{os.getpid()}"
+        )
+        try:
+            shutil.copy2(host_registry_path, backup)
+        except Exception:
+            raise SystemExit(32)
+        updated = dict(projects)
+        updated[repository] = project
+        temporary = host_registry_path.with_name(f".{host_registry_path.name}.{os.getpid()}.tmp")
+        payload = json.dumps(updated, indent=2, sort_keys=True) + "\n"
+        try:
+            with temporary.open("w", encoding="utf-8") as handle:
+                handle.write(payload)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.chmod(temporary, stat.S_IMODE(registry_stat.st_mode))
+            os.replace(temporary, host_registry_path)
+            directory_fd = os.open(str(host_registry_path.parent), os.O_RDONLY)
+            try:
+                os.fsync(directory_fd)
+            finally:
+                os.close(directory_fd)
+        except Exception:
+            try:
+                temporary.unlink(missing_ok=True)
+            except Exception:
+                pass
+            raise SystemExit(34)
+
+    if existing_state is None:
+        state_root.mkdir(parents=True, exist_ok=True)
+        if state_root.is_symlink() or not state_root.is_dir():
+            raise SystemExit(50)
+        temporary_state = state_path.with_name(f".{state_path.name}.{os.getpid()}.tmp")
+        payload = {
+            "repository": repository,
+            "revision": legacy_revision,
+            "status": "success",
+        }
+        try:
+            with temporary_state.open("x", encoding="utf-8") as handle:
+                handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.chmod(temporary_state, 0o600)
+            os.replace(temporary_state, state_path)
+            directory_fd = os.open(str(state_root), os.O_RDONLY)
+            try:
+                os.fsync(directory_fd)
+            finally:
+                os.close(directory_fd)
+        except Exception:
+            try:
+                temporary_state.unlink(missing_ok=True)
+            except Exception:
+                pass
+            raise SystemExit(51)
+
+    try:
+        controller_path = Path("/opt/gitea-deploy/deploy.py")
+        if controller_path.is_symlink() or not controller_path.is_file():
+            raise RuntimeError("controller module is unavailable")
+        controller_spec = importlib.util.spec_from_file_location(
+            "appops_locked_privatemuse_bootstrap", controller_path
+        )
+        if controller_spec is None or controller_spec.loader is None:
+            raise RuntimeError("controller module cannot be loaded")
+        controller = importlib.util.module_from_spec(controller_spec)
+        controller_spec.loader.exec_module(controller)
+        controller.bootstrap_legacy_state(repository, project, legacy_revision)
+    except Exception:
+        raise SystemExit(52)
+
+    try:
+        final_state = json.loads(state_path.read_text(encoding="utf-8"))
+    except Exception:
+        raise SystemExit(53)
+    if not (
+        isinstance(final_state, dict)
+        and final_state.get("schema_version") == 3
+        and final_state.get("repository") == repository
+        and final_state.get("live_revision") == legacy_revision
+        and final_state.get("live_status") == "healthy"
+        and final_state.get("live_legacy_provenance") is True
+    ):
+        raise SystemExit(53)
+
+print("enrolled", flush=True)
+'''
+
+
+class PrivateMuseDeploymentEnrollment:
+    """One-time, fail-closed enrollment of the existing PrivateMuse production runtime."""
+
+    def __init__(self, portainer: PortainerConnector | None = None):
+        self.portainer = portainer or PortainerConnector()
+
+    def run(self) -> dict[str, Any]:
+        status = self.portainer.get_container_status(_CONTROLLER)
+        if status.get("status") != "ok":
+            return {
+                "status": "controller_unavailable",
+                "reason": redact_message(
+                    str(status.get("reason") or status.get("status") or "unavailable")
+                ),
+            }
+        controller = dict(status.get("container") or {})
+        if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
+            return {
+                "status": "controller_unavailable",
+                "reason": "deployment controller is not running",
+            }
+        container_id = str(controller.get("id") or "")
+        config = self.portainer._config()
+        if not config or not container_id:
+            return {
+                "status": "connector_unavailable",
+                "reason": "Portainer deployment control is unavailable",
+            }
+
+        command = [
+            "python3",
+            "-c",
+            _ENROLL_SCRIPT,
+            _REPOSITORY,
+            _LEGACY_REVISION,
+            json.dumps(_PROJECT, separators=(",", ":"), sort_keys=True),
+        ]
+        endpoint = quote(str(config.endpoint_id), safe="")
+        safe_container = quote(container_id, safe="")
+        try:
+            created = self.portainer._request(
+                "POST",
+                f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
+                json={
+                    "AttachStdout": False,
+                    "AttachStderr": False,
+                    "Tty": False,
+                    "Cmd": command,
+                },
+            ).json()
+            exec_id = str(created.get("Id") or "")
+            if not exec_id:
+                return {"status": "error", "reason": "PrivateMuse enrollment could not start"}
+            safe_exec = quote(exec_id, safe="")
+            self.portainer._request(
+                "POST",
+                f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
+                json={"Detach": True, "Tty": False},
+            )
+
+            deadline = time.monotonic() + 120
+            exit_code: int | None = None
+            while time.monotonic() < deadline:
+                inspected = self.portainer._request(
+                    "GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
+                ).json()
+                if not bool(inspected.get("Running")):
+                    raw_exit = inspected.get("ExitCode")
+                    exit_code = int(raw_exit) if raw_exit is not None else -1
+                    break
+                time.sleep(0.25)
+        except (RuntimeError, ValueError, TypeError) as exc:
+            return {"status": "error", "reason": redact_message(str(exc))}
+
+        if exit_code is None:
+            return {"status": "error", "reason": "PrivateMuse enrollment timed out"}
+        reasons = {
+            20: "deployment projects registry is unavailable or unsafe",
+            21: "deployment projects registry is not a regular file",
+            22: "deployment projects registry does not contain an object",
+            23: "PrivateMuse already has a different deployment target",
+            24: "mounted PrivateMuse bare repository is unavailable",
+            25: "verified PrivateMuse legacy revision cannot be resolved",
+            26: "PrivateMuse main cannot be resolved",
+            27: "verified PrivateMuse legacy revision is not an ancestor of main",
+            28: "required PrivateMuse production Compose file is absent from main",
+            29: "deployment registry lock file cannot be opened",
+            30: "deployment registry lock cannot be acquired",
+            31: "deployment projects registry cannot be read as JSON",
+            32: "deployment projects registry backup cannot be created",
+            34: "deployment projects registry replacement failed",
+            36: "deployment projects registry host alias is unavailable or mismatched",
+            40: "PrivateMuse fixed enrollment contract was altered",
+            41: "PrivateMuse production environment file is unavailable or unsafe",
+            42: "PrivateMuse deployment state file is unavailable or invalid",
+            43: "PrivateMuse deployment state conflicts with the verified legacy baseline",
+            44: "PrivateMuse Compose project inventory could not be read",
+            45: "PrivateMuse Compose project contains an unexpected container",
+            46: "required PrivateMuse production container is missing or unhealthy",
+            47: "stale PrivateMuse audio container could not be inspected",
+            48: "a stale PrivateMuse audio provider is still running and was not removed",
+            49: "a verified stopped PrivateMuse audio container could not be removed",
+            50: "deployment state directory is unavailable or unsafe",
+            51: "verified PrivateMuse legacy state could not be written durably",
+            52: "deployment controller rejected PrivateMuse legacy-state bootstrap",
+            53: "PrivateMuse legacy-state bootstrap did not produce the expected managed state",
+            54: "PrivateMuse deployment target is not an object",
+            55: "PrivateMuse deployment target is owned by the controller v2 managed-target contract",
+            60: "PrivateMuse deployment target differs in branch",
+            61: "PrivateMuse deployment target differs in slug",
+            62: "PrivateMuse deployment target differs in source_dir",
+            63: "PrivateMuse deployment target differs in exact_source",
+            64: "PrivateMuse deployment target differs in env_source",
+            65: "PrivateMuse deployment target differs in compose_files",
+            66: "PrivateMuse deployment target differs in compose_project",
+            67: "PrivateMuse deployment target differs in containers",
+            68: "PrivateMuse deployment target differs in verify_oci_revision",
+            69: "PrivateMuse deployment target differs in health_timeout",
+            70: "PrivateMuse deployment target differs in environment",
+            71: "PrivateMuse deployment target matches canonical fields but has additional or invalid controller metadata",
+        }
+        if exit_code != 0:
+            return {
+                "status": "validation_failed",
+                "reason": reasons.get(
+                    exit_code,
+                    "PrivateMuse enrollment failed closed before a production application deploy",
+                ),
+            }
+        return {
+            "status": "enrolled",
+            "repository": _REPOSITORY,
+            "legacy_revision": _LEGACY_REVISION,
+            "compose_project": _PROJECT["compose_project"],
+            "containers": list(_PROJECT["containers"]),
+        }