Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d

backend/app/services/deployment_target_registry.py

@@ -0,0 +1,537 @@
+from __future__ import annotations
+
+import json
+import re
+import time
+from typing import Any
+from urllib.parse import quote
+
+from ..connectors.portainer import PortainerConnector
+from ..core.redaction import redact_message
+
+_REPOSITORY = re.compile(r"^jens/[a-z0-9_.-]+$")
+_REVISION = re.compile(r"^[0-9a-f]{40}$")
+_BRANCH = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,119}$")
+_SLUG = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,62}$")
+_COMPOSE_PROJECT = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
+_CONTAINER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
+_CONTROLLER = "gitea-deploy-control"
+_MAX_CONTAINERS = 32
+
+# Fixed application-owned mutation. Callers never provide a command or host
+# path. Repository-derived topology is validated before it reaches this script,
+# then re-bound to the exact immutable Git revision inside the controller.
+_REGISTER_SCRIPT = r'''
+import fcntl
+import json
+import os
+from pathlib import Path
+import shutil
+import stat
+import subprocess
+import sys
+import time
+
+path = Path("/opt/gitea-deploy/projects.json")
+host_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
+lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
+repository, revision, branch, raw_project = sys.argv[1:5]
+project = json.loads(raw_project)
+
+if path.is_symlink() or not path.is_file():
+    raise SystemExit(20)
+info = path.stat()
+if not stat.S_ISREG(info.st_mode):
+    raise SystemExit(21)
+
+try:
+    if host_path.is_symlink() or not host_path.is_file() or not os.path.samefile(path, host_path):
+        raise SystemExit(36)
+except OSError:
+    raise SystemExit(36)
+mutation_path = host_path
+
+if project.get("branch") != branch:
+    raise SystemExit(37)
+compose_files = project.get("compose_files")
+if not isinstance(compose_files, list) or len(compose_files) != 1:
+    raise SystemExit(38)
+compose_file = compose_files[0]
+if not isinstance(compose_file, str) or not compose_file:
+    raise SystemExit(38)
+
+owner, name = repository.split("/", 1)
+bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
+if not bare_repo.is_dir():
+    raise SystemExit(24)
+
+def git_run(*args):
+    return subprocess.run(
+        ["git", f"--git-dir={bare_repo}", *args],
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        timeout=10,
+        check=False,
+    )
+
+try:
+    commit_result = git_run("rev-parse", f"{revision}^{{commit}}")
+except Exception:
+    raise SystemExit(25)
+commit = commit_result.stdout.strip()
+if commit_result.returncode != 0 or commit != revision:
+    raise SystemExit(25)
+
+try:
+    branch_result = git_run("rev-parse", f"refs/heads/{branch}")
+except Exception:
+    raise SystemExit(26)
+branch_head = branch_result.stdout.strip()
+if branch_result.returncode != 0 or not branch_head:
+    raise SystemExit(26)
+
+try:
+    ancestry = git_run("merge-base", "--is-ancestor", revision, branch_head)
+except Exception:
+    raise SystemExit(27)
+if ancestry.returncode != 0:
+    raise SystemExit(27)
+
+try:
+    compose = git_run("cat-file", "-e", f"{revision}:{compose_file}")
+except Exception:
+    raise SystemExit(28)
+if compose.returncode != 0:
+    raise SystemExit(28)
+
+try:
+    lock_handle = lock_path.open("a+", encoding="utf-8")
+except Exception:
+    raise SystemExit(29)
+
+with lock_handle as lock:
+    try:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+    except Exception:
+        raise SystemExit(30)
+
+    try:
+        projects = json.loads(mutation_path.read_text(encoding="utf-8"))
+    except Exception:
+        raise SystemExit(31)
+    if not isinstance(projects, dict):
+        raise SystemExit(22)
+
+    current = projects.get(repository)
+    changed = current is None
+    if current is not None:
+        if not isinstance(current, dict):
+            raise SystemExit(23)
+        allowed_extra_keys = {
+            "env_source",
+            "environment",
+            "gitea_repository_id",
+            "gitea_webhook_secret_file",
+            "validation",
+        }
+        extra_keys = set(current) - set(project)
+        if not extra_keys.issubset(allowed_extra_keys):
+            raise SystemExit(23)
+        canonical_current = {key: current.get(key) for key in project}
+        if canonical_current != project:
+            raise SystemExit(23)
+        env_source = current.get("env_source")
+        if env_source is not None:
+            if (
+                not isinstance(env_source, str)
+                or not env_source.startswith("/srv/example/appdata/")
+                or len(env_source) > 512
+                or ".." in Path(env_source).parts
+            ):
+                raise SystemExit(23)
+        environment = current.get("environment")
+        if environment is not None:
+            if not isinstance(environment, dict) or len(environment) > 64:
+                raise SystemExit(23)
+            for key, value in environment.items():
+                if (
+                    not isinstance(key, str)
+                    or not key
+                    or len(key) > 128
+                    or not isinstance(value, str)
+                    or len(value) > 4096
+                ):
+                    raise SystemExit(23)
+        runnerless_keys = {
+            "gitea_repository_id",
+            "gitea_webhook_secret_file",
+            "validation",
+        }
+        present_runnerless_keys = runnerless_keys.intersection(current)
+        if present_runnerless_keys and present_runnerless_keys != runnerless_keys:
+            raise SystemExit(23)
+        repository_id = current.get("gitea_repository_id")
+        if repository_id is not None and (
+            not isinstance(repository_id, int)
+            or isinstance(repository_id, bool)
+            or repository_id <= 0
+        ):
+            raise SystemExit(23)
+        webhook_secret_file = current.get("gitea_webhook_secret_file")
+        if webhook_secret_file is not None:
+            expected_secret_file = repository.replace("/", "__")
+            if webhook_secret_file != expected_secret_file:
+                raise SystemExit(23)
+        validation = current.get("validation")
+        if validation is not None:
+            expected_validation = {
+                "validation_gate_version": 1,
+                "required": True,
+                "contexts": ["Managed validation / full (push)"],
+            "timeout_seconds": 3600 if repository == "jens/mediarr" else 1800,
+                "poll_interval_seconds": 5,
+                "max_poll_interval_seconds": 30,
+            }
+            if validation != expected_validation:
+                raise SystemExit(23)
+
+    if changed:
+        backup = mutation_path.with_name(
+            f"projects.json.appops-backup-{int(time.time())}-{os.getpid()}"
+        )
+        try:
+            shutil.copy2(mutation_path, backup)
+        except Exception:
+            raise SystemExit(32)
+
+        updated = dict(projects)
+        updated[repository] = project
+        temporary = mutation_path.with_name(f".{mutation_path.name}.{os.getpid()}.tmp")
+        payload = json.dumps(updated, indent=2, sort_keys=True) + "\n"
+        try:
+            with temporary.open("w", encoding="utf-8") as handle:
+                handle.write(payload)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.chmod(temporary, stat.S_IMODE(info.st_mode))
+        except Exception:
+            try:
+                temporary.unlink(missing_ok=True)
+            except Exception:
+                pass
+            raise SystemExit(33)
+
+        try:
+            os.replace(temporary, mutation_path)
+        except Exception:
+            try:
+                temporary.unlink(missing_ok=True)
+            except Exception:
+                pass
+            raise SystemExit(34)
+
+        directory_fd = None
+        try:
+            directory_fd = os.open(str(mutation_path.parent), os.O_RDONLY)
+            os.fsync(directory_fd)
+        except Exception:
+            raise SystemExit(35)
+        finally:
+            if directory_fd is not None:
+                os.close(directory_fd)
+
+    print("registered" if changed else "already-registered", flush=True)
+'''
+
+
+def _safe_compose_file(value: str) -> str:
+    path = str(value or "").strip().replace("\\", "/")
+    if not path or path.startswith("/") or len(path) > 240:
+        raise ValueError("Compose file must be a bounded repository-relative path")
+    parts = path.split("/")
+    if any(part in {"", ".", ".."} for part in parts):
+        raise ValueError("Compose file path cannot contain empty or traversal segments")
+    if not path.lower().endswith((".yml", ".yaml")):
+        raise ValueError("Compose file must use a .yml or .yaml suffix")
+    return path
+
+
+class DeploymentTargetRegistry:
+    """Narrow bridge for exact-SHA Compose deployment target onboarding."""
+
+    def __init__(self, portainer: PortainerConnector | None = None):
+        self.portainer = portainer or PortainerConnector()
+
+    @staticmethod
+    def _slug(repository: str) -> str:
+        return repository.split("/", 1)[1]
+
+    @staticmethod
+    def default_compose_project(repository: str) -> str:
+        raw = DeploymentTargetRegistry._slug(repository).lower()
+        value = re.sub(r"[^a-z0-9_-]+", "-", raw).strip("-_")
+        if not value:
+            raise ValueError("Repository name cannot be converted to a Compose project")
+        return value[:63]
+
+    @staticmethod
+    def _project(
+        repository: str,
+        *,
+        branch: str,
+        compose_file: str,
+        compose_project: str,
+        containers: list[str],
+        health_timeout: int,
+        verify_oci_revision: bool,
+    ) -> dict[str, object]:
+        slug = DeploymentTargetRegistry._slug(repository)
+        return {
+            "branch": branch,
+            "slug": slug,
+            "source_dir": f"/mnt/cache/appdata/gitea-deploy-runners/sources/{slug}",
+            "exact_source": True,
+            "compose_files": [compose_file],
+            "compose_project": compose_project,
+            "containers": containers,
+            "verify_oci_revision": verify_oci_revision,
+            "health_timeout": health_timeout,
+        }
+
+    @staticmethod
+    def validate(
+        repository: str,
+        revision: str,
+        *,
+        branch: str = "main",
+        compose_file: str = "docker-compose.yml",
+        compose_project: str | None = None,
+        containers: list[str] | None = None,
+        health_timeout: int = 180,
+    ) -> tuple[str, str, str, str, str, list[str], int]:
+        normalized_repository = str(repository or "").strip().lower()
+        normalized_revision = str(revision or "").strip().lower()
+        normalized_branch = str(branch or "").strip()
+        if not _REPOSITORY.fullmatch(normalized_repository):
+            raise ValueError("Repository must be a canonical Jens owner/name identifier")
+        slug = DeploymentTargetRegistry._slug(normalized_repository)
+        if not _SLUG.fullmatch(slug):
+            raise ValueError("Repository name cannot be used as a bounded deployment slug")
+        if not _REVISION.fullmatch(normalized_revision):
+            raise ValueError("Revision must be a full lowercase 40-character Git SHA")
+        if (
+            not _BRANCH.fullmatch(normalized_branch)
+            or ".." in normalized_branch
+            or "//" in normalized_branch
+            or normalized_branch.endswith("/")
+        ):
+            raise ValueError("Default branch is outside the bounded Git ref contract")
+        normalized_compose_file = _safe_compose_file(compose_file)
+        normalized_project = str(
+            compose_project
+            or DeploymentTargetRegistry.default_compose_project(normalized_repository)
+        ).strip().lower()
+        if not _COMPOSE_PROJECT.fullmatch(normalized_project):
+            raise ValueError("Compose project is outside the bounded naming contract")
+        normalized_containers = [str(item or "").strip() for item in (containers or [slug])]
+        if not normalized_containers or len(normalized_containers) > _MAX_CONTAINERS:
+            raise ValueError("Deployment target must contain between 1 and 32 containers")
+        if any(not _CONTAINER.fullmatch(item) for item in normalized_containers):
+            raise ValueError("Container name is outside the bounded naming contract")
+        if len(set(normalized_containers)) != len(normalized_containers):
+            raise ValueError("Container names must be unique")
+        timeout = int(health_timeout)
+        if timeout < 30 or timeout > 900:
+            raise ValueError("Health timeout must be between 30 and 900 seconds")
+        return (
+            normalized_repository,
+            normalized_revision,
+            normalized_branch,
+            normalized_compose_file,
+            normalized_project,
+            normalized_containers,
+            timeout,
+        )
+
+    def register_compose_target(
+        self,
+        repository: str,
+        revision: str,
+        *,
+        branch: str = "main",
+        compose_file: str = "docker-compose.yml",
+        compose_project: str | None = None,
+        containers: list[str] | None = None,
+        health_timeout: int = 180,
+        verify_oci_revision: bool = False,
+    ) -> dict[str, Any]:
+        try:
+            (
+                repository,
+                revision,
+                branch,
+                compose_file,
+                compose_project,
+                containers,
+                health_timeout,
+            ) = self.validate(
+                repository,
+                revision,
+                branch=branch,
+                compose_file=compose_file,
+                compose_project=compose_project,
+                containers=containers,
+                health_timeout=health_timeout,
+            )
+        except (TypeError, ValueError) as exc:
+            return {"status": "invalid", "reason": str(exc)}
+
+        status = self.portainer.get_container_status(_CONTROLLER)
+        if status.get("status") != "ok":
+            return {
+                "status": "controller_unavailable",
+                "reason": redact_message(
+                    str(status.get("reason") or status.get("status") or "unavailable")
+                ),
+            }
+        controller = dict(status.get("container") or {})
+        if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
+            return {
+                "status": "controller_unavailable",
+                "reason": "deployment controller is not running",
+            }
+        container_id = str(controller.get("id") or "")
+        config = self.portainer._config()
+        if not config or not container_id:
+            return {
+                "status": "connector_unavailable",
+                "reason": "Portainer deployment control is unavailable",
+            }
+
+        project = self._project(
+            repository,
+            branch=branch,
+            compose_file=compose_file,
+            compose_project=compose_project,
+            containers=containers,
+            health_timeout=health_timeout,
+            verify_oci_revision=bool(verify_oci_revision),
+        )
+        command = [
+            "python3",
+            "-c",
+            _REGISTER_SCRIPT,
+            repository,
+            revision,
+            branch,
+            json.dumps(project, separators=(",", ":"), sort_keys=True),
+        ]
+        endpoint = quote(str(config.endpoint_id), safe="")
+        safe_container = quote(container_id, safe="")
+        try:
+            created = self.portainer._request(
+                "POST",
+                f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
+                json={
+                    "AttachStdout": False,
+                    "AttachStderr": False,
+                    "Tty": False,
+                    "Cmd": command,
+                },
+            ).json()
+            exec_id = str(created.get("Id") or "")
+            if not exec_id:
+                return {
+                    "status": "error",
+                    "reason": "deployment target registration could not start",
+                }
+            safe_exec = quote(exec_id, safe="")
+            self.portainer._request(
+                "POST",
+                f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
+                json={"Detach": True, "Tty": False},
+            )
+
+            deadline = time.monotonic() + 25
+            exit_code: int | None = None
+            while time.monotonic() < deadline:
+                inspected = self.portainer._request(
+                    "GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
+                ).json()
+                if not bool(inspected.get("Running")):
+                    raw_exit = inspected.get("ExitCode")
+                    exit_code = int(raw_exit) if raw_exit is not None else -1
+                    break
+                time.sleep(0.25)
+        except (RuntimeError, ValueError, TypeError) as exc:
+            return {"status": "error", "reason": redact_message(str(exc))}
+
+        if exit_code is None:
+            return {
+                "status": "error",
+                "reason": "deployment target registration timed out",
+            }
+        if exit_code == 23:
+            return {
+                "status": "conflict",
+                "reason": "repository already has a different deployment target",
+            }
+        preflight_reasons = {
+            20: "deployment projects registry is unavailable or unsafe",
+            21: "deployment projects registry is not a regular file",
+            22: "deployment projects registry does not contain an object",
+            24: "mounted bare repository is unavailable",
+            25: "exact revision cannot be resolved in the mounted bare repository",
+            26: "default branch cannot be resolved in the mounted bare repository",
+            27: "exact revision is not an ancestor of the default branch in the mounted bare repository",
+            28: "configured Compose file is absent at the exact revision",
+            29: "deployment registry lock file cannot be opened",
+            30: "deployment registry lock cannot be acquired",
+            31: "deployment projects registry cannot be read as JSON",
+            32: "deployment projects registry backup cannot be created",
+            33: "deployment projects registry replacement file cannot be durably written",
+            34: "deployment projects registry replacement cannot be installed atomically",
+            35: "deployment projects registry directory durability sync failed after replacement",
+            36: "deployment projects registry host alias is unavailable or does not match the controller registry",
+            37: "controller project branch does not match the requested default branch",
+            38: "controller project Compose contract is invalid",
+        }
+        if exit_code in preflight_reasons:
+            return {
+                "status": "validation_failed",
+                "reason": preflight_reasons[exit_code],
+            }
+        if exit_code != 0:
+            return {
+                "status": "validation_failed",
+                "reason": "controller rejected the bounded repository/revision preflight; projects.json was not changed",
+            }
+        return {
+            "status": "registered",
+            "repository": repository,
+            "revision": revision,
+            "branch": branch,
+            "slug": self._slug(repository),
+            "compose_file": compose_file,
+            "compose_project": compose_project,
+            "container": containers[0] if len(containers) == 1 else None,
+            "containers": containers,
+        }
+
+    def register_single_container(
+        self,
+        repository: str,
+        revision: str,
+        *,
+        branch: str = "main",
+        compose_file: str = "docker-compose.yml",
+    ) -> dict[str, Any]:
+        slug = str(repository or "").strip().lower().split("/", 1)[-1]
+        return self.register_compose_target(
+            repository,
+            revision,
+            branch=branch,
+            compose_file=compose_file,
+            containers=[slug],
+        )