Files
Unraid-AppOps-Gateway-Public/backend/app/services/deployment_target_registry.py
T

700 lines
29 KiB
Python

from __future__ import annotations
import json
import re
import time
from typing import Any
from urllib.parse import quote
from ..connectors.portainer import PortainerConnector
from ..core.redaction import redact_message
_REPOSITORY = re.compile(r"^jens/[a-z0-9_.-]+$")
_REVISION = re.compile(r"^[0-9a-f]{40}$")
_BRANCH = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,119}$")
_SLUG = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,62}$")
_COMPOSE_PROJECT = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
_CONTAINER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
_CONTROLLER = "gitea-deploy-control"
_MAX_CONTAINERS = 32
# Fixed application-owned mutation. Callers never provide a command or host
# path. Repository-derived topology is validated before it reaches this script,
# then re-bound to the exact immutable Git revision inside the controller.
_REGISTER_SCRIPT = r"""
import fcntl
import json
import os
from pathlib import Path
import re
import stat
import subprocess
import sys
import time
path = Path("/controller-config/projects.json")
host_path = Path("/controller-hub-releases/controller-config/projects.json")
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
repository, revision, branch, raw_project = sys.argv[1:5]
project = json.loads(raw_project)
requested_project = project
strengthening_deadline = time.monotonic() + 20
def _validation_compatible(repository, validation):
expected_validation = {
"validation_gate_version": 1,
"required": True,
"contexts": ["Managed validation / full (push)"],
"timeout_seconds": 3600 if repository == "jens/mediarr" else 1800,
"poll_interval_seconds": 5,
"max_poll_interval_seconds": 30,
}
if not isinstance(validation, dict) or validation.get("required") is not True or any(
type(validation.get(key)) is not int
for key in ("validation_gate_version", "timeout_seconds", "poll_interval_seconds", "max_poll_interval_seconds")
):
return False
if validation == expected_validation:
return True
return repository == "jens/mediarr" and validation == {
**expected_validation,
"contexts": ["CI / gates (push)", "Managed validation / full (push)"],
}
def _source_verified_strengthening():
# Only an exact protected-main descriptor can strengthen an existing target.
# The controller reader uses fixed credential files and private stdin.
if branch != "main" or time.monotonic() >= strengthening_deadline:
return False
try:
fresh = git_run("rev-parse", "refs/heads/main")
if fresh.returncode or fresh.stdout.strip() != revision:
return False
result = git_run("show", f"{revision}:.itworx/deployment-target.json")
if result.returncode or len(result.stdout.encode("utf-8")) > 16384:
return False
def unique(pairs):
value = {}
for key, item in pairs:
if key in value:
raise ValueError("duplicate descriptor field")
value[key] = item
return value
descriptor = json.loads(result.stdout, object_pairs_hook=unique)
allowed = {"apiVersion", "applicationId", "composeFile", "composeProject",
"healthTimeout", "primaryService", "verifyOciRevision"}
if (not isinstance(descriptor, dict) or set(descriptor) - allowed
or descriptor.get("apiVersion") != "itworx.deployment-target/v1"
or descriptor.get("verifyOciRevision") is not True
or descriptor.get("composeFile") != compose_file
or descriptor.get("composeProject", descriptor.get("applicationId")) != requested_project["compose_project"]
or type(descriptor.get("healthTimeout", 180)) is not int
or descriptor.get("healthTimeout", 180) != requested_project["health_timeout"]
or not isinstance(descriptor.get("applicationId"), str)
or not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", descriptor["applicationId"])
or not isinstance(descriptor.get("primaryService", descriptor["applicationId"]), str)
or not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", descriptor.get("primaryService", descriptor["applicationId"]))):
return False
sys.path.insert(0, "/opt/gitea-deploy")
from status_server import _registered_legacy_api_reader
protected = _registered_legacy_api_reader(repository)(
f"/repos/{repository}/branches/main", query={},
)
return (time.monotonic() < strengthening_deadline
and isinstance(protected, dict) and protected.get("protected") is True
and isinstance(protected.get("commit"), dict)
and protected["commit"].get("id") == revision)
except Exception:
return False
for directory in (path.parent, host_path.parent, host_path.parent.parent):
info = directory.lstat()
if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
raise SystemExit(21)
for registry_path in (path, host_path):
info = registry_path.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or stat.S_IMODE(info.st_mode) != 0o600:
raise SystemExit(21)
try:
if host_path.is_symlink() or not host_path.is_file() or not os.path.samefile(path, host_path):
raise SystemExit(36)
except OSError:
raise SystemExit(36)
mutation_path = host_path
if project.get("branch") != branch:
raise SystemExit(37)
compose_files = project.get("compose_files")
if not isinstance(compose_files, list) or len(compose_files) != 1:
raise SystemExit(38)
compose_file = compose_files[0]
if not isinstance(compose_file, str) or not compose_file:
raise SystemExit(38)
owner, name = repository.split("/", 1)
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
if not bare_repo.is_dir():
raise SystemExit(24)
def git_run(*args):
# Trust only this validated controller-owned mount, preserving Git's normal
# checks elsewhere. Always read original objects rather than replacement refs.
for directory in (bare_repo.parent.parent, bare_repo.parent, bare_repo):
info = directory.lstat()
if (not stat.S_ISDIR(info.st_mode) or info.st_uid not in {0, 1000}
or info.st_mode & 0o022):
raise ValueError("unsafe mounted repository")
return subprocess.run(
["git", "--no-replace-objects", "-c", f"safe.directory={bare_repo}",
f"--git-dir={bare_repo}", *args],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
timeout=10,
check=False,
)
try:
commit_result = git_run("rev-parse", f"{revision}^{{commit}}")
except Exception:
raise SystemExit(25)
commit = commit_result.stdout.strip()
if commit_result.returncode != 0 or commit != revision:
raise SystemExit(25)
try:
branch_result = git_run("rev-parse", f"refs/heads/{branch}")
except Exception:
raise SystemExit(26)
branch_head = branch_result.stdout.strip()
if branch_result.returncode != 0 or not branch_head:
raise SystemExit(26)
try:
ancestry = git_run("merge-base", "--is-ancestor", revision, branch_head)
except Exception:
raise SystemExit(27)
if ancestry.returncode != 0:
raise SystemExit(27)
try:
compose = git_run("cat-file", "-e", f"{revision}:{compose_file}")
except Exception:
raise SystemExit(28)
if compose.returncode != 0:
raise SystemExit(28)
try:
lock_handle = lock_path.open("a+", encoding="utf-8")
except Exception:
raise SystemExit(29)
with lock_handle as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX)
except Exception:
raise SystemExit(30)
try:
registry_info = mutation_path.lstat()
registry_identity = (registry_info.st_dev, registry_info.st_ino, registry_info.st_mode,
registry_info.st_uid, registry_info.st_gid, registry_info.st_mtime_ns,
registry_info.st_ctime_ns, registry_info.st_size)
if not stat.S_ISREG(registry_info.st_mode) or registry_info.st_uid != 0 or stat.S_IMODE(registry_info.st_mode) != 0o600:
raise SystemExit(31)
descriptor = os.open(mutation_path, os.O_RDONLY | os.O_NOFOLLOW)
with os.fdopen(descriptor, "rb") as stream:
original = stream.read(4 * 1024 * 1024 + 1)
if len(original) > 4 * 1024 * 1024:
raise SystemExit(31)
projects = json.loads(original)
except Exception:
raise SystemExit(31)
if not isinstance(projects, dict):
raise SystemExit(22)
current = projects.get(repository)
changed = current is None
if current is not None:
if not isinstance(current, dict):
raise SystemExit(23)
allowed_extra_keys = {
"env_source",
"environment",
"gitea_repository_id",
"gitea_webhook_secret_file",
"validation",
"legacy_containers",
}
extra_keys = set(current) - set(project)
if not extra_keys.issubset(allowed_extra_keys):
raise SystemExit(23)
immutable_keys = set(project) - {"source_dir", "exact_source", "verify_oci_revision", "health_timeout"}
if any(current.get(key) != project[key] for key in immutable_keys):
raise SystemExit(23)
# Preserve the registered source mode and fixed historical appdata path.
# Registration must never silently migrate an existing runtime's source.
source = current.get("source_dir")
known_sources = {project["source_dir"], f"/srv/example/appdata/{project['slug']}/source"}
exact_source = current.get("exact_source", False)
verification = current.get("verify_oci_revision", False)
health_timeout = current.get("health_timeout", 180)
if (not isinstance(source, str) or source not in known_sources or type(exact_source) is not bool
or type(verification) is not bool or type(project["verify_oci_revision"]) is not bool
or type(health_timeout) is not int or health_timeout != project["health_timeout"]
or verification and project["verify_oci_revision"] is not True):
raise SystemExit(23)
legacy_containers = current.get("legacy_containers")
if legacy_containers is not None and (
not isinstance(legacy_containers, list) or len(legacy_containers) > 32
or any(not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", name)
for name in legacy_containers)
or len(set(legacy_containers)) != len(legacy_containers)
or set(legacy_containers).intersection(project["containers"])
):
raise SystemExit(23)
env_source = current.get("env_source")
if env_source is not None:
if (
not isinstance(env_source, str)
or not env_source.startswith("/srv/example/appdata/")
or len(env_source) > 512
or ".." in Path(env_source).parts
):
raise SystemExit(23)
environment = current.get("environment")
if environment is not None:
if not isinstance(environment, dict) or len(environment) > 64:
raise SystemExit(23)
for key, value in environment.items():
if (
not isinstance(key, str)
or not key
or len(key) > 128
or not isinstance(value, str)
or len(value) > 4096
):
raise SystemExit(23)
runnerless_keys = {
"gitea_repository_id",
"gitea_webhook_secret_file",
"validation",
}
present_runnerless_keys = runnerless_keys.intersection(current)
if present_runnerless_keys and present_runnerless_keys != runnerless_keys:
raise SystemExit(23)
repository_id = current.get("gitea_repository_id")
if repository_id is not None and (
not isinstance(repository_id, int)
or isinstance(repository_id, bool)
or repository_id <= 0
):
raise SystemExit(23)
webhook_secret_file = current.get("gitea_webhook_secret_file")
if webhook_secret_file is not None:
expected_secret_file = repository.replace("/", "__")
if webhook_secret_file != expected_secret_file:
raise SystemExit(23)
validation = current.get("validation")
if validation is not None:
if not _validation_compatible(repository, validation):
raise SystemExit(23)
strengthened = not verification and project["verify_oci_revision"] is True
if strengthened and not _source_verified_strengthening():
raise SystemExit(39)
reconciled = dict(current)
if strengthened:
reconciled["verify_oci_revision"] = True
changed = reconciled != current
project = reconciled
if changed:
try:
backup_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state/registry-backups")
for directory in (backup_root.parent, backup_root.parent.parent):
directory_info = directory.lstat()
if not stat.S_ISDIR(directory_info.st_mode) or directory_info.st_uid != 0 or directory_info.st_mode & 0o022:
raise SystemExit(32)
backup_root.mkdir(mode=0o700, exist_ok=True)
directory_info = backup_root.lstat()
if not stat.S_ISDIR(directory_info.st_mode) or directory_info.st_uid != 0 or stat.S_IMODE(directory_info.st_mode) != 0o700:
raise SystemExit(32)
backup = backup_root / f"projects.json.appops-backup-{time.time_ns()}-{os.getpid()}"
fd = os.open(backup, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, "wb") as handle:
handle.write(original)
handle.flush()
os.fsync(handle.fileno())
for directory in (backup_root.parent, backup_root):
fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
os.fsync(fd)
finally:
os.close(fd)
except Exception:
raise SystemExit(32)
updated = dict(projects)
updated[repository] = project
temporary = mutation_path.with_name(f".{mutation_path.name}.{os.getpid()}.tmp")
payload = json.dumps(updated, indent=2, sort_keys=True) + "\n"
try:
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
handle.write(payload)
handle.flush()
os.fsync(handle.fileno())
except Exception:
try:
temporary.unlink(missing_ok=True)
except Exception:
pass
raise SystemExit(33)
try:
if current is not None and strengthened and not _source_verified_strengthening():
temporary.unlink(missing_ok=True)
raise SystemExit(39)
latest = mutation_path.lstat()
latest_identity = (latest.st_dev, latest.st_ino, latest.st_mode, latest.st_uid,
latest.st_gid, latest.st_mtime_ns, latest.st_ctime_ns, latest.st_size)
if latest_identity != registry_identity or mutation_path.read_bytes() != original or not os.path.samefile(path, mutation_path):
raise SystemExit(34)
if current is not None and strengthened and time.monotonic() >= strengthening_deadline:
temporary.unlink(missing_ok=True)
raise SystemExit(39)
os.replace(temporary, mutation_path)
except Exception:
try:
temporary.unlink(missing_ok=True)
except Exception:
pass
raise SystemExit(34)
directory_fd = None
try:
directory_fd = os.open(str(mutation_path.parent), os.O_RDONLY)
os.fsync(directory_fd)
except Exception:
raise SystemExit(35)
finally:
if directory_fd is not None:
os.close(directory_fd)
print("registered" if changed else "already-registered", flush=True)
"""
def _safe_compose_file(value: str) -> str:
path = str(value or "").strip().replace("\\", "/")
if not path or path.startswith("/") or len(path) > 240:
raise ValueError("Compose file must be a bounded repository-relative path")
parts = path.split("/")
if any(part in {"", ".", ".."} for part in parts):
raise ValueError("Compose file path cannot contain empty or traversal segments")
if not path.lower().endswith((".yml", ".yaml")):
raise ValueError("Compose file must use a .yml or .yaml suffix")
return path
class DeploymentTargetRegistry:
"""Narrow bridge for exact-SHA Compose deployment target onboarding."""
def __init__(self, portainer: PortainerConnector | None = None):
self.portainer = portainer or PortainerConnector()
@staticmethod
def _slug(repository: str) -> str:
return repository.split("/", 1)[1]
@staticmethod
def default_compose_project(repository: str) -> str:
raw = DeploymentTargetRegistry._slug(repository).lower()
value = re.sub(r"[^a-z0-9_-]+", "-", raw).strip("-_")
if not value:
raise ValueError("Repository name cannot be converted to a Compose project")
return value[:63]
@staticmethod
def _project(
repository: str,
*,
branch: str,
compose_file: str,
compose_project: str,
containers: list[str],
health_timeout: int,
verify_oci_revision: bool,
) -> dict[str, object]:
slug = DeploymentTargetRegistry._slug(repository)
return {
"branch": branch,
"slug": slug,
"source_dir": f"/mnt/cache/appdata/gitea-deploy-runners/sources/{slug}",
"exact_source": True,
"compose_files": [compose_file],
"compose_project": compose_project,
"containers": containers,
"verify_oci_revision": verify_oci_revision,
"health_timeout": health_timeout,
}
@staticmethod
def validate(
repository: str,
revision: str,
*,
branch: str = "main",
compose_file: str = "docker-compose.yml",
compose_project: str | None = None,
containers: list[str] | None = None,
health_timeout: int = 180,
) -> tuple[str, str, str, str, str, list[str], int]:
normalized_repository = str(repository or "").strip().lower()
normalized_revision = str(revision or "").strip().lower()
normalized_branch = str(branch or "").strip()
if not _REPOSITORY.fullmatch(normalized_repository):
raise ValueError(
"Repository must be a canonical Jens owner/name identifier"
)
slug = DeploymentTargetRegistry._slug(normalized_repository)
if not _SLUG.fullmatch(slug):
raise ValueError(
"Repository name cannot be used as a bounded deployment slug"
)
if not _REVISION.fullmatch(normalized_revision):
raise ValueError("Revision must be a full lowercase 40-character Git SHA")
if (
not _BRANCH.fullmatch(normalized_branch)
or ".." in normalized_branch
or "//" in normalized_branch
or normalized_branch.endswith("/")
):
raise ValueError("Default branch is outside the bounded Git ref contract")
normalized_compose_file = _safe_compose_file(compose_file)
normalized_project = (
str(
compose_project
or DeploymentTargetRegistry.default_compose_project(
normalized_repository
)
)
.strip()
.lower()
)
if not _COMPOSE_PROJECT.fullmatch(normalized_project):
raise ValueError("Compose project is outside the bounded naming contract")
normalized_containers = [
str(item or "").strip() for item in (containers or [slug])
]
if not normalized_containers or len(normalized_containers) > _MAX_CONTAINERS:
raise ValueError(
"Deployment target must contain between 1 and 32 containers"
)
if any(not _CONTAINER.fullmatch(item) for item in normalized_containers):
raise ValueError("Container name is outside the bounded naming contract")
if len(set(normalized_containers)) != len(normalized_containers):
raise ValueError("Container names must be unique")
timeout = int(health_timeout)
if timeout < 30 or timeout > 900:
raise ValueError("Health timeout must be between 30 and 900 seconds")
return (
normalized_repository,
normalized_revision,
normalized_branch,
normalized_compose_file,
normalized_project,
normalized_containers,
timeout,
)
def register_compose_target(
self,
repository: str,
revision: str,
*,
branch: str = "main",
compose_file: str = "docker-compose.yml",
compose_project: str | None = None,
containers: list[str] | None = None,
health_timeout: int = 180,
verify_oci_revision: bool = False,
) -> dict[str, Any]:
try:
if type(verify_oci_revision) is not bool:
raise ValueError("OCI revision verification must be a boolean")
(
repository,
revision,
branch,
compose_file,
compose_project,
containers,
health_timeout,
) = self.validate(
repository,
revision,
branch=branch,
compose_file=compose_file,
compose_project=compose_project,
containers=containers,
health_timeout=health_timeout,
)
except (TypeError, ValueError) as exc:
return {"status": "invalid", "reason": str(exc)}
status = self.portainer.get_container_status(_CONTROLLER)
if status.get("status") != "ok":
return {
"status": "controller_unavailable",
"reason": redact_message(
str(status.get("reason") or status.get("status") or "unavailable")
),
}
controller = dict(status.get("container") or {})
if (
controller.get("name") != _CONTROLLER
or controller.get("state") != "running"
):
return {
"status": "controller_unavailable",
"reason": "deployment controller is not running",
}
container_id = str(controller.get("id") or "")
config = self.portainer._config()
if not config or not container_id:
return {
"status": "connector_unavailable",
"reason": "Portainer deployment control is unavailable",
}
project = self._project(
repository,
branch=branch,
compose_file=compose_file,
compose_project=compose_project,
containers=containers,
health_timeout=health_timeout,
verify_oci_revision=bool(verify_oci_revision),
)
command = [
"python3",
"-c",
_REGISTER_SCRIPT,
repository,
revision,
branch,
json.dumps(project, separators=(",", ":"), sort_keys=True),
]
endpoint = quote(str(config.endpoint_id), safe="")
safe_container = quote(container_id, safe="")
try:
created = self.portainer._request(
"POST",
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
json={
"AttachStdout": False,
"AttachStderr": False,
"Tty": False,
"Cmd": command,
},
).json()
exec_id = str(created.get("Id") or "")
if not exec_id:
return {
"status": "error",
"reason": "deployment target registration could not start",
}
safe_exec = quote(exec_id, safe="")
self.portainer._request(
"POST",
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
json={"Detach": True, "Tty": False},
)
deadline = time.monotonic() + 25
exit_code: int | None = None
while time.monotonic() < deadline:
inspected = self.portainer._request(
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
).json()
if not bool(inspected.get("Running")):
raw_exit = inspected.get("ExitCode")
exit_code = int(raw_exit) if raw_exit is not None else -1
break
time.sleep(0.25)
except (RuntimeError, ValueError, TypeError) as exc:
return {"status": "error", "reason": redact_message(str(exc))}
if exit_code is None:
return {
"status": "error",
"reason": "deployment target registration timed out",
}
if exit_code == 23:
return {
"status": "conflict",
"reason": "repository already has a different deployment target",
}
preflight_reasons = {
20: "deployment projects registry is unavailable or unsafe",
21: "deployment projects registry is not a regular file",
22: "deployment projects registry does not contain an object",
24: "mounted bare repository is unavailable",
25: "exact revision cannot be resolved in the mounted bare repository",
26: "default branch cannot be resolved in the mounted bare repository",
27: "exact revision is not an ancestor of the default branch in the mounted bare repository",
28: "configured Compose file is absent at the exact revision",
29: "deployment registry lock file cannot be opened",
30: "deployment registry lock cannot be acquired",
31: "deployment projects registry cannot be read as JSON",
32: "deployment projects registry backup cannot be created",
33: "deployment projects registry replacement file cannot be durably written",
34: "deployment projects registry replacement cannot be installed atomically",
35: "deployment projects registry directory durability sync failed after replacement",
36: "deployment projects registry host alias is unavailable or does not match the controller registry",
37: "controller project branch does not match the requested default branch",
38: "controller project Compose contract is invalid",
39: "existing target verification strengthening requires its exact protected-main source descriptor",
}
if exit_code in preflight_reasons:
return {
"status": "validation_failed",
"reason": preflight_reasons[exit_code],
}
if exit_code != 0:
return {
"status": "validation_failed",
"reason": "controller rejected the bounded repository/revision preflight; projects.json was not changed",
}
return {
"status": "registered",
"repository": repository,
"revision": revision,
"branch": branch,
"slug": self._slug(repository),
"compose_file": compose_file,
"compose_project": compose_project,
"container": containers[0] if len(containers) == 1 else None,
"containers": containers,
}
def register_single_container(
self,
repository: str,
revision: str,
*,
branch: str = "main",
compose_file: str = "docker-compose.yml",
) -> dict[str, Any]:
slug = str(repository or "").strip().lower().split("/", 1)[-1]
return self.register_compose_target(
repository,
revision,
branch=branch,
compose_file=compose_file,
containers=[slug],
)