547 lines
21 KiB
Python
547 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
from typing import Any
|
|
from urllib.parse import quote
|
|
|
|
from ..connectors.portainer import PortainerConnector
|
|
from ..core.redaction import redact_message
|
|
|
|
_CONTROLLER = "gitea-deploy-control"
|
|
_REPOSITORY = "jens/privatemuse"
|
|
_LEGACY_REVISION = "715f7d152d35745521a2e014e334a1509bdce6a2"
|
|
|
|
_PROJECT: dict[str, object] = {
|
|
"branch": "main",
|
|
"slug": "privatemuse",
|
|
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/privatemuse",
|
|
"exact_source": True,
|
|
"env_source": "/srv/example/appdata/privatemuse/.env",
|
|
"compose_files": ["docker-compose.yml", "docker-compose.unraid.yml"],
|
|
"compose_project": "privatemuse",
|
|
"containers": [
|
|
"privatemuse-backend-1",
|
|
"privatemuse-frontend-1",
|
|
"privatemuse-audio-enhancement-1",
|
|
],
|
|
"verify_oci_revision": False,
|
|
"health_timeout": 180,
|
|
"environment": {"PRIVATEMUSE_RELEASE": "{short_revision}"},
|
|
}
|
|
|
|
# This is a fixed, application-owned one-time migration. Callers cannot supply
|
|
# commands, paths, container names, Compose files or deployment JSON. It uses
|
|
# the same controller-owned registry, global production lock and official
|
|
# deploy.py bootstrap-legacy path as the deployment broker itself.
|
|
_ENROLL_SCRIPT = r'''
|
|
import fcntl
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
registry_path = Path("/controller-config/projects.json")
|
|
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
|
|
state_root = Path("/mnt/cache/appdata/gitea-deploy-runners/state")
|
|
state_path = state_root / "privatemuse.json"
|
|
project_lock_path = state_root / "locks" / "privatemuse.lock"
|
|
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
|
repository, legacy_revision, raw_project = sys.argv[1:4]
|
|
project = json.loads(raw_project)
|
|
|
|
expected_project = {
|
|
"branch": "main",
|
|
"slug": "privatemuse",
|
|
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/privatemuse",
|
|
"exact_source": True,
|
|
"env_source": "/srv/example/appdata/privatemuse/.env",
|
|
"compose_files": ["docker-compose.yml", "docker-compose.unraid.yml"],
|
|
"compose_project": "privatemuse",
|
|
"containers": [
|
|
"privatemuse-backend-1",
|
|
"privatemuse-frontend-1",
|
|
"privatemuse-audio-enhancement-1",
|
|
],
|
|
"verify_oci_revision": False,
|
|
"health_timeout": 180,
|
|
"environment": {"PRIVATEMUSE_RELEASE": "{short_revision}"},
|
|
}
|
|
expected_legacy = "715f7d152d35745521a2e014e334a1509bdce6a2"
|
|
stale_audio = {
|
|
"privatemuse-audio-runtime-base-1",
|
|
"privatemuse-audio-speech-1",
|
|
"privatemuse-audio-music-1",
|
|
"privatemuse-audio-sound-1",
|
|
"privatemuse-audio-transcription-1",
|
|
"privatemuse-audio-lipsync-1",
|
|
}
|
|
|
|
if repository != "jens/privatemuse" or legacy_revision != expected_legacy or project != expected_project:
|
|
raise SystemExit(40)
|
|
if registry_path.is_symlink() or not registry_path.is_file():
|
|
raise SystemExit(20)
|
|
registry_stat = registry_path.stat()
|
|
if not stat.S_ISREG(registry_stat.st_mode):
|
|
raise SystemExit(21)
|
|
try:
|
|
if (
|
|
host_registry_path.is_symlink()
|
|
or not host_registry_path.is_file()
|
|
or not os.path.samefile(registry_path, host_registry_path)
|
|
):
|
|
raise SystemExit(36)
|
|
except OSError:
|
|
raise SystemExit(36)
|
|
|
|
env_source = Path(str(project["env_source"]))
|
|
if env_source.is_symlink() or not env_source.is_file():
|
|
raise SystemExit(41)
|
|
|
|
owner, name = repository.split("/", 1)
|
|
bare_repo = Path("/gitea-repositories") / owner / f"{name}.git"
|
|
if not bare_repo.is_dir():
|
|
raise SystemExit(24)
|
|
|
|
def git_run(*args):
|
|
return subprocess.run(
|
|
["git", f"--git-dir={bare_repo}", *args],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
|
|
legacy = git_run("rev-parse", f"{legacy_revision}^{{commit}}")
|
|
if legacy.returncode != 0 or legacy.stdout.strip() != legacy_revision:
|
|
raise SystemExit(25)
|
|
main = git_run("rev-parse", "refs/heads/main")
|
|
main_revision = main.stdout.strip()
|
|
if main.returncode != 0 or len(main_revision) != 40:
|
|
raise SystemExit(26)
|
|
if git_run("merge-base", "--is-ancestor", legacy_revision, main_revision).returncode != 0:
|
|
raise SystemExit(27)
|
|
for compose_file in project["compose_files"]:
|
|
if git_run("cat-file", "-e", f"{main_revision}:{compose_file}").returncode != 0:
|
|
raise SystemExit(28)
|
|
|
|
try:
|
|
if state_root.is_symlink() or not state_root.is_dir():
|
|
raise SystemExit(50)
|
|
project_lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if project_lock_path.parent.is_symlink() or project_lock_path.is_symlink():
|
|
raise SystemExit(29)
|
|
project_lock_handle = project_lock_path.open("a+", encoding="utf-8")
|
|
lock_handle = lock_path.open("a+", encoding="utf-8")
|
|
except Exception:
|
|
raise SystemExit(29)
|
|
|
|
with project_lock_handle as project_lock, lock_handle as lock:
|
|
try:
|
|
fcntl.flock(project_lock, fcntl.LOCK_EX)
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
except Exception:
|
|
raise SystemExit(30)
|
|
|
|
try:
|
|
projects = json.loads(host_registry_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(31)
|
|
if not isinstance(projects, dict):
|
|
raise SystemExit(22)
|
|
|
|
current = projects.get(repository)
|
|
canonical_target_match = current == project
|
|
if current is not None and current != project:
|
|
if not isinstance(current, dict):
|
|
raise SystemExit(54)
|
|
if current.get("appops_managed_target") is True:
|
|
raise SystemExit(55)
|
|
target_fields = [
|
|
"branch",
|
|
"slug",
|
|
"source_dir",
|
|
"exact_source",
|
|
"env_source",
|
|
"compose_files",
|
|
"compose_project",
|
|
"containers",
|
|
"verify_oci_revision",
|
|
"health_timeout",
|
|
"environment",
|
|
]
|
|
for index, key in enumerate(target_fields, start=60):
|
|
if key not in current or current.get(key) != project[key]:
|
|
raise SystemExit(index)
|
|
extra_keys = set(current) - set(project)
|
|
allowed_metadata_keys = {
|
|
"gitea_repository_id",
|
|
"gitea_webhook_secret_file",
|
|
"validation",
|
|
}
|
|
expected_validation = {
|
|
"validation_gate_version": 1,
|
|
"required": True,
|
|
"contexts": ["Managed validation / full (push)"],
|
|
"timeout_seconds": 1800,
|
|
"poll_interval_seconds": 5,
|
|
"max_poll_interval_seconds": 30,
|
|
}
|
|
repository_id = current.get("gitea_repository_id")
|
|
if (
|
|
extra_keys != allowed_metadata_keys
|
|
or not isinstance(repository_id, int)
|
|
or isinstance(repository_id, bool)
|
|
or repository_id <= 0
|
|
or current.get("gitea_webhook_secret_file") != "jens__privatemuse"
|
|
or current.get("validation") != expected_validation
|
|
):
|
|
raise SystemExit(71)
|
|
canonical_target_match = True
|
|
|
|
existing_state = None
|
|
if state_path.exists():
|
|
if state_path.is_symlink() or not state_path.is_file():
|
|
raise SystemExit(42)
|
|
try:
|
|
existing_state = json.loads(state_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(42)
|
|
if not isinstance(existing_state, dict):
|
|
raise SystemExit(42)
|
|
if existing_state.get("schema_version") == 3:
|
|
if existing_state.get("repository") != repository:
|
|
raise SystemExit(43)
|
|
if not canonical_target_match:
|
|
raise SystemExit(23)
|
|
print("already-enrolled", flush=True)
|
|
raise SystemExit(0)
|
|
if not (
|
|
existing_state.get("repository") == repository
|
|
and existing_state.get("status") == "success"
|
|
and existing_state.get("revision") == legacy_revision
|
|
and "schema_version" not in existing_state
|
|
and "live_receipt" not in existing_state
|
|
and "live_legacy_provenance" not in existing_state
|
|
):
|
|
raise SystemExit(43)
|
|
|
|
expected_names = set(project["containers"])
|
|
observed_result = subprocess.run(
|
|
[
|
|
"docker",
|
|
"ps",
|
|
"--all",
|
|
"--filter",
|
|
f"label=com.docker.compose.project={project['compose_project']}",
|
|
"--format",
|
|
"{{.Names}}",
|
|
],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=15,
|
|
check=False,
|
|
)
|
|
if observed_result.returncode != 0:
|
|
raise SystemExit(44)
|
|
observed_names = {line.strip() for line in observed_result.stdout.splitlines() if line.strip()}
|
|
unexpected = observed_names - expected_names - stale_audio
|
|
if unexpected:
|
|
raise SystemExit(45)
|
|
|
|
for required in sorted(expected_names):
|
|
inspected = subprocess.run(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
"{{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{index .Config.Labels \"com.docker.compose.project\"}}",
|
|
required,
|
|
],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
if inspected.returncode != 0:
|
|
raise SystemExit(46)
|
|
try:
|
|
running, health, compose_project = inspected.stdout.strip().split("|", 2)
|
|
except ValueError:
|
|
raise SystemExit(46)
|
|
if running != "true" or health in {"starting", "unhealthy"} or compose_project != "privatemuse":
|
|
raise SystemExit(46)
|
|
|
|
# Remove only known, stopped provider containers. Images, named volumes,
|
|
# model directories and persistent media are deliberately left untouched.
|
|
for stale in sorted(observed_names & stale_audio):
|
|
inspected = subprocess.run(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
"{{.State.Running}}|{{index .Config.Labels \"com.docker.compose.project\"}}",
|
|
stale,
|
|
],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
if inspected.returncode != 0:
|
|
raise SystemExit(47)
|
|
try:
|
|
running, compose_project = inspected.stdout.strip().split("|", 1)
|
|
except ValueError:
|
|
raise SystemExit(47)
|
|
if running == "true" or compose_project != "privatemuse":
|
|
raise SystemExit(48)
|
|
removed = subprocess.run(
|
|
["docker", "rm", stale],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=20,
|
|
check=False,
|
|
)
|
|
if removed.returncode != 0:
|
|
raise SystemExit(49)
|
|
|
|
if current is None:
|
|
backup = host_registry_path.with_name(
|
|
f"projects.json.privatemuse-backup-{int(time.time())}-{os.getpid()}"
|
|
)
|
|
try:
|
|
shutil.copy2(host_registry_path, backup)
|
|
except Exception:
|
|
raise SystemExit(32)
|
|
updated = dict(projects)
|
|
updated[repository] = project
|
|
temporary = host_registry_path.with_name(f".{host_registry_path.name}.{os.getpid()}.tmp")
|
|
payload = json.dumps(updated, indent=2, sort_keys=True) + "\n"
|
|
try:
|
|
with temporary.open("w", encoding="utf-8") as handle:
|
|
handle.write(payload)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.chmod(temporary, stat.S_IMODE(registry_stat.st_mode))
|
|
os.replace(temporary, host_registry_path)
|
|
directory_fd = os.open(str(host_registry_path.parent), os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
except Exception:
|
|
try:
|
|
temporary.unlink(missing_ok=True)
|
|
except Exception:
|
|
pass
|
|
raise SystemExit(34)
|
|
|
|
if existing_state is None:
|
|
state_root.mkdir(parents=True, exist_ok=True)
|
|
if state_root.is_symlink() or not state_root.is_dir():
|
|
raise SystemExit(50)
|
|
temporary_state = state_path.with_name(f".{state_path.name}.{os.getpid()}.tmp")
|
|
payload = {
|
|
"repository": repository,
|
|
"revision": legacy_revision,
|
|
"status": "success",
|
|
}
|
|
try:
|
|
with temporary_state.open("x", encoding="utf-8") as handle:
|
|
handle.write(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.chmod(temporary_state, 0o600)
|
|
os.replace(temporary_state, state_path)
|
|
directory_fd = os.open(str(state_root), os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
except Exception:
|
|
try:
|
|
temporary_state.unlink(missing_ok=True)
|
|
except Exception:
|
|
pass
|
|
raise SystemExit(51)
|
|
|
|
try:
|
|
controller_path = Path("/opt/gitea-deploy/deploy.py")
|
|
if controller_path.is_symlink() or not controller_path.is_file():
|
|
raise RuntimeError("controller module is unavailable")
|
|
controller_spec = importlib.util.spec_from_file_location(
|
|
"appops_locked_privatemuse_bootstrap", controller_path
|
|
)
|
|
if controller_spec is None or controller_spec.loader is None:
|
|
raise RuntimeError("controller module cannot be loaded")
|
|
controller = importlib.util.module_from_spec(controller_spec)
|
|
controller_spec.loader.exec_module(controller)
|
|
controller.bootstrap_legacy_state(repository, project, legacy_revision)
|
|
except Exception:
|
|
raise SystemExit(52)
|
|
|
|
try:
|
|
final_state = json.loads(state_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(53)
|
|
if not (
|
|
isinstance(final_state, dict)
|
|
and final_state.get("schema_version") == 3
|
|
and final_state.get("repository") == repository
|
|
and final_state.get("live_revision") == legacy_revision
|
|
and final_state.get("live_status") == "healthy"
|
|
and final_state.get("live_legacy_provenance") is True
|
|
):
|
|
raise SystemExit(53)
|
|
|
|
print("enrolled", flush=True)
|
|
'''
|
|
|
|
|
|
class PrivateMuseDeploymentEnrollment:
|
|
"""One-time, fail-closed enrollment of the existing PrivateMuse production runtime."""
|
|
|
|
def __init__(self, portainer: PortainerConnector | None = None):
|
|
self.portainer = portainer or PortainerConnector()
|
|
|
|
def run(self) -> dict[str, Any]:
|
|
status = self.portainer.get_container_status(_CONTROLLER)
|
|
if status.get("status") != "ok":
|
|
return {
|
|
"status": "controller_unavailable",
|
|
"reason": redact_message(
|
|
str(status.get("reason") or status.get("status") or "unavailable")
|
|
),
|
|
}
|
|
controller = dict(status.get("container") or {})
|
|
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
|
return {
|
|
"status": "controller_unavailable",
|
|
"reason": "deployment controller is not running",
|
|
}
|
|
container_id = str(controller.get("id") or "")
|
|
config = self.portainer._config()
|
|
if not config or not container_id:
|
|
return {
|
|
"status": "connector_unavailable",
|
|
"reason": "Portainer deployment control is unavailable",
|
|
}
|
|
|
|
command = [
|
|
"python3",
|
|
"-c",
|
|
_ENROLL_SCRIPT,
|
|
_REPOSITORY,
|
|
_LEGACY_REVISION,
|
|
json.dumps(_PROJECT, separators=(",", ":"), sort_keys=True),
|
|
]
|
|
endpoint = quote(str(config.endpoint_id), safe="")
|
|
safe_container = quote(container_id, safe="")
|
|
try:
|
|
created = self.portainer._request(
|
|
"POST",
|
|
f"/api/endpoints/{endpoint}/docker/containers/{safe_container}/exec",
|
|
json={
|
|
"AttachStdout": False,
|
|
"AttachStderr": False,
|
|
"Tty": False,
|
|
"Cmd": command,
|
|
},
|
|
).json()
|
|
exec_id = str(created.get("Id") or "")
|
|
if not exec_id:
|
|
return {"status": "error", "reason": "PrivateMuse enrollment could not start"}
|
|
safe_exec = quote(exec_id, safe="")
|
|
self.portainer._request(
|
|
"POST",
|
|
f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/start",
|
|
json={"Detach": True, "Tty": False},
|
|
)
|
|
|
|
deadline = time.monotonic() + 120
|
|
exit_code: int | None = None
|
|
while time.monotonic() < deadline:
|
|
inspected = self.portainer._request(
|
|
"GET", f"/api/endpoints/{endpoint}/docker/exec/{safe_exec}/json"
|
|
).json()
|
|
if not bool(inspected.get("Running")):
|
|
raw_exit = inspected.get("ExitCode")
|
|
exit_code = int(raw_exit) if raw_exit is not None else -1
|
|
break
|
|
time.sleep(0.25)
|
|
except (RuntimeError, ValueError, TypeError) as exc:
|
|
return {"status": "error", "reason": redact_message(str(exc))}
|
|
|
|
if exit_code is None:
|
|
return {"status": "error", "reason": "PrivateMuse enrollment timed out"}
|
|
reasons = {
|
|
20: "deployment projects registry is unavailable or unsafe",
|
|
21: "deployment projects registry is not a regular file",
|
|
22: "deployment projects registry does not contain an object",
|
|
23: "PrivateMuse already has a different deployment target",
|
|
24: "mounted PrivateMuse bare repository is unavailable",
|
|
25: "verified PrivateMuse legacy revision cannot be resolved",
|
|
26: "PrivateMuse main cannot be resolved",
|
|
27: "verified PrivateMuse legacy revision is not an ancestor of main",
|
|
28: "required PrivateMuse production Compose file is absent from main",
|
|
29: "deployment registry lock file cannot be opened",
|
|
30: "deployment registry lock cannot be acquired",
|
|
31: "deployment projects registry cannot be read as JSON",
|
|
32: "deployment projects registry backup cannot be created",
|
|
34: "deployment projects registry replacement failed",
|
|
36: "deployment projects registry host alias is unavailable or mismatched",
|
|
40: "PrivateMuse fixed enrollment contract was altered",
|
|
41: "PrivateMuse production environment file is unavailable or unsafe",
|
|
42: "PrivateMuse deployment state file is unavailable or invalid",
|
|
43: "PrivateMuse deployment state conflicts with the verified legacy baseline",
|
|
44: "PrivateMuse Compose project inventory could not be read",
|
|
45: "PrivateMuse Compose project contains an unexpected container",
|
|
46: "required PrivateMuse production container is missing or unhealthy",
|
|
47: "stale PrivateMuse audio container could not be inspected",
|
|
48: "a stale PrivateMuse audio provider is still running and was not removed",
|
|
49: "a verified stopped PrivateMuse audio container could not be removed",
|
|
50: "deployment state directory is unavailable or unsafe",
|
|
51: "verified PrivateMuse legacy state could not be written durably",
|
|
52: "deployment controller rejected PrivateMuse legacy-state bootstrap",
|
|
53: "PrivateMuse legacy-state bootstrap did not produce the expected managed state",
|
|
54: "PrivateMuse deployment target is not an object",
|
|
55: "PrivateMuse deployment target is owned by the controller v2 managed-target contract",
|
|
60: "PrivateMuse deployment target differs in branch",
|
|
61: "PrivateMuse deployment target differs in slug",
|
|
62: "PrivateMuse deployment target differs in source_dir",
|
|
63: "PrivateMuse deployment target differs in exact_source",
|
|
64: "PrivateMuse deployment target differs in env_source",
|
|
65: "PrivateMuse deployment target differs in compose_files",
|
|
66: "PrivateMuse deployment target differs in compose_project",
|
|
67: "PrivateMuse deployment target differs in containers",
|
|
68: "PrivateMuse deployment target differs in verify_oci_revision",
|
|
69: "PrivateMuse deployment target differs in health_timeout",
|
|
70: "PrivateMuse deployment target differs in environment",
|
|
71: "PrivateMuse deployment target matches canonical fields but has additional or invalid controller metadata",
|
|
}
|
|
if exit_code != 0:
|
|
return {
|
|
"status": "validation_failed",
|
|
"reason": reasons.get(
|
|
exit_code,
|
|
"PrivateMuse enrollment failed closed before a production application deploy",
|
|
),
|
|
}
|
|
return {
|
|
"status": "enrolled",
|
|
"repository": _REPOSITORY,
|
|
"legacy_revision": _LEGACY_REVISION,
|
|
"compose_project": _PROJECT["compose_project"],
|
|
"containers": list(_PROJECT["containers"]),
|
|
}
|