Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d

backend/app/services/workmesh_first_deployment.py

@@ -0,0 +1,588 @@
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import quote
+
+from sqlmodel import Session
+
+from ..connectors.portainer import PortainerConnector
+from ..core.redaction import redact_message
+from ..core.time import utc_now
+from ..db.models import AppSetting, JobRun
+from .job_runner import set_job_progress
+
+_CONTROLLER = "gitea-deploy-control"
+_REPOSITORY = "jens/workmesh"
+_STATE_KEY = "deployment:workmesh:first-deploy:v1"
+_RELEASE_REVISION = "7dfbdb7d25e0efd70c53106de1d36d3fff6e3a05"
+_CONTAINERS = (
+    "workmesh-postgres-1",
+    "workmesh-mock-ats-1",
+    "workmesh-api-1",
+    "workmesh-n8n-1",
+    "workmesh-workmesh-1",
+)
+_PORTS = (1255, 1256, 1257, 1258)
+
+
+# Fixed first-deploy bootstrap. Callers provide no repository, path, command,
+# port, environment value, or secret. Secret material is generated inside the
+# root-owned deployment controller and is never attached to exec output.
+_FIRST_DEPLOY_SCRIPT = r"""
+import fcntl
+import json
+import os
+from pathlib import Path
+import secrets
+import shutil
+import stat
+import subprocess
+import sys
+import time
+
+repository = "jens/workmesh"
+expected_release_revision = "7dfbdb7d25e0efd70c53106de1d36d3fff6e3a05"
+slug = "workmesh"
+compose_project = "workmesh"
+containers = [
+    "workmesh-postgres-1",
+    "workmesh-mock-ats-1",
+    "workmesh-api-1",
+    "workmesh-n8n-1",
+    "workmesh-workmesh-1",
+]
+ports = {1255, 1256, 1257, 1258}
+controller_root = Path("/opt/gitea-deploy")
+registry_path = controller_root / "projects.json"
+host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")
+lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
+env_dir = Path("/srv/example/appdata/workmesh")
+env_path = env_dir / ".env"
+bare_repo = Path("/gitea-repositories/jens/workmesh.git")
+source_dir = Path("/mnt/cache/appdata/gitea-deploy-runners/sources/workmesh")
+deploy_py = controller_root / "deploy.py"
+
+base_project = {
+    "branch": "main",
+    "slug": slug,
+    "source_dir": str(source_dir),
+    "exact_source": True,
+    "compose_files": ["compose.yaml"],
+    "compose_project": compose_project,
+    "containers": containers,
+    "verify_oci_revision": False,
+    "health_timeout": 300,
+}
+project = {**base_project, "env_source": str(env_path)}
+
+for path in (registry_path, host_registry_path, lock_path, bare_repo, deploy_py):
+    if path.is_symlink():
+        raise SystemExit(20)
+if not registry_path.is_file() or not host_registry_path.is_file() or not deploy_py.is_file():
+    raise SystemExit(20)
+if not bare_repo.is_dir():
+    raise SystemExit(21)
+try:
+    if not os.path.samefile(registry_path, host_registry_path):
+        raise SystemExit(22)
+except OSError:
+    raise SystemExit(22)
+
+
+def git(*args):
+    return subprocess.run(
+        ["git", f"--git-dir={bare_repo}", *args],
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        timeout=15,
+        check=False,
+    )
+
+
+main = git("rev-parse", "refs/heads/main")
+revision = main.stdout.strip()
+if main.returncode != 0 or len(revision) != 40 or revision != expected_release_revision:
+    raise SystemExit(23)
+for required in ("compose.yaml", "VERSION", ".itworx/deployment-target.json"):
+    if git("cat-file", "-e", f"{revision}:{required}").returncode != 0:
+        raise SystemExit(24)
+version = git("show", f"{revision}:VERSION")
+if version.returncode != 0 or version.stdout.strip() != "0.1.0":
+    raise SystemExit(25)
+descriptor = git("show", f"{revision}:.itworx/deployment-target.json")
+try:
+    descriptor_payload = json.loads(descriptor.stdout)
+except Exception:
+    raise SystemExit(26)
+if descriptor_payload != {
+    "apiVersion": "itworx.deployment-target/v1",
+    "applicationId": "workmesh",
+    "composeFile": "compose.yaml",
+    "composeProject": "workmesh",
+    "healthTimeout": 300,
+    "verifyOciRevision": False,
+}:
+    raise SystemExit(26)
+
+# Refuse to take host ports from any non-WorkMesh container, including stopped
+# containers that would become conflicting if restarted later.
+listed = subprocess.run(
+    ["docker", "ps", "-aq"],
+    text=True,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.DEVNULL,
+    timeout=20,
+    check=False,
+)
+if listed.returncode != 0:
+    raise SystemExit(27)
+for container_id in [line.strip() for line in listed.stdout.splitlines() if line.strip()]:
+    inspected = subprocess.run(
+        ["docker", "inspect", container_id],
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        timeout=10,
+        check=False,
+    )
+    if inspected.returncode != 0:
+        raise SystemExit(27)
+    try:
+        item = json.loads(inspected.stdout)[0]
+    except Exception:
+        raise SystemExit(27)
+    labels = dict(dict(item.get("Config") or {}).get("Labels") or {})
+    if labels.get("com.docker.compose.project") == compose_project:
+        continue
+    bindings = dict(dict(item.get("HostConfig") or {}).get("PortBindings") or {})
+    for values in bindings.values():
+        for binding in values or []:
+            raw = str(dict(binding).get("HostPort") or "")
+            if raw.isdigit() and int(raw) in ports:
+                raise SystemExit(28)
+
+required_fixed = {
+    "WORKMESH_ENV": "production",
+    "WORKMESH_API_PORT": "1256",
+    "WORKMESH_WEB_PORT": "1255",
+    "WORKMESH_BIND_HOST": "127.0.0.1",
+    "MOCK_ATS_PORT": "1258",
+    "WORKMESH_POSTGRES_DB": "workmesh",
+    "WORKMESH_POSTGRES_USER": "workmesh",
+    "N8N_BASE_URL": "http://n8n:5678",
+    "WORKMESH_API_BASE_URL": "http://api:8000",
+    "N8N_PORT": "1257",
+    "MOCK_ATS_BASE_URL": "http://mock-ats:8010",
+    "DEMO_CONTROLS_ENABLED": "true",
+    "AI_PROVIDER": "disabled",
+    "AI_MODEL": "qwen3.5:9b",
+    "AI_TIMEOUT_SECONDS": "30",
+    "OLLAMA_BASE_URL": "http://host.docker.internal:11434",
+    "OPENAI_COMPATIBLE_BASE_URL": "",
+    "OPENAI_COMPATIBLE_API_KEY": "",
+    "KNOWLEDGE_PROVIDER": "fixture",
+    "RAGCORE_BASE_URL": "",
+    "RAGCORE_TOKEN": "",
+    "RAGCORE_SPACE_IDS": "",
+    "RAGCORE_PROFILE": "hybrid-v1",
+    "RAGCORE_TIMEOUT_SECONDS": "10",
+    "NEXT_PUBLIC_WORKMESH_API_URL": "http://127.0.0.1:1256",
+    "CORS_ORIGINS": "http://localhost:1255,http://127.0.0.1:1255",
+}
+secret_keys = (
+    "WORKMESH_POSTGRES_PASSWORD",
+    "WORKMESH_N8N_CALLBACK_SECRET",
+    "N8N_ENCRYPTION_KEY",
+)
+
+
+def parse_env(path):
+    values = {}
+    for line in path.read_text(encoding="utf-8").splitlines():
+        if not line or line.startswith("#"):
+            continue
+        if "=" not in line:
+            raise SystemExit(31)
+        key, value = line.split("=", 1)
+        if key in values or not key:
+            raise SystemExit(31)
+        values[key] = value
+    return values
+
+
+if env_path.exists():
+    if env_path.is_symlink() or not env_path.is_file():
+        raise SystemExit(29)
+    info = env_path.stat()
+    if info.st_uid != 0 or stat.S_IMODE(info.st_mode) & 0o077:
+        raise SystemExit(30)
+    env = parse_env(env_path)
+    if set(env) != set(required_fixed) | set(secret_keys):
+        raise SystemExit(31)
+    if any(env.get(key) != value for key, value in required_fixed.items()):
+        raise SystemExit(32)
+    if any(len(env.get(key, "")) < 32 or "change-me" in env.get(key, "").lower() for key in secret_keys):
+        raise SystemExit(33)
+else:
+    env_dir.mkdir(parents=True, exist_ok=True)
+    if env_dir.is_symlink() or not env_dir.is_dir():
+        raise SystemExit(29)
+    generated = {key: secrets.token_urlsafe(48) for key in secret_keys}
+    env = {**required_fixed, **generated}
+    temporary_env = env_dir / f".env.{os.getpid()}.tmp"
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    descriptor_fd = os.open(temporary_env, flags, 0o600)
+    try:
+        with os.fdopen(descriptor_fd, "w", encoding="utf-8") as handle:
+            for key in sorted(env):
+                handle.write(f"{key}={env[key]}\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary_env, env_path)
+        os.chmod(env_path, 0o600)
+        directory_fd = os.open(env_dir, os.O_RDONLY)
+        try:
+            os.fsync(directory_fd)
+        finally:
+            os.close(directory_fd)
+    finally:
+        try:
+            temporary_env.unlink(missing_ok=True)
+        except Exception:
+            pass
+
+try:
+    lock_handle = lock_path.open("a+", encoding="utf-8")
+except Exception:
+    raise SystemExit(34)
+with lock_handle as lock:
+    try:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+    except Exception:
+        raise SystemExit(34)
+    try:
+        projects = json.loads(host_registry_path.read_text(encoding="utf-8"))
+    except Exception:
+        raise SystemExit(35)
+    if not isinstance(projects, dict):
+        raise SystemExit(35)
+    current = projects.get(repository)
+    if current is not None:
+        if not isinstance(current, dict):
+            raise SystemExit(36)
+        allowed_metadata = {"gitea_repository_id", "gitea_webhook_secret_file", "validation"}
+        if set(current) - set(project) - allowed_metadata:
+            raise SystemExit(36)
+        for key, value in base_project.items():
+            if current.get(key) != value:
+                raise SystemExit(36)
+        current_env = current.get("env_source")
+        if current_env not in {None, str(env_path)}:
+            raise SystemExit(36)
+        updated_project = {**current, **project}
+    else:
+        updated_project = project
+
+    if current != updated_project:
+        backup = host_registry_path.with_name(
+            f"projects.json.workmesh-backup-{int(time.time())}-{os.getpid()}"
+        )
+        shutil.copy2(host_registry_path, backup)
+        updated = dict(projects)
+        updated[repository] = updated_project
+        temporary = host_registry_path.with_name(f".{host_registry_path.name}.{os.getpid()}.tmp")
+        payload = json.dumps(updated, indent=2, sort_keys=True) + "\n"
+        with temporary.open("w", encoding="utf-8") as handle:
+            handle.write(payload)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.chmod(temporary, stat.S_IMODE(host_registry_path.stat().st_mode))
+        os.replace(temporary, host_registry_path)
+        directory_fd = os.open(host_registry_path.parent, os.O_RDONLY)
+        try:
+            os.fsync(directory_fd)
+        finally:
+            os.close(directory_fd)
+
+# The official controller is the sole deployment executor. It revalidates the
+# exact revision, source tree, Compose topology, health, receipt and rollback.
+preflight = subprocess.run(
+    ["python3", str(deploy_py), "preflight", repository, revision],
+    stdout=subprocess.DEVNULL,
+    stderr=subprocess.DEVNULL,
+    timeout=300,
+    check=False,
+)
+if preflight.returncode != 0:
+    raise SystemExit(40)
+# Refuse branch drift between preflight and the mutating deployment.
+main_again = git("rev-parse", "refs/heads/main")
+if main_again.returncode != 0 or main_again.stdout.strip() != revision:
+    raise SystemExit(41)
+deployed = subprocess.run(
+    ["python3", str(deploy_py), "deploy", repository, revision],
+    stdout=subprocess.DEVNULL,
+    stderr=subprocess.DEVNULL,
+    timeout=1800,
+    check=False,
+)
+if deployed.returncode != 0:
+    raise SystemExit(42)
+raise SystemExit(0)
+"""
+
+
+class WorkMeshFirstDeployment:
+    def __init__(self, portainer: PortainerConnector | None = None):
+        self.portainer = portainer or PortainerConnector()
+
+    @staticmethod
+    def _state(session: Session) -> AppSetting:
+        row = session.get(AppSetting, _STATE_KEY)
+        if row:
+            return row
+        row = AppSetting(
+            key=_STATE_KEY,
+            value_json={"state": "idle", "version": 1},
+            is_secret=False,
+        )
+        session.add(row)
+        session.commit()
+        session.refresh(row)
+        return row
+
+    @staticmethod
+    def tracked_job_id(session: Session) -> str | None:
+        row = session.get(AppSetting, _STATE_KEY)
+        state = dict(row.value_json or {}) if row else {}
+        job_id = str(state.get("jobId") or "")
+        return job_id or None
+
+    @staticmethod
+    def _result_for_exit(exit_code: int) -> dict[str, Any]:
+        if exit_code == 0:
+            return {
+                "status": "deployed",
+                "repository": _REPOSITORY,
+                "revision": _RELEASE_REVISION,
+                "compose_project": "workmesh",
+                "containers": list(_CONTAINERS),
+                "ports": list(_PORTS),
+            }
+        reasons = {
+            20: "deployment controller registry or executable is unsafe or unavailable",
+            21: "mounted WorkMesh bare repository is unavailable",
+            22: "deployment registry host alias does not match the controller registry",
+            23: "WorkMesh protected main is not the fixed v0.1.0 release revision",
+            24: "required WorkMesh deployment artifacts are absent from protected main",
+            25: "WorkMesh VERSION is not 0.1.0",
+            26: "WorkMesh deployment descriptor differs from the fixed first-deploy contract",
+            27: "Docker inventory could not be inspected safely",
+            28: "one of the reserved WorkMesh loopback ports is already owned by another container",
+            29: "WorkMesh environment directory or file is unsafe",
+            30: "WorkMesh environment file permissions are too broad",
+            31: "WorkMesh environment file keys are outside the fixed contract",
+            32: "WorkMesh environment fixed values differ from the reviewed contract",
+            33: "WorkMesh environment secrets are missing, weak, or development placeholders",
+            34: "deployment registry lock could not be acquired",
+            35: "deployment project registry is invalid",
+            36: "an existing WorkMesh deployment target conflicts with the fixed contract",
+            40: "official WorkMesh deployment preflight failed",
+            41: "WorkMesh protected main changed between preflight and deployment",
+            42: "official WorkMesh first deployment failed or rolled back",
+        }
+        return {
+            "status": "failed",
+            "reason": reasons.get(exit_code, "WorkMesh first deployment failed closed"),
+        }
+
+    def run(self, session: Session, *, job: JobRun) -> dict[str, Any]:
+        row = self._state(session)
+        state = dict(row.value_json or {})
+        if state.get("state") == "completed" and isinstance(state.get("result"), dict):
+            return dict(state["result"])
+
+        execution_id = str(state.get("executionId") or "")
+        if state.get("state") == "creating":
+            return {
+                "status": "outcome_unknown",
+                "reason": "WorkMesh deployment exec creation has an unknown outcome; do not retry",
+            }
+        if state.get("state") == "created" and execution_id:
+            return {
+                "status": "outcome_unknown",
+                "reason": (
+                    "WorkMesh deployment execution was created but its start boundary "
+                    "was not durably confirmed; do not retry"
+                ),
+                "execution_id": execution_id,
+            }
+
+        controller_status = self.portainer.get_container_status(_CONTROLLER)
+        if controller_status.get("status") != "ok":
+            return {
+                "status": "outcome_unknown" if state.get("state") == "started" else "error",
+                "reason": "deployment controller is unavailable",
+                **({"execution_id": execution_id} if execution_id else {}),
+            }
+        controller = dict(controller_status.get("container") or {})
+        if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
+            return {
+                "status": "outcome_unknown" if state.get("state") == "started" else "error",
+                "reason": "deployment controller is not running",
+                **({"execution_id": execution_id} if execution_id else {}),
+            }
+        controller_id = str(controller.get("id") or "")
+        config = self.portainer._config()
+        if not config or not controller_id:
+            return {
+                "status": "outcome_unknown" if state.get("state") == "started" else "error",
+                "reason": "Portainer deployment control is unavailable",
+                **({"execution_id": execution_id} if execution_id else {}),
+            }
+
+        original_controller_id = str(state.get("controllerId") or "")
+        endpoint = quote(str(config.endpoint_id), safe="")
+
+        if state.get("state") == "started" and execution_id:
+            if original_controller_id != controller_id:
+                return {
+                    "status": "outcome_unknown",
+                    "reason": "deployment controller identity changed while first-deploy outcome was unknown",
+                    "execution_id": execution_id,
+                }
+            try:
+                inspected = self.portainer._request(
+                    "GET",
+                    f"/api/endpoints/{endpoint}/docker/exec/{quote(execution_id, safe='')}/json",
+                ).json()
+            except (RuntimeError, ValueError, TypeError) as exc:
+                return {
+                    "status": "outcome_unknown",
+                    "reason": f"existing WorkMesh deployment outcome remains unknown: {redact_message(str(exc))}",
+                    "execution_id": execution_id,
+                }
+            if (
+                str(inspected.get("ID") or "") != execution_id
+                or str(inspected.get("ContainerID") or "") != controller_id
+            ):
+                return {
+                    "status": "outcome_unknown",
+                    "reason": "WorkMesh deployment execution identity could not be verified",
+                    "execution_id": execution_id,
+                }
+            if bool(inspected.get("Running")):
+                if job.status not in {"succeeded", "failed", "cancelled"}:
+                    set_job_progress(session, job, max(job.progress, 50), "Official WorkMesh first deployment is still running.")
+                return {"status": "in_progress", "execution_id": execution_id}
+            raw_exit = inspected.get("ExitCode")
+            exit_code = int(raw_exit) if raw_exit is not None else -1
+            result = self._result_for_exit(exit_code)
+            row.value_json = {
+                **state,
+                "state": "completed",
+                "result": result,
+                "completedAt": utc_now().isoformat(),
+            }
+            row.updated_at = utc_now()
+            session.add(row)
+            session.commit()
+            return result
+
+        if state.get("state") not in {"idle", None}:
+            return {
+                "status": "outcome_unknown",
+                "reason": "WorkMesh first-deploy state is not safely restartable",
+                **({"execution_id": execution_id} if execution_id else {}),
+            }
+
+        safe_controller = quote(controller_id, safe="")
+        row.value_json = {
+            "state": "creating",
+            "version": 1,
+            "jobId": job.id,
+            "controllerId": controller_id,
+            "createdAt": utc_now().isoformat(),
+        }
+        row.updated_at = utc_now()
+        session.add(row)
+        session.commit()
+        try:
+            created = self.portainer._request(
+                "POST",
+                f"/api/endpoints/{endpoint}/docker/containers/{safe_controller}/exec",
+                json={
+                    "AttachStdout": False,
+                    "AttachStderr": False,
+                    "Tty": False,
+                    "Cmd": ["python3", "-c", _FIRST_DEPLOY_SCRIPT],
+                },
+            ).json()
+            execution_id = str(created.get("Id") or "")
+            if not execution_id:
+                return {
+                    "status": "outcome_unknown",
+                    "reason": "WorkMesh deployment exec creation identity is unknown; do not retry",
+                }
+            row.value_json = {
+                "state": "created",
+                "version": 1,
+                "jobId": job.id,
+                "executionId": execution_id,
+                "controllerId": controller_id,
+                "createdAt": utc_now().isoformat(),
+            }
+            row.updated_at = utc_now()
+            session.add(row)
+            session.commit()
+            self.portainer._request(
+                "POST",
+                f"/api/endpoints/{endpoint}/docker/exec/{quote(execution_id, safe='')}/start",
+                json={"Detach": True, "Tty": False},
+            )
+            row.value_json = {
+                **dict(row.value_json or {}),
+                "state": "started",
+                "startedAt": utc_now().isoformat(),
+            }
+            row.updated_at = utc_now()
+            session.add(row)
+            session.commit()
+        except (RuntimeError, ValueError, TypeError) as exc:
+            return {
+                "status": "outcome_unknown",
+                "reason": f"WorkMesh deployment exec outcome is unknown: {redact_message(str(exc))}",
+                **({"execution_id": execution_id} if execution_id else {}),
+            }
+        set_job_progress(session, job, 25, "Official WorkMesh first deployment started.")
+        return {"status": "in_progress", "execution_id": execution_id}
+
+
+
+def workmesh_first_deploy_job_cancellation_reason(
+    session: Session,
+    job: JobRun,
+) -> str | None:
+    # A running fixed-target deployment may have reached the controller even
+    # when its state row was lost or does not yet name this job. Keep both
+    # cancellation entry points fail-closed until execution is reconciled.
+    if (
+        job.action_id == "deployment.target.deploy_workmesh"
+        and job.target == _REPOSITORY
+        and job.status == "running"
+    ):
+        return (
+            "WorkMesh first deployment may own an unresolved controller execution and "
+            "cannot be cancelled until that exact execution is reconciled."
+        )
+    row = session.get(AppSetting, _STATE_KEY)
+    state = dict(row.value_json or {}) if row else {}
+    if str(state.get("jobId") or "") != job.id:
+        return None
+    if state.get("state") in {"created", "started"}:
+        return (
+            "WorkMesh first deployment owns an active controller execution and "
+            "cannot be cancelled until that exact execution is reconciled."
+        )
+    return None