from __future__ import annotations from typing import Any from urllib.parse import quote from sqlmodel import Session from ..connectors.portainer import PortainerConnector from ..core.redaction import redact_message from ..core.time import utc_now from ..db.models import AppSetting, JobRun from .job_runner import set_job_progress _CONTROLLER = "gitea-deploy-control" _REPOSITORY = "jens/workmesh" _STATE_KEY = "deployment:workmesh:first-deploy:v1" _RELEASE_REVISION = "7dfbdb7d25e0efd70c53106de1d36d3fff6e3a05" _CONTAINERS = ( "workmesh-postgres-1", "workmesh-mock-ats-1", "workmesh-api-1", "workmesh-n8n-1", "workmesh-workmesh-1", ) _PORTS = (1255, 1256, 1257, 1258) # Fixed first-deploy bootstrap. Callers provide no repository, path, command, # port, environment value, or secret. Secret material is generated inside the # root-owned deployment controller and is never attached to exec output. _FIRST_DEPLOY_SCRIPT = r""" import fcntl import json import os from pathlib import Path import secrets import shutil import stat import subprocess import sys import time repository = "jens/workmesh" expected_release_revision = "7dfbdb7d25e0efd70c53106de1d36d3fff6e3a05" slug = "workmesh" compose_project = "workmesh" containers = [ "workmesh-postgres-1", "workmesh-mock-ats-1", "workmesh-api-1", "workmesh-n8n-1", "workmesh-workmesh-1", ] ports = {1255, 1256, 1257, 1258} controller_root = Path("/opt/gitea-deploy") registry_path = Path("/controller-config/projects.json") host_registry_path = Path("/controller-hub-releases/controller-config/projects.json") lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock") env_dir = Path("/srv/example/appdata/workmesh") env_path = env_dir / ".env" bare_repo = Path("/gitea-repositories/jens/workmesh.git") source_dir = Path("/mnt/cache/appdata/gitea-deploy-runners/sources/workmesh") deploy_py = controller_root / "deploy.py" base_project = { "branch": "main", "slug": slug, "source_dir": str(source_dir), "exact_source": True, "compose_files": ["compose.yaml"], "compose_project": compose_project, "containers": containers, "verify_oci_revision": False, "health_timeout": 300, } project = {**base_project, "env_source": str(env_path)} for path in (registry_path, host_registry_path, lock_path, bare_repo, deploy_py): if path.is_symlink(): raise SystemExit(20) if not registry_path.is_file() or not host_registry_path.is_file() or not deploy_py.is_file(): raise SystemExit(20) if not bare_repo.is_dir(): raise SystemExit(21) try: if not os.path.samefile(registry_path, host_registry_path): raise SystemExit(22) except OSError: raise SystemExit(22) def git(*args): return subprocess.run( ["git", f"--git-dir={bare_repo}", *args], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=15, check=False, ) main = git("rev-parse", "refs/heads/main") revision = main.stdout.strip() if main.returncode != 0 or len(revision) != 40 or revision != expected_release_revision: raise SystemExit(23) for required in ("compose.yaml", "VERSION", ".itworx/deployment-target.json"): if git("cat-file", "-e", f"{revision}:{required}").returncode != 0: raise SystemExit(24) version = git("show", f"{revision}:VERSION") if version.returncode != 0 or version.stdout.strip() != "0.1.0": raise SystemExit(25) descriptor = git("show", f"{revision}:.itworx/deployment-target.json") try: descriptor_payload = json.loads(descriptor.stdout) except Exception: raise SystemExit(26) if descriptor_payload != { "apiVersion": "itworx.deployment-target/v1", "applicationId": "workmesh", "composeFile": "compose.yaml", "composeProject": "workmesh", "healthTimeout": 300, "verifyOciRevision": False, }: raise SystemExit(26) # Refuse to take host ports from any non-WorkMesh container, including stopped # containers that would become conflicting if restarted later. listed = subprocess.run( ["docker", "ps", "-aq"], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=20, check=False, ) if listed.returncode != 0: raise SystemExit(27) for container_id in [line.strip() for line in listed.stdout.splitlines() if line.strip()]: inspected = subprocess.run( ["docker", "inspect", container_id], text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10, check=False, ) if inspected.returncode != 0: raise SystemExit(27) try: item = json.loads(inspected.stdout)[0] except Exception: raise SystemExit(27) labels = dict(dict(item.get("Config") or {}).get("Labels") or {}) if labels.get("com.docker.compose.project") == compose_project: continue bindings = dict(dict(item.get("HostConfig") or {}).get("PortBindings") or {}) for values in bindings.values(): for binding in values or []: raw = str(dict(binding).get("HostPort") or "") if raw.isdigit() and int(raw) in ports: raise SystemExit(28) required_fixed = { "WORKMESH_ENV": "production", "WORKMESH_API_PORT": "1256", "WORKMESH_WEB_PORT": "1255", "WORKMESH_BIND_HOST": "127.0.0.1", "MOCK_ATS_PORT": "1258", "WORKMESH_POSTGRES_DB": "workmesh", "WORKMESH_POSTGRES_USER": "workmesh", "N8N_BASE_URL": "http://n8n:5678", "WORKMESH_API_BASE_URL": "http://api:8000", "N8N_PORT": "1257", "MOCK_ATS_BASE_URL": "http://mock-ats:8010", "DEMO_CONTROLS_ENABLED": "true", "AI_PROVIDER": "disabled", "AI_MODEL": "qwen3.5:9b", "AI_TIMEOUT_SECONDS": "30", "OLLAMA_BASE_URL": "http://host.docker.internal:11434", "OPENAI_COMPATIBLE_BASE_URL": "", "OPENAI_COMPATIBLE_API_KEY": "", "KNOWLEDGE_PROVIDER": "fixture", "RAGCORE_BASE_URL": "", "RAGCORE_TOKEN": "", "RAGCORE_SPACE_IDS": "", "RAGCORE_PROFILE": "hybrid-v1", "RAGCORE_TIMEOUT_SECONDS": "10", "NEXT_PUBLIC_WORKMESH_API_URL": "http://127.0.0.1:1256", "CORS_ORIGINS": "http://localhost:1255,http://127.0.0.1:1255", } secret_keys = ( "WORKMESH_POSTGRES_PASSWORD", "WORKMESH_N8N_CALLBACK_SECRET", "N8N_ENCRYPTION_KEY", ) def parse_env(path): values = {} for line in path.read_text(encoding="utf-8").splitlines(): if not line or line.startswith("#"): continue if "=" not in line: raise SystemExit(31) key, value = line.split("=", 1) if key in values or not key: raise SystemExit(31) values[key] = value return values if env_path.exists(): if env_path.is_symlink() or not env_path.is_file(): raise SystemExit(29) info = env_path.stat() if info.st_uid != 0 or stat.S_IMODE(info.st_mode) & 0o077: raise SystemExit(30) env = parse_env(env_path) if set(env) != set(required_fixed) | set(secret_keys): raise SystemExit(31) if any(env.get(key) != value for key, value in required_fixed.items()): raise SystemExit(32) if any(len(env.get(key, "")) < 32 or "change-me" in env.get(key, "").lower() for key in secret_keys): raise SystemExit(33) else: env_dir.mkdir(parents=True, exist_ok=True) if env_dir.is_symlink() or not env_dir.is_dir(): raise SystemExit(29) generated = {key: secrets.token_urlsafe(48) for key in secret_keys} env = {**required_fixed, **generated} temporary_env = env_dir / f".env.{os.getpid()}.tmp" flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL descriptor_fd = os.open(temporary_env, flags, 0o600) try: with os.fdopen(descriptor_fd, "w", encoding="utf-8") as handle: for key in sorted(env): handle.write(f"{key}={env[key]}\n") handle.flush() os.fsync(handle.fileno()) os.replace(temporary_env, env_path) os.chmod(env_path, 0o600) directory_fd = os.open(env_dir, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) finally: try: temporary_env.unlink(missing_ok=True) except Exception: pass try: lock_handle = lock_path.open("a+", encoding="utf-8") except Exception: raise SystemExit(34) with lock_handle as lock: try: fcntl.flock(lock, fcntl.LOCK_EX) except Exception: raise SystemExit(34) try: projects = json.loads(host_registry_path.read_text(encoding="utf-8")) except Exception: raise SystemExit(35) if not isinstance(projects, dict): raise SystemExit(35) current = projects.get(repository) if current is not None: if not isinstance(current, dict): raise SystemExit(36) allowed_metadata = {"gitea_repository_id", "gitea_webhook_secret_file", "validation"} if set(current) - set(project) - allowed_metadata: raise SystemExit(36) for key, value in base_project.items(): if current.get(key) != value: raise SystemExit(36) current_env = current.get("env_source") if current_env not in {None, str(env_path)}: raise SystemExit(36) updated_project = {**current, **project} else: updated_project = project if current != updated_project: backup = host_registry_path.with_name( f"projects.json.workmesh-backup-{int(time.time())}-{os.getpid()}" ) shutil.copy2(host_registry_path, backup) updated = dict(projects) updated[repository] = updated_project temporary = host_registry_path.with_name(f".{host_registry_path.name}.{os.getpid()}.tmp") payload = json.dumps(updated, indent=2, sort_keys=True) + "\n" with temporary.open("w", encoding="utf-8") as handle: handle.write(payload) handle.flush() os.fsync(handle.fileno()) os.chmod(temporary, stat.S_IMODE(host_registry_path.stat().st_mode)) os.replace(temporary, host_registry_path) directory_fd = os.open(host_registry_path.parent, os.O_RDONLY) try: os.fsync(directory_fd) finally: os.close(directory_fd) # The official controller is the sole deployment executor. It revalidates the # exact revision, source tree, Compose topology, health, receipt and rollback. preflight = subprocess.run( ["python3", str(deploy_py), "preflight", repository, revision], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=300, check=False, ) if preflight.returncode != 0: raise SystemExit(40) # Refuse branch drift between preflight and the mutating deployment. main_again = git("rev-parse", "refs/heads/main") if main_again.returncode != 0 or main_again.stdout.strip() != revision: raise SystemExit(41) deployed = subprocess.run( ["python3", str(deploy_py), "deploy", repository, revision], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=1800, check=False, ) if deployed.returncode != 0: raise SystemExit(42) raise SystemExit(0) """ class WorkMeshFirstDeployment: def __init__(self, portainer: PortainerConnector | None = None): self.portainer = portainer or PortainerConnector() @staticmethod def _state(session: Session) -> AppSetting: row = session.get(AppSetting, _STATE_KEY) if row: return row row = AppSetting( key=_STATE_KEY, value_json={"state": "idle", "version": 1}, is_secret=False, ) session.add(row) session.commit() session.refresh(row) return row @staticmethod def tracked_job_id(session: Session) -> str | None: row = session.get(AppSetting, _STATE_KEY) state = dict(row.value_json or {}) if row else {} job_id = str(state.get("jobId") or "") return job_id or None @staticmethod def _result_for_exit(exit_code: int) -> dict[str, Any]: if exit_code == 0: return { "status": "deployed", "repository": _REPOSITORY, "revision": _RELEASE_REVISION, "compose_project": "workmesh", "containers": list(_CONTAINERS), "ports": list(_PORTS), } reasons = { 20: "deployment controller registry or executable is unsafe or unavailable", 21: "mounted WorkMesh bare repository is unavailable", 22: "deployment registry host alias does not match the controller registry", 23: "WorkMesh protected main is not the fixed v0.1.0 release revision", 24: "required WorkMesh deployment artifacts are absent from protected main", 25: "WorkMesh VERSION is not 0.1.0", 26: "WorkMesh deployment descriptor differs from the fixed first-deploy contract", 27: "Docker inventory could not be inspected safely", 28: "one of the reserved WorkMesh loopback ports is already owned by another container", 29: "WorkMesh environment directory or file is unsafe", 30: "WorkMesh environment file permissions are too broad", 31: "WorkMesh environment file keys are outside the fixed contract", 32: "WorkMesh environment fixed values differ from the reviewed contract", 33: "WorkMesh environment secrets are missing, weak, or development placeholders", 34: "deployment registry lock could not be acquired", 35: "deployment project registry is invalid", 36: "an existing WorkMesh deployment target conflicts with the fixed contract", 40: "official WorkMesh deployment preflight failed", 41: "WorkMesh protected main changed between preflight and deployment", 42: "official WorkMesh first deployment failed or rolled back", } return { "status": "failed", "reason": reasons.get(exit_code, "WorkMesh first deployment failed closed"), } def run(self, session: Session, *, job: JobRun) -> dict[str, Any]: row = self._state(session) state = dict(row.value_json or {}) if state.get("state") == "completed" and isinstance(state.get("result"), dict): return dict(state["result"]) execution_id = str(state.get("executionId") or "") if state.get("state") == "creating": return { "status": "outcome_unknown", "reason": "WorkMesh deployment exec creation has an unknown outcome; do not retry", } if state.get("state") == "created" and execution_id: return { "status": "outcome_unknown", "reason": ( "WorkMesh deployment execution was created but its start boundary " "was not durably confirmed; do not retry" ), "execution_id": execution_id, } controller_status = self.portainer.get_container_status(_CONTROLLER) if controller_status.get("status") != "ok": return { "status": "outcome_unknown" if state.get("state") == "started" else "error", "reason": "deployment controller is unavailable", **({"execution_id": execution_id} if execution_id else {}), } controller = dict(controller_status.get("container") or {}) if controller.get("name") != _CONTROLLER or controller.get("state") != "running": return { "status": "outcome_unknown" if state.get("state") == "started" else "error", "reason": "deployment controller is not running", **({"execution_id": execution_id} if execution_id else {}), } controller_id = str(controller.get("id") or "") config = self.portainer._config() if not config or not controller_id: return { "status": "outcome_unknown" if state.get("state") == "started" else "error", "reason": "Portainer deployment control is unavailable", **({"execution_id": execution_id} if execution_id else {}), } original_controller_id = str(state.get("controllerId") or "") endpoint = quote(str(config.endpoint_id), safe="") if state.get("state") == "started" and execution_id: if original_controller_id != controller_id: return { "status": "outcome_unknown", "reason": "deployment controller identity changed while first-deploy outcome was unknown", "execution_id": execution_id, } try: inspected = self.portainer._request( "GET", f"/api/endpoints/{endpoint}/docker/exec/{quote(execution_id, safe='')}/json", ).json() except (RuntimeError, ValueError, TypeError) as exc: return { "status": "outcome_unknown", "reason": f"existing WorkMesh deployment outcome remains unknown: {redact_message(str(exc))}", "execution_id": execution_id, } if ( str(inspected.get("ID") or "") != execution_id or str(inspected.get("ContainerID") or "") != controller_id ): return { "status": "outcome_unknown", "reason": "WorkMesh deployment execution identity could not be verified", "execution_id": execution_id, } if bool(inspected.get("Running")): if job.status not in {"succeeded", "failed", "cancelled"}: set_job_progress(session, job, max(job.progress, 50), "Official WorkMesh first deployment is still running.") return {"status": "in_progress", "execution_id": execution_id} raw_exit = inspected.get("ExitCode") exit_code = int(raw_exit) if raw_exit is not None else -1 result = self._result_for_exit(exit_code) row.value_json = { **state, "state": "completed", "result": result, "completedAt": utc_now().isoformat(), } row.updated_at = utc_now() session.add(row) session.commit() return result if state.get("state") not in {"idle", None}: return { "status": "outcome_unknown", "reason": "WorkMesh first-deploy state is not safely restartable", **({"execution_id": execution_id} if execution_id else {}), } safe_controller = quote(controller_id, safe="") row.value_json = { "state": "creating", "version": 1, "jobId": job.id, "controllerId": controller_id, "createdAt": utc_now().isoformat(), } row.updated_at = utc_now() session.add(row) session.commit() try: created = self.portainer._request( "POST", f"/api/endpoints/{endpoint}/docker/containers/{safe_controller}/exec", json={ "AttachStdout": False, "AttachStderr": False, "Tty": False, "Cmd": ["python3", "-c", _FIRST_DEPLOY_SCRIPT], }, ).json() execution_id = str(created.get("Id") or "") if not execution_id: return { "status": "outcome_unknown", "reason": "WorkMesh deployment exec creation identity is unknown; do not retry", } row.value_json = { "state": "created", "version": 1, "jobId": job.id, "executionId": execution_id, "controllerId": controller_id, "createdAt": utc_now().isoformat(), } row.updated_at = utc_now() session.add(row) session.commit() self.portainer._request( "POST", f"/api/endpoints/{endpoint}/docker/exec/{quote(execution_id, safe='')}/start", json={"Detach": True, "Tty": False}, ) row.value_json = { **dict(row.value_json or {}), "state": "started", "startedAt": utc_now().isoformat(), } row.updated_at = utc_now() session.add(row) session.commit() except (RuntimeError, ValueError, TypeError) as exc: return { "status": "outcome_unknown", "reason": f"WorkMesh deployment exec outcome is unknown: {redact_message(str(exc))}", **({"execution_id": execution_id} if execution_id else {}), } set_job_progress(session, job, 25, "Official WorkMesh first deployment started.") return {"status": "in_progress", "execution_id": execution_id} def workmesh_first_deploy_job_cancellation_reason( session: Session, job: JobRun, ) -> str | None: # A running fixed-target deployment may have reached the controller even # when its state row was lost or does not yet name this job. Keep both # cancellation entry points fail-closed until execution is reconciled. if ( job.action_id == "deployment.target.deploy_workmesh" and job.target == _REPOSITORY and job.status == "running" ): return ( "WorkMesh first deployment may own an unresolved controller execution and " "cannot be cancelled until that exact execution is reconciled." ) row = session.get(AppSetting, _STATE_KEY) state = dict(row.value_json or {}) if row else {} if str(state.get("jobId") or "") != job.id: return None if state.get("state") in {"created", "started"}: return ( "WorkMesh first deployment owns an active controller execution and " "cannot be cancelled until that exact execution is reconciled." ) return None