589 lines
22 KiB
Python
589 lines
22 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from urllib.parse import quote
|
|
|
|
from sqlmodel import Session
|
|
|
|
from ..connectors.portainer import PortainerConnector
|
|
from ..core.redaction import redact_message
|
|
from ..core.time import utc_now
|
|
from ..db.models import AppSetting, JobRun
|
|
from .job_runner import set_job_progress
|
|
|
|
_CONTROLLER = "gitea-deploy-control"
|
|
_REPOSITORY = "jens/workmesh"
|
|
_STATE_KEY = "deployment:workmesh:first-deploy:v1"
|
|
_RELEASE_REVISION = "7dfbdb7d25e0efd70c53106de1d36d3fff6e3a05"
|
|
_CONTAINERS = (
|
|
"workmesh-postgres-1",
|
|
"workmesh-mock-ats-1",
|
|
"workmesh-api-1",
|
|
"workmesh-n8n-1",
|
|
"workmesh-workmesh-1",
|
|
)
|
|
_PORTS = (1255, 1256, 1257, 1258)
|
|
|
|
|
|
# Fixed first-deploy bootstrap. Callers provide no repository, path, command,
|
|
# port, environment value, or secret. Secret material is generated inside the
|
|
# root-owned deployment controller and is never attached to exec output.
|
|
_FIRST_DEPLOY_SCRIPT = r"""
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
repository = "jens/workmesh"
|
|
expected_release_revision = "7dfbdb7d25e0efd70c53106de1d36d3fff6e3a05"
|
|
slug = "workmesh"
|
|
compose_project = "workmesh"
|
|
containers = [
|
|
"workmesh-postgres-1",
|
|
"workmesh-mock-ats-1",
|
|
"workmesh-api-1",
|
|
"workmesh-n8n-1",
|
|
"workmesh-workmesh-1",
|
|
]
|
|
ports = {1255, 1256, 1257, 1258}
|
|
controller_root = Path("/opt/gitea-deploy")
|
|
registry_path = Path("/controller-config/projects.json")
|
|
host_registry_path = Path("/controller-hub-releases/controller-config/projects.json")
|
|
lock_path = Path("/mnt/cache/appdata/gitea-deploy-runners/deploy.lock")
|
|
env_dir = Path("/srv/example/appdata/workmesh")
|
|
env_path = env_dir / ".env"
|
|
bare_repo = Path("/gitea-repositories/jens/workmesh.git")
|
|
source_dir = Path("/mnt/cache/appdata/gitea-deploy-runners/sources/workmesh")
|
|
deploy_py = controller_root / "deploy.py"
|
|
|
|
base_project = {
|
|
"branch": "main",
|
|
"slug": slug,
|
|
"source_dir": str(source_dir),
|
|
"exact_source": True,
|
|
"compose_files": ["compose.yaml"],
|
|
"compose_project": compose_project,
|
|
"containers": containers,
|
|
"verify_oci_revision": False,
|
|
"health_timeout": 300,
|
|
}
|
|
project = {**base_project, "env_source": str(env_path)}
|
|
|
|
for path in (registry_path, host_registry_path, lock_path, bare_repo, deploy_py):
|
|
if path.is_symlink():
|
|
raise SystemExit(20)
|
|
if not registry_path.is_file() or not host_registry_path.is_file() or not deploy_py.is_file():
|
|
raise SystemExit(20)
|
|
if not bare_repo.is_dir():
|
|
raise SystemExit(21)
|
|
try:
|
|
if not os.path.samefile(registry_path, host_registry_path):
|
|
raise SystemExit(22)
|
|
except OSError:
|
|
raise SystemExit(22)
|
|
|
|
|
|
def git(*args):
|
|
return subprocess.run(
|
|
["git", f"--git-dir={bare_repo}", *args],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=15,
|
|
check=False,
|
|
)
|
|
|
|
|
|
main = git("rev-parse", "refs/heads/main")
|
|
revision = main.stdout.strip()
|
|
if main.returncode != 0 or len(revision) != 40 or revision != expected_release_revision:
|
|
raise SystemExit(23)
|
|
for required in ("compose.yaml", "VERSION", ".itworx/deployment-target.json"):
|
|
if git("cat-file", "-e", f"{revision}:{required}").returncode != 0:
|
|
raise SystemExit(24)
|
|
version = git("show", f"{revision}:VERSION")
|
|
if version.returncode != 0 or version.stdout.strip() != "0.1.0":
|
|
raise SystemExit(25)
|
|
descriptor = git("show", f"{revision}:.itworx/deployment-target.json")
|
|
try:
|
|
descriptor_payload = json.loads(descriptor.stdout)
|
|
except Exception:
|
|
raise SystemExit(26)
|
|
if descriptor_payload != {
|
|
"apiVersion": "itworx.deployment-target/v1",
|
|
"applicationId": "workmesh",
|
|
"composeFile": "compose.yaml",
|
|
"composeProject": "workmesh",
|
|
"healthTimeout": 300,
|
|
"verifyOciRevision": False,
|
|
}:
|
|
raise SystemExit(26)
|
|
|
|
# Refuse to take host ports from any non-WorkMesh container, including stopped
|
|
# containers that would become conflicting if restarted later.
|
|
listed = subprocess.run(
|
|
["docker", "ps", "-aq"],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=20,
|
|
check=False,
|
|
)
|
|
if listed.returncode != 0:
|
|
raise SystemExit(27)
|
|
for container_id in [line.strip() for line in listed.stdout.splitlines() if line.strip()]:
|
|
inspected = subprocess.run(
|
|
["docker", "inspect", container_id],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
if inspected.returncode != 0:
|
|
raise SystemExit(27)
|
|
try:
|
|
item = json.loads(inspected.stdout)[0]
|
|
except Exception:
|
|
raise SystemExit(27)
|
|
labels = dict(dict(item.get("Config") or {}).get("Labels") or {})
|
|
if labels.get("com.docker.compose.project") == compose_project:
|
|
continue
|
|
bindings = dict(dict(item.get("HostConfig") or {}).get("PortBindings") or {})
|
|
for values in bindings.values():
|
|
for binding in values or []:
|
|
raw = str(dict(binding).get("HostPort") or "")
|
|
if raw.isdigit() and int(raw) in ports:
|
|
raise SystemExit(28)
|
|
|
|
required_fixed = {
|
|
"WORKMESH_ENV": "production",
|
|
"WORKMESH_API_PORT": "1256",
|
|
"WORKMESH_WEB_PORT": "1255",
|
|
"WORKMESH_BIND_HOST": "127.0.0.1",
|
|
"MOCK_ATS_PORT": "1258",
|
|
"WORKMESH_POSTGRES_DB": "workmesh",
|
|
"WORKMESH_POSTGRES_USER": "workmesh",
|
|
"N8N_BASE_URL": "http://n8n:5678",
|
|
"WORKMESH_API_BASE_URL": "http://api:8000",
|
|
"N8N_PORT": "1257",
|
|
"MOCK_ATS_BASE_URL": "http://mock-ats:8010",
|
|
"DEMO_CONTROLS_ENABLED": "true",
|
|
"AI_PROVIDER": "disabled",
|
|
"AI_MODEL": "qwen3.5:9b",
|
|
"AI_TIMEOUT_SECONDS": "30",
|
|
"OLLAMA_BASE_URL": "http://host.docker.internal:11434",
|
|
"OPENAI_COMPATIBLE_BASE_URL": "",
|
|
"OPENAI_COMPATIBLE_API_KEY": "",
|
|
"KNOWLEDGE_PROVIDER": "fixture",
|
|
"RAGCORE_BASE_URL": "",
|
|
"RAGCORE_TOKEN": "",
|
|
"RAGCORE_SPACE_IDS": "",
|
|
"RAGCORE_PROFILE": "hybrid-v1",
|
|
"RAGCORE_TIMEOUT_SECONDS": "10",
|
|
"NEXT_PUBLIC_WORKMESH_API_URL": "http://127.0.0.1:1256",
|
|
"CORS_ORIGINS": "http://localhost:1255,http://127.0.0.1:1255",
|
|
}
|
|
secret_keys = (
|
|
"WORKMESH_POSTGRES_PASSWORD",
|
|
"WORKMESH_N8N_CALLBACK_SECRET",
|
|
"N8N_ENCRYPTION_KEY",
|
|
)
|
|
|
|
|
|
def parse_env(path):
|
|
values = {}
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
if "=" not in line:
|
|
raise SystemExit(31)
|
|
key, value = line.split("=", 1)
|
|
if key in values or not key:
|
|
raise SystemExit(31)
|
|
values[key] = value
|
|
return values
|
|
|
|
|
|
if env_path.exists():
|
|
if env_path.is_symlink() or not env_path.is_file():
|
|
raise SystemExit(29)
|
|
info = env_path.stat()
|
|
if info.st_uid != 0 or stat.S_IMODE(info.st_mode) & 0o077:
|
|
raise SystemExit(30)
|
|
env = parse_env(env_path)
|
|
if set(env) != set(required_fixed) | set(secret_keys):
|
|
raise SystemExit(31)
|
|
if any(env.get(key) != value for key, value in required_fixed.items()):
|
|
raise SystemExit(32)
|
|
if any(len(env.get(key, "")) < 32 or "change-me" in env.get(key, "").lower() for key in secret_keys):
|
|
raise SystemExit(33)
|
|
else:
|
|
env_dir.mkdir(parents=True, exist_ok=True)
|
|
if env_dir.is_symlink() or not env_dir.is_dir():
|
|
raise SystemExit(29)
|
|
generated = {key: secrets.token_urlsafe(48) for key in secret_keys}
|
|
env = {**required_fixed, **generated}
|
|
temporary_env = env_dir / f".env.{os.getpid()}.tmp"
|
|
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
descriptor_fd = os.open(temporary_env, flags, 0o600)
|
|
try:
|
|
with os.fdopen(descriptor_fd, "w", encoding="utf-8") as handle:
|
|
for key in sorted(env):
|
|
handle.write(f"{key}={env[key]}\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temporary_env, env_path)
|
|
os.chmod(env_path, 0o600)
|
|
directory_fd = os.open(env_dir, os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
finally:
|
|
try:
|
|
temporary_env.unlink(missing_ok=True)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
lock_handle = lock_path.open("a+", encoding="utf-8")
|
|
except Exception:
|
|
raise SystemExit(34)
|
|
with lock_handle as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
except Exception:
|
|
raise SystemExit(34)
|
|
try:
|
|
projects = json.loads(host_registry_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
raise SystemExit(35)
|
|
if not isinstance(projects, dict):
|
|
raise SystemExit(35)
|
|
current = projects.get(repository)
|
|
if current is not None:
|
|
if not isinstance(current, dict):
|
|
raise SystemExit(36)
|
|
allowed_metadata = {"gitea_repository_id", "gitea_webhook_secret_file", "validation"}
|
|
if set(current) - set(project) - allowed_metadata:
|
|
raise SystemExit(36)
|
|
for key, value in base_project.items():
|
|
if current.get(key) != value:
|
|
raise SystemExit(36)
|
|
current_env = current.get("env_source")
|
|
if current_env not in {None, str(env_path)}:
|
|
raise SystemExit(36)
|
|
updated_project = {**current, **project}
|
|
else:
|
|
updated_project = project
|
|
|
|
if current != updated_project:
|
|
backup = host_registry_path.with_name(
|
|
f"projects.json.workmesh-backup-{int(time.time())}-{os.getpid()}"
|
|
)
|
|
shutil.copy2(host_registry_path, backup)
|
|
updated = dict(projects)
|
|
updated[repository] = updated_project
|
|
temporary = host_registry_path.with_name(f".{host_registry_path.name}.{os.getpid()}.tmp")
|
|
payload = json.dumps(updated, indent=2, sort_keys=True) + "\n"
|
|
with temporary.open("w", encoding="utf-8") as handle:
|
|
handle.write(payload)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.chmod(temporary, stat.S_IMODE(host_registry_path.stat().st_mode))
|
|
os.replace(temporary, host_registry_path)
|
|
directory_fd = os.open(host_registry_path.parent, os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory_fd)
|
|
finally:
|
|
os.close(directory_fd)
|
|
|
|
# The official controller is the sole deployment executor. It revalidates the
|
|
# exact revision, source tree, Compose topology, health, receipt and rollback.
|
|
preflight = subprocess.run(
|
|
["python3", str(deploy_py), "preflight", repository, revision],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=300,
|
|
check=False,
|
|
)
|
|
if preflight.returncode != 0:
|
|
raise SystemExit(40)
|
|
# Refuse branch drift between preflight and the mutating deployment.
|
|
main_again = git("rev-parse", "refs/heads/main")
|
|
if main_again.returncode != 0 or main_again.stdout.strip() != revision:
|
|
raise SystemExit(41)
|
|
deployed = subprocess.run(
|
|
["python3", str(deploy_py), "deploy", repository, revision],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=1800,
|
|
check=False,
|
|
)
|
|
if deployed.returncode != 0:
|
|
raise SystemExit(42)
|
|
raise SystemExit(0)
|
|
"""
|
|
|
|
|
|
class WorkMeshFirstDeployment:
|
|
def __init__(self, portainer: PortainerConnector | None = None):
|
|
self.portainer = portainer or PortainerConnector()
|
|
|
|
@staticmethod
|
|
def _state(session: Session) -> AppSetting:
|
|
row = session.get(AppSetting, _STATE_KEY)
|
|
if row:
|
|
return row
|
|
row = AppSetting(
|
|
key=_STATE_KEY,
|
|
value_json={"state": "idle", "version": 1},
|
|
is_secret=False,
|
|
)
|
|
session.add(row)
|
|
session.commit()
|
|
session.refresh(row)
|
|
return row
|
|
|
|
@staticmethod
|
|
def tracked_job_id(session: Session) -> str | None:
|
|
row = session.get(AppSetting, _STATE_KEY)
|
|
state = dict(row.value_json or {}) if row else {}
|
|
job_id = str(state.get("jobId") or "")
|
|
return job_id or None
|
|
|
|
@staticmethod
|
|
def _result_for_exit(exit_code: int) -> dict[str, Any]:
|
|
if exit_code == 0:
|
|
return {
|
|
"status": "deployed",
|
|
"repository": _REPOSITORY,
|
|
"revision": _RELEASE_REVISION,
|
|
"compose_project": "workmesh",
|
|
"containers": list(_CONTAINERS),
|
|
"ports": list(_PORTS),
|
|
}
|
|
reasons = {
|
|
20: "deployment controller registry or executable is unsafe or unavailable",
|
|
21: "mounted WorkMesh bare repository is unavailable",
|
|
22: "deployment registry host alias does not match the controller registry",
|
|
23: "WorkMesh protected main is not the fixed v0.1.0 release revision",
|
|
24: "required WorkMesh deployment artifacts are absent from protected main",
|
|
25: "WorkMesh VERSION is not 0.1.0",
|
|
26: "WorkMesh deployment descriptor differs from the fixed first-deploy contract",
|
|
27: "Docker inventory could not be inspected safely",
|
|
28: "one of the reserved WorkMesh loopback ports is already owned by another container",
|
|
29: "WorkMesh environment directory or file is unsafe",
|
|
30: "WorkMesh environment file permissions are too broad",
|
|
31: "WorkMesh environment file keys are outside the fixed contract",
|
|
32: "WorkMesh environment fixed values differ from the reviewed contract",
|
|
33: "WorkMesh environment secrets are missing, weak, or development placeholders",
|
|
34: "deployment registry lock could not be acquired",
|
|
35: "deployment project registry is invalid",
|
|
36: "an existing WorkMesh deployment target conflicts with the fixed contract",
|
|
40: "official WorkMesh deployment preflight failed",
|
|
41: "WorkMesh protected main changed between preflight and deployment",
|
|
42: "official WorkMesh first deployment failed or rolled back",
|
|
}
|
|
return {
|
|
"status": "failed",
|
|
"reason": reasons.get(exit_code, "WorkMesh first deployment failed closed"),
|
|
}
|
|
|
|
def run(self, session: Session, *, job: JobRun) -> dict[str, Any]:
|
|
row = self._state(session)
|
|
state = dict(row.value_json or {})
|
|
if state.get("state") == "completed" and isinstance(state.get("result"), dict):
|
|
return dict(state["result"])
|
|
|
|
execution_id = str(state.get("executionId") or "")
|
|
if state.get("state") == "creating":
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"reason": "WorkMesh deployment exec creation has an unknown outcome; do not retry",
|
|
}
|
|
if state.get("state") == "created" and execution_id:
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"reason": (
|
|
"WorkMesh deployment execution was created but its start boundary "
|
|
"was not durably confirmed; do not retry"
|
|
),
|
|
"execution_id": execution_id,
|
|
}
|
|
|
|
controller_status = self.portainer.get_container_status(_CONTROLLER)
|
|
if controller_status.get("status") != "ok":
|
|
return {
|
|
"status": "outcome_unknown" if state.get("state") == "started" else "error",
|
|
"reason": "deployment controller is unavailable",
|
|
**({"execution_id": execution_id} if execution_id else {}),
|
|
}
|
|
controller = dict(controller_status.get("container") or {})
|
|
if controller.get("name") != _CONTROLLER or controller.get("state") != "running":
|
|
return {
|
|
"status": "outcome_unknown" if state.get("state") == "started" else "error",
|
|
"reason": "deployment controller is not running",
|
|
**({"execution_id": execution_id} if execution_id else {}),
|
|
}
|
|
controller_id = str(controller.get("id") or "")
|
|
config = self.portainer._config()
|
|
if not config or not controller_id:
|
|
return {
|
|
"status": "outcome_unknown" if state.get("state") == "started" else "error",
|
|
"reason": "Portainer deployment control is unavailable",
|
|
**({"execution_id": execution_id} if execution_id else {}),
|
|
}
|
|
|
|
original_controller_id = str(state.get("controllerId") or "")
|
|
endpoint = quote(str(config.endpoint_id), safe="")
|
|
|
|
if state.get("state") == "started" and execution_id:
|
|
if original_controller_id != controller_id:
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"reason": "deployment controller identity changed while first-deploy outcome was unknown",
|
|
"execution_id": execution_id,
|
|
}
|
|
try:
|
|
inspected = self.portainer._request(
|
|
"GET",
|
|
f"/api/endpoints/{endpoint}/docker/exec/{quote(execution_id, safe='')}/json",
|
|
).json()
|
|
except (RuntimeError, ValueError, TypeError) as exc:
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"reason": f"existing WorkMesh deployment outcome remains unknown: {redact_message(str(exc))}",
|
|
"execution_id": execution_id,
|
|
}
|
|
if (
|
|
str(inspected.get("ID") or "") != execution_id
|
|
or str(inspected.get("ContainerID") or "") != controller_id
|
|
):
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"reason": "WorkMesh deployment execution identity could not be verified",
|
|
"execution_id": execution_id,
|
|
}
|
|
if bool(inspected.get("Running")):
|
|
if job.status not in {"succeeded", "failed", "cancelled"}:
|
|
set_job_progress(session, job, max(job.progress, 50), "Official WorkMesh first deployment is still running.")
|
|
return {"status": "in_progress", "execution_id": execution_id}
|
|
raw_exit = inspected.get("ExitCode")
|
|
exit_code = int(raw_exit) if raw_exit is not None else -1
|
|
result = self._result_for_exit(exit_code)
|
|
row.value_json = {
|
|
**state,
|
|
"state": "completed",
|
|
"result": result,
|
|
"completedAt": utc_now().isoformat(),
|
|
}
|
|
row.updated_at = utc_now()
|
|
session.add(row)
|
|
session.commit()
|
|
return result
|
|
|
|
if state.get("state") not in {"idle", None}:
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"reason": "WorkMesh first-deploy state is not safely restartable",
|
|
**({"execution_id": execution_id} if execution_id else {}),
|
|
}
|
|
|
|
safe_controller = quote(controller_id, safe="")
|
|
row.value_json = {
|
|
"state": "creating",
|
|
"version": 1,
|
|
"jobId": job.id,
|
|
"controllerId": controller_id,
|
|
"createdAt": utc_now().isoformat(),
|
|
}
|
|
row.updated_at = utc_now()
|
|
session.add(row)
|
|
session.commit()
|
|
try:
|
|
created = self.portainer._request(
|
|
"POST",
|
|
f"/api/endpoints/{endpoint}/docker/containers/{safe_controller}/exec",
|
|
json={
|
|
"AttachStdout": False,
|
|
"AttachStderr": False,
|
|
"Tty": False,
|
|
"Cmd": ["python3", "-c", _FIRST_DEPLOY_SCRIPT],
|
|
},
|
|
).json()
|
|
execution_id = str(created.get("Id") or "")
|
|
if not execution_id:
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"reason": "WorkMesh deployment exec creation identity is unknown; do not retry",
|
|
}
|
|
row.value_json = {
|
|
"state": "created",
|
|
"version": 1,
|
|
"jobId": job.id,
|
|
"executionId": execution_id,
|
|
"controllerId": controller_id,
|
|
"createdAt": utc_now().isoformat(),
|
|
}
|
|
row.updated_at = utc_now()
|
|
session.add(row)
|
|
session.commit()
|
|
self.portainer._request(
|
|
"POST",
|
|
f"/api/endpoints/{endpoint}/docker/exec/{quote(execution_id, safe='')}/start",
|
|
json={"Detach": True, "Tty": False},
|
|
)
|
|
row.value_json = {
|
|
**dict(row.value_json or {}),
|
|
"state": "started",
|
|
"startedAt": utc_now().isoformat(),
|
|
}
|
|
row.updated_at = utc_now()
|
|
session.add(row)
|
|
session.commit()
|
|
except (RuntimeError, ValueError, TypeError) as exc:
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"reason": f"WorkMesh deployment exec outcome is unknown: {redact_message(str(exc))}",
|
|
**({"execution_id": execution_id} if execution_id else {}),
|
|
}
|
|
set_job_progress(session, job, 25, "Official WorkMesh first deployment started.")
|
|
return {"status": "in_progress", "execution_id": execution_id}
|
|
|
|
|
|
|
|
def workmesh_first_deploy_job_cancellation_reason(
|
|
session: Session,
|
|
job: JobRun,
|
|
) -> str | None:
|
|
# A running fixed-target deployment may have reached the controller even
|
|
# when its state row was lost or does not yet name this job. Keep both
|
|
# cancellation entry points fail-closed until execution is reconciled.
|
|
if (
|
|
job.action_id == "deployment.target.deploy_workmesh"
|
|
and job.target == _REPOSITORY
|
|
and job.status == "running"
|
|
):
|
|
return (
|
|
"WorkMesh first deployment may own an unresolved controller execution and "
|
|
"cannot be cancelled until that exact execution is reconciled."
|
|
)
|
|
row = session.get(AppSetting, _STATE_KEY)
|
|
state = dict(row.value_json or {}) if row else {}
|
|
if str(state.get("jobId") or "") != job.id:
|
|
return None
|
|
if state.get("state") in {"created", "started"}:
|
|
return (
|
|
"WorkMesh first deployment owns an active controller execution and "
|
|
"cannot be cancelled until that exact execution is reconciled."
|
|
)
|
|
return None
|