388 lines
16 KiB
Python
388 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
import time
|
|
from datetime import datetime
|
|
from typing import Any
|
|
from urllib.parse import quote, urlsplit, urlunsplit
|
|
|
|
import httpx
|
|
|
|
from ..connectors.portainer import PortainerConnector
|
|
from ..core.config import settings
|
|
from ..core.redaction import redact_message
|
|
from .deployment_broker import DeploymentBrokerClient
|
|
|
|
SOURCE_REPOSITORY = "jens/unraid-appops-gateway"
|
|
GITEA_OWNER = "Jens"
|
|
GITEA_REPOSITORY = "Unraid-AppOps-Gateway"
|
|
REQUIRED_PUSH_CONTEXT = "Managed validation / full (push)"
|
|
HELPER_NAME = "itworx-controller-artifact-promoter"
|
|
CONTROLLER_CONTAINER = "gitea-deploy-control"
|
|
CONTROLLER_VERSION = "2.1.13"
|
|
HELPER_LABEL = "itworx.controller.promoter"
|
|
_HELPER_TIMEOUT_SECONDS = 240
|
|
_REVISION = re.compile(r"^[0-9a-f]{40}$")
|
|
_IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$")
|
|
_TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$")
|
|
|
|
|
|
class ControllerArtifactPromotionError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def helper_container_spec(*, revision: str, image_id: str) -> dict[str, Any]:
|
|
if not _REVISION.fullmatch(revision):
|
|
raise ValueError("controller_promoter_revision_invalid")
|
|
if not _IMAGE_ID.fullmatch(image_id):
|
|
raise ValueError("controller_promoter_image_invalid")
|
|
|
|
command = f"""set -Eeuo pipefail
|
|
umask 077
|
|
revision='{revision}'
|
|
version='{CONTROLLER_VERSION}'
|
|
repo='/gitea-repositories/jens/unraid-appops-gateway.git'
|
|
work="$(mktemp -d -p /tmp 'itworx-controller-artifact-promoter-{revision[:12]}.XXXXXX')"
|
|
[[ -d "$work" && ! -L "$work" && "$(stat -c '%u:%a' "$work")" == '0:700' ]]
|
|
cleanup() {{ rm -rf -- "$work"; }}
|
|
trap cleanup EXIT
|
|
command -v git >/dev/null
|
|
command -v docker >/dev/null
|
|
docker compose version >/dev/null
|
|
command -v python3 >/dev/null
|
|
command -v tar >/dev/null
|
|
command -v curl >/dev/null
|
|
git --git-dir="$repo" cat-file -e "$revision^{{commit}}"
|
|
tree="$(git --git-dir="$repo" rev-parse "$revision^{{tree}}")"
|
|
git --git-dir="$repo" archive --format=tar "$revision" \
|
|
ops/gitea-deploy-broker \
|
|
scripts/prepare_controller_artifact.py \
|
|
scripts/reconcile_controller_state.py \
|
|
scripts/run_repopulse_adoption.py \
|
|
scripts/prepare_controller_hub_release_volume.py \
|
|
scripts/vendor \
|
|
scripts/upgrade_deployment_controller.sh > "$work/controller.tar"
|
|
chmod 0600 "$work/controller.tar"
|
|
archive_sha="$(sha256sum "$work/controller.tar" | cut -d ' ' -f 1)"
|
|
git --git-dir="$repo" show "$revision:scripts/upgrade_deployment_controller.sh" > "$work/upgrade.sh"
|
|
chmod 0700 "$work/upgrade.sh"
|
|
bash "$work/upgrade.sh" \
|
|
--archive "$work/controller.tar" \
|
|
--archive-sha256 "$archive_sha" \
|
|
--source-repository '{SOURCE_REPOSITORY}' \
|
|
--source-revision "$revision" \
|
|
--source-tree "$tree" \
|
|
--controller-version "$version" \
|
|
--image-id '{image_id}' \
|
|
--confirm UPGRADE_DEPLOYMENT_CONTROLLER_V2
|
|
"""
|
|
|
|
return {
|
|
"Image": image_id,
|
|
"User": "0:0",
|
|
"Cmd": ["/bin/bash", "-lc", command],
|
|
"Labels": {
|
|
HELPER_LABEL: "true",
|
|
"itworx.source.repository": SOURCE_REPOSITORY,
|
|
"itworx.source.revision": revision,
|
|
"itworx.controller.version": CONTROLLER_VERSION,
|
|
},
|
|
"HostConfig": {
|
|
"Binds": [
|
|
"/mnt/cache:/mnt/cache",
|
|
"/mnt/user:/mnt/user",
|
|
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
|
|
"/var/lib/docker:/var/lib/docker:ro",
|
|
"/var/run/docker.sock:/var/run/docker.sock",
|
|
],
|
|
"NetworkMode": "host",
|
|
"Privileged": False,
|
|
"SecurityOpt": ["no-new-privileges:true"],
|
|
"RestartPolicy": {"Name": "no"},
|
|
},
|
|
}
|
|
|
|
|
|
class ControllerArtifactPromoter:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
portainer: PortainerConnector | None = None,
|
|
broker: DeploymentBrokerClient | None = None,
|
|
) -> None:
|
|
self.portainer = portainer or PortainerConnector()
|
|
self.broker = broker or DeploymentBrokerClient()
|
|
|
|
@staticmethod
|
|
def _gitea_base_url() -> str:
|
|
raw = str(settings.apis.get("GITEA_URL") or "").strip().rstrip("/")
|
|
parsed = urlsplit(raw)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username:
|
|
raise ControllerArtifactPromotionError("controller_promoter_gitea_url_invalid")
|
|
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", ""))
|
|
|
|
@staticmethod
|
|
def _gitea_token() -> str:
|
|
token = str(settings.apis.get("GITEA_ADMIN_TOKEN") or "").strip()
|
|
if not token:
|
|
raise ControllerArtifactPromotionError("controller_promoter_gitea_token_missing")
|
|
return token
|
|
|
|
@staticmethod
|
|
def _status_key(item: dict[str, Any]) -> tuple[datetime, int] | None:
|
|
raw = str(item.get("updated_at") or item.get("created_at") or "")
|
|
if not _TIMESTAMP.fullmatch(raw):
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
identifier = item.get("id")
|
|
return parsed, identifier if isinstance(identifier, int) and not isinstance(identifier, bool) else -1
|
|
|
|
def _validated_main_revision(self) -> str:
|
|
branch = "main"
|
|
base = self._gitea_base_url()
|
|
headers = {
|
|
"Authorization": f"token {self._gitea_token()}",
|
|
"Accept": "application/json",
|
|
"User-Agent": "itworx-controller-artifact-promoter/1",
|
|
}
|
|
timeout = max(2.0, min(20.0, float(settings.apis.get("GITEA_TIMEOUT_SECONDS") or 8)))
|
|
verify = bool(settings.apis.get("GITEA_VERIFY_TLS", settings.verify_tls))
|
|
owner = quote(GITEA_OWNER, safe="")
|
|
repository = quote(GITEA_REPOSITORY, safe="")
|
|
try:
|
|
with httpx.Client(timeout=timeout, verify=verify, follow_redirects=False) as client:
|
|
branch_response = client.get(
|
|
f"{base}/api/v1/repos/{owner}/{repository}/branches/{branch}",
|
|
headers=headers,
|
|
)
|
|
if branch_response.status_code != 200:
|
|
raise ControllerArtifactPromotionError("controller_promoter_main_unavailable")
|
|
branch_payload = branch_response.json()
|
|
commit = branch_payload.get("commit") if isinstance(branch_payload, dict) else None
|
|
revision = str(commit.get("id") or "") if isinstance(commit, dict) else ""
|
|
if (
|
|
not isinstance(branch_payload, dict)
|
|
or branch_payload.get("name") != branch
|
|
or branch_payload.get("protected") is not True
|
|
or not _REVISION.fullmatch(revision)
|
|
):
|
|
raise ControllerArtifactPromotionError("controller_promoter_main_not_protected")
|
|
|
|
status_response = client.get(
|
|
f"{base}/api/v1/repos/{owner}/{repository}/statuses/{revision}",
|
|
params={"limit": 100, "page": 1},
|
|
headers=headers,
|
|
)
|
|
if status_response.status_code != 200:
|
|
raise ControllerArtifactPromotionError("controller_promoter_status_unavailable")
|
|
statuses = status_response.json()
|
|
except ControllerArtifactPromotionError:
|
|
raise
|
|
except (httpx.RequestError, ValueError) as exc:
|
|
raise ControllerArtifactPromotionError(
|
|
"controller_promoter_gitea_unavailable"
|
|
) from exc
|
|
|
|
if not isinstance(statuses, list) or len(statuses) > 100:
|
|
raise ControllerArtifactPromotionError("controller_promoter_status_invalid")
|
|
latest: tuple[tuple[datetime, int], str] | None = None
|
|
for item in statuses:
|
|
if not isinstance(item, dict) or item.get("context") != REQUIRED_PUSH_CONTEXT:
|
|
continue
|
|
state = str(item.get("status") or "")
|
|
if state not in {"pending", "success", "failure", "error", "warning", "neutral", "cancelled"}:
|
|
continue
|
|
key = self._status_key(item)
|
|
if key is not None and (latest is None or key > latest[0]):
|
|
latest = (key, state)
|
|
if latest is None or latest[1] != "success":
|
|
raise ControllerArtifactPromotionError("controller_promoter_main_not_validated")
|
|
return revision
|
|
|
|
def _portainer_endpoint(self) -> str:
|
|
config = self.portainer._config()
|
|
if config is None or not config.endpoint_id:
|
|
raise ControllerArtifactPromotionError("controller_promoter_portainer_unavailable")
|
|
return config.endpoint_id
|
|
|
|
def _controller_image_id(self) -> str:
|
|
listed = self.portainer.list_containers()
|
|
containers = listed.get("containers") if isinstance(listed, dict) else None
|
|
candidate = next(
|
|
(
|
|
item
|
|
for item in containers or []
|
|
if isinstance(item, dict) and item.get("name") == CONTROLLER_CONTAINER
|
|
),
|
|
None,
|
|
)
|
|
if candidate is None or candidate.get("state") != "running":
|
|
raise ControllerArtifactPromotionError("controller_promoter_controller_unavailable")
|
|
identifier = str(candidate.get("id") or "")
|
|
endpoint = self._portainer_endpoint()
|
|
try:
|
|
response = self.portainer._request(
|
|
"GET",
|
|
f"/api/endpoints/{endpoint}/docker/containers/{quote(identifier, safe='')}/json",
|
|
)
|
|
payload = response.json()
|
|
except (RuntimeError, ValueError) as exc:
|
|
raise ControllerArtifactPromotionError(
|
|
"controller_promoter_controller_inspect_failed"
|
|
) from exc
|
|
image_id = str(payload.get("Image") or "") if isinstance(payload, dict) else ""
|
|
if not _IMAGE_ID.fullmatch(image_id):
|
|
raise ControllerArtifactPromotionError("controller_promoter_controller_image_unpinned")
|
|
return image_id
|
|
|
|
def _existing_helper(self) -> dict[str, Any] | None:
|
|
listed = self.portainer.list_containers()
|
|
containers = listed.get("containers") if isinstance(listed, dict) else None
|
|
return next(
|
|
(
|
|
item
|
|
for item in containers or []
|
|
if isinstance(item, dict) and item.get("name") == HELPER_NAME
|
|
),
|
|
None,
|
|
)
|
|
|
|
def _remove_helper(self, identifier: str) -> None:
|
|
endpoint = self._portainer_endpoint()
|
|
self.portainer._request(
|
|
"DELETE",
|
|
f"/api/endpoints/{endpoint}/docker/containers/{quote(identifier, safe='')}",
|
|
params={"v": "1", "force": "0"},
|
|
)
|
|
|
|
def _run_helper(self, *, revision: str, image_id: str) -> dict[str, Any]:
|
|
endpoint = self._portainer_endpoint()
|
|
existing = self._existing_helper()
|
|
if existing is not None:
|
|
identifier = str(existing.get("id") or "")
|
|
if existing.get("state") == "running":
|
|
raise ControllerArtifactPromotionError("controller_promoter_already_running")
|
|
if identifier:
|
|
try:
|
|
inspect = self.portainer._request(
|
|
"GET",
|
|
f"/api/endpoints/{endpoint}/docker/containers/{quote(identifier, safe='')}/json",
|
|
).json()
|
|
except (RuntimeError, ValueError) as exc:
|
|
raise ControllerArtifactPromotionError(
|
|
"controller_promoter_helper_inspect_failed"
|
|
) from exc
|
|
labels = ((inspect.get("Config") or {}).get("Labels") or {}) if isinstance(inspect, dict) else {}
|
|
if not isinstance(labels, dict) or labels.get(HELPER_LABEL) != "true":
|
|
raise ControllerArtifactPromotionError("controller_promoter_name_conflict")
|
|
self._remove_helper(identifier)
|
|
|
|
spec = helper_container_spec(revision=revision, image_id=image_id)
|
|
try:
|
|
created_response = self.portainer._request(
|
|
"POST",
|
|
f"/api/endpoints/{endpoint}/docker/containers/create",
|
|
params={"name": HELPER_NAME},
|
|
json=spec,
|
|
)
|
|
created = created_response.json()
|
|
helper_id = str(created.get("Id") or "") if isinstance(created, dict) else ""
|
|
if not helper_id:
|
|
raise ControllerArtifactPromotionError("controller_promoter_create_failed")
|
|
self.portainer._request(
|
|
"POST",
|
|
f"/api/endpoints/{endpoint}/docker/containers/{quote(helper_id, safe='')}/start",
|
|
)
|
|
except ControllerArtifactPromotionError:
|
|
raise
|
|
except (RuntimeError, ValueError) as exc:
|
|
raise ControllerArtifactPromotionError("controller_promoter_start_failed") from exc
|
|
|
|
deadline = time.monotonic() + _HELPER_TIMEOUT_SECONDS
|
|
state: dict[str, Any] = {}
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
payload = self.portainer._request(
|
|
"GET",
|
|
f"/api/endpoints/{endpoint}/docker/containers/{quote(helper_id, safe='')}/json",
|
|
).json()
|
|
except (RuntimeError, ValueError) as exc:
|
|
raise ControllerArtifactPromotionError(
|
|
"controller_promoter_helper_inspect_failed"
|
|
) from exc
|
|
state = payload.get("State") if isinstance(payload, dict) and isinstance(payload.get("State"), dict) else {}
|
|
if state.get("Running") is False:
|
|
break
|
|
time.sleep(2)
|
|
else:
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"revision": revision,
|
|
"reason": "controller_promoter_timeout",
|
|
"helperId": helper_id,
|
|
}
|
|
|
|
exit_code = state.get("ExitCode")
|
|
logs = self.portainer.get_container_logs(helper_id, tail=200)
|
|
log_lines = logs.get("logs") if isinstance(logs, dict) and isinstance(logs.get("logs"), list) else []
|
|
tail = "\n".join(str(line) for line in log_lines[-40:])[-4000:]
|
|
if exit_code != 0:
|
|
try:
|
|
self._remove_helper(helper_id)
|
|
except RuntimeError:
|
|
pass
|
|
return {
|
|
"status": "failed",
|
|
"revision": revision,
|
|
"reason": "controller_promoter_helper_failed",
|
|
"exitCode": exit_code,
|
|
"logTail": redact_message(tail),
|
|
}
|
|
|
|
try:
|
|
self._remove_helper(helper_id)
|
|
except RuntimeError:
|
|
pass
|
|
return {"status": "promoted", "revision": revision, "logTail": redact_message(tail)}
|
|
|
|
def run(self) -> dict[str, Any]:
|
|
revision = self._validated_main_revision()
|
|
current = self.broker.controller_health()
|
|
artifact = current.get("artifact") if isinstance(current, dict) else None
|
|
if (
|
|
isinstance(artifact, dict)
|
|
and artifact.get("sourceRevision") == revision
|
|
and artifact.get("controllerVersion") == CONTROLLER_VERSION
|
|
):
|
|
return {
|
|
"status": "already_applied",
|
|
"revision": revision,
|
|
"controllerVersion": CONTROLLER_VERSION,
|
|
}
|
|
|
|
result = self._run_helper(revision=revision, image_id=self._controller_image_id())
|
|
if result.get("status") != "promoted":
|
|
return result
|
|
|
|
verified = self.broker.controller_health()
|
|
artifact = verified.get("artifact") if isinstance(verified, dict) else None
|
|
if (
|
|
not isinstance(artifact, dict)
|
|
or artifact.get("sourceRevision") != revision
|
|
or artifact.get("controllerVersion") != CONTROLLER_VERSION
|
|
):
|
|
return {
|
|
"status": "outcome_unknown",
|
|
"revision": revision,
|
|
"reason": "controller_promoter_postcheck_failed",
|
|
}
|
|
return {
|
|
"status": "promoted",
|
|
"revision": revision,
|
|
"controllerVersion": CONTROLLER_VERSION,
|
|
"artifactSha256": artifact.get("artifactSha256"),
|
|
}
|