612 lines
25 KiB
Python
612 lines
25 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from typing import Any
|
|
from urllib.parse import quote, urlsplit, urlunsplit
|
|
|
|
import httpx
|
|
import yaml
|
|
from sqlmodel import Session, select
|
|
|
|
from ..core.config import settings
|
|
from ..core.redaction import redact_message
|
|
from ..core.time import utc_now
|
|
from ..db.models import AppSetting, Policy
|
|
from .deployment_target_registry import DeploymentTargetRegistry
|
|
|
|
_CATALOG_KEY = "deployment:gitea-project-catalog:v1"
|
|
_OWNER = "Jens"
|
|
_DESCRIPTOR = ".itworx/deployment-target.json"
|
|
_COMPOSE_CANDIDATES = (
|
|
"docker-compose.yml",
|
|
"docker-compose.yaml",
|
|
"compose.yml",
|
|
"compose.yaml",
|
|
)
|
|
# Must remain identical to DeploymentPlanV1.application_id.
|
|
_APP_ID = re.compile(r"^[a-z][a-z0-9-]{1,62}$")
|
|
_TRANSIENT_STATES = {"gitea_unavailable", "controller_unavailable", "registration_error"}
|
|
_RETRY_STATES = _TRANSIENT_STATES | {"controller_conflict", "policy_conflict"}
|
|
_DEPLOYMENT_ACTIONS = ("deployment.apply", "deployment.rollback", "deployment.runnerless.provision")
|
|
|
|
|
|
class GiteaProjectAutoRegistration:
|
|
"""Keep AppOps' project catalog aligned with the owned Gitea portfolio.
|
|
|
|
Every repository is recorded in the catalog. A repository becomes an
|
|
executable deployment target only when an exact revision exposes a bounded
|
|
Compose contract compatible with DeploymentPlanV1. Exact deployment
|
|
policies are created only after the controller target has been registered
|
|
successfully. Existing targets and policies are never overwritten.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
target_registry: DeploymentTargetRegistry | None = None,
|
|
owner: str = _OWNER,
|
|
) -> None:
|
|
self.target_registry = target_registry or DeploymentTargetRegistry()
|
|
self.owner = owner
|
|
|
|
@staticmethod
|
|
def _base_url() -> str:
|
|
raw = str(settings.apis.get("GITEA_URL") or "").strip().rstrip("/")
|
|
parsed = urlsplit(raw)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username:
|
|
raise RuntimeError("invalid_gitea_url")
|
|
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", ""))
|
|
|
|
@staticmethod
|
|
def _token() -> str:
|
|
token = str(settings.apis.get("GITEA_ADMIN_TOKEN") or "").strip()
|
|
if not token:
|
|
raise RuntimeError("missing_gitea_admin_token")
|
|
return token
|
|
|
|
def _client(self) -> httpx.Client:
|
|
timeout = max(2.0, min(20.0, float(settings.apis.get("GITEA_TIMEOUT_SECONDS") or 8)))
|
|
return httpx.Client(
|
|
timeout=timeout,
|
|
verify=bool(settings.apis.get("GITEA_VERIFY_TLS", settings.verify_tls)),
|
|
headers={
|
|
"Authorization": f"token {self._token()}",
|
|
"Accept": "application/json",
|
|
},
|
|
)
|
|
|
|
def _get(
|
|
self,
|
|
client: httpx.Client,
|
|
path: str,
|
|
*,
|
|
params: dict[str, Any] | None = None,
|
|
allow_not_found: bool = False,
|
|
) -> Any:
|
|
try:
|
|
response = client.get(f"{self._base_url()}{path}", params=params)
|
|
except (httpx.RequestError, ValueError) as exc:
|
|
raise RuntimeError(redact_message(str(exc))) from exc
|
|
if allow_not_found and response.status_code == 404:
|
|
return None
|
|
if response.status_code in {401, 403}:
|
|
raise RuntimeError("gitea_admin_access_denied")
|
|
if response.status_code >= 400:
|
|
raise RuntimeError(f"gitea_project_catalog_http_{response.status_code}")
|
|
try:
|
|
return response.json()
|
|
except ValueError as exc:
|
|
raise RuntimeError("gitea_project_catalog_invalid_response") from exc
|
|
|
|
def _repositories(self, client: httpx.Client) -> list[dict[str, Any]]:
|
|
owner = quote(self.owner, safe="")
|
|
repositories: list[dict[str, Any]] = []
|
|
for page in range(1, 11):
|
|
payload = self._get(
|
|
client,
|
|
f"/api/v1/users/{owner}/repos",
|
|
params={"page": page, "limit": 50},
|
|
)
|
|
if not isinstance(payload, list):
|
|
raise RuntimeError("gitea_project_catalog_invalid_repository_list")
|
|
items = [item for item in payload if isinstance(item, dict)]
|
|
repositories.extend(items)
|
|
if len(items) < 50:
|
|
break
|
|
return repositories[:500]
|
|
|
|
def _branch_head(
|
|
self,
|
|
client: httpx.Client,
|
|
*,
|
|
repository_name: str,
|
|
branch: str,
|
|
) -> str:
|
|
owner = quote(self.owner, safe="")
|
|
repository = quote(repository_name, safe="")
|
|
branch_ref = quote(branch, safe="")
|
|
payload = self._get(
|
|
client,
|
|
f"/api/v1/repos/{owner}/{repository}/branches/{branch_ref}",
|
|
)
|
|
commit = payload.get("commit") if isinstance(payload, dict) else None
|
|
revision = ""
|
|
if isinstance(commit, dict):
|
|
revision = str(commit.get("id") or commit.get("sha") or "").strip().lower()
|
|
if not re.fullmatch(r"[0-9a-f]{40}", revision):
|
|
raise RuntimeError("gitea_default_branch_head_unavailable")
|
|
return revision
|
|
|
|
def _contents(
|
|
self,
|
|
client: httpx.Client,
|
|
*,
|
|
repository_name: str,
|
|
revision: str,
|
|
path: str = "",
|
|
allow_not_found: bool = False,
|
|
) -> Any:
|
|
owner = quote(self.owner, safe="")
|
|
repository = quote(repository_name, safe="")
|
|
suffix = f"/{quote(path, safe='/')}" if path else ""
|
|
return self._get(
|
|
client,
|
|
f"/api/v1/repos/{owner}/{repository}/contents{suffix}",
|
|
params={"ref": revision},
|
|
allow_not_found=allow_not_found,
|
|
)
|
|
|
|
def _read_text(
|
|
self,
|
|
client: httpx.Client,
|
|
*,
|
|
repository_name: str,
|
|
revision: str,
|
|
path: str,
|
|
allow_not_found: bool = False,
|
|
) -> str | None:
|
|
payload = self._contents(
|
|
client,
|
|
repository_name=repository_name,
|
|
revision=revision,
|
|
path=path,
|
|
allow_not_found=allow_not_found,
|
|
)
|
|
if payload is None:
|
|
return None
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError("gitea_project_file_invalid_response")
|
|
if str(payload.get("encoding") or "").lower() != "base64":
|
|
raise RuntimeError("gitea_project_file_encoding_unsupported")
|
|
content = str(payload.get("content") or "").replace("\n", "")
|
|
try:
|
|
raw = base64.b64decode(content, validate=True)
|
|
return raw.decode("utf-8")
|
|
except (ValueError, UnicodeDecodeError) as exc:
|
|
raise RuntimeError("gitea_project_file_decode_failed") from exc
|
|
|
|
@staticmethod
|
|
def _application_id(repository_name: str) -> str:
|
|
raw = repository_name.lower()
|
|
value = re.sub(r"[^a-z0-9]+", "-", raw).strip("-")[:63]
|
|
if not _APP_ID.fullmatch(value):
|
|
raise ValueError("repository name cannot be converted to a DeploymentPlanV1 application id")
|
|
return value
|
|
|
|
@staticmethod
|
|
def _safe_descriptor(payload: Any, repository_name: str) -> dict[str, Any]:
|
|
if payload is None:
|
|
return {}
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("deployment descriptor must contain a JSON object")
|
|
allowed = {
|
|
"apiVersion",
|
|
"applicationId",
|
|
"composeFile",
|
|
"composeProject",
|
|
"healthTimeout",
|
|
"verifyOciRevision",
|
|
}
|
|
unknown = set(payload) - allowed
|
|
if unknown:
|
|
raise ValueError("deployment descriptor contains unsupported fields")
|
|
if payload.get("apiVersion") not in {None, "itworx.deployment-target/v1"}:
|
|
raise ValueError("deployment descriptor apiVersion is unsupported")
|
|
application_id = str(
|
|
payload.get("applicationId")
|
|
or GiteaProjectAutoRegistration._application_id(repository_name)
|
|
).strip().lower()
|
|
if not _APP_ID.fullmatch(application_id):
|
|
raise ValueError("deployment descriptor applicationId is invalid")
|
|
compose_file = str(payload.get("composeFile") or "").strip()
|
|
compose_project = str(payload.get("composeProject") or application_id).strip().lower()
|
|
if compose_project != application_id:
|
|
raise ValueError("deployment descriptor composeProject must equal applicationId")
|
|
timeout = int(payload.get("healthTimeout") or 180)
|
|
return {
|
|
"applicationId": application_id,
|
|
"composeFile": compose_file,
|
|
"composeProject": compose_project,
|
|
"healthTimeout": timeout,
|
|
"verifyOciRevision": bool(payload.get("verifyOciRevision", False)),
|
|
}
|
|
|
|
def _compose_contract(
|
|
self,
|
|
client: httpx.Client,
|
|
*,
|
|
repository_name: str,
|
|
revision: str,
|
|
) -> dict[str, Any] | None:
|
|
root = self._contents(
|
|
client,
|
|
repository_name=repository_name,
|
|
revision=revision,
|
|
)
|
|
if not isinstance(root, list):
|
|
raise RuntimeError("gitea_project_root_invalid_response")
|
|
names = {
|
|
str(item.get("name") or "")
|
|
for item in root
|
|
if isinstance(item, dict)
|
|
}
|
|
|
|
descriptor: dict[str, Any] = {}
|
|
descriptor_present = False
|
|
if ".itworx" in names:
|
|
raw_descriptor = self._read_text(
|
|
client,
|
|
repository_name=repository_name,
|
|
revision=revision,
|
|
path=_DESCRIPTOR,
|
|
allow_not_found=True,
|
|
)
|
|
if raw_descriptor is not None:
|
|
descriptor_present = True
|
|
try:
|
|
descriptor = self._safe_descriptor(json.loads(raw_descriptor), repository_name)
|
|
except (json.JSONDecodeError, TypeError, ValueError) as exc:
|
|
raise ValueError(f"invalid deployment descriptor: {exc}") from exc
|
|
|
|
compose_file = str(descriptor.get("composeFile") or "")
|
|
if not compose_file:
|
|
compose_file = next((candidate for candidate in _COMPOSE_CANDIDATES if candidate in names), "")
|
|
if not compose_file:
|
|
return None
|
|
|
|
raw_compose = self._read_text(
|
|
client,
|
|
repository_name=repository_name,
|
|
revision=revision,
|
|
path=compose_file,
|
|
)
|
|
if raw_compose is None:
|
|
raise ValueError("configured Compose file is unavailable")
|
|
try:
|
|
compose = yaml.safe_load(raw_compose)
|
|
except yaml.YAMLError as exc:
|
|
raise ValueError("Compose file is not valid YAML") from exc
|
|
services = compose.get("services") if isinstance(compose, dict) else None
|
|
if not isinstance(services, dict) or not services:
|
|
raise ValueError("Compose file does not define services")
|
|
if len(services) > 8:
|
|
if descriptor_present:
|
|
raise ValueError("deployment descriptor exceeds DeploymentPlanV1's eight-service bound")
|
|
return None
|
|
|
|
service_names = [str(name).strip() for name in services]
|
|
if any(not name for name in service_names):
|
|
raise ValueError("Compose service name is invalid")
|
|
|
|
if descriptor_present:
|
|
application_id = str(descriptor["applicationId"])
|
|
if application_id not in service_names:
|
|
raise ValueError("deployment descriptor applicationId must name a Compose service")
|
|
else:
|
|
repository_id = self._application_id(repository_name)
|
|
compose_name = str(compose.get("name") or "").strip().lower() if isinstance(compose, dict) else ""
|
|
candidates: list[str] = []
|
|
for candidate in (repository_id, compose_name):
|
|
if candidate and _APP_ID.fullmatch(candidate) and candidate in service_names:
|
|
if candidate not in candidates:
|
|
candidates.append(candidate)
|
|
if len(service_names) == 1 and _APP_ID.fullmatch(service_names[0]):
|
|
if service_names[0] not in candidates:
|
|
candidates.append(service_names[0])
|
|
if len(candidates) != 1:
|
|
# The repository remains cataloged, but deployment requires an
|
|
# explicit bounded descriptor to select its primary service.
|
|
return None
|
|
application_id = candidates[0]
|
|
|
|
compose_project = application_id
|
|
containers: list[str] = []
|
|
for service_name, raw_service in services.items():
|
|
service = raw_service if isinstance(raw_service, dict) else {}
|
|
explicit_name = str(service.get("container_name") or "").strip()
|
|
container = explicit_name or f"{compose_project}-{service_name}-1"
|
|
containers.append(container)
|
|
|
|
return {
|
|
"applicationId": application_id,
|
|
"composeFile": compose_file,
|
|
"composeProject": compose_project,
|
|
"containers": containers,
|
|
"serviceNames": service_names,
|
|
"healthTimeout": int(descriptor.get("healthTimeout") or 180),
|
|
"verifyOciRevision": bool(descriptor.get("verifyOciRevision", False)),
|
|
}
|
|
|
|
@staticmethod
|
|
def _policy_id(application_id: str, action: str) -> str:
|
|
digest = hashlib.sha256(f"{action}:{application_id}".encode("utf-8")).hexdigest()[:20]
|
|
suffix = action.rsplit(".", 1)[-1]
|
|
return f"gitea-auto-{application_id[:32]}-{suffix}-{digest}"
|
|
|
|
@staticmethod
|
|
def _existing_exact_policy_ids(
|
|
session: Session,
|
|
*,
|
|
application_id: str,
|
|
) -> tuple[bool, list[str]]:
|
|
identifiers: list[str] = []
|
|
for action in _DEPLOYMENT_ACTIONS:
|
|
existing = session.exec(
|
|
select(Policy).where(
|
|
(Policy.action == action) & (Policy.target == application_id)
|
|
)
|
|
).first()
|
|
if existing is None or existing.manual != "allowed":
|
|
return False, []
|
|
identifiers.append(existing.id)
|
|
return True, identifiers
|
|
|
|
@staticmethod
|
|
def _ensure_exact_policies(
|
|
session: Session,
|
|
*,
|
|
application_id: str,
|
|
repository: str,
|
|
revision: str,
|
|
) -> tuple[bool, list[str], int]:
|
|
existing_by_action: dict[str, Policy | None] = {}
|
|
for action in _DEPLOYMENT_ACTIONS:
|
|
existing = session.exec(
|
|
select(Policy).where(
|
|
(Policy.action == action) & (Policy.target == application_id)
|
|
)
|
|
).first()
|
|
if existing is not None and existing.manual != "allowed":
|
|
return False, [], 0
|
|
existing_by_action[action] = existing
|
|
|
|
identifiers: list[str] = []
|
|
created = 0
|
|
for action in _DEPLOYMENT_ACTIONS:
|
|
existing = existing_by_action[action]
|
|
if existing is not None:
|
|
identifiers.append(existing.id)
|
|
continue
|
|
policy_id = GiteaProjectAutoRegistration._policy_id(application_id, action)
|
|
row = Policy(
|
|
id=policy_id,
|
|
action=action,
|
|
target=application_id,
|
|
voice="blocked",
|
|
manual="allowed",
|
|
risk="high",
|
|
reason="Gitea auto-registration generated exact-target deployment policy.",
|
|
provenance_json={
|
|
"createdBy": "gitea-project-auto-registration/v1",
|
|
"repository": repository,
|
|
"sourceRevision": revision,
|
|
"createdAt": utc_now().isoformat(),
|
|
"policyTemplateVersion": "exact-target/v1",
|
|
},
|
|
)
|
|
session.add(row)
|
|
identifiers.append(policy_id)
|
|
created += 1
|
|
session.commit()
|
|
return True, identifiers, created
|
|
|
|
@staticmethod
|
|
def catalog(session: Session) -> dict[str, Any]:
|
|
row = session.get(AppSetting, _CATALOG_KEY)
|
|
if row is None or row.is_secret:
|
|
return {
|
|
"status": "never_synced",
|
|
"owner": _OWNER,
|
|
"projects": [],
|
|
"summary": {"total": 0},
|
|
}
|
|
return dict(row.value_json or {})
|
|
|
|
@staticmethod
|
|
def _marker(repo: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"updatedAt": str(repo.get("updated_at") or ""),
|
|
"defaultBranch": str(repo.get("default_branch") or ""),
|
|
"archived": bool(repo.get("archived")),
|
|
"empty": bool(repo.get("empty")),
|
|
"private": bool(repo.get("private")),
|
|
}
|
|
|
|
def sync(self, session: Session, *, force: bool = False) -> dict[str, Any]:
|
|
previous = self.catalog(session)
|
|
previous_items = {
|
|
str(item.get("repository") or ""): item
|
|
for item in previous.get("projects", [])
|
|
if isinstance(item, dict)
|
|
}
|
|
try:
|
|
client = self._client()
|
|
with client:
|
|
repositories = self._repositories(client)
|
|
projects: list[dict[str, Any]] = []
|
|
controller_registered = 0
|
|
policies_created = 0
|
|
changed = 0
|
|
for repo in sorted(
|
|
repositories,
|
|
key=lambda item: str(item.get("full_name") or item.get("name") or "").casefold(),
|
|
):
|
|
name = str(repo.get("name") or "").strip()
|
|
if not name:
|
|
continue
|
|
repository = f"jens/{name.lower()}"
|
|
marker = self._marker(repo)
|
|
prior = previous_items.get(repository)
|
|
if (
|
|
not force
|
|
and prior is not None
|
|
and all(prior.get(key) == value for key, value in marker.items())
|
|
and str(prior.get("state") or "") not in _RETRY_STATES
|
|
):
|
|
projects.append(dict(prior))
|
|
continue
|
|
|
|
changed += 1
|
|
item: dict[str, Any] = {
|
|
"repository": repository,
|
|
"name": name,
|
|
**marker,
|
|
"state": "known",
|
|
"deploymentRegistered": False,
|
|
"policyIds": [],
|
|
}
|
|
if marker["archived"]:
|
|
item["state"] = "archived"
|
|
projects.append(item)
|
|
continue
|
|
branch = str(marker["defaultBranch"] or "").strip()
|
|
if marker["empty"] or not branch:
|
|
item["state"] = "empty"
|
|
projects.append(item)
|
|
continue
|
|
|
|
try:
|
|
revision = self._branch_head(
|
|
client,
|
|
repository_name=name,
|
|
branch=branch,
|
|
)
|
|
item["revision"] = revision
|
|
contract = self._compose_contract(
|
|
client,
|
|
repository_name=name,
|
|
revision=revision,
|
|
)
|
|
if contract is None:
|
|
item["state"] = "known_unconfigured"
|
|
projects.append(item)
|
|
continue
|
|
application_id = str(contract["applicationId"])
|
|
item.update(
|
|
{
|
|
"applicationId": application_id,
|
|
"composeFile": contract["composeFile"],
|
|
"composeProject": contract["composeProject"],
|
|
"containers": list(contract["containers"]),
|
|
"serviceNames": list(contract["serviceNames"]),
|
|
}
|
|
)
|
|
result = self.target_registry.register_compose_target(
|
|
repository,
|
|
revision,
|
|
branch=branch,
|
|
compose_file=str(contract["composeFile"]),
|
|
compose_project=str(contract["composeProject"]),
|
|
containers=list(contract["containers"]),
|
|
health_timeout=int(contract["healthTimeout"]),
|
|
verify_oci_revision=bool(contract["verifyOciRevision"]),
|
|
)
|
|
result_status = str(result.get("status") or "error")
|
|
if result_status == "registered":
|
|
policy_ok, policy_ids, created = self._ensure_exact_policies(
|
|
session,
|
|
application_id=application_id,
|
|
repository=repository,
|
|
revision=revision,
|
|
)
|
|
policies_created += created
|
|
if policy_ok:
|
|
item["state"] = "registered"
|
|
item["deploymentRegistered"] = True
|
|
item["policyIds"] = policy_ids
|
|
controller_registered += 1
|
|
else:
|
|
item["state"] = "policy_conflict"
|
|
elif result_status == "conflict":
|
|
policy_ok, policy_ids = self._existing_exact_policy_ids(
|
|
session,
|
|
application_id=application_id,
|
|
)
|
|
if policy_ok:
|
|
item["state"] = "registered_existing"
|
|
item["deploymentRegistered"] = True
|
|
item["policyIds"] = policy_ids
|
|
else:
|
|
item["state"] = "controller_conflict"
|
|
item["reason"] = str(result.get("reason") or "")[:240]
|
|
elif result_status in {"controller_unavailable", "connector_unavailable"}:
|
|
item["state"] = "controller_unavailable"
|
|
item["reason"] = str(result.get("reason") or "")[:240]
|
|
else:
|
|
item["state"] = "registration_error"
|
|
item["reason"] = str(result.get("reason") or result_status)[:240]
|
|
except (RuntimeError, TypeError, ValueError, yaml.YAMLError) as exc:
|
|
item["state"] = "registration_error"
|
|
item["reason"] = redact_message(str(exc))[:240]
|
|
projects.append(item)
|
|
except (RuntimeError, ValueError) as exc:
|
|
return {
|
|
**previous,
|
|
"status": "gitea_unavailable",
|
|
"reason": redact_message(str(exc))[:240],
|
|
}
|
|
|
|
summary = {
|
|
"total": len(projects),
|
|
"registered": sum(
|
|
str(item.get("state")) in {"registered", "registered_existing"}
|
|
for item in projects
|
|
),
|
|
"knownUnconfigured": sum(
|
|
str(item.get("state")) == "known_unconfigured" for item in projects
|
|
),
|
|
"archived": sum(str(item.get("state")) == "archived" for item in projects),
|
|
"empty": sum(str(item.get("state")) == "empty" for item in projects),
|
|
"conflicts": sum(
|
|
str(item.get("state")) in {"controller_conflict", "policy_conflict"}
|
|
for item in projects
|
|
),
|
|
"errors": sum(
|
|
str(item.get("state")) in _TRANSIENT_STATES for item in projects
|
|
),
|
|
"changedThisSync": changed,
|
|
"controllerTargetsRegisteredThisSync": controller_registered,
|
|
"policiesCreatedThisSync": policies_created,
|
|
}
|
|
value = {
|
|
"status": "ok",
|
|
"owner": self.owner,
|
|
"syncedAt": utc_now().isoformat(),
|
|
"automatic": True,
|
|
"projects": projects,
|
|
"summary": summary,
|
|
}
|
|
row = session.get(AppSetting, _CATALOG_KEY) or AppSetting(
|
|
key=_CATALOG_KEY,
|
|
is_secret=False,
|
|
)
|
|
row.value_json = value
|
|
row.is_secret = False
|
|
row.updated_at = utc_now()
|
|
session.add(row)
|
|
session.commit()
|
|
return value
|