497 lines
18 KiB
Python
497 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from threading import Lock
|
|
from time import monotonic
|
|
from typing import Any
|
|
from urllib.parse import urlsplit, urlunsplit
|
|
|
|
import httpx
|
|
|
|
from ..core.config import settings
|
|
from ..core.redaction import redact_message
|
|
from .base import BaseConnector
|
|
|
|
_CACHE_SECONDS = 15
|
|
_WARNING_AGE_SECONDS = 180
|
|
_CRITICAL_AGE_SECONDS = 600
|
|
_ACTIVE_WARNING_AGE_SECONDS = 1_800
|
|
_ACTIVE_CRITICAL_AGE_SECONDS = 7_200
|
|
_QUEUE_SAMPLE_LIMIT = 50
|
|
_CACHE_LOCK = Lock()
|
|
_CACHE: tuple[float, dict[str, Any]] | None = None
|
|
_DEFAULT_RUNNER_TARGETS = {
|
|
"linux-validation": "itworx-validation-runner-2",
|
|
"docker-validation": "itworx-docker-runner",
|
|
"browser-validation": "itworx-browser-runner",
|
|
"security-validation": "itworx-security-runner",
|
|
"mcp-validation": "itworx-security-runner",
|
|
"unraid-deploy": "itworx-security-runner",
|
|
}
|
|
_DEFAULT_RUNNER_TARGET_CANDIDATES = {
|
|
"linux-validation": ("itworx-validation-runner-2", "itworx-validation-runner-3"),
|
|
"docker-validation": (
|
|
"itworx-docker-runner",
|
|
"itworx-validation-runner-2",
|
|
"itworx-security-runner",
|
|
),
|
|
"browser-validation": ("itworx-browser-runner",),
|
|
"security-validation": ("itworx-security-runner",),
|
|
"mcp-validation": ("itworx-security-runner",),
|
|
"unraid-deploy": ("itworx-security-runner",),
|
|
}
|
|
|
|
|
|
class GiteaActionsConnector(BaseConnector):
|
|
"""Read a bounded Gitea Actions control-plane snapshot; never execute jobs."""
|
|
|
|
name = "Gitea Actions"
|
|
|
|
def is_configured(self) -> bool:
|
|
return bool(
|
|
settings.apis.get("GITEA_URL") and settings.apis.get("GITEA_ADMIN_TOKEN")
|
|
)
|
|
|
|
def _base_url(self) -> str:
|
|
raw = str(settings.apis.get("GITEA_URL") or "").strip().rstrip("/")
|
|
parsed = urlsplit(raw)
|
|
if (
|
|
parsed.scheme not in {"http", "https"}
|
|
or not parsed.netloc
|
|
or parsed.username
|
|
):
|
|
raise RuntimeError("invalid_gitea_url")
|
|
return urlunsplit(
|
|
(parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", "")
|
|
)
|
|
|
|
def _request(
|
|
self, path: str, params: dict[str, Any] | None = None
|
|
) -> dict[str, Any]:
|
|
if not self.is_configured():
|
|
raise RuntimeError("missing_config")
|
|
token = str(settings.apis.get("GITEA_ADMIN_TOKEN") or "")
|
|
timeout = max(
|
|
2.0,
|
|
min(20.0, float(settings.apis.get("GITEA_TIMEOUT_SECONDS") or 8)),
|
|
)
|
|
try:
|
|
with httpx.Client(
|
|
timeout=timeout,
|
|
verify=bool(settings.apis.get("GITEA_VERIFY_TLS", settings.verify_tls)),
|
|
) as client:
|
|
response = client.get(
|
|
f"{self._base_url()}{path}",
|
|
params=params,
|
|
headers={
|
|
"Authorization": f"token {token}",
|
|
"Accept": "application/json",
|
|
},
|
|
)
|
|
except (httpx.RequestError, ValueError) as exc:
|
|
raise RuntimeError(redact_message(str(exc))) from exc
|
|
if response.status_code in {401, 403}:
|
|
raise RuntimeError("gitea_admin_access_denied")
|
|
if response.status_code >= 400:
|
|
raise RuntimeError(f"gitea_actions_http_{response.status_code}")
|
|
try:
|
|
payload = response.json()
|
|
except ValueError as exc:
|
|
raise RuntimeError("gitea_actions_invalid_response") from exc
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError("gitea_actions_invalid_response") # noqa: TRY004
|
|
return payload
|
|
|
|
@staticmethod
|
|
def _timestamp(value: Any) -> datetime | None:
|
|
if not isinstance(value, str) or not value:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
|
|
|
@staticmethod
|
|
def _labels(value: Any) -> list[str]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
labels = []
|
|
for item in value:
|
|
label = item.get("name") if isinstance(item, dict) else item
|
|
text = str(label or "").strip()
|
|
if text:
|
|
labels.append(text[:120])
|
|
return labels
|
|
|
|
@staticmethod
|
|
def _configured_restart_targets() -> dict[str, str]:
|
|
raw = str(settings.apis.get("GITEA_RUNNER_TARGETS_JSON") or "{}")
|
|
try:
|
|
payload = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
payload = {}
|
|
return (
|
|
{
|
|
str(label)[:120]: str(target)[:240]
|
|
for label, target in payload.items()
|
|
if str(label).strip() and str(target).strip()
|
|
}
|
|
if isinstance(payload, dict)
|
|
else {}
|
|
)
|
|
|
|
@classmethod
|
|
def _restart_targets(cls) -> dict[str, str]:
|
|
return {**_DEFAULT_RUNNER_TARGETS, **cls._configured_restart_targets()}
|
|
|
|
@classmethod
|
|
def _target_for_label(
|
|
cls,
|
|
label: str,
|
|
runners: list[dict[str, Any]],
|
|
targets: dict[str, str],
|
|
) -> str | None:
|
|
"""Keep explicit mappings authoritative; prove only built-in defaults."""
|
|
configured = cls._configured_restart_targets()
|
|
if label in configured:
|
|
return configured[label]
|
|
|
|
compatible = {
|
|
str(runner.get("name") or "")
|
|
for runner in runners
|
|
if label in runner.get("labels", []) and str(runner.get("name") or "")
|
|
}
|
|
for candidate in _DEFAULT_RUNNER_TARGET_CANDIDATES.get(
|
|
label, (targets.get(label, ""),)
|
|
):
|
|
if candidate and candidate in compatible:
|
|
return candidate
|
|
return None
|
|
|
|
def snapshot(self, *, refresh: bool = False) -> dict[str, Any]:
|
|
global _CACHE
|
|
now_monotonic = monotonic()
|
|
with _CACHE_LOCK:
|
|
if (
|
|
not refresh
|
|
and _CACHE is not None
|
|
and now_monotonic - _CACHE[0] < _CACHE_SECONDS
|
|
):
|
|
cached = copy.deepcopy(_CACHE[1])
|
|
cached["cache"] = {"hit": True, "ttlSeconds": _CACHE_SECONDS}
|
|
return cached
|
|
|
|
if not self.is_configured():
|
|
return {
|
|
"status": "missing_config",
|
|
"severity": "unknown",
|
|
"generatedAt": datetime.now(UTC).isoformat(),
|
|
"runners": [],
|
|
"queue": {
|
|
"queued": 0,
|
|
"inProgress": 0,
|
|
"oldestQueuedAgeSeconds": None,
|
|
"oldestInProgressAgeSeconds": None,
|
|
"sampleTruncated": False,
|
|
"blockedByRunnerLabels": [],
|
|
"staleAssignmentRunnerLabels": [],
|
|
},
|
|
"recommendedAction": None,
|
|
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
|
}
|
|
|
|
try:
|
|
runner_payload = self._request(
|
|
"/api/v1/admin/actions/runners", {"disabled": "false"}
|
|
)
|
|
queued_payload = self._request(
|
|
"/api/v1/admin/actions/jobs",
|
|
{
|
|
"status": "queued",
|
|
"page": 1,
|
|
"limit": 100,
|
|
"sort": "id",
|
|
"order": "asc",
|
|
},
|
|
)
|
|
pending_payload = self._request(
|
|
"/api/v1/admin/actions/jobs",
|
|
{
|
|
"status": "pending",
|
|
"page": 1,
|
|
"limit": 100,
|
|
"sort": "id",
|
|
"order": "asc",
|
|
},
|
|
)
|
|
active_payload = self._request(
|
|
"/api/v1/admin/actions/jobs",
|
|
{"status": "in_progress", "page": 1, "limit": 100},
|
|
)
|
|
except RuntimeError as exc:
|
|
return {
|
|
"status": "error",
|
|
"severity": "warning",
|
|
"reason": redact_message(str(exc))[:240],
|
|
"generatedAt": datetime.now(UTC).isoformat(),
|
|
"runners": [],
|
|
"queue": {
|
|
"queued": 0,
|
|
"inProgress": 0,
|
|
"oldestQueuedAgeSeconds": None,
|
|
"oldestInProgressAgeSeconds": None,
|
|
"sampleTruncated": False,
|
|
"blockedByRunnerLabels": [],
|
|
"staleAssignmentRunnerLabels": [],
|
|
},
|
|
"recommendedAction": None,
|
|
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
|
}
|
|
|
|
raw_runners = runner_payload.get("runners")
|
|
runner_items = raw_runners if isinstance(raw_runners, list) else []
|
|
runners = [
|
|
{
|
|
"id": int(item.get("id", 0) or 0),
|
|
"name": str(item.get("name") or "unknown")[:160],
|
|
"status": str(item.get("status") or "unknown")[:32],
|
|
"busy": bool(item.get("busy")),
|
|
"disabled": bool(item.get("disabled")),
|
|
"labels": self._labels(item.get("labels")),
|
|
}
|
|
for item in runner_items
|
|
if isinstance(item, dict)
|
|
]
|
|
queued_jobs = [
|
|
item
|
|
for payload in (queued_payload, pending_payload)
|
|
for item in (
|
|
payload.get("jobs") if isinstance(payload.get("jobs"), list) else []
|
|
)
|
|
if isinstance(item, dict)
|
|
]
|
|
active_jobs = (
|
|
active_payload.get("jobs")
|
|
if isinstance(active_payload.get("jobs"), list)
|
|
else []
|
|
)
|
|
now = datetime.now(UTC)
|
|
queued_times = [self._timestamp(item.get("created_at")) for item in queued_jobs]
|
|
queued_times = [item for item in queued_times if item is not None]
|
|
oldest_age = max(
|
|
(int((now - item).total_seconds()) for item in queued_times), default=None
|
|
)
|
|
active_times = [
|
|
self._timestamp(item.get("started_at"))
|
|
or self._timestamp(item.get("created_at"))
|
|
for item in active_jobs
|
|
]
|
|
active_times = [item for item in active_times if item is not None]
|
|
oldest_active_age = max(
|
|
(int((now - item).total_seconds()) for item in active_times), default=None
|
|
)
|
|
stale_active_jobs = []
|
|
for job in active_jobs:
|
|
created_at = self._timestamp(job.get("started_at")) or self._timestamp(
|
|
job.get("created_at")
|
|
)
|
|
if (
|
|
created_at is not None
|
|
and int((now - created_at).total_seconds())
|
|
>= _ACTIVE_CRITICAL_AGE_SECONDS
|
|
):
|
|
stale_active_jobs.append(job)
|
|
|
|
online_runners = [
|
|
item
|
|
for item in runners
|
|
if item["status"] == "online" and not item["disabled"]
|
|
]
|
|
blocked_jobs = []
|
|
stale_assignment_jobs: list[dict[str, Any]] = []
|
|
queue_items = []
|
|
for job in queued_jobs:
|
|
required = set(self._labels(job.get("labels")))
|
|
matching = [
|
|
runner
|
|
for runner in online_runners
|
|
if required.issubset(set(runner["labels"]))
|
|
]
|
|
if required and not matching:
|
|
blocked_jobs.append(job)
|
|
created = self._timestamp(job.get("created_at"))
|
|
age = int((now - created).total_seconds()) if created else None
|
|
all_matching_busy = bool(matching) and all(
|
|
bool(runner["busy"]) for runner in matching
|
|
)
|
|
reason = (
|
|
"no_online_runner_matches_required_capability"
|
|
if required and not matching
|
|
else "all_matching_runners_are_busy"
|
|
if all_matching_busy
|
|
else "awaiting_scheduler_assignment"
|
|
)
|
|
if (
|
|
required
|
|
and matching
|
|
and not all_matching_busy
|
|
and age is not None
|
|
and age >= _CRITICAL_AGE_SECONDS
|
|
):
|
|
stale_assignment_jobs.append(
|
|
{
|
|
"job": job,
|
|
"required": sorted(required),
|
|
"matching": matching,
|
|
"ageSeconds": age,
|
|
}
|
|
)
|
|
if len(queue_items) < _QUEUE_SAMPLE_LIMIT:
|
|
queue_items.append(
|
|
{
|
|
"id": int(job.get("id", 0) or 0),
|
|
"name": str(job.get("name") or "job")[:160],
|
|
"labels": sorted(required)[:20],
|
|
"ageSeconds": age,
|
|
"matchingRunners": [runner["name"] for runner in matching][:20],
|
|
"reason": reason,
|
|
}
|
|
)
|
|
|
|
severity = "healthy"
|
|
if (
|
|
(oldest_age or 0) >= _CRITICAL_AGE_SECONDS
|
|
or (oldest_active_age or 0) >= _ACTIVE_CRITICAL_AGE_SECONDS
|
|
or blocked_jobs
|
|
):
|
|
severity = "critical"
|
|
elif (
|
|
(oldest_age or 0) >= _WARNING_AGE_SECONDS
|
|
or (oldest_active_age or 0) >= _ACTIVE_WARNING_AGE_SECONDS
|
|
or len(online_runners) < len(runners)
|
|
):
|
|
severity = "warning"
|
|
|
|
targets = self._restart_targets()
|
|
blocked_labels = [
|
|
label for job in blocked_jobs for label in self._labels(job.get("labels"))
|
|
]
|
|
stale_active_labels = [
|
|
label
|
|
for job in stale_active_jobs
|
|
for label in self._labels(job.get("labels"))
|
|
]
|
|
stale_assignment_labels = sorted(
|
|
{
|
|
label
|
|
for candidate in stale_assignment_jobs
|
|
for label in candidate["required"]
|
|
}
|
|
)
|
|
|
|
recommended_action: dict[str, Any] | None = None
|
|
for candidate in sorted(
|
|
stale_assignment_jobs,
|
|
key=lambda item: int(item["ageSeconds"]),
|
|
reverse=True,
|
|
):
|
|
job = candidate["job"]
|
|
matching_by_name = {
|
|
str(runner.get("name") or ""): runner for runner in candidate["matching"]
|
|
}
|
|
for label in candidate["required"]:
|
|
target = self._target_for_label(label, candidate["matching"], targets)
|
|
if not target:
|
|
continue
|
|
target_runner = matching_by_name.get(target)
|
|
if target_runner is not None and bool(target_runner.get("busy")):
|
|
continue
|
|
recommended_action = {
|
|
"intent": "runner.recover",
|
|
"target": target,
|
|
"requiresApproval": False,
|
|
"autoExecute": True,
|
|
"reason": "Gitea Actions scheduler left a compatible idle runner unassigned for at least ten minutes.",
|
|
"evidence": {
|
|
"jobId": int(job.get("id", 0) or 0),
|
|
"label": label,
|
|
"minimumAgeSeconds": _CRITICAL_AGE_SECONDS,
|
|
},
|
|
}
|
|
break
|
|
if recommended_action is not None:
|
|
break
|
|
|
|
if recommended_action is None:
|
|
restart_target = None
|
|
restart_reason = ""
|
|
for label in [*blocked_labels, *stale_active_labels]:
|
|
target = self._target_for_label(label, runners, targets)
|
|
if target:
|
|
restart_target = target
|
|
restart_reason = (
|
|
"Gitea Actions queue is stalled without a matching online runner."
|
|
if label in blocked_labels
|
|
else "A Gitea Actions job exceeded the two-hour guardrail on an explicitly mapped runner."
|
|
)
|
|
break
|
|
recommended_action = (
|
|
{
|
|
"intent": "container.restart",
|
|
"target": restart_target,
|
|
"requiresApproval": True,
|
|
"autoExecute": False,
|
|
"reason": restart_reason,
|
|
}
|
|
if severity == "critical" and restart_target
|
|
else None
|
|
)
|
|
|
|
result = {
|
|
"status": "ok",
|
|
"severity": severity,
|
|
"generatedAt": now.isoformat(),
|
|
"runners": runners,
|
|
"summary": {
|
|
"total": len(runners),
|
|
"online": len(online_runners),
|
|
"busy": sum(bool(item["busy"]) for item in runners),
|
|
"offline": sum(item["status"] != "online" for item in runners),
|
|
},
|
|
"queue": {
|
|
"queued": int(queued_payload.get("total_count", 0) or 0)
|
|
+ int(pending_payload.get("total_count", 0) or 0),
|
|
"inProgress": int(
|
|
active_payload.get("total_count", len(active_jobs)) or 0
|
|
),
|
|
"oldestQueuedAgeSeconds": oldest_age,
|
|
"oldestInProgressAgeSeconds": oldest_active_age,
|
|
"sampleTruncated": (
|
|
int(queued_payload.get("total_count", 0) or 0)
|
|
> len(
|
|
queued_payload.get("jobs")
|
|
if isinstance(queued_payload.get("jobs"), list)
|
|
else []
|
|
)
|
|
or int(pending_payload.get("total_count", 0) or 0)
|
|
> len(
|
|
pending_payload.get("jobs")
|
|
if isinstance(pending_payload.get("jobs"), list)
|
|
else []
|
|
)
|
|
),
|
|
"blockedByRunnerLabels": sorted(set(blocked_labels))[:20],
|
|
"staleActiveRunnerLabels": sorted(set(stale_active_labels))[:20],
|
|
"staleAssignmentRunnerLabels": stale_assignment_labels[:20],
|
|
"items": queue_items,
|
|
},
|
|
"recommendedAction": recommended_action,
|
|
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
|
}
|
|
with _CACHE_LOCK:
|
|
_CACHE = (now_monotonic, copy.deepcopy(result))
|
|
return result
|