Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d
This commit is contained in:
Public Source Publisher committed
commit
1dfbca08a1
528 files changed
+131054
No files matched your search
@@ -0,0 +1,496 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from threading import Lock
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from ..core.config import settings
|
||||
from ..core.redaction import redact_message
|
||||
from .base import BaseConnector
|
||||
|
||||
_CACHE_SECONDS = 15
|
||||
_WARNING_AGE_SECONDS = 180
|
||||
_CRITICAL_AGE_SECONDS = 600
|
||||
_ACTIVE_WARNING_AGE_SECONDS = 1_800
|
||||
_ACTIVE_CRITICAL_AGE_SECONDS = 7_200
|
||||
_QUEUE_SAMPLE_LIMIT = 50
|
||||
_CACHE_LOCK = Lock()
|
||||
_CACHE: tuple[float, dict[str, Any]] | None = None
|
||||
_DEFAULT_RUNNER_TARGETS = {
|
||||
"linux-validation": "itworx-validation-runner-2",
|
||||
"docker-validation": "itworx-docker-runner",
|
||||
"browser-validation": "itworx-browser-runner",
|
||||
"security-validation": "itworx-security-runner",
|
||||
"mcp-validation": "itworx-security-runner",
|
||||
"unraid-deploy": "itworx-security-runner",
|
||||
}
|
||||
_DEFAULT_RUNNER_TARGET_CANDIDATES = {
|
||||
"linux-validation": ("itworx-validation-runner-2", "itworx-validation-runner-3"),
|
||||
"docker-validation": (
|
||||
"itworx-docker-runner",
|
||||
"itworx-validation-runner-2",
|
||||
"itworx-security-runner",
|
||||
),
|
||||
"browser-validation": ("itworx-browser-runner",),
|
||||
"security-validation": ("itworx-security-runner",),
|
||||
"mcp-validation": ("itworx-security-runner",),
|
||||
"unraid-deploy": ("itworx-security-runner",),
|
||||
}
|
||||
|
||||
|
||||
class GiteaActionsConnector(BaseConnector):
|
||||
"""Read a bounded Gitea Actions control-plane snapshot; never execute jobs."""
|
||||
|
||||
name = "Gitea Actions"
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return bool(
|
||||
settings.apis.get("GITEA_URL") and settings.apis.get("GITEA_ADMIN_TOKEN")
|
||||
)
|
||||
|
||||
def _base_url(self) -> str:
|
||||
raw = str(settings.apis.get("GITEA_URL") or "").strip().rstrip("/")
|
||||
parsed = urlsplit(raw)
|
||||
if (
|
||||
parsed.scheme not in {"http", "https"}
|
||||
or not parsed.netloc
|
||||
or parsed.username
|
||||
):
|
||||
raise RuntimeError("invalid_gitea_url")
|
||||
return urlunsplit(
|
||||
(parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", "")
|
||||
)
|
||||
|
||||
def _request(
|
||||
self, path: str, params: dict[str, Any] | None = None
|
||||
) -> dict[str, Any]:
|
||||
if not self.is_configured():
|
||||
raise RuntimeError("missing_config")
|
||||
token = str(settings.apis.get("GITEA_ADMIN_TOKEN") or "")
|
||||
timeout = max(
|
||||
2.0,
|
||||
min(20.0, float(settings.apis.get("GITEA_TIMEOUT_SECONDS") or 8)),
|
||||
)
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=timeout,
|
||||
verify=bool(settings.apis.get("GITEA_VERIFY_TLS", settings.verify_tls)),
|
||||
) as client:
|
||||
response = client.get(
|
||||
f"{self._base_url()}{path}",
|
||||
params=params,
|
||||
headers={
|
||||
"Authorization": f"token {token}",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
except (httpx.RequestError, ValueError) as exc:
|
||||
raise RuntimeError(redact_message(str(exc))) from exc
|
||||
if response.status_code in {401, 403}:
|
||||
raise RuntimeError("gitea_admin_access_denied")
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"gitea_actions_http_{response.status_code}")
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("gitea_actions_invalid_response") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("gitea_actions_invalid_response") # noqa: TRY004
|
||||
return payload
|
||||
|
||||
@staticmethod
|
||||
def _timestamp(value: Any) -> datetime | None:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
|
||||
@staticmethod
|
||||
def _labels(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
labels = []
|
||||
for item in value:
|
||||
label = item.get("name") if isinstance(item, dict) else item
|
||||
text = str(label or "").strip()
|
||||
if text:
|
||||
labels.append(text[:120])
|
||||
return labels
|
||||
|
||||
@staticmethod
|
||||
def _configured_restart_targets() -> dict[str, str]:
|
||||
raw = str(settings.apis.get("GITEA_RUNNER_TARGETS_JSON") or "{}")
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
payload = {}
|
||||
return (
|
||||
{
|
||||
str(label)[:120]: str(target)[:240]
|
||||
for label, target in payload.items()
|
||||
if str(label).strip() and str(target).strip()
|
||||
}
|
||||
if isinstance(payload, dict)
|
||||
else {}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _restart_targets(cls) -> dict[str, str]:
|
||||
return {**_DEFAULT_RUNNER_TARGETS, **cls._configured_restart_targets()}
|
||||
|
||||
@classmethod
|
||||
def _target_for_label(
|
||||
cls,
|
||||
label: str,
|
||||
runners: list[dict[str, Any]],
|
||||
targets: dict[str, str],
|
||||
) -> str | None:
|
||||
"""Keep explicit mappings authoritative; prove only built-in defaults."""
|
||||
configured = cls._configured_restart_targets()
|
||||
if label in configured:
|
||||
return configured[label]
|
||||
|
||||
compatible = {
|
||||
str(runner.get("name") or "")
|
||||
for runner in runners
|
||||
if label in runner.get("labels", []) and str(runner.get("name") or "")
|
||||
}
|
||||
for candidate in _DEFAULT_RUNNER_TARGET_CANDIDATES.get(
|
||||
label, (targets.get(label, ""),)
|
||||
):
|
||||
if candidate and candidate in compatible:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
def snapshot(self, *, refresh: bool = False) -> dict[str, Any]:
|
||||
global _CACHE
|
||||
now_monotonic = monotonic()
|
||||
with _CACHE_LOCK:
|
||||
if (
|
||||
not refresh
|
||||
and _CACHE is not None
|
||||
and now_monotonic - _CACHE[0] < _CACHE_SECONDS
|
||||
):
|
||||
cached = copy.deepcopy(_CACHE[1])
|
||||
cached["cache"] = {"hit": True, "ttlSeconds": _CACHE_SECONDS}
|
||||
return cached
|
||||
|
||||
if not self.is_configured():
|
||||
return {
|
||||
"status": "missing_config",
|
||||
"severity": "unknown",
|
||||
"generatedAt": datetime.now(UTC).isoformat(),
|
||||
"runners": [],
|
||||
"queue": {
|
||||
"queued": 0,
|
||||
"inProgress": 0,
|
||||
"oldestQueuedAgeSeconds": None,
|
||||
"oldestInProgressAgeSeconds": None,
|
||||
"sampleTruncated": False,
|
||||
"blockedByRunnerLabels": [],
|
||||
"staleAssignmentRunnerLabels": [],
|
||||
},
|
||||
"recommendedAction": None,
|
||||
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
||||
}
|
||||
|
||||
try:
|
||||
runner_payload = self._request(
|
||||
"/api/v1/admin/actions/runners", {"disabled": "false"}
|
||||
)
|
||||
queued_payload = self._request(
|
||||
"/api/v1/admin/actions/jobs",
|
||||
{
|
||||
"status": "queued",
|
||||
"page": 1,
|
||||
"limit": 100,
|
||||
"sort": "id",
|
||||
"order": "asc",
|
||||
},
|
||||
)
|
||||
pending_payload = self._request(
|
||||
"/api/v1/admin/actions/jobs",
|
||||
{
|
||||
"status": "pending",
|
||||
"page": 1,
|
||||
"limit": 100,
|
||||
"sort": "id",
|
||||
"order": "asc",
|
||||
},
|
||||
)
|
||||
active_payload = self._request(
|
||||
"/api/v1/admin/actions/jobs",
|
||||
{"status": "in_progress", "page": 1, "limit": 100},
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
return {
|
||||
"status": "error",
|
||||
"severity": "warning",
|
||||
"reason": redact_message(str(exc))[:240],
|
||||
"generatedAt": datetime.now(UTC).isoformat(),
|
||||
"runners": [],
|
||||
"queue": {
|
||||
"queued": 0,
|
||||
"inProgress": 0,
|
||||
"oldestQueuedAgeSeconds": None,
|
||||
"oldestInProgressAgeSeconds": None,
|
||||
"sampleTruncated": False,
|
||||
"blockedByRunnerLabels": [],
|
||||
"staleAssignmentRunnerLabels": [],
|
||||
},
|
||||
"recommendedAction": None,
|
||||
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
||||
}
|
||||
|
||||
raw_runners = runner_payload.get("runners")
|
||||
runner_items = raw_runners if isinstance(raw_runners, list) else []
|
||||
runners = [
|
||||
{
|
||||
"id": int(item.get("id", 0) or 0),
|
||||
"name": str(item.get("name") or "unknown")[:160],
|
||||
"status": str(item.get("status") or "unknown")[:32],
|
||||
"busy": bool(item.get("busy")),
|
||||
"disabled": bool(item.get("disabled")),
|
||||
"labels": self._labels(item.get("labels")),
|
||||
}
|
||||
for item in runner_items
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
queued_jobs = [
|
||||
item
|
||||
for payload in (queued_payload, pending_payload)
|
||||
for item in (
|
||||
payload.get("jobs") if isinstance(payload.get("jobs"), list) else []
|
||||
)
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
active_jobs = (
|
||||
active_payload.get("jobs")
|
||||
if isinstance(active_payload.get("jobs"), list)
|
||||
else []
|
||||
)
|
||||
now = datetime.now(UTC)
|
||||
queued_times = [self._timestamp(item.get("created_at")) for item in queued_jobs]
|
||||
queued_times = [item for item in queued_times if item is not None]
|
||||
oldest_age = max(
|
||||
(int((now - item).total_seconds()) for item in queued_times), default=None
|
||||
)
|
||||
active_times = [
|
||||
self._timestamp(item.get("started_at"))
|
||||
or self._timestamp(item.get("created_at"))
|
||||
for item in active_jobs
|
||||
]
|
||||
active_times = [item for item in active_times if item is not None]
|
||||
oldest_active_age = max(
|
||||
(int((now - item).total_seconds()) for item in active_times), default=None
|
||||
)
|
||||
stale_active_jobs = []
|
||||
for job in active_jobs:
|
||||
created_at = self._timestamp(job.get("started_at")) or self._timestamp(
|
||||
job.get("created_at")
|
||||
)
|
||||
if (
|
||||
created_at is not None
|
||||
and int((now - created_at).total_seconds())
|
||||
>= _ACTIVE_CRITICAL_AGE_SECONDS
|
||||
):
|
||||
stale_active_jobs.append(job)
|
||||
|
||||
online_runners = [
|
||||
item
|
||||
for item in runners
|
||||
if item["status"] == "online" and not item["disabled"]
|
||||
]
|
||||
blocked_jobs = []
|
||||
stale_assignment_jobs: list[dict[str, Any]] = []
|
||||
queue_items = []
|
||||
for job in queued_jobs:
|
||||
required = set(self._labels(job.get("labels")))
|
||||
matching = [
|
||||
runner
|
||||
for runner in online_runners
|
||||
if required.issubset(set(runner["labels"]))
|
||||
]
|
||||
if required and not matching:
|
||||
blocked_jobs.append(job)
|
||||
created = self._timestamp(job.get("created_at"))
|
||||
age = int((now - created).total_seconds()) if created else None
|
||||
all_matching_busy = bool(matching) and all(
|
||||
bool(runner["busy"]) for runner in matching
|
||||
)
|
||||
reason = (
|
||||
"no_online_runner_matches_required_capability"
|
||||
if required and not matching
|
||||
else "all_matching_runners_are_busy"
|
||||
if all_matching_busy
|
||||
else "awaiting_scheduler_assignment"
|
||||
)
|
||||
if (
|
||||
required
|
||||
and matching
|
||||
and not all_matching_busy
|
||||
and age is not None
|
||||
and age >= _CRITICAL_AGE_SECONDS
|
||||
):
|
||||
stale_assignment_jobs.append(
|
||||
{
|
||||
"job": job,
|
||||
"required": sorted(required),
|
||||
"matching": matching,
|
||||
"ageSeconds": age,
|
||||
}
|
||||
)
|
||||
if len(queue_items) < _QUEUE_SAMPLE_LIMIT:
|
||||
queue_items.append(
|
||||
{
|
||||
"id": int(job.get("id", 0) or 0),
|
||||
"name": str(job.get("name") or "job")[:160],
|
||||
"labels": sorted(required)[:20],
|
||||
"ageSeconds": age,
|
||||
"matchingRunners": [runner["name"] for runner in matching][:20],
|
||||
"reason": reason,
|
||||
}
|
||||
)
|
||||
|
||||
severity = "healthy"
|
||||
if (
|
||||
(oldest_age or 0) >= _CRITICAL_AGE_SECONDS
|
||||
or (oldest_active_age or 0) >= _ACTIVE_CRITICAL_AGE_SECONDS
|
||||
or blocked_jobs
|
||||
):
|
||||
severity = "critical"
|
||||
elif (
|
||||
(oldest_age or 0) >= _WARNING_AGE_SECONDS
|
||||
or (oldest_active_age or 0) >= _ACTIVE_WARNING_AGE_SECONDS
|
||||
or len(online_runners) < len(runners)
|
||||
):
|
||||
severity = "warning"
|
||||
|
||||
targets = self._restart_targets()
|
||||
blocked_labels = [
|
||||
label for job in blocked_jobs for label in self._labels(job.get("labels"))
|
||||
]
|
||||
stale_active_labels = [
|
||||
label
|
||||
for job in stale_active_jobs
|
||||
for label in self._labels(job.get("labels"))
|
||||
]
|
||||
stale_assignment_labels = sorted(
|
||||
{
|
||||
label
|
||||
for candidate in stale_assignment_jobs
|
||||
for label in candidate["required"]
|
||||
}
|
||||
)
|
||||
|
||||
recommended_action: dict[str, Any] | None = None
|
||||
for candidate in sorted(
|
||||
stale_assignment_jobs,
|
||||
key=lambda item: int(item["ageSeconds"]),
|
||||
reverse=True,
|
||||
):
|
||||
job = candidate["job"]
|
||||
matching_by_name = {
|
||||
str(runner.get("name") or ""): runner for runner in candidate["matching"]
|
||||
}
|
||||
for label in candidate["required"]:
|
||||
target = self._target_for_label(label, candidate["matching"], targets)
|
||||
if not target:
|
||||
continue
|
||||
target_runner = matching_by_name.get(target)
|
||||
if target_runner is not None and bool(target_runner.get("busy")):
|
||||
continue
|
||||
recommended_action = {
|
||||
"intent": "runner.recover",
|
||||
"target": target,
|
||||
"requiresApproval": False,
|
||||
"autoExecute": True,
|
||||
"reason": "Gitea Actions scheduler left a compatible idle runner unassigned for at least ten minutes.",
|
||||
"evidence": {
|
||||
"jobId": int(job.get("id", 0) or 0),
|
||||
"label": label,
|
||||
"minimumAgeSeconds": _CRITICAL_AGE_SECONDS,
|
||||
},
|
||||
}
|
||||
break
|
||||
if recommended_action is not None:
|
||||
break
|
||||
|
||||
if recommended_action is None:
|
||||
restart_target = None
|
||||
restart_reason = ""
|
||||
for label in [*blocked_labels, *stale_active_labels]:
|
||||
target = self._target_for_label(label, runners, targets)
|
||||
if target:
|
||||
restart_target = target
|
||||
restart_reason = (
|
||||
"Gitea Actions queue is stalled without a matching online runner."
|
||||
if label in blocked_labels
|
||||
else "A Gitea Actions job exceeded the two-hour guardrail on an explicitly mapped runner."
|
||||
)
|
||||
break
|
||||
recommended_action = (
|
||||
{
|
||||
"intent": "container.restart",
|
||||
"target": restart_target,
|
||||
"requiresApproval": True,
|
||||
"autoExecute": False,
|
||||
"reason": restart_reason,
|
||||
}
|
||||
if severity == "critical" and restart_target
|
||||
else None
|
||||
)
|
||||
|
||||
result = {
|
||||
"status": "ok",
|
||||
"severity": severity,
|
||||
"generatedAt": now.isoformat(),
|
||||
"runners": runners,
|
||||
"summary": {
|
||||
"total": len(runners),
|
||||
"online": len(online_runners),
|
||||
"busy": sum(bool(item["busy"]) for item in runners),
|
||||
"offline": sum(item["status"] != "online" for item in runners),
|
||||
},
|
||||
"queue": {
|
||||
"queued": int(queued_payload.get("total_count", 0) or 0)
|
||||
+ int(pending_payload.get("total_count", 0) or 0),
|
||||
"inProgress": int(
|
||||
active_payload.get("total_count", len(active_jobs)) or 0
|
||||
),
|
||||
"oldestQueuedAgeSeconds": oldest_age,
|
||||
"oldestInProgressAgeSeconds": oldest_active_age,
|
||||
"sampleTruncated": (
|
||||
int(queued_payload.get("total_count", 0) or 0)
|
||||
> len(
|
||||
queued_payload.get("jobs")
|
||||
if isinstance(queued_payload.get("jobs"), list)
|
||||
else []
|
||||
)
|
||||
or int(pending_payload.get("total_count", 0) or 0)
|
||||
> len(
|
||||
pending_payload.get("jobs")
|
||||
if isinstance(pending_payload.get("jobs"), list)
|
||||
else []
|
||||
)
|
||||
),
|
||||
"blockedByRunnerLabels": sorted(set(blocked_labels))[:20],
|
||||
"staleActiveRunnerLabels": sorted(set(stale_active_labels))[:20],
|
||||
"staleAssignmentRunnerLabels": stale_assignment_labels[:20],
|
||||
"items": queue_items,
|
||||
},
|
||||
"recommendedAction": recommended_action,
|
||||
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
||||
}
|
||||
with _CACHE_LOCK:
|
||||
_CACHE = (now_monotonic, copy.deepcopy(result))
|
||||
return result
|
||||
Reference in new issue
Block a user