781 lines
29 KiB
Python
781 lines
29 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from threading import Lock
|
|
from time import monotonic
|
|
from typing import Any
|
|
from urllib.parse import urlsplit, urlunsplit
|
|
|
|
import httpx
|
|
|
|
from ..core.config import settings
|
|
from ..core.redaction import redact_message
|
|
from .base import BaseConnector
|
|
|
|
_CACHE_SECONDS = 15
|
|
_WARNING_AGE_SECONDS = 180
|
|
_CRITICAL_AGE_SECONDS = 600
|
|
_ACTIVE_WARNING_AGE_SECONDS = 1_800
|
|
_ACTIVE_CRITICAL_AGE_SECONDS = 7_200
|
|
_QUEUE_SAMPLE_LIMIT = 50
|
|
_RECOVERY_MIN_OBSERVATION_SECONDS = 600
|
|
_RECOVERY_MAX_SAMPLE_GAP_SECONDS = 45
|
|
_CACHE_LOCK = Lock()
|
|
_CACHE: tuple[float, dict[str, Any]] | None = None
|
|
_RECOVERY_OBSERVATIONS: dict[tuple[Any, ...], tuple[float, float]] = {}
|
|
_RECOVERY_SOURCE: str | None = None
|
|
_SNAPSHOT_GENERATION = 0
|
|
_COMMITTED_GENERATION = 0
|
|
_DEFAULT_RUNNER_TARGETS = {
|
|
"linux-validation": "itworx-validation-runner-2",
|
|
"docker-validation": "itworx-docker-runner",
|
|
"browser-validation": "itworx-browser-runner",
|
|
"security-validation": "itworx-security-runner",
|
|
"mcp-validation": "itworx-security-runner",
|
|
"unraid-deploy": "itworx-security-runner",
|
|
}
|
|
_DEFAULT_RUNNER_TARGET_CANDIDATES = {
|
|
"linux-validation": ("itworx-validation-runner-2", "itworx-validation-runner-3"),
|
|
"docker-validation": (
|
|
"itworx-docker-runner",
|
|
"itworx-validation-runner-2",
|
|
"itworx-security-runner",
|
|
),
|
|
"browser-validation": ("itworx-browser-runner",),
|
|
"security-validation": ("itworx-security-runner",),
|
|
"mcp-validation": ("itworx-security-runner",),
|
|
"unraid-deploy": ("itworx-security-runner",),
|
|
}
|
|
|
|
|
|
def _is_count(value: Any) -> bool:
|
|
return type(value) is int and value >= 0
|
|
|
|
|
|
def _is_identifier(value: Any) -> bool:
|
|
return type(value) is int and value > 0
|
|
|
|
|
|
def _is_label_list(value: Any) -> bool:
|
|
def valid_label(text: Any) -> bool:
|
|
return (
|
|
isinstance(text, str)
|
|
and len(text) <= 120
|
|
and bool(text.strip())
|
|
and text == text.strip()
|
|
)
|
|
|
|
return isinstance(value, list) and all(
|
|
valid_label(item)
|
|
or (isinstance(item, dict) and valid_label(item.get("name")))
|
|
for item in value
|
|
)
|
|
|
|
|
|
def _complete_rows(payload: Any, row_key: str) -> list[dict[str, Any]] | None:
|
|
if not isinstance(payload, dict):
|
|
return None
|
|
rows = payload.get(row_key)
|
|
total = payload.get("total_count")
|
|
if not isinstance(rows, list) or not _is_count(total) or total != len(rows):
|
|
return None
|
|
if not all(isinstance(row, dict) for row in rows):
|
|
return None
|
|
return rows
|
|
|
|
|
|
def _valid_runner_inventory(payload: Any) -> list[dict[str, Any]] | None:
|
|
runners = _complete_rows(payload, "runners")
|
|
if runners is None:
|
|
return None
|
|
ids: set[int] = set()
|
|
for runner in runners:
|
|
runner_id = runner.get("id")
|
|
if (
|
|
not _is_identifier(runner_id)
|
|
or runner_id in ids
|
|
or not isinstance(runner.get("name"), str)
|
|
or not runner["name"].strip()
|
|
or not isinstance(runner.get("status"), str)
|
|
or not runner["status"].strip()
|
|
or type(runner.get("busy")) is not bool
|
|
or type(runner.get("disabled")) is not bool
|
|
or not _is_label_list(runner.get("labels"))
|
|
):
|
|
return None
|
|
ids.add(runner_id)
|
|
return runners
|
|
|
|
|
|
def _valid_job_inventory(
|
|
payload: Any, expected_status: str, seen_ids: set[int]
|
|
) -> list[dict[str, Any]] | None:
|
|
jobs = _complete_rows(payload, "jobs")
|
|
if jobs is None:
|
|
return None
|
|
local_ids: set[int] = set()
|
|
for job in jobs:
|
|
job_id = job.get("id")
|
|
if (
|
|
not _is_identifier(job_id)
|
|
or job_id in local_ids
|
|
or job_id in seen_ids
|
|
or not (
|
|
job.get("status") == expected_status
|
|
or (
|
|
expected_status == "pending"
|
|
and job.get("status") == "requested"
|
|
)
|
|
)
|
|
or not _is_label_list(job.get("labels"))
|
|
):
|
|
return None
|
|
for field in (
|
|
"run_id",
|
|
"run_number",
|
|
"workflow_id",
|
|
"attempt",
|
|
"run_attempt",
|
|
):
|
|
value = job.get(field)
|
|
if value is not None and not _is_identifier(value):
|
|
return None
|
|
attempt = job.get("attempt")
|
|
run_attempt = job.get("run_attempt")
|
|
if attempt is not None and run_attempt is not None and attempt != run_attempt:
|
|
return None
|
|
if expected_status == "in_progress" and not _is_identifier(
|
|
job.get("runner_id")
|
|
):
|
|
return None
|
|
local_ids.add(job_id)
|
|
seen_ids.update(local_ids)
|
|
return jobs
|
|
|
|
|
|
def _reset_recovery_state_locked(*, clear_cache: bool = True) -> None:
|
|
global _CACHE
|
|
_RECOVERY_OBSERVATIONS.clear()
|
|
if clear_cache:
|
|
_CACHE = None
|
|
|
|
|
|
def _observe_recovery_candidates(
|
|
generation: int,
|
|
origin: str,
|
|
candidates: list[dict[str, Any]],
|
|
sample_started: float,
|
|
sample_finished: float,
|
|
*,
|
|
complete: bool,
|
|
) -> list[dict[str, Any]] | None:
|
|
global _RECOVERY_SOURCE, _COMMITTED_GENERATION
|
|
with _CACHE_LOCK:
|
|
if generation != _SNAPSHOT_GENERATION:
|
|
return None
|
|
_COMMITTED_GENERATION = generation
|
|
if _RECOVERY_SOURCE != origin:
|
|
_RECOVERY_SOURCE = origin
|
|
_reset_recovery_state_locked(clear_cache=False)
|
|
if (
|
|
not complete
|
|
or sample_finished < sample_started
|
|
or sample_finished - sample_started > _RECOVERY_MAX_SAMPLE_GAP_SECONDS
|
|
or len(candidates) > 500
|
|
):
|
|
_reset_recovery_state_locked(clear_cache=False)
|
|
return []
|
|
|
|
observations: dict[tuple[Any, ...], tuple[float, float]] = {}
|
|
matured: list[dict[str, Any]] = []
|
|
for candidate in candidates:
|
|
key = candidate["key"]
|
|
previous = _RECOVERY_OBSERVATIONS.get(key)
|
|
gap = sample_finished - previous[1] if previous is not None else None
|
|
if gap is not None and 0 <= gap <= _RECOVERY_MAX_SAMPLE_GAP_SECONDS:
|
|
first_seen = previous[0]
|
|
else:
|
|
first_seen = sample_finished
|
|
observations[key] = (first_seen, sample_finished)
|
|
observed_seconds = max(0, int(sample_finished - first_seen))
|
|
if observed_seconds >= _RECOVERY_MIN_OBSERVATION_SECONDS:
|
|
matured.append(
|
|
{
|
|
**candidate,
|
|
"observedClaimableSeconds": observed_seconds,
|
|
}
|
|
)
|
|
_RECOVERY_OBSERVATIONS.clear()
|
|
_RECOVERY_OBSERVATIONS.update(observations)
|
|
return matured
|
|
class GiteaActionsConnector(BaseConnector):
|
|
"""Read a bounded Gitea Actions control-plane snapshot; never execute jobs."""
|
|
|
|
name = "Gitea Actions"
|
|
|
|
def is_configured(self) -> bool:
|
|
return bool(
|
|
settings.apis.get("GITEA_URL") and settings.apis.get("GITEA_ADMIN_TOKEN")
|
|
)
|
|
|
|
def _base_url(self) -> str:
|
|
raw = str(settings.apis.get("GITEA_URL") or "").strip().rstrip("/")
|
|
parsed = urlsplit(raw)
|
|
if (
|
|
parsed.scheme not in {"http", "https"}
|
|
or not parsed.netloc
|
|
or parsed.username
|
|
):
|
|
raise RuntimeError("invalid_gitea_url")
|
|
return urlunsplit(
|
|
(parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", "")
|
|
)
|
|
|
|
def _request(
|
|
self, path: str, params: dict[str, Any] | None = None
|
|
) -> dict[str, Any]:
|
|
if not self.is_configured():
|
|
raise RuntimeError("missing_config")
|
|
token = str(settings.apis.get("GITEA_ADMIN_TOKEN") or "")
|
|
timeout = max(
|
|
2.0,
|
|
min(20.0, float(settings.apis.get("GITEA_TIMEOUT_SECONDS") or 8)),
|
|
)
|
|
try:
|
|
with httpx.Client(
|
|
timeout=timeout,
|
|
verify=bool(settings.apis.get("GITEA_VERIFY_TLS", settings.verify_tls)),
|
|
) as client:
|
|
response = client.get(
|
|
f"{self._base_url()}{path}",
|
|
params=params,
|
|
headers={
|
|
"Authorization": f"token {token}",
|
|
"Accept": "application/json",
|
|
},
|
|
)
|
|
except (httpx.RequestError, ValueError) as exc:
|
|
raise RuntimeError(redact_message(str(exc))) from exc
|
|
if response.status_code in {401, 403}:
|
|
raise RuntimeError("gitea_admin_access_denied")
|
|
if response.status_code >= 400:
|
|
raise RuntimeError(f"gitea_actions_http_{response.status_code}")
|
|
try:
|
|
payload = response.json()
|
|
except ValueError as exc:
|
|
raise RuntimeError("gitea_actions_invalid_response") from exc
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError("gitea_actions_invalid_response") # noqa: TRY004
|
|
return payload
|
|
|
|
@staticmethod
|
|
def _timestamp(value: Any) -> datetime | None:
|
|
if not isinstance(value, str) or not value:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
|
|
|
@staticmethod
|
|
def _labels(value: Any) -> list[str]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
labels = []
|
|
for item in value:
|
|
label = item.get("name") if isinstance(item, dict) else item
|
|
text = str(label or "").strip()
|
|
if text:
|
|
labels.append(text[:120])
|
|
return labels
|
|
|
|
@staticmethod
|
|
def _configured_restart_targets() -> dict[str, str]:
|
|
raw = str(settings.apis.get("GITEA_RUNNER_TARGETS_JSON") or "{}")
|
|
try:
|
|
payload = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
payload = {}
|
|
return (
|
|
{
|
|
str(label)[:120]: str(target)[:240]
|
|
for label, target in payload.items()
|
|
if str(label).strip() and str(target).strip()
|
|
}
|
|
if isinstance(payload, dict)
|
|
else {}
|
|
)
|
|
|
|
@classmethod
|
|
def _restart_targets(cls) -> dict[str, str]:
|
|
return {**_DEFAULT_RUNNER_TARGETS, **cls._configured_restart_targets()}
|
|
|
|
@classmethod
|
|
def _target_for_label(
|
|
cls,
|
|
label: str,
|
|
runners: list[dict[str, Any]],
|
|
targets: dict[str, str],
|
|
) -> str | None:
|
|
"""Keep explicit mappings authoritative; prove only built-in defaults."""
|
|
configured = cls._configured_restart_targets()
|
|
if label in configured:
|
|
return configured[label]
|
|
|
|
compatible = {
|
|
str(runner.get("name") or "")
|
|
for runner in runners
|
|
if label in runner.get("labels", []) and str(runner.get("name") or "")
|
|
}
|
|
for candidate in _DEFAULT_RUNNER_TARGET_CANDIDATES.get(
|
|
label, (targets.get(label, ""),)
|
|
):
|
|
if candidate and candidate in compatible:
|
|
return candidate
|
|
return None
|
|
|
|
@staticmethod
|
|
def _empty_snapshot(status: str, severity: str) -> dict[str, Any]:
|
|
return {
|
|
"status": status,
|
|
"severity": severity,
|
|
"generatedAt": datetime.now(UTC).isoformat(),
|
|
"runners": [],
|
|
"queue": {
|
|
"queued": 0,
|
|
"inProgress": 0,
|
|
"oldestQueuedAgeSeconds": None,
|
|
"oldestInProgressAgeSeconds": None,
|
|
"sampleTruncated": False,
|
|
"blockedByRunnerLabels": [],
|
|
"staleActiveRunnerLabels": [],
|
|
"staleAssignmentRunnerLabels": [],
|
|
"items": [],
|
|
},
|
|
"recommendedAction": None,
|
|
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
|
}
|
|
|
|
def snapshot(self, *, refresh: bool = False) -> dict[str, Any]:
|
|
global _CACHE, _RECOVERY_SOURCE, _SNAPSHOT_GENERATION, _COMMITTED_GENERATION
|
|
if not self.is_configured():
|
|
with _CACHE_LOCK:
|
|
_SNAPSHOT_GENERATION += 1
|
|
_COMMITTED_GENERATION = _SNAPSHOT_GENERATION
|
|
_RECOVERY_SOURCE = None
|
|
_reset_recovery_state_locked()
|
|
return self._empty_snapshot("missing_config", "unknown")
|
|
|
|
try:
|
|
origin = self._base_url()
|
|
except RuntimeError as exc:
|
|
with _CACHE_LOCK:
|
|
_SNAPSHOT_GENERATION += 1
|
|
_COMMITTED_GENERATION = _SNAPSHOT_GENERATION
|
|
_RECOVERY_SOURCE = None
|
|
_reset_recovery_state_locked()
|
|
result = self._empty_snapshot("error", "warning")
|
|
result["reason"] = redact_message(str(exc))[:240]
|
|
return result
|
|
|
|
now_monotonic = monotonic()
|
|
with _CACHE_LOCK:
|
|
if _RECOVERY_SOURCE is not None and _RECOVERY_SOURCE != origin:
|
|
_RECOVERY_SOURCE = origin
|
|
_reset_recovery_state_locked()
|
|
if (
|
|
not refresh
|
|
and _CACHE is not None
|
|
and now_monotonic - _CACHE[0] < _CACHE_SECONDS
|
|
):
|
|
cached = copy.deepcopy(_CACHE[1])
|
|
action = cached.get("recommendedAction")
|
|
if isinstance(action, dict) and action.get("intent") == "runner.recover":
|
|
cached["recommendedAction"] = None
|
|
cached.get("queue", {})["staleAssignmentRunnerLabels"] = []
|
|
cached["cache"] = {"hit": True, "ttlSeconds": _CACHE_SECONDS}
|
|
return cached
|
|
_SNAPSHOT_GENERATION += 1
|
|
generation = _SNAPSHOT_GENERATION
|
|
sample_started = now_monotonic
|
|
|
|
try:
|
|
runner_payload = self._request(
|
|
"/api/v1/admin/actions/runners", {"disabled": "false"}
|
|
)
|
|
queued_payload = self._request(
|
|
"/api/v1/admin/actions/jobs",
|
|
{
|
|
"status": "queued",
|
|
"page": 1,
|
|
"limit": 100,
|
|
"sort": "id",
|
|
"order": "asc",
|
|
},
|
|
)
|
|
# Keep pending inventory in the all-or-error fetch, but it is not
|
|
# runnable queue pressure and must not drive runner diagnostics.
|
|
pending_payload = self._request(
|
|
"/api/v1/admin/actions/jobs",
|
|
{
|
|
"status": "pending",
|
|
"page": 1,
|
|
"limit": 100,
|
|
"sort": "id",
|
|
"order": "asc",
|
|
},
|
|
)
|
|
active_payload = self._request(
|
|
"/api/v1/admin/actions/jobs",
|
|
{"status": "in_progress", "page": 1, "limit": 100},
|
|
)
|
|
except RuntimeError as exc:
|
|
with _CACHE_LOCK:
|
|
if generation == _SNAPSHOT_GENERATION:
|
|
_COMMITTED_GENERATION = generation
|
|
_RECOVERY_SOURCE = origin
|
|
_reset_recovery_state_locked()
|
|
result = self._empty_snapshot("error", "warning")
|
|
result["reason"] = redact_message(str(exc))[:240]
|
|
return result
|
|
|
|
sample_finished = monotonic()
|
|
|
|
recovery_runners = _valid_runner_inventory(runner_payload)
|
|
seen_job_ids: set[int] = set()
|
|
recovery_queued_jobs = _valid_job_inventory(
|
|
queued_payload, "queued", seen_job_ids
|
|
)
|
|
recovery_pending_jobs = _valid_job_inventory(
|
|
pending_payload, "pending", seen_job_ids
|
|
)
|
|
recovery_active_jobs = _valid_job_inventory(
|
|
active_payload, "in_progress", seen_job_ids
|
|
)
|
|
recovery_inventory_complete = all(
|
|
rows is not None
|
|
for rows in (
|
|
recovery_runners,
|
|
recovery_queued_jobs,
|
|
recovery_pending_jobs,
|
|
recovery_active_jobs,
|
|
)
|
|
)
|
|
runner_payload = runner_payload if isinstance(runner_payload, dict) else {}
|
|
queued_payload = queued_payload if isinstance(queued_payload, dict) else {}
|
|
active_payload = active_payload if isinstance(active_payload, dict) else {}
|
|
|
|
def display_int(value: Any) -> int:
|
|
try:
|
|
return int(value or 0)
|
|
except (TypeError, ValueError, OverflowError):
|
|
return 0
|
|
|
|
def display_count(value: Any, fallback: int = 0) -> int:
|
|
return value if _is_count(value) else fallback
|
|
|
|
raw_runners = runner_payload.get("runners")
|
|
runner_items = raw_runners if isinstance(raw_runners, list) else []
|
|
runners = [
|
|
{
|
|
"id": display_int(item.get("id", 0)),
|
|
"name": str(item.get("name") or "unknown")[:160],
|
|
"status": str(item.get("status") or "unknown")[:32],
|
|
"busy": bool(item.get("busy")),
|
|
"disabled": bool(item.get("disabled")),
|
|
"labels": self._labels(item.get("labels")),
|
|
}
|
|
for item in runner_items
|
|
if isinstance(item, dict)
|
|
]
|
|
queued_jobs = [
|
|
item
|
|
for item in (
|
|
queued_payload.get("jobs")
|
|
if isinstance(queued_payload.get("jobs"), list)
|
|
else []
|
|
)
|
|
if isinstance(item, dict)
|
|
]
|
|
raw_active_jobs = (
|
|
active_payload.get("jobs")
|
|
if isinstance(active_payload.get("jobs"), list)
|
|
else []
|
|
)
|
|
active_jobs = [item for item in raw_active_jobs if isinstance(item, dict)]
|
|
now = datetime.now(UTC)
|
|
queued_times = [self._timestamp(item.get("created_at")) for item in queued_jobs]
|
|
queued_times = [item for item in queued_times if item is not None]
|
|
oldest_age = max(
|
|
(int((now - item).total_seconds()) for item in queued_times), default=None
|
|
)
|
|
active_times = [
|
|
self._timestamp(item.get("started_at"))
|
|
or self._timestamp(item.get("created_at"))
|
|
for item in active_jobs
|
|
]
|
|
active_times = [item for item in active_times if item is not None]
|
|
oldest_active_age = max(
|
|
(int((now - item).total_seconds()) for item in active_times), default=None
|
|
)
|
|
stale_active_jobs = []
|
|
for job in active_jobs:
|
|
created_at = self._timestamp(job.get("started_at")) or self._timestamp(
|
|
job.get("created_at")
|
|
)
|
|
if (
|
|
created_at is not None
|
|
and int((now - created_at).total_seconds())
|
|
>= _ACTIVE_CRITICAL_AGE_SECONDS
|
|
):
|
|
stale_active_jobs.append(job)
|
|
|
|
online_runners = [
|
|
item
|
|
for item in runners
|
|
if item["status"] == "online" and not item["disabled"]
|
|
]
|
|
blocked_jobs = []
|
|
eligible_recovery_candidates: list[dict[str, Any]] = []
|
|
queue_items = []
|
|
targets = self._restart_targets()
|
|
active_runner_ids = {
|
|
job["runner_id"] for job in recovery_active_jobs or []
|
|
}
|
|
for job in queued_jobs:
|
|
required = set(self._labels(job.get("labels")))
|
|
matching = [
|
|
runner
|
|
for runner in online_runners
|
|
if required.issubset(set(runner["labels"]))
|
|
]
|
|
if required and not matching:
|
|
blocked_jobs.append(job)
|
|
created = self._timestamp(job.get("created_at"))
|
|
age = int((now - created).total_seconds()) if created else None
|
|
all_matching_busy = bool(matching) and all(
|
|
bool(runner["busy"]) for runner in matching
|
|
)
|
|
reason = (
|
|
"no_online_runner_matches_required_capability"
|
|
if required and not matching
|
|
else "all_matching_runners_are_busy"
|
|
if all_matching_busy
|
|
else "awaiting_scheduler_assignment"
|
|
)
|
|
if recovery_inventory_complete and required:
|
|
for label in sorted(required):
|
|
target = self._target_for_label(label, online_runners, targets)
|
|
if not target:
|
|
continue
|
|
exact_runners = [
|
|
runner for runner in runners if runner["name"] == target
|
|
]
|
|
if len(exact_runners) != 1:
|
|
continue
|
|
target_runner = exact_runners[0]
|
|
target_labels = tuple(sorted(target_runner["labels"]))
|
|
if (
|
|
not required.issubset(set(target_labels))
|
|
or target_runner["status"] != "online"
|
|
or target_runner["disabled"] is not False
|
|
or target_runner["busy"] is not False
|
|
or target_runner["id"] in active_runner_ids
|
|
):
|
|
continue
|
|
key = (
|
|
origin,
|
|
job["id"],
|
|
tuple(
|
|
(field, job[field])
|
|
for field in (
|
|
"run_id",
|
|
"run_number",
|
|
"workflow_id",
|
|
"attempt",
|
|
"run_attempt",
|
|
)
|
|
if job.get(field) is not None
|
|
),
|
|
tuple(sorted(required)),
|
|
label,
|
|
target,
|
|
target_runner["id"],
|
|
target_runner["name"],
|
|
target_labels,
|
|
)
|
|
eligible_recovery_candidates.append(
|
|
{
|
|
"key": key,
|
|
"job": job,
|
|
"required": sorted(required),
|
|
"label": label,
|
|
"target": target,
|
|
}
|
|
)
|
|
if len(queue_items) < _QUEUE_SAMPLE_LIMIT:
|
|
queue_items.append(
|
|
{
|
|
"id": display_int(job.get("id", 0)),
|
|
"name": str(job.get("name") or "job")[:160],
|
|
"labels": sorted(required)[:20],
|
|
"ageSeconds": age,
|
|
"matchingRunners": [runner["name"] for runner in matching][:20],
|
|
"reason": reason,
|
|
}
|
|
)
|
|
|
|
try:
|
|
current_origin = self._base_url() if self.is_configured() else None
|
|
except RuntimeError:
|
|
current_origin = None
|
|
source_matches = current_origin == origin
|
|
matured_recovery_candidates = _observe_recovery_candidates(
|
|
generation,
|
|
origin,
|
|
eligible_recovery_candidates,
|
|
sample_started,
|
|
sample_finished,
|
|
complete=recovery_inventory_complete and source_matches,
|
|
)
|
|
if matured_recovery_candidates is None:
|
|
matured_recovery_candidates = []
|
|
|
|
severity = "healthy"
|
|
if (
|
|
(oldest_age or 0) >= _CRITICAL_AGE_SECONDS
|
|
or (oldest_active_age or 0) >= _ACTIVE_CRITICAL_AGE_SECONDS
|
|
or blocked_jobs
|
|
):
|
|
severity = "critical"
|
|
elif (
|
|
(oldest_age or 0) >= _WARNING_AGE_SECONDS
|
|
or (oldest_active_age or 0) >= _ACTIVE_WARNING_AGE_SECONDS
|
|
or len(online_runners) < len(runners)
|
|
):
|
|
severity = "warning"
|
|
|
|
blocked_labels = [
|
|
label for job in blocked_jobs for label in self._labels(job.get("labels"))
|
|
]
|
|
stale_active_labels = [
|
|
label
|
|
for job in stale_active_jobs
|
|
for label in self._labels(job.get("labels"))
|
|
]
|
|
stale_assignment_labels = sorted(
|
|
{
|
|
label
|
|
for candidate in matured_recovery_candidates
|
|
for label in candidate["required"]
|
|
}
|
|
)
|
|
|
|
recommended_action: dict[str, Any] | None = None
|
|
for candidate in sorted(
|
|
matured_recovery_candidates,
|
|
key=lambda item: item["observedClaimableSeconds"],
|
|
reverse=True,
|
|
):
|
|
job = candidate["job"]
|
|
recommended_action = {
|
|
"intent": "runner.recover",
|
|
"target": candidate["target"],
|
|
"requiresApproval": False,
|
|
"autoExecute": False,
|
|
"reason": "Gitea Actions scheduler left a compatible idle runner unassigned for at least ten minutes.",
|
|
"evidence": {
|
|
"jobId": display_int(job.get("id", 0)),
|
|
"label": candidate["label"],
|
|
"observedClaimableSeconds": candidate[
|
|
"observedClaimableSeconds"
|
|
],
|
|
},
|
|
}
|
|
break
|
|
|
|
if recommended_action is None:
|
|
restart_target = None
|
|
restart_reason = ""
|
|
for label in [*blocked_labels, *stale_active_labels]:
|
|
target = self._target_for_label(label, runners, targets)
|
|
if target:
|
|
restart_target = target
|
|
restart_reason = (
|
|
"Gitea Actions queue is stalled without a matching online runner."
|
|
if label in blocked_labels
|
|
else "A Gitea Actions job exceeded the two-hour guardrail on an explicitly mapped runner."
|
|
)
|
|
break
|
|
recommended_action = (
|
|
{
|
|
"intent": "container.restart",
|
|
"target": restart_target,
|
|
"requiresApproval": True,
|
|
"autoExecute": False,
|
|
"reason": restart_reason,
|
|
}
|
|
if severity == "critical" and restart_target
|
|
else None
|
|
)
|
|
|
|
result = {
|
|
"status": "ok",
|
|
"severity": severity,
|
|
"generatedAt": now.isoformat(),
|
|
"runners": runners,
|
|
"summary": {
|
|
"total": len(runners),
|
|
"online": len(online_runners),
|
|
"busy": sum(bool(item["busy"]) for item in runners),
|
|
"offline": sum(item["status"] != "online" for item in runners),
|
|
},
|
|
"queue": {
|
|
"queued": display_count(queued_payload.get("total_count", 0)),
|
|
"inProgress": display_count(
|
|
active_payload.get("total_count", len(active_jobs))
|
|
),
|
|
"oldestQueuedAgeSeconds": oldest_age,
|
|
"oldestInProgressAgeSeconds": oldest_active_age,
|
|
"sampleTruncated": (
|
|
display_count(queued_payload.get("total_count", 0))
|
|
> len(
|
|
queued_payload.get("jobs")
|
|
if isinstance(queued_payload.get("jobs"), list)
|
|
else []
|
|
)
|
|
),
|
|
"blockedByRunnerLabels": sorted(set(blocked_labels))[:20],
|
|
"staleActiveRunnerLabels": sorted(set(stale_active_labels))[:20],
|
|
"staleAssignmentRunnerLabels": stale_assignment_labels[:20],
|
|
"items": queue_items,
|
|
},
|
|
"recommendedAction": recommended_action,
|
|
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
|
}
|
|
try:
|
|
final_origin = self._base_url() if self.is_configured() else None
|
|
except RuntimeError:
|
|
final_origin = None
|
|
with _CACHE_LOCK:
|
|
if final_origin != origin and generation == _SNAPSHOT_GENERATION:
|
|
_SNAPSHOT_GENERATION += 1
|
|
_COMMITTED_GENERATION = _SNAPSHOT_GENERATION
|
|
_RECOVERY_SOURCE = final_origin
|
|
_reset_recovery_state_locked()
|
|
if (
|
|
final_origin == origin
|
|
and generation == _SNAPSHOT_GENERATION
|
|
and _COMMITTED_GENERATION == generation
|
|
):
|
|
_CACHE = (sample_finished, copy.deepcopy(result))
|
|
else:
|
|
action = result.get("recommendedAction")
|
|
if isinstance(action, dict) and action.get("intent") == "runner.recover":
|
|
result["recommendedAction"] = None
|
|
result["queue"]["staleAssignmentRunnerLabels"] = []
|
|
return result
|