from __future__ import annotations import copy import json from datetime import UTC, datetime from threading import Lock from time import monotonic from typing import Any from urllib.parse import urlsplit, urlunsplit import httpx from ..core.config import settings from ..core.redaction import redact_message from .base import BaseConnector _CACHE_SECONDS = 15 _WARNING_AGE_SECONDS = 180 _CRITICAL_AGE_SECONDS = 600 _ACTIVE_WARNING_AGE_SECONDS = 1_800 _ACTIVE_CRITICAL_AGE_SECONDS = 7_200 _QUEUE_SAMPLE_LIMIT = 50 _RECOVERY_MIN_OBSERVATION_SECONDS = 600 _RECOVERY_MAX_SAMPLE_GAP_SECONDS = 45 _CACHE_LOCK = Lock() _CACHE: tuple[float, dict[str, Any]] | None = None _RECOVERY_OBSERVATIONS: dict[tuple[Any, ...], tuple[float, float]] = {} _RECOVERY_SOURCE: str | None = None _SNAPSHOT_GENERATION = 0 _COMMITTED_GENERATION = 0 _DEFAULT_RUNNER_TARGETS = { "linux-validation": "itworx-validation-runner-2", "docker-validation": "itworx-docker-runner", "browser-validation": "itworx-browser-runner", "security-validation": "itworx-security-runner", "mcp-validation": "itworx-security-runner", "unraid-deploy": "itworx-security-runner", } _DEFAULT_RUNNER_TARGET_CANDIDATES = { "linux-validation": ("itworx-validation-runner-2", "itworx-validation-runner-3"), "docker-validation": ( "itworx-docker-runner", "itworx-validation-runner-2", "itworx-security-runner", ), "browser-validation": ("itworx-browser-runner",), "security-validation": ("itworx-security-runner",), "mcp-validation": ("itworx-security-runner",), "unraid-deploy": ("itworx-security-runner",), } def _is_count(value: Any) -> bool: return type(value) is int and value >= 0 def _is_identifier(value: Any) -> bool: return type(value) is int and value > 0 def _is_label_list(value: Any) -> bool: def valid_label(text: Any) -> bool: return ( isinstance(text, str) and len(text) <= 120 and bool(text.strip()) and text == text.strip() ) return isinstance(value, list) and all( valid_label(item) or (isinstance(item, dict) and valid_label(item.get("name"))) for item in value ) def _complete_rows(payload: Any, row_key: str) -> list[dict[str, Any]] | None: if not isinstance(payload, dict): return None rows = payload.get(row_key) total = payload.get("total_count") if not isinstance(rows, list) or not _is_count(total) or total != len(rows): return None if not all(isinstance(row, dict) for row in rows): return None return rows def _valid_runner_inventory(payload: Any) -> list[dict[str, Any]] | None: runners = _complete_rows(payload, "runners") if runners is None: return None ids: set[int] = set() for runner in runners: runner_id = runner.get("id") if ( not _is_identifier(runner_id) or runner_id in ids or not isinstance(runner.get("name"), str) or not runner["name"].strip() or not isinstance(runner.get("status"), str) or not runner["status"].strip() or type(runner.get("busy")) is not bool or type(runner.get("disabled")) is not bool or not _is_label_list(runner.get("labels")) ): return None ids.add(runner_id) return runners def _valid_job_inventory( payload: Any, expected_status: str, seen_ids: set[int] ) -> list[dict[str, Any]] | None: jobs = _complete_rows(payload, "jobs") if jobs is None: return None local_ids: set[int] = set() for job in jobs: job_id = job.get("id") if ( not _is_identifier(job_id) or job_id in local_ids or job_id in seen_ids or not ( job.get("status") == expected_status or ( expected_status == "pending" and job.get("status") == "requested" ) ) or not _is_label_list(job.get("labels")) ): return None for field in ( "run_id", "run_number", "workflow_id", "attempt", "run_attempt", ): value = job.get(field) if value is not None and not _is_identifier(value): return None attempt = job.get("attempt") run_attempt = job.get("run_attempt") if attempt is not None and run_attempt is not None and attempt != run_attempt: return None if expected_status == "in_progress" and not _is_identifier( job.get("runner_id") ): return None local_ids.add(job_id) seen_ids.update(local_ids) return jobs def _reset_recovery_state_locked(*, clear_cache: bool = True) -> None: global _CACHE _RECOVERY_OBSERVATIONS.clear() if clear_cache: _CACHE = None def _observe_recovery_candidates( generation: int, origin: str, candidates: list[dict[str, Any]], sample_started: float, sample_finished: float, *, complete: bool, ) -> list[dict[str, Any]] | None: global _RECOVERY_SOURCE, _COMMITTED_GENERATION with _CACHE_LOCK: if generation != _SNAPSHOT_GENERATION: return None _COMMITTED_GENERATION = generation if _RECOVERY_SOURCE != origin: _RECOVERY_SOURCE = origin _reset_recovery_state_locked(clear_cache=False) if ( not complete or sample_finished < sample_started or sample_finished - sample_started > _RECOVERY_MAX_SAMPLE_GAP_SECONDS or len(candidates) > 500 ): _reset_recovery_state_locked(clear_cache=False) return [] observations: dict[tuple[Any, ...], tuple[float, float]] = {} matured: list[dict[str, Any]] = [] for candidate in candidates: key = candidate["key"] previous = _RECOVERY_OBSERVATIONS.get(key) gap = sample_finished - previous[1] if previous is not None else None if gap is not None and 0 <= gap <= _RECOVERY_MAX_SAMPLE_GAP_SECONDS: first_seen = previous[0] else: first_seen = sample_finished observations[key] = (first_seen, sample_finished) observed_seconds = max(0, int(sample_finished - first_seen)) if observed_seconds >= _RECOVERY_MIN_OBSERVATION_SECONDS: matured.append( { **candidate, "observedClaimableSeconds": observed_seconds, } ) _RECOVERY_OBSERVATIONS.clear() _RECOVERY_OBSERVATIONS.update(observations) return matured class GiteaActionsConnector(BaseConnector): """Read a bounded Gitea Actions control-plane snapshot; never execute jobs.""" name = "Gitea Actions" def is_configured(self) -> bool: return bool( settings.apis.get("GITEA_URL") and settings.apis.get("GITEA_ADMIN_TOKEN") ) def _base_url(self) -> str: raw = str(settings.apis.get("GITEA_URL") or "").strip().rstrip("/") parsed = urlsplit(raw) if ( parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username ): raise RuntimeError("invalid_gitea_url") return urlunsplit( (parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", "") ) def _request( self, path: str, params: dict[str, Any] | None = None ) -> dict[str, Any]: if not self.is_configured(): raise RuntimeError("missing_config") token = str(settings.apis.get("GITEA_ADMIN_TOKEN") or "") timeout = max( 2.0, min(20.0, float(settings.apis.get("GITEA_TIMEOUT_SECONDS") or 8)), ) try: with httpx.Client( timeout=timeout, verify=bool(settings.apis.get("GITEA_VERIFY_TLS", settings.verify_tls)), ) as client: response = client.get( f"{self._base_url()}{path}", params=params, headers={ "Authorization": f"token {token}", "Accept": "application/json", }, ) except (httpx.RequestError, ValueError) as exc: raise RuntimeError(redact_message(str(exc))) from exc if response.status_code in {401, 403}: raise RuntimeError("gitea_admin_access_denied") if response.status_code >= 400: raise RuntimeError(f"gitea_actions_http_{response.status_code}") try: payload = response.json() except ValueError as exc: raise RuntimeError("gitea_actions_invalid_response") from exc if not isinstance(payload, dict): raise RuntimeError("gitea_actions_invalid_response") # noqa: TRY004 return payload @staticmethod def _timestamp(value: Any) -> datetime | None: if not isinstance(value, str) or not value: return None try: parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: return None return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) @staticmethod def _labels(value: Any) -> list[str]: if not isinstance(value, list): return [] labels = [] for item in value: label = item.get("name") if isinstance(item, dict) else item text = str(label or "").strip() if text: labels.append(text[:120]) return labels @staticmethod def _configured_restart_targets() -> dict[str, str]: raw = str(settings.apis.get("GITEA_RUNNER_TARGETS_JSON") or "{}") try: payload = json.loads(raw) except json.JSONDecodeError: payload = {} return ( { str(label)[:120]: str(target)[:240] for label, target in payload.items() if str(label).strip() and str(target).strip() } if isinstance(payload, dict) else {} ) @classmethod def _restart_targets(cls) -> dict[str, str]: return {**_DEFAULT_RUNNER_TARGETS, **cls._configured_restart_targets()} @classmethod def _target_for_label( cls, label: str, runners: list[dict[str, Any]], targets: dict[str, str], ) -> str | None: """Keep explicit mappings authoritative; prove only built-in defaults.""" configured = cls._configured_restart_targets() if label in configured: return configured[label] compatible = { str(runner.get("name") or "") for runner in runners if label in runner.get("labels", []) and str(runner.get("name") or "") } for candidate in _DEFAULT_RUNNER_TARGET_CANDIDATES.get( label, (targets.get(label, ""),) ): if candidate and candidate in compatible: return candidate return None @staticmethod def _empty_snapshot(status: str, severity: str) -> dict[str, Any]: return { "status": status, "severity": severity, "generatedAt": datetime.now(UTC).isoformat(), "runners": [], "inventory": {"scope": "all_registrations", "complete": False}, "queue": { "queued": 0, "inProgress": 0, "oldestQueuedAgeSeconds": None, "oldestInProgressAgeSeconds": None, "sampleTruncated": False, "blockedByRunnerLabels": [], "staleActiveRunnerLabels": [], "staleAssignmentRunnerLabels": [], "items": [], }, "recommendedAction": None, "cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS}, } def snapshot(self, *, refresh: bool = False) -> dict[str, Any]: global _CACHE, _RECOVERY_SOURCE, _SNAPSHOT_GENERATION, _COMMITTED_GENERATION if not self.is_configured(): with _CACHE_LOCK: _SNAPSHOT_GENERATION += 1 _COMMITTED_GENERATION = _SNAPSHOT_GENERATION _RECOVERY_SOURCE = None _reset_recovery_state_locked() return self._empty_snapshot("missing_config", "unknown") try: origin = self._base_url() except RuntimeError as exc: with _CACHE_LOCK: _SNAPSHOT_GENERATION += 1 _COMMITTED_GENERATION = _SNAPSHOT_GENERATION _RECOVERY_SOURCE = None _reset_recovery_state_locked() result = self._empty_snapshot("error", "warning") result["reason"] = redact_message(str(exc))[:240] return result now_monotonic = monotonic() with _CACHE_LOCK: if _RECOVERY_SOURCE is not None and _RECOVERY_SOURCE != origin: _RECOVERY_SOURCE = origin _reset_recovery_state_locked() if ( not refresh and _CACHE is not None and now_monotonic - _CACHE[0] < _CACHE_SECONDS ): cached = copy.deepcopy(_CACHE[1]) action = cached.get("recommendedAction") if isinstance(action, dict) and action.get("intent") == "runner.recover": cached["recommendedAction"] = None cached.get("queue", {})["staleAssignmentRunnerLabels"] = [] cached["cache"] = {"hit": True, "ttlSeconds": _CACHE_SECONDS} return cached _SNAPSHOT_GENERATION += 1 generation = _SNAPSHOT_GENERATION sample_started = now_monotonic try: runner_payload = self._request( "/api/v1/admin/actions/runners", {"page": 1, "limit": 100} ) queued_payload = self._request( "/api/v1/admin/actions/jobs", { "status": "queued", "page": 1, "limit": 100, "sort": "id", "order": "asc", }, ) # Keep pending inventory in the all-or-error fetch, but it is not # runnable queue pressure and must not drive runner diagnostics. pending_payload = self._request( "/api/v1/admin/actions/jobs", { "status": "pending", "page": 1, "limit": 100, "sort": "id", "order": "asc", }, ) active_payload = self._request( "/api/v1/admin/actions/jobs", {"status": "in_progress", "page": 1, "limit": 100}, ) except RuntimeError as exc: with _CACHE_LOCK: if generation == _SNAPSHOT_GENERATION: _COMMITTED_GENERATION = generation _RECOVERY_SOURCE = origin _reset_recovery_state_locked() result = self._empty_snapshot("error", "warning") result["reason"] = redact_message(str(exc))[:240] return result sample_finished = monotonic() recovery_runners = _valid_runner_inventory(runner_payload) seen_job_ids: set[int] = set() recovery_queued_jobs = _valid_job_inventory( queued_payload, "queued", seen_job_ids ) recovery_pending_jobs = _valid_job_inventory( pending_payload, "pending", seen_job_ids ) recovery_active_jobs = _valid_job_inventory( active_payload, "in_progress", seen_job_ids ) recovery_inventory_complete = all( rows is not None for rows in ( recovery_runners, recovery_queued_jobs, recovery_pending_jobs, recovery_active_jobs, ) ) runner_payload = runner_payload if isinstance(runner_payload, dict) else {} queued_payload = queued_payload if isinstance(queued_payload, dict) else {} active_payload = active_payload if isinstance(active_payload, dict) else {} def display_int(value: Any) -> int: try: return int(value or 0) except (TypeError, ValueError, OverflowError): return 0 def display_count(value: Any, fallback: int = 0) -> int: return value if _is_count(value) else fallback raw_runners = runner_payload.get("runners") runner_items = raw_runners if isinstance(raw_runners, list) else [] runners = [ { "id": display_int(item.get("id", 0)), "name": str(item.get("name") or "unknown")[:160], "status": str(item.get("status") or "unknown")[:32], "busy": bool(item.get("busy")), "disabled": bool(item.get("disabled")), "labels": self._labels(item.get("labels")), } for item in runner_items if isinstance(item, dict) ] queued_jobs = [ item for item in ( queued_payload.get("jobs") if isinstance(queued_payload.get("jobs"), list) else [] ) if isinstance(item, dict) ] raw_active_jobs = ( active_payload.get("jobs") if isinstance(active_payload.get("jobs"), list) else [] ) active_jobs = [item for item in raw_active_jobs if isinstance(item, dict)] now = datetime.now(UTC) queued_times = [self._timestamp(item.get("created_at")) for item in queued_jobs] queued_times = [item for item in queued_times if item is not None] oldest_age = max( (int((now - item).total_seconds()) for item in queued_times), default=None ) active_times = [ self._timestamp(item.get("started_at")) or self._timestamp(item.get("created_at")) for item in active_jobs ] active_times = [item for item in active_times if item is not None] oldest_active_age = max( (int((now - item).total_seconds()) for item in active_times), default=None ) stale_active_jobs = [] for job in active_jobs: created_at = self._timestamp(job.get("started_at")) or self._timestamp( job.get("created_at") ) if ( created_at is not None and int((now - created_at).total_seconds()) >= _ACTIVE_CRITICAL_AGE_SECONDS ): stale_active_jobs.append(job) online_runners = [ item for item in runners if item["status"] == "online" and not item["disabled"] ] blocked_jobs = [] eligible_recovery_candidates: list[dict[str, Any]] = [] queue_items = [] targets = self._restart_targets() active_runner_ids = { job["runner_id"] for job in recovery_active_jobs or [] } for job in queued_jobs: required = set(self._labels(job.get("labels"))) matching = [ runner for runner in online_runners if required.issubset(set(runner["labels"])) ] if required and not matching: blocked_jobs.append(job) created = self._timestamp(job.get("created_at")) age = int((now - created).total_seconds()) if created else None all_matching_busy = bool(matching) and all( bool(runner["busy"]) for runner in matching ) reason = ( "no_online_runner_matches_required_capability" if required and not matching else "all_matching_runners_are_busy" if all_matching_busy else "awaiting_scheduler_assignment" ) if recovery_inventory_complete and required: for label in sorted(required): target = self._target_for_label(label, online_runners, targets) if not target: continue exact_runners = [ runner for runner in runners if runner["name"] == target ] if len(exact_runners) != 1: continue target_runner = exact_runners[0] target_labels = tuple(sorted(target_runner["labels"])) if ( not required.issubset(set(target_labels)) or target_runner["status"] != "online" or target_runner["disabled"] is not False or target_runner["busy"] is not False or target_runner["id"] in active_runner_ids ): continue key = ( origin, job["id"], tuple( (field, job[field]) for field in ( "run_id", "run_number", "workflow_id", "attempt", "run_attempt", ) if job.get(field) is not None ), tuple(sorted(required)), label, target, target_runner["id"], target_runner["name"], target_labels, ) eligible_recovery_candidates.append( { "key": key, "job": job, "required": sorted(required), "label": label, "target": target, } ) if len(queue_items) < _QUEUE_SAMPLE_LIMIT: queue_items.append( { "id": display_int(job.get("id", 0)), "name": str(job.get("name") or "job")[:160], "labels": sorted(required)[:20], "ageSeconds": age, "matchingRunners": [runner["name"] for runner in matching][:20], "reason": reason, } ) try: current_origin = self._base_url() if self.is_configured() else None except RuntimeError: current_origin = None source_matches = current_origin == origin matured_recovery_candidates = _observe_recovery_candidates( generation, origin, eligible_recovery_candidates, sample_started, sample_finished, complete=recovery_inventory_complete and source_matches, ) if matured_recovery_candidates is None: matured_recovery_candidates = [] severity = "healthy" if ( (oldest_age or 0) >= _CRITICAL_AGE_SECONDS or (oldest_active_age or 0) >= _ACTIVE_CRITICAL_AGE_SECONDS or blocked_jobs ): severity = "critical" elif ( (oldest_age or 0) >= _WARNING_AGE_SECONDS or (oldest_active_age or 0) >= _ACTIVE_WARNING_AGE_SECONDS or len(online_runners) < sum(not item["disabled"] for item in runners) ): severity = "warning" blocked_labels = [ label for job in blocked_jobs for label in self._labels(job.get("labels")) ] stale_active_labels = [ label for job in stale_active_jobs for label in self._labels(job.get("labels")) ] stale_assignment_labels = sorted( { label for candidate in matured_recovery_candidates for label in candidate["required"] } ) recommended_action: dict[str, Any] | None = None for candidate in sorted( matured_recovery_candidates, key=lambda item: item["observedClaimableSeconds"], reverse=True, ): job = candidate["job"] recommended_action = { "intent": "runner.recover", "target": candidate["target"], "requiresApproval": False, "autoExecute": False, "reason": "Gitea Actions scheduler left a compatible idle runner unassigned for at least ten minutes.", "evidence": { "jobId": display_int(job.get("id", 0)), "label": candidate["label"], "observedClaimableSeconds": candidate[ "observedClaimableSeconds" ], }, } break if recommended_action is None: restart_target = None restart_reason = "" for label in [*blocked_labels, *stale_active_labels]: target = self._target_for_label(label, runners, targets) # A disabled registration can be an operator pause or a fenced # active lease. Neither is permission to restart its container. if any(item["name"] == target and item["disabled"] for item in runners): continue if target: restart_target = target restart_reason = ( "Gitea Actions queue is stalled without a matching online runner." if label in blocked_labels else "A Gitea Actions job exceeded the two-hour guardrail on an explicitly mapped runner." ) break recommended_action = ( { "intent": "container.restart", "target": restart_target, "requiresApproval": True, "autoExecute": False, "reason": restart_reason, } if severity == "critical" and restart_target else None ) result = { "status": "ok", "severity": severity, "generatedAt": now.isoformat(), "runners": runners, "inventory": { "scope": "all_registrations", "complete": recovery_runners is not None, }, "summary": { "total": len(runners), "online": len(online_runners), "busy": sum(bool(item["busy"]) for item in runners), "offline": sum(item["status"] != "online" for item in runners), "disabled": sum(item["disabled"] for item in runners), "available": sum(not item["busy"] for item in online_runners), }, "queue": { "queued": display_count(queued_payload.get("total_count", 0)), "inProgress": display_count( active_payload.get("total_count", len(active_jobs)) ), "oldestQueuedAgeSeconds": oldest_age, "oldestInProgressAgeSeconds": oldest_active_age, "sampleTruncated": ( display_count(queued_payload.get("total_count", 0)) > len( queued_payload.get("jobs") if isinstance(queued_payload.get("jobs"), list) else [] ) ), "blockedByRunnerLabels": sorted(set(blocked_labels))[:20], "staleActiveRunnerLabels": sorted(set(stale_active_labels))[:20], "staleAssignmentRunnerLabels": stale_assignment_labels[:20], "items": queue_items, }, "recommendedAction": recommended_action, "cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS}, } try: final_origin = self._base_url() if self.is_configured() else None except RuntimeError: final_origin = None with _CACHE_LOCK: if final_origin != origin and generation == _SNAPSHOT_GENERATION: _SNAPSHOT_GENERATION += 1 _COMMITTED_GENERATION = _SNAPSHOT_GENERATION _RECOVERY_SOURCE = final_origin _reset_recovery_state_locked() if ( final_origin == origin and generation == _SNAPSHOT_GENERATION and _COMMITTED_GENERATION == generation ): _CACHE = (sample_finished, copy.deepcopy(result)) else: action = result.get("recommendedAction") if isinstance(action, dict) and action.get("intent") == "runner.recover": result["recommendedAction"] = None result["queue"]["staleAssignmentRunnerLabels"] = [] return result