Jens/Unraid-AppOps-Gateway-Public
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Publish curated Unraid-AppOps-Gateway source from bc001a6df859f7c9836fa145afb739d1ceebc524

Browse Source
This commit is contained in:
Public Source Publisher committed 2026-10-07 23:44:08 +00:00
1 parent 631b846397
commit ab3fa553f0
15 files changed
+3687 -227
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/app/connectors/gitea_actions.py
+379 -91
View File
@@ -20,8 +20,14 @@ _CRITICAL_AGE_SECONDS = 600
_ACTIVE_WARNING_AGE_SECONDS = 1_800
_ACTIVE_CRITICAL_AGE_SECONDS = 7_200
_QUEUE_SAMPLE_LIMIT = 50
_RECOVERY_MIN_OBSERVATION_SECONDS = 600
_RECOVERY_MAX_SAMPLE_GAP_SECONDS = 45
_CACHE_LOCK = Lock()
_CACHE: tuple[float, dict[str, Any]] | None = None
_RECOVERY_OBSERVATIONS: dict[tuple[Any, ...], tuple[float, float]] = {}
_RECOVERY_SOURCE: str | None = None
_SNAPSHOT_GENERATION = 0
_COMMITTED_GENERATION = 0
_DEFAULT_RUNNER_TARGETS = {
"linux-validation": "itworx-validation-runner-2",
"docker-validation": "itworx-docker-runner",
@@ -44,6 +50,166 @@ _DEFAULT_RUNNER_TARGET_CANDIDATES = {
}
def _is_count(value: Any) -> bool:
return type(value) is int and value >= 0
def _is_identifier(value: Any) -> bool:
return type(value) is int and value > 0
def _is_label_list(value: Any) -> bool:
def valid_label(text: Any) -> bool:
return (
isinstance(text, str)
and len(text) <= 120
and bool(text.strip())
and text == text.strip()
)
return isinstance(value, list) and all(
valid_label(item)
or (isinstance(item, dict) and valid_label(item.get("name")))
for item in value
)
def _complete_rows(payload: Any, row_key: str) -> list[dict[str, Any]] | None:
if not isinstance(payload, dict):
return None
rows = payload.get(row_key)
total = payload.get("total_count")
if not isinstance(rows, list) or not _is_count(total) or total != len(rows):
return None
if not all(isinstance(row, dict) for row in rows):
return None
return rows
def _valid_runner_inventory(payload: Any) -> list[dict[str, Any]] | None:
runners = _complete_rows(payload, "runners")
if runners is None:
return None
ids: set[int] = set()
for runner in runners:
runner_id = runner.get("id")
if (
not _is_identifier(runner_id)
or runner_id in ids
or not isinstance(runner.get("name"), str)
or not runner["name"].strip()
or not isinstance(runner.get("status"), str)
or not runner["status"].strip()
or type(runner.get("busy")) is not bool
or type(runner.get("disabled")) is not bool
or not _is_label_list(runner.get("labels"))
):
return None
ids.add(runner_id)
return runners
def _valid_job_inventory(
payload: Any, expected_status: str, seen_ids: set[int]
) -> list[dict[str, Any]] | None:
jobs = _complete_rows(payload, "jobs")
if jobs is None:
return None
local_ids: set[int] = set()
for job in jobs:
job_id = job.get("id")
if (
not _is_identifier(job_id)
or job_id in local_ids
or job_id in seen_ids
or not (
job.get("status") == expected_status
or (
expected_status == "pending"
and job.get("status") == "requested"
)
)
or not _is_label_list(job.get("labels"))
):
return None
for field in (
"run_id",
"run_number",
"workflow_id",
"attempt",
"run_attempt",
):
value = job.get(field)
if value is not None and not _is_identifier(value):
return None
attempt = job.get("attempt")
run_attempt = job.get("run_attempt")
if attempt is not None and run_attempt is not None and attempt != run_attempt:
return None
if expected_status == "in_progress" and not _is_identifier(
job.get("runner_id")
):
return None
local_ids.add(job_id)
seen_ids.update(local_ids)
return jobs
def _reset_recovery_state_locked(*, clear_cache: bool = True) -> None:
global _CACHE
_RECOVERY_OBSERVATIONS.clear()
if clear_cache:
_CACHE = None
def _observe_recovery_candidates(
generation: int,
origin: str,
candidates: list[dict[str, Any]],
sample_started: float,
sample_finished: float,
*,
complete: bool,
) -> list[dict[str, Any]] | None:
global _RECOVERY_SOURCE, _COMMITTED_GENERATION
with _CACHE_LOCK:
if generation != _SNAPSHOT_GENERATION:
return None
_COMMITTED_GENERATION = generation
if _RECOVERY_SOURCE != origin:
_RECOVERY_SOURCE = origin
_reset_recovery_state_locked(clear_cache=False)
if (
not complete
or sample_finished < sample_started
or sample_finished - sample_started > _RECOVERY_MAX_SAMPLE_GAP_SECONDS
or len(candidates) > 500
):
_reset_recovery_state_locked(clear_cache=False)
return []
observations: dict[tuple[Any, ...], tuple[float, float]] = {}
matured: list[dict[str, Any]] = []
for candidate in candidates:
key = candidate["key"]
previous = _RECOVERY_OBSERVATIONS.get(key)
gap = sample_finished - previous[1] if previous is not None else None
if gap is not None and 0 <= gap <= _RECOVERY_MAX_SAMPLE_GAP_SECONDS:
first_seen = previous[0]
else:
first_seen = sample_finished
observations[key] = (first_seen, sample_finished)
observed_seconds = max(0, int(sample_finished - first_seen))
if observed_seconds >= _RECOVERY_MIN_OBSERVATION_SECONDS:
matured.append(
{
**candidate,
"observedClaimableSeconds": observed_seconds,
}
)
_RECOVERY_OBSERVATIONS.clear()
_RECOVERY_OBSERVATIONS.update(observations)
return matured
class GiteaActionsConnector(BaseConnector):
"""Read a bounded Gitea Actions control-plane snapshot; never execute jobs."""
@@ -171,37 +337,70 @@ class GiteaActionsConnector(BaseConnector):
return candidate
return None
@staticmethod
def _empty_snapshot(status: str, severity: str) -> dict[str, Any]:
return {
"status": status,
"severity": severity,
"generatedAt": datetime.now(UTC).isoformat(),
"runners": [],
"queue": {
"queued": 0,
"inProgress": 0,
"oldestQueuedAgeSeconds": None,
"oldestInProgressAgeSeconds": None,
"sampleTruncated": False,
"blockedByRunnerLabels": [],
"staleActiveRunnerLabels": [],
"staleAssignmentRunnerLabels": [],
"items": [],
},
"recommendedAction": None,
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
}
def snapshot(self, *, refresh: bool = False) -> dict[str, Any]:
global _CACHE
global _CACHE, _RECOVERY_SOURCE, _SNAPSHOT_GENERATION, _COMMITTED_GENERATION
if not self.is_configured():
with _CACHE_LOCK:
_SNAPSHOT_GENERATION += 1
_COMMITTED_GENERATION = _SNAPSHOT_GENERATION
_RECOVERY_SOURCE = None
_reset_recovery_state_locked()
return self._empty_snapshot("missing_config", "unknown")
try:
origin = self._base_url()
except RuntimeError as exc:
with _CACHE_LOCK:
_SNAPSHOT_GENERATION += 1
_COMMITTED_GENERATION = _SNAPSHOT_GENERATION
_RECOVERY_SOURCE = None
_reset_recovery_state_locked()
result = self._empty_snapshot("error", "warning")
result["reason"] = redact_message(str(exc))[:240]
return result
now_monotonic = monotonic()
with _CACHE_LOCK:
if _RECOVERY_SOURCE is not None and _RECOVERY_SOURCE != origin:
_RECOVERY_SOURCE = origin
_reset_recovery_state_locked()
if (
not refresh
and _CACHE is not None
and now_monotonic - _CACHE[0] < _CACHE_SECONDS
):
cached = copy.deepcopy(_CACHE[1])
action = cached.get("recommendedAction")
if isinstance(action, dict) and action.get("intent") == "runner.recover":
cached["recommendedAction"] = None
cached.get("queue", {})["staleAssignmentRunnerLabels"] = []
cached["cache"] = {"hit": True, "ttlSeconds": _CACHE_SECONDS}
return cached
if not self.is_configured():
return {
"status": "missing_config",
"severity": "unknown",
"generatedAt": datetime.now(UTC).isoformat(),
"runners": [],
"queue": {
"queued": 0,
"inProgress": 0,
"oldestQueuedAgeSeconds": None,
"oldestInProgressAgeSeconds": None,
"sampleTruncated": False,
"blockedByRunnerLabels": [],
"staleAssignmentRunnerLabels": [],
},
"recommendedAction": None,
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
}
_SNAPSHOT_GENERATION += 1
generation = _SNAPSHOT_GENERATION
sample_started = now_monotonic
try:
runner_payload = self._request(
@@ -219,7 +418,7 @@ class GiteaActionsConnector(BaseConnector):
)
# Keep pending inventory in the all-or-error fetch, but it is not
# runnable queue pressure and must not drive runner diagnostics.
self._request(
pending_payload = self._request(
"/api/v1/admin/actions/jobs",
{
"status": "pending",
@@ -234,30 +433,55 @@ class GiteaActionsConnector(BaseConnector):
{"status": "in_progress", "page": 1, "limit": 100},
)
except RuntimeError as exc:
return {
"status": "error",
"severity": "warning",
"reason": redact_message(str(exc))[:240],
"generatedAt": datetime.now(UTC).isoformat(),
"runners": [],
"queue": {
"queued": 0,
"inProgress": 0,
"oldestQueuedAgeSeconds": None,
"oldestInProgressAgeSeconds": None,
"sampleTruncated": False,
"blockedByRunnerLabels": [],
"staleAssignmentRunnerLabels": [],
},
"recommendedAction": None,
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
}
with _CACHE_LOCK:
if generation == _SNAPSHOT_GENERATION:
_COMMITTED_GENERATION = generation
_RECOVERY_SOURCE = origin
_reset_recovery_state_locked()
result = self._empty_snapshot("error", "warning")
result["reason"] = redact_message(str(exc))[:240]
return result
sample_finished = monotonic()
recovery_runners = _valid_runner_inventory(runner_payload)
seen_job_ids: set[int] = set()
recovery_queued_jobs = _valid_job_inventory(
queued_payload, "queued", seen_job_ids
)
recovery_pending_jobs = _valid_job_inventory(
pending_payload, "pending", seen_job_ids
)
recovery_active_jobs = _valid_job_inventory(
active_payload, "in_progress", seen_job_ids
)
recovery_inventory_complete = all(
rows is not None
for rows in (
recovery_runners,
recovery_queued_jobs,
recovery_pending_jobs,
recovery_active_jobs,
)
)
runner_payload = runner_payload if isinstance(runner_payload, dict) else {}
queued_payload = queued_payload if isinstance(queued_payload, dict) else {}
active_payload = active_payload if isinstance(active_payload, dict) else {}
def display_int(value: Any) -> int:
try:
return int(value or 0)
except (TypeError, ValueError, OverflowError):
return 0
def display_count(value: Any, fallback: int = 0) -> int:
return value if _is_count(value) else fallback
raw_runners = runner_payload.get("runners")
runner_items = raw_runners if isinstance(raw_runners, list) else []
runners = [
{
"id": int(item.get("id", 0) or 0),
"id": display_int(item.get("id", 0)),
"name": str(item.get("name") or "unknown")[:160],
"status": str(item.get("status") or "unknown")[:32],
"busy": bool(item.get("busy")),
@@ -276,11 +500,12 @@ class GiteaActionsConnector(BaseConnector):
)
if isinstance(item, dict)
]
active_jobs = (
raw_active_jobs = (
active_payload.get("jobs")
if isinstance(active_payload.get("jobs"), list)
else []
)
active_jobs = [item for item in raw_active_jobs if isinstance(item, dict)]
now = datetime.now(UTC)
queued_times = [self._timestamp(item.get("created_at")) for item in queued_jobs]
queued_times = [item for item in queued_times if item is not None]
@@ -314,8 +539,12 @@ class GiteaActionsConnector(BaseConnector):
if item["status"] == "online" and not item["disabled"]
]
blocked_jobs = []
stale_assignment_jobs: list[dict[str, Any]] = []
eligible_recovery_candidates: list[dict[str, Any]] = []
queue_items = []
targets = self._restart_targets()
active_runner_ids = {
job["runner_id"] for job in recovery_active_jobs or []
}
for job in queued_jobs:
required = set(self._labels(job.get("labels")))
matching = [
@@ -337,25 +566,60 @@ class GiteaActionsConnector(BaseConnector):
if all_matching_busy
else "awaiting_scheduler_assignment"
)
if (
required
and matching
and not all_matching_busy
and age is not None
and age >= _CRITICAL_AGE_SECONDS
):
stale_assignment_jobs.append(
{
"job": job,
"required": sorted(required),
"matching": matching,
"ageSeconds": age,
}
)
if recovery_inventory_complete and required:
for label in sorted(required):
target = self._target_for_label(label, online_runners, targets)
if not target:
continue
exact_runners = [
runner for runner in runners if runner["name"] == target
]
if len(exact_runners) != 1:
continue
target_runner = exact_runners[0]
target_labels = tuple(sorted(target_runner["labels"]))
if (
not required.issubset(set(target_labels))
or target_runner["status"] != "online"
or target_runner["disabled"] is not False
or target_runner["busy"] is not False
or target_runner["id"] in active_runner_ids
):
continue
key = (
origin,
job["id"],
tuple(
(field, job[field])
for field in (
"run_id",
"run_number",
"workflow_id",
"attempt",
"run_attempt",
)
if job.get(field) is not None
),
tuple(sorted(required)),
label,
target,
target_runner["id"],
target_runner["name"],
target_labels,
)
eligible_recovery_candidates.append(
{
"key": key,
"job": job,
"required": sorted(required),
"label": label,
"target": target,
}
)
if len(queue_items) < _QUEUE_SAMPLE_LIMIT:
queue_items.append(
{
"id": int(job.get("id", 0) or 0),
"id": display_int(job.get("id", 0)),
"name": str(job.get("name") or "job")[:160],
"labels": sorted(required)[:20],
"ageSeconds": age,
@@ -364,6 +628,22 @@ class GiteaActionsConnector(BaseConnector):
}
)
try:
current_origin = self._base_url() if self.is_configured() else None
except RuntimeError:
current_origin = None
source_matches = current_origin == origin
matured_recovery_candidates = _observe_recovery_candidates(
generation,
origin,
eligible_recovery_candidates,
sample_started,
sample_finished,
complete=recovery_inventory_complete and source_matches,
)
if matured_recovery_candidates is None:
matured_recovery_candidates = []
severity = "healthy"
if (
(oldest_age or 0) >= _CRITICAL_AGE_SECONDS
@@ -378,7 +658,6 @@ class GiteaActionsConnector(BaseConnector):
):
severity = "warning"
targets = self._restart_targets()
blocked_labels = [
label for job in blocked_jobs for label in self._labels(job.get("labels"))
]
@@ -390,43 +669,33 @@ class GiteaActionsConnector(BaseConnector):
stale_assignment_labels = sorted(
{
label
for candidate in stale_assignment_jobs
for candidate in matured_recovery_candidates
for label in candidate["required"]
}
)
recommended_action: dict[str, Any] | None = None
for candidate in sorted(
stale_assignment_jobs,
key=lambda item: int(item["ageSeconds"]),
matured_recovery_candidates,
key=lambda item: item["observedClaimableSeconds"],
reverse=True,
):
job = candidate["job"]
matching_by_name = {
str(runner.get("name") or ""): runner for runner in candidate["matching"]
recommended_action = {
"intent": "runner.recover",
"target": candidate["target"],
"requiresApproval": False,
"autoExecute": False,
"reason": "Gitea Actions scheduler left a compatible idle runner unassigned for at least ten minutes.",
"evidence": {
"jobId": display_int(job.get("id", 0)),
"label": candidate["label"],
"observedClaimableSeconds": candidate[
"observedClaimableSeconds"
],
},
}
for label in candidate["required"]:
target = self._target_for_label(label, candidate["matching"], targets)
if not target:
continue
target_runner = matching_by_name.get(target)
if target_runner is not None and bool(target_runner.get("busy")):
continue
recommended_action = {
"intent": "runner.recover",
"target": target,
"requiresApproval": False,
"autoExecute": True,
"reason": "Gitea Actions scheduler left a compatible idle runner unassigned for at least ten minutes.",
"evidence": {
"jobId": int(job.get("id", 0) or 0),
"label": label,
"minimumAgeSeconds": _CRITICAL_AGE_SECONDS,
},
}
break
if recommended_action is not None:
break
break
if recommended_action is None:
restart_target = None
@@ -465,14 +734,14 @@ class GiteaActionsConnector(BaseConnector):
"offline": sum(item["status"] != "online" for item in runners),
},
"queue": {
"queued": int(queued_payload.get("total_count", 0) or 0),
"inProgress": int(
active_payload.get("total_count", len(active_jobs)) or 0
"queued": display_count(queued_payload.get("total_count", 0)),
"inProgress": display_count(
active_payload.get("total_count", len(active_jobs))
),
"oldestQueuedAgeSeconds": oldest_age,
"oldestInProgressAgeSeconds": oldest_active_age,
"sampleTruncated": (
int(queued_payload.get("total_count", 0) or 0)
display_count(queued_payload.get("total_count", 0))
> len(
queued_payload.get("jobs")
if isinstance(queued_payload.get("jobs"), list)
@@ -487,6 +756,25 @@ class GiteaActionsConnector(BaseConnector):
"recommendedAction": recommended_action,
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
}
try:
final_origin = self._base_url() if self.is_configured() else None
except RuntimeError:
final_origin = None
with _CACHE_LOCK:
_CACHE = (now_monotonic, copy.deepcopy(result))
if final_origin != origin and generation == _SNAPSHOT_GENERATION:
_SNAPSHOT_GENERATION += 1
_COMMITTED_GENERATION = _SNAPSHOT_GENERATION
_RECOVERY_SOURCE = final_origin
_reset_recovery_state_locked()
if (
final_origin == origin
and generation == _SNAPSHOT_GENERATION
and _COMMITTED_GENERATION == generation
):
_CACHE = (sample_finished, copy.deepcopy(result))
else:
action = result.get("recommendedAction")
if isinstance(action, dict) and action.get("intent") == "runner.recover":
result["recommendedAction"] = None
result["queue"]["staleAssignmentRunnerLabels"] = []
return result