Publish curated Unraid-AppOps-Gateway source from bc001a6df859f7c9836fa145afb739d1ceebc524
This commit is contained in:
1 parent
631b846397
commit
ab3fa553f0
15 files changed
+3687
-227
No files matched your search
@@ -20,8 +20,14 @@ _CRITICAL_AGE_SECONDS = 600
|
||||
_ACTIVE_WARNING_AGE_SECONDS = 1_800
|
||||
_ACTIVE_CRITICAL_AGE_SECONDS = 7_200
|
||||
_QUEUE_SAMPLE_LIMIT = 50
|
||||
_RECOVERY_MIN_OBSERVATION_SECONDS = 600
|
||||
_RECOVERY_MAX_SAMPLE_GAP_SECONDS = 45
|
||||
_CACHE_LOCK = Lock()
|
||||
_CACHE: tuple[float, dict[str, Any]] | None = None
|
||||
_RECOVERY_OBSERVATIONS: dict[tuple[Any, ...], tuple[float, float]] = {}
|
||||
_RECOVERY_SOURCE: str | None = None
|
||||
_SNAPSHOT_GENERATION = 0
|
||||
_COMMITTED_GENERATION = 0
|
||||
_DEFAULT_RUNNER_TARGETS = {
|
||||
"linux-validation": "itworx-validation-runner-2",
|
||||
"docker-validation": "itworx-docker-runner",
|
||||
@@ -44,6 +50,166 @@ _DEFAULT_RUNNER_TARGET_CANDIDATES = {
|
||||
}
|
||||
|
||||
|
||||
def _is_count(value: Any) -> bool:
|
||||
return type(value) is int and value >= 0
|
||||
|
||||
|
||||
def _is_identifier(value: Any) -> bool:
|
||||
return type(value) is int and value > 0
|
||||
|
||||
|
||||
def _is_label_list(value: Any) -> bool:
|
||||
def valid_label(text: Any) -> bool:
|
||||
return (
|
||||
isinstance(text, str)
|
||||
and len(text) <= 120
|
||||
and bool(text.strip())
|
||||
and text == text.strip()
|
||||
)
|
||||
|
||||
return isinstance(value, list) and all(
|
||||
valid_label(item)
|
||||
or (isinstance(item, dict) and valid_label(item.get("name")))
|
||||
for item in value
|
||||
)
|
||||
|
||||
|
||||
def _complete_rows(payload: Any, row_key: str) -> list[dict[str, Any]] | None:
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
rows = payload.get(row_key)
|
||||
total = payload.get("total_count")
|
||||
if not isinstance(rows, list) or not _is_count(total) or total != len(rows):
|
||||
return None
|
||||
if not all(isinstance(row, dict) for row in rows):
|
||||
return None
|
||||
return rows
|
||||
|
||||
|
||||
def _valid_runner_inventory(payload: Any) -> list[dict[str, Any]] | None:
|
||||
runners = _complete_rows(payload, "runners")
|
||||
if runners is None:
|
||||
return None
|
||||
ids: set[int] = set()
|
||||
for runner in runners:
|
||||
runner_id = runner.get("id")
|
||||
if (
|
||||
not _is_identifier(runner_id)
|
||||
or runner_id in ids
|
||||
or not isinstance(runner.get("name"), str)
|
||||
or not runner["name"].strip()
|
||||
or not isinstance(runner.get("status"), str)
|
||||
or not runner["status"].strip()
|
||||
or type(runner.get("busy")) is not bool
|
||||
or type(runner.get("disabled")) is not bool
|
||||
or not _is_label_list(runner.get("labels"))
|
||||
):
|
||||
return None
|
||||
ids.add(runner_id)
|
||||
return runners
|
||||
|
||||
|
||||
def _valid_job_inventory(
|
||||
payload: Any, expected_status: str, seen_ids: set[int]
|
||||
) -> list[dict[str, Any]] | None:
|
||||
jobs = _complete_rows(payload, "jobs")
|
||||
if jobs is None:
|
||||
return None
|
||||
local_ids: set[int] = set()
|
||||
for job in jobs:
|
||||
job_id = job.get("id")
|
||||
if (
|
||||
not _is_identifier(job_id)
|
||||
or job_id in local_ids
|
||||
or job_id in seen_ids
|
||||
or not (
|
||||
job.get("status") == expected_status
|
||||
or (
|
||||
expected_status == "pending"
|
||||
and job.get("status") == "requested"
|
||||
)
|
||||
)
|
||||
or not _is_label_list(job.get("labels"))
|
||||
):
|
||||
return None
|
||||
for field in (
|
||||
"run_id",
|
||||
"run_number",
|
||||
"workflow_id",
|
||||
"attempt",
|
||||
"run_attempt",
|
||||
):
|
||||
value = job.get(field)
|
||||
if value is not None and not _is_identifier(value):
|
||||
return None
|
||||
attempt = job.get("attempt")
|
||||
run_attempt = job.get("run_attempt")
|
||||
if attempt is not None and run_attempt is not None and attempt != run_attempt:
|
||||
return None
|
||||
if expected_status == "in_progress" and not _is_identifier(
|
||||
job.get("runner_id")
|
||||
):
|
||||
return None
|
||||
local_ids.add(job_id)
|
||||
seen_ids.update(local_ids)
|
||||
return jobs
|
||||
|
||||
|
||||
def _reset_recovery_state_locked(*, clear_cache: bool = True) -> None:
|
||||
global _CACHE
|
||||
_RECOVERY_OBSERVATIONS.clear()
|
||||
if clear_cache:
|
||||
_CACHE = None
|
||||
|
||||
|
||||
def _observe_recovery_candidates(
|
||||
generation: int,
|
||||
origin: str,
|
||||
candidates: list[dict[str, Any]],
|
||||
sample_started: float,
|
||||
sample_finished: float,
|
||||
*,
|
||||
complete: bool,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
global _RECOVERY_SOURCE, _COMMITTED_GENERATION
|
||||
with _CACHE_LOCK:
|
||||
if generation != _SNAPSHOT_GENERATION:
|
||||
return None
|
||||
_COMMITTED_GENERATION = generation
|
||||
if _RECOVERY_SOURCE != origin:
|
||||
_RECOVERY_SOURCE = origin
|
||||
_reset_recovery_state_locked(clear_cache=False)
|
||||
if (
|
||||
not complete
|
||||
or sample_finished < sample_started
|
||||
or sample_finished - sample_started > _RECOVERY_MAX_SAMPLE_GAP_SECONDS
|
||||
or len(candidates) > 500
|
||||
):
|
||||
_reset_recovery_state_locked(clear_cache=False)
|
||||
return []
|
||||
|
||||
observations: dict[tuple[Any, ...], tuple[float, float]] = {}
|
||||
matured: list[dict[str, Any]] = []
|
||||
for candidate in candidates:
|
||||
key = candidate["key"]
|
||||
previous = _RECOVERY_OBSERVATIONS.get(key)
|
||||
gap = sample_finished - previous[1] if previous is not None else None
|
||||
if gap is not None and 0 <= gap <= _RECOVERY_MAX_SAMPLE_GAP_SECONDS:
|
||||
first_seen = previous[0]
|
||||
else:
|
||||
first_seen = sample_finished
|
||||
observations[key] = (first_seen, sample_finished)
|
||||
observed_seconds = max(0, int(sample_finished - first_seen))
|
||||
if observed_seconds >= _RECOVERY_MIN_OBSERVATION_SECONDS:
|
||||
matured.append(
|
||||
{
|
||||
**candidate,
|
||||
"observedClaimableSeconds": observed_seconds,
|
||||
}
|
||||
)
|
||||
_RECOVERY_OBSERVATIONS.clear()
|
||||
_RECOVERY_OBSERVATIONS.update(observations)
|
||||
return matured
|
||||
class GiteaActionsConnector(BaseConnector):
|
||||
"""Read a bounded Gitea Actions control-plane snapshot; never execute jobs."""
|
||||
|
||||
@@ -171,37 +337,70 @@ class GiteaActionsConnector(BaseConnector):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _empty_snapshot(status: str, severity: str) -> dict[str, Any]:
|
||||
return {
|
||||
"status": status,
|
||||
"severity": severity,
|
||||
"generatedAt": datetime.now(UTC).isoformat(),
|
||||
"runners": [],
|
||||
"queue": {
|
||||
"queued": 0,
|
||||
"inProgress": 0,
|
||||
"oldestQueuedAgeSeconds": None,
|
||||
"oldestInProgressAgeSeconds": None,
|
||||
"sampleTruncated": False,
|
||||
"blockedByRunnerLabels": [],
|
||||
"staleActiveRunnerLabels": [],
|
||||
"staleAssignmentRunnerLabels": [],
|
||||
"items": [],
|
||||
},
|
||||
"recommendedAction": None,
|
||||
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
||||
}
|
||||
|
||||
def snapshot(self, *, refresh: bool = False) -> dict[str, Any]:
|
||||
global _CACHE
|
||||
global _CACHE, _RECOVERY_SOURCE, _SNAPSHOT_GENERATION, _COMMITTED_GENERATION
|
||||
if not self.is_configured():
|
||||
with _CACHE_LOCK:
|
||||
_SNAPSHOT_GENERATION += 1
|
||||
_COMMITTED_GENERATION = _SNAPSHOT_GENERATION
|
||||
_RECOVERY_SOURCE = None
|
||||
_reset_recovery_state_locked()
|
||||
return self._empty_snapshot("missing_config", "unknown")
|
||||
|
||||
try:
|
||||
origin = self._base_url()
|
||||
except RuntimeError as exc:
|
||||
with _CACHE_LOCK:
|
||||
_SNAPSHOT_GENERATION += 1
|
||||
_COMMITTED_GENERATION = _SNAPSHOT_GENERATION
|
||||
_RECOVERY_SOURCE = None
|
||||
_reset_recovery_state_locked()
|
||||
result = self._empty_snapshot("error", "warning")
|
||||
result["reason"] = redact_message(str(exc))[:240]
|
||||
return result
|
||||
|
||||
now_monotonic = monotonic()
|
||||
with _CACHE_LOCK:
|
||||
if _RECOVERY_SOURCE is not None and _RECOVERY_SOURCE != origin:
|
||||
_RECOVERY_SOURCE = origin
|
||||
_reset_recovery_state_locked()
|
||||
if (
|
||||
not refresh
|
||||
and _CACHE is not None
|
||||
and now_monotonic - _CACHE[0] < _CACHE_SECONDS
|
||||
):
|
||||
cached = copy.deepcopy(_CACHE[1])
|
||||
action = cached.get("recommendedAction")
|
||||
if isinstance(action, dict) and action.get("intent") == "runner.recover":
|
||||
cached["recommendedAction"] = None
|
||||
cached.get("queue", {})["staleAssignmentRunnerLabels"] = []
|
||||
cached["cache"] = {"hit": True, "ttlSeconds": _CACHE_SECONDS}
|
||||
return cached
|
||||
|
||||
if not self.is_configured():
|
||||
return {
|
||||
"status": "missing_config",
|
||||
"severity": "unknown",
|
||||
"generatedAt": datetime.now(UTC).isoformat(),
|
||||
"runners": [],
|
||||
"queue": {
|
||||
"queued": 0,
|
||||
"inProgress": 0,
|
||||
"oldestQueuedAgeSeconds": None,
|
||||
"oldestInProgressAgeSeconds": None,
|
||||
"sampleTruncated": False,
|
||||
"blockedByRunnerLabels": [],
|
||||
"staleAssignmentRunnerLabels": [],
|
||||
},
|
||||
"recommendedAction": None,
|
||||
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
||||
}
|
||||
_SNAPSHOT_GENERATION += 1
|
||||
generation = _SNAPSHOT_GENERATION
|
||||
sample_started = now_monotonic
|
||||
|
||||
try:
|
||||
runner_payload = self._request(
|
||||
@@ -219,7 +418,7 @@ class GiteaActionsConnector(BaseConnector):
|
||||
)
|
||||
# Keep pending inventory in the all-or-error fetch, but it is not
|
||||
# runnable queue pressure and must not drive runner diagnostics.
|
||||
self._request(
|
||||
pending_payload = self._request(
|
||||
"/api/v1/admin/actions/jobs",
|
||||
{
|
||||
"status": "pending",
|
||||
@@ -234,30 +433,55 @@ class GiteaActionsConnector(BaseConnector):
|
||||
{"status": "in_progress", "page": 1, "limit": 100},
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
return {
|
||||
"status": "error",
|
||||
"severity": "warning",
|
||||
"reason": redact_message(str(exc))[:240],
|
||||
"generatedAt": datetime.now(UTC).isoformat(),
|
||||
"runners": [],
|
||||
"queue": {
|
||||
"queued": 0,
|
||||
"inProgress": 0,
|
||||
"oldestQueuedAgeSeconds": None,
|
||||
"oldestInProgressAgeSeconds": None,
|
||||
"sampleTruncated": False,
|
||||
"blockedByRunnerLabels": [],
|
||||
"staleAssignmentRunnerLabels": [],
|
||||
},
|
||||
"recommendedAction": None,
|
||||
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
||||
}
|
||||
with _CACHE_LOCK:
|
||||
if generation == _SNAPSHOT_GENERATION:
|
||||
_COMMITTED_GENERATION = generation
|
||||
_RECOVERY_SOURCE = origin
|
||||
_reset_recovery_state_locked()
|
||||
result = self._empty_snapshot("error", "warning")
|
||||
result["reason"] = redact_message(str(exc))[:240]
|
||||
return result
|
||||
|
||||
sample_finished = monotonic()
|
||||
|
||||
recovery_runners = _valid_runner_inventory(runner_payload)
|
||||
seen_job_ids: set[int] = set()
|
||||
recovery_queued_jobs = _valid_job_inventory(
|
||||
queued_payload, "queued", seen_job_ids
|
||||
)
|
||||
recovery_pending_jobs = _valid_job_inventory(
|
||||
pending_payload, "pending", seen_job_ids
|
||||
)
|
||||
recovery_active_jobs = _valid_job_inventory(
|
||||
active_payload, "in_progress", seen_job_ids
|
||||
)
|
||||
recovery_inventory_complete = all(
|
||||
rows is not None
|
||||
for rows in (
|
||||
recovery_runners,
|
||||
recovery_queued_jobs,
|
||||
recovery_pending_jobs,
|
||||
recovery_active_jobs,
|
||||
)
|
||||
)
|
||||
runner_payload = runner_payload if isinstance(runner_payload, dict) else {}
|
||||
queued_payload = queued_payload if isinstance(queued_payload, dict) else {}
|
||||
active_payload = active_payload if isinstance(active_payload, dict) else {}
|
||||
|
||||
def display_int(value: Any) -> int:
|
||||
try:
|
||||
return int(value or 0)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return 0
|
||||
|
||||
def display_count(value: Any, fallback: int = 0) -> int:
|
||||
return value if _is_count(value) else fallback
|
||||
|
||||
raw_runners = runner_payload.get("runners")
|
||||
runner_items = raw_runners if isinstance(raw_runners, list) else []
|
||||
runners = [
|
||||
{
|
||||
"id": int(item.get("id", 0) or 0),
|
||||
"id": display_int(item.get("id", 0)),
|
||||
"name": str(item.get("name") or "unknown")[:160],
|
||||
"status": str(item.get("status") or "unknown")[:32],
|
||||
"busy": bool(item.get("busy")),
|
||||
@@ -276,11 +500,12 @@ class GiteaActionsConnector(BaseConnector):
|
||||
)
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
active_jobs = (
|
||||
raw_active_jobs = (
|
||||
active_payload.get("jobs")
|
||||
if isinstance(active_payload.get("jobs"), list)
|
||||
else []
|
||||
)
|
||||
active_jobs = [item for item in raw_active_jobs if isinstance(item, dict)]
|
||||
now = datetime.now(UTC)
|
||||
queued_times = [self._timestamp(item.get("created_at")) for item in queued_jobs]
|
||||
queued_times = [item for item in queued_times if item is not None]
|
||||
@@ -314,8 +539,12 @@ class GiteaActionsConnector(BaseConnector):
|
||||
if item["status"] == "online" and not item["disabled"]
|
||||
]
|
||||
blocked_jobs = []
|
||||
stale_assignment_jobs: list[dict[str, Any]] = []
|
||||
eligible_recovery_candidates: list[dict[str, Any]] = []
|
||||
queue_items = []
|
||||
targets = self._restart_targets()
|
||||
active_runner_ids = {
|
||||
job["runner_id"] for job in recovery_active_jobs or []
|
||||
}
|
||||
for job in queued_jobs:
|
||||
required = set(self._labels(job.get("labels")))
|
||||
matching = [
|
||||
@@ -337,25 +566,60 @@ class GiteaActionsConnector(BaseConnector):
|
||||
if all_matching_busy
|
||||
else "awaiting_scheduler_assignment"
|
||||
)
|
||||
if (
|
||||
required
|
||||
and matching
|
||||
and not all_matching_busy
|
||||
and age is not None
|
||||
and age >= _CRITICAL_AGE_SECONDS
|
||||
):
|
||||
stale_assignment_jobs.append(
|
||||
{
|
||||
"job": job,
|
||||
"required": sorted(required),
|
||||
"matching": matching,
|
||||
"ageSeconds": age,
|
||||
}
|
||||
)
|
||||
if recovery_inventory_complete and required:
|
||||
for label in sorted(required):
|
||||
target = self._target_for_label(label, online_runners, targets)
|
||||
if not target:
|
||||
continue
|
||||
exact_runners = [
|
||||
runner for runner in runners if runner["name"] == target
|
||||
]
|
||||
if len(exact_runners) != 1:
|
||||
continue
|
||||
target_runner = exact_runners[0]
|
||||
target_labels = tuple(sorted(target_runner["labels"]))
|
||||
if (
|
||||
not required.issubset(set(target_labels))
|
||||
or target_runner["status"] != "online"
|
||||
or target_runner["disabled"] is not False
|
||||
or target_runner["busy"] is not False
|
||||
or target_runner["id"] in active_runner_ids
|
||||
):
|
||||
continue
|
||||
key = (
|
||||
origin,
|
||||
job["id"],
|
||||
tuple(
|
||||
(field, job[field])
|
||||
for field in (
|
||||
"run_id",
|
||||
"run_number",
|
||||
"workflow_id",
|
||||
"attempt",
|
||||
"run_attempt",
|
||||
)
|
||||
if job.get(field) is not None
|
||||
),
|
||||
tuple(sorted(required)),
|
||||
label,
|
||||
target,
|
||||
target_runner["id"],
|
||||
target_runner["name"],
|
||||
target_labels,
|
||||
)
|
||||
eligible_recovery_candidates.append(
|
||||
{
|
||||
"key": key,
|
||||
"job": job,
|
||||
"required": sorted(required),
|
||||
"label": label,
|
||||
"target": target,
|
||||
}
|
||||
)
|
||||
if len(queue_items) < _QUEUE_SAMPLE_LIMIT:
|
||||
queue_items.append(
|
||||
{
|
||||
"id": int(job.get("id", 0) or 0),
|
||||
"id": display_int(job.get("id", 0)),
|
||||
"name": str(job.get("name") or "job")[:160],
|
||||
"labels": sorted(required)[:20],
|
||||
"ageSeconds": age,
|
||||
@@ -364,6 +628,22 @@ class GiteaActionsConnector(BaseConnector):
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
current_origin = self._base_url() if self.is_configured() else None
|
||||
except RuntimeError:
|
||||
current_origin = None
|
||||
source_matches = current_origin == origin
|
||||
matured_recovery_candidates = _observe_recovery_candidates(
|
||||
generation,
|
||||
origin,
|
||||
eligible_recovery_candidates,
|
||||
sample_started,
|
||||
sample_finished,
|
||||
complete=recovery_inventory_complete and source_matches,
|
||||
)
|
||||
if matured_recovery_candidates is None:
|
||||
matured_recovery_candidates = []
|
||||
|
||||
severity = "healthy"
|
||||
if (
|
||||
(oldest_age or 0) >= _CRITICAL_AGE_SECONDS
|
||||
@@ -378,7 +658,6 @@ class GiteaActionsConnector(BaseConnector):
|
||||
):
|
||||
severity = "warning"
|
||||
|
||||
targets = self._restart_targets()
|
||||
blocked_labels = [
|
||||
label for job in blocked_jobs for label in self._labels(job.get("labels"))
|
||||
]
|
||||
@@ -390,43 +669,33 @@ class GiteaActionsConnector(BaseConnector):
|
||||
stale_assignment_labels = sorted(
|
||||
{
|
||||
label
|
||||
for candidate in stale_assignment_jobs
|
||||
for candidate in matured_recovery_candidates
|
||||
for label in candidate["required"]
|
||||
}
|
||||
)
|
||||
|
||||
recommended_action: dict[str, Any] | None = None
|
||||
for candidate in sorted(
|
||||
stale_assignment_jobs,
|
||||
key=lambda item: int(item["ageSeconds"]),
|
||||
matured_recovery_candidates,
|
||||
key=lambda item: item["observedClaimableSeconds"],
|
||||
reverse=True,
|
||||
):
|
||||
job = candidate["job"]
|
||||
matching_by_name = {
|
||||
str(runner.get("name") or ""): runner for runner in candidate["matching"]
|
||||
recommended_action = {
|
||||
"intent": "runner.recover",
|
||||
"target": candidate["target"],
|
||||
"requiresApproval": False,
|
||||
"autoExecute": False,
|
||||
"reason": "Gitea Actions scheduler left a compatible idle runner unassigned for at least ten minutes.",
|
||||
"evidence": {
|
||||
"jobId": display_int(job.get("id", 0)),
|
||||
"label": candidate["label"],
|
||||
"observedClaimableSeconds": candidate[
|
||||
"observedClaimableSeconds"
|
||||
],
|
||||
},
|
||||
}
|
||||
for label in candidate["required"]:
|
||||
target = self._target_for_label(label, candidate["matching"], targets)
|
||||
if not target:
|
||||
continue
|
||||
target_runner = matching_by_name.get(target)
|
||||
if target_runner is not None and bool(target_runner.get("busy")):
|
||||
continue
|
||||
recommended_action = {
|
||||
"intent": "runner.recover",
|
||||
"target": target,
|
||||
"requiresApproval": False,
|
||||
"autoExecute": True,
|
||||
"reason": "Gitea Actions scheduler left a compatible idle runner unassigned for at least ten minutes.",
|
||||
"evidence": {
|
||||
"jobId": int(job.get("id", 0) or 0),
|
||||
"label": label,
|
||||
"minimumAgeSeconds": _CRITICAL_AGE_SECONDS,
|
||||
},
|
||||
}
|
||||
break
|
||||
if recommended_action is not None:
|
||||
break
|
||||
break
|
||||
|
||||
if recommended_action is None:
|
||||
restart_target = None
|
||||
@@ -465,14 +734,14 @@ class GiteaActionsConnector(BaseConnector):
|
||||
"offline": sum(item["status"] != "online" for item in runners),
|
||||
},
|
||||
"queue": {
|
||||
"queued": int(queued_payload.get("total_count", 0) or 0),
|
||||
"inProgress": int(
|
||||
active_payload.get("total_count", len(active_jobs)) or 0
|
||||
"queued": display_count(queued_payload.get("total_count", 0)),
|
||||
"inProgress": display_count(
|
||||
active_payload.get("total_count", len(active_jobs))
|
||||
),
|
||||
"oldestQueuedAgeSeconds": oldest_age,
|
||||
"oldestInProgressAgeSeconds": oldest_active_age,
|
||||
"sampleTruncated": (
|
||||
int(queued_payload.get("total_count", 0) or 0)
|
||||
display_count(queued_payload.get("total_count", 0))
|
||||
> len(
|
||||
queued_payload.get("jobs")
|
||||
if isinstance(queued_payload.get("jobs"), list)
|
||||
@@ -487,6 +756,25 @@ class GiteaActionsConnector(BaseConnector):
|
||||
"recommendedAction": recommended_action,
|
||||
"cache": {"hit": False, "ttlSeconds": _CACHE_SECONDS},
|
||||
}
|
||||
try:
|
||||
final_origin = self._base_url() if self.is_configured() else None
|
||||
except RuntimeError:
|
||||
final_origin = None
|
||||
with _CACHE_LOCK:
|
||||
_CACHE = (now_monotonic, copy.deepcopy(result))
|
||||
if final_origin != origin and generation == _SNAPSHOT_GENERATION:
|
||||
_SNAPSHOT_GENERATION += 1
|
||||
_COMMITTED_GENERATION = _SNAPSHOT_GENERATION
|
||||
_RECOVERY_SOURCE = final_origin
|
||||
_reset_recovery_state_locked()
|
||||
if (
|
||||
final_origin == origin
|
||||
and generation == _SNAPSHOT_GENERATION
|
||||
and _COMMITTED_GENERATION == generation
|
||||
):
|
||||
_CACHE = (sample_finished, copy.deepcopy(result))
|
||||
else:
|
||||
action = result.get("recommendedAction")
|
||||
if isinstance(action, dict) and action.get("intent") == "runner.recover":
|
||||
result["recommendedAction"] = None
|
||||
result["queue"]["staleAssignmentRunnerLabels"] = []
|
||||
return result
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import UTC, datetime
|
||||
from functools import wraps
|
||||
@@ -759,6 +760,38 @@ def appops_registered_legacy_delivery(repository: str, revision: str, request_id
|
||||
}))
|
||||
|
||||
|
||||
@appops_mcp.tool(name="appops.projectbrain_delivery.preflight", annotations=_READ_ONLY_TOOL)
|
||||
@_tracked("appops.projectbrain_delivery.preflight")
|
||||
def appops_projectbrain_delivery_preflight(revision: str) -> dict[str, Any]:
|
||||
"""Read the fixed ProjectBrain retry evidence without changing delivery or deployment state."""
|
||||
if not isinstance(revision, str) or not re.fullmatch(r"[0-9a-f]{40}", revision):
|
||||
raise ValueError("ProjectBrain delivery revision is invalid")
|
||||
return _json(DeploymentBrokerClient().projectbrain_delivery_operation(
|
||||
"preflight", {"revision": revision},
|
||||
))
|
||||
|
||||
|
||||
@appops_mcp.tool(name="appops.projectbrain_delivery.revalidate", annotations=_OPERATION_RUN_TOOL)
|
||||
@_tracked("appops.projectbrain_delivery.revalidate")
|
||||
def appops_projectbrain_delivery_revalidate(
|
||||
revision: str, expected_preflight_sha256: str, idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Queue only the existing ProjectBrain main push after the fixed evidence fingerprint is rechecked."""
|
||||
if (not isinstance(revision, str) or not re.fullmatch(r"[0-9a-f]{40}", revision)
|
||||
or not isinstance(expected_preflight_sha256, str)
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", expected_preflight_sha256)
|
||||
or not isinstance(idempotency_key, str)
|
||||
or not re.fullmatch(r"[A-Za-z0-9._:-]{8,128}", idempotency_key)):
|
||||
raise ValueError("ProjectBrain revalidation contract is invalid")
|
||||
return _json(DeploymentBrokerClient().projectbrain_delivery_operation(
|
||||
"revalidate", {
|
||||
"revision": revision,
|
||||
"expectedPreflightSha256": expected_preflight_sha256,
|
||||
"idempotencyKey": idempotency_key,
|
||||
},
|
||||
))
|
||||
|
||||
|
||||
@appops_mcp.tool(name="server.operation.execute", annotations=_LEGACY_APPROVAL_EXECUTION_TOOL)
|
||||
@_tracked("server.operation.execute")
|
||||
def execute_operation(
|
||||
|
||||
@@ -294,6 +294,28 @@ class DeploymentBrokerClient:
|
||||
error_response_fields=frozenset({"noMutation"}),
|
||||
)
|
||||
|
||||
def projectbrain_delivery_operation(self, stage: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Use the fixed ProjectBrain retry gate; callers cannot select a repository or runtime."""
|
||||
if stage not in {"preflight", "revalidate"} or not isinstance(payload, dict):
|
||||
raise ValueError("ProjectBrain delivery stage is invalid")
|
||||
if stage == "preflight":
|
||||
if (set(payload) != {"revision"} or not isinstance(payload.get("revision"), str)
|
||||
or not re.fullmatch(r"[0-9a-f]{40}", payload["revision"])):
|
||||
raise ValueError("ProjectBrain preflight contract is invalid")
|
||||
elif (set(payload) != {"revision", "expectedPreflightSha256", "idempotencyKey"}
|
||||
or not isinstance(payload.get("revision"), str)
|
||||
or not re.fullmatch(r"[0-9a-f]{40}", payload["revision"])
|
||||
or not isinstance(payload.get("expectedPreflightSha256"), str)
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", payload["expectedPreflightSha256"])
|
||||
or not isinstance(payload.get("idempotencyKey"), str)
|
||||
or not re.fullmatch(r"[A-Za-z0-9._:-]{8,128}", payload["idempotencyKey"])):
|
||||
raise ValueError("ProjectBrain revalidation contract is invalid")
|
||||
return self._request(
|
||||
"POST", f"/v1/appops/projectbrain-delivery/{stage}",
|
||||
secret=settings.mcp_token or "", payload=payload,
|
||||
error_response_fields=frozenset({"noMutation"}),
|
||||
)
|
||||
|
||||
def registered_legacy_operation(self, stage: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Signed fixed-stage metadata adoption/recovery for existing registered targets."""
|
||||
if stage not in {"observe", "baseline", "preflight", "adopt", "status", "rollback", "delivery"}:
|
||||
|
||||
Reference in new issue
Block a user