Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d

backend/tests/test_runner_scheduler_liveness_recovery.py

@@ -0,0 +1,101 @@
+from datetime import UTC, datetime, timedelta
+
+from backend.app.connectors import gitea_actions
+from backend.app.connectors.gitea_actions import GiteaActionsConnector
+from backend.app.core.config import settings
+
+
+def _configure(monkeypatch) -> GiteaActionsConnector:
+    monkeypatch.setitem(settings.apis, "GITEA_URL", "https://gitea.test")
+    monkeypatch.setitem(settings.apis, "GITEA_ADMIN_TOKEN", "fixture-secret")
+    monkeypatch.setitem(
+        settings.apis,
+        "GITEA_RUNNER_TARGETS_JSON",
+        '{"linux-validation":"itworx-validation-runner-2",'
+        '"mcp-validation":"itworx-security-runner"}',
+    )
+    monkeypatch.setattr(gitea_actions, "_CACHE", None)
+    return GiteaActionsConnector()
+
+
+def _requests(created_at: str, *, busy: bool = False):
+    def fake_request(path, params=None):
+        params = params or {}
+        if path.endswith("/runners"):
+            return {
+                "total_count": 1,
+                "runners": [
+                    {
+                        "id": 17,
+                        "name": "validation-secondary",
+                        "status": "online",
+                        "busy": busy,
+                        "disabled": False,
+                        "labels": ["linux-validation", "ubuntu-latest"],
+                    }
+                ],
+            }
+        if params.get("status") == "queued":
+            return {
+                "total_count": 1,
+                "jobs": [
+                    {
+                        "id": 91,
+                        "name": "full",
+                        "status": "queued",
+                        "created_at": created_at,
+                        "labels": ["linux-validation"],
+                    }
+                ],
+            }
+        return {"total_count": 0, "jobs": []}
+
+    return fake_request
+
+
+def test_old_unassigned_job_on_idle_matching_runner_is_recovery_candidate(monkeypatch):
+    connector = _configure(monkeypatch)
+    created_at = (datetime.now(UTC) - timedelta(minutes=12)).isoformat()
+    monkeypatch.setattr(connector, "_request", _requests(created_at))
+
+    result = connector.snapshot(refresh=True)
+
+    assert result["severity"] == "critical"
+    assert result["queue"]["items"][0]["reason"] == "awaiting_scheduler_assignment"
+    assert result["queue"]["staleAssignmentRunnerLabels"] == ["linux-validation"]
+    assert result["recommendedAction"] == {
+        "intent": "runner.recover",
+        "target": "itworx-validation-runner-2",
+        "requiresApproval": False,
+        "autoExecute": True,
+        "reason": "Gitea Actions scheduler left a compatible idle runner unassigned for at least ten minutes.",
+        "evidence": {
+            "jobId": 91,
+            "label": "linux-validation",
+            "minimumAgeSeconds": 600,
+        },
+    }
+
+
+def test_recent_unassigned_job_does_not_trigger_recovery(monkeypatch):
+    connector = _configure(monkeypatch)
+    created_at = (datetime.now(UTC) - timedelta(minutes=2)).isoformat()
+    monkeypatch.setattr(connector, "_request", _requests(created_at))
+
+    result = connector.snapshot(refresh=True)
+
+    assert result["queue"]["items"][0]["reason"] == "awaiting_scheduler_assignment"
+    assert result["queue"]["staleAssignmentRunnerLabels"] == []
+    assert result["recommendedAction"] is None
+
+
+def test_busy_matching_runner_is_never_restarted_for_waiting_queue(monkeypatch):
+    connector = _configure(monkeypatch)
+    created_at = (datetime.now(UTC) - timedelta(minutes=12)).isoformat()
+    monkeypatch.setattr(connector, "_request", _requests(created_at, busy=True))
+
+    result = connector.snapshot(refresh=True)
+
+    assert result["queue"]["items"][0]["reason"] == "all_matching_runners_are_busy"
+    assert result["queue"]["staleAssignmentRunnerLabels"] == []
+    assert result["recommendedAction"] is None