Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d

backend/tests/test_runner_fleet.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py"
+
+
+@pytest.fixture
+def fleet_module():
+    spec = importlib.util.spec_from_file_location("appops_runner_fleet_test", FLEET_PATH)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_cleanup_candidates_exclude_unknown_attached_and_labeled_networks(
+    fleet_module, monkeypatch
+):
+    service = fleet_module.RunnerFleetService()
+    now = 1_800_000_000
+    monkeypatch.setattr(fleet_module.time, "time", lambda: now)
+    old = "2026-01-01T00:00:00Z"
+    monkeypatch.setattr(
+        service,
+        "_network_inventory",
+        lambda: (
+            [
+                {
+                    "id": "a" * 64,
+                    "name": "GITEA-ACTIONS-TASK-12-WORKFLOW-ci-JOB-test-network",
+                    "created": old,
+                    "attached": 0,
+                    "labels": {},
+                },
+                {
+                    "id": "b" * 64,
+                    "name": "production_default",
+                    "created": old,
+                    "attached": 0,
+                    "labels": {},
+                },
+                {
+                    "id": "c" * 64,
+                    "name": "GITEA-ACTIONS-TASK-13-WORKFLOW-ci-JOB-test-network",
+                    "created": old,
+                    "attached": 1,
+                    "labels": {},
+                },
+                {
+                    "id": "d" * 64,
+                    "name": "GITEA-ACTIONS-TASK-14-WORKFLOW-ci-JOB-test-network",
+                    "created": old,
+                    "attached": 0,
+                    "labels": {"com.example.keep": "true"},
+                },
+            ],
+            2048,
+        ),
+    )
+
+    candidates = service.cleanup_candidates()
+
+    assert [candidate["id"] for candidate in candidates] == ["a" * 64]
+    assert candidates[0]["reason"] == "empty_gitea_actions_task_network_in_dedicated_dind"
+
+
+def test_automatic_cycle_only_deletes_exact_preview_candidates(fleet_module, monkeypatch):
+    service = fleet_module.RunnerFleetService()
+    candidate = {
+        "id": "a" * 64,
+        "name": "GITEA-ACTIONS-TASK-12-WORKFLOW-ci-JOB-test-network",
+        "ageSeconds": 3600,
+        "reason": "empty_gitea_actions_task_network_in_dedicated_dind",
+    }
+    snapshot = {
+        "status": "healthy",
+        "networks": {"cleanupCandidates": [candidate]},
+        "storage": {"status": "healthy"},
+        "topology": {
+            "dind": {
+                "name": "gitea-runner-dind",
+                "running": True,
+                "health": "healthy",
+                "startedAt": "2026-09-16T18:00:00Z",
+                "restartPolicy": "unless-stopped",
+            },
+            "runners": [],
+        },
+    }
+    calls: list[list[str]] = []
+    monkeypatch.setattr(service, "snapshot", lambda: snapshot)
+    monkeypatch.setattr(service, "_dind", lambda args, timeout=20: calls.append(args) or "")
+    monkeypatch.setattr(service, "_record", lambda **_kwargs: None)
+
+    result = service.automatic_cycle()
+
+    assert result["removedNetworks"] == 1
+    assert calls == [["network", "rm", "a" * 64]]
+
+
+def test_recovery_budget_stops_restart_loops(fleet_module, monkeypatch):
+    service = fleet_module.RunnerFleetService()
+    now = 1_800_000_000
+    monkeypatch.setattr(fleet_module.time, "time", lambda: now)
+    monkeypatch.setattr(
+        service,
+        "_events",
+        lambda: [
+            {
+                "action": "restart",
+                "target": "itworx-security-runner",
+                "at": "2027-01-15T08:00:00Z",
+            }
+            for _ in range(fleet_module.RECOVERY_BUDGET)
+        ],
+    )
+
+    assert service._recovery_available("itworx-security-runner") is False
+
+
+def test_fresh_starting_runner_is_not_restarted_before_health_grace_expires(
+    fleet_module,
+):
+    fresh = {
+        "name": "itworx-validation-runner-2",
+        "running": True,
+        "health": "starting",
+        "startedAt": "1970-01-01T00:16:00Z",
+    }
+    stale = {**fresh, "startedAt": "1970-01-01T00:13:00Z"}
+
+    assert fleet_module._container_is_healthy(fresh, now=1_000)
+    assert not fleet_module._container_is_healthy(stale, now=1_000)
+
+
+def test_automatic_cycle_starts_only_an_exited_known_runner(fleet_module, monkeypatch):
+    service = fleet_module.RunnerFleetService()
+    snapshot = {
+        "status": "healthy",
+        "networks": {"cleanupCandidates": []},
+        "storage": {"status": "healthy"},
+        "topology": {
+            "dind": {
+                "name": "gitea-runner-dind",
+                "running": True,
+                "health": "healthy",
+                "startedAt": "2026-09-16T18:00:00Z",
+                "restartPolicy": "unless-stopped",
+            },
+            "runners": [
+                {
+                    "name": "itworx-validation-runner-2",
+                    "running": False,
+                    "health": "unhealthy",
+                },
+                {
+                    "name": "itworx-security-runner",
+                    "running": True,
+                    "health": "unhealthy",
+                },
+            ]
+        },
+    }
+    commands: list[list[str]] = []
+    events: list[dict[str, object]] = []
+    monkeypatch.setattr(service, "snapshot", lambda: snapshot)
+    monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
+    monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
+    monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
+
+    service.automatic_cycle()
+
+    assert commands == [["docker", "start", "itworx-validation-runner-2"]]
+    assert {event["action"] for event in events} == {"start", "restart.deferred"}