Publish curated Unraid-AppOps-Gateway source from 3c5fb0c6d1f4e693e4a0fa50e7151509b4450786

PUBLIC_SOURCE_EXPORT.md

@@ -1,3 +1,3 @@
 # Public source export
 
-This source snapshot was generated from the private canonical repository at revision `a898651d5411fca177825aa908a157a34f4ce4a8`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
+This source snapshot was generated from the private canonical repository at revision `3c5fb0c6d1f4e693e4a0fa50e7151509b4450786`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.

PUBLIC_SOURCE_MANIFEST.json

@@ -1,6 +1,6 @@
 {
   "schemaVersion": 1,
-  "sourceRevision": "a898651d5411fca177825aa908a157a34f4ce4a8",
+  "sourceRevision": "3c5fb0c6d1f4e693e4a0fa50e7151509b4450786",
   "files": [
     {
       "path": ".dockerignore",
@@ -489,8 +489,8 @@
     },
     {
       "path": "backend/app/services/controller_artifact_promoter.py",
-      "sha256": "557cc8dfc7fd5f8625dc36eab9bdef730b7a4cc8fa9bf273954b4aabae190ab0",
+      "sha256": "a193123dd59de03f19cc14e51e9551e0ab316b741cc8b417ed530f86c539dc13",
-      "bytes": 16381
+      "bytes": 16370
     },
     {
       "path": "backend/app/services/controller_bootstrap_import.py",
@@ -797,6 +797,11 @@
       "sha256": "80c9444a354c1bee6114ae004b0c7d9c1e72834afd2b2dc04ed7c6b738119150",
       "bytes": 47199
     },
+    {
+      "path": "backend/tests/test_appops_live_state_recovery.py",
+      "sha256": "33b5fa25e441128f84c4e518cef803fa9229ffe04a6ec4e980a3bb2707175959",
+      "bytes": 12368
+    },
     {
       "path": "backend/tests/test_appops_self_upgrade_controller.py",
       "sha256": "ad6d8fba2213c1c8018bc9ff1c2ebd61684792c27fa00a07195c5d6aea99d69f",
@@ -1207,6 +1212,11 @@
       "sha256": "a442a54748caf77f912d05b7ed9d68645db502d1a2a96cd1838501343bccdf90",
       "bytes": 5966
     },
+    {
+      "path": "backend/tests/test_runner_fleet_cutover_safety.py",
+      "sha256": "85f93b1e70114ea1b86bdc4adeb05ea18b91e2886c5ed0dd44474991ee3ddda9",
+      "bytes": 15917
+    },
     {
       "path": "backend/tests/test_runner_fleet_scheduler_recovery.py",
       "sha256": "567c59526c430c61c7823a984fb8c2a7eb17c5614e3e0a28a2b25b9af85fc55f",

backend/app/services/controller_artifact_promoter.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import json
 import re
 import time
 from datetime import datetime
@@ -20,7 +19,7 @@ GITEA_REPOSITORY = "Unraid-AppOps-Gateway"
 REQUIRED_PUSH_CONTEXT = "Managed validation / full (push)"
 HELPER_NAME = "itworx-controller-artifact-promoter"
 CONTROLLER_CONTAINER = "gitea-deploy-control"
-CONTROLLER_VERSION = "2.1.5"
+CONTROLLER_VERSION = "2.1.13"
 HELPER_LABEL = "itworx.controller.promoter"
 _HELPER_TIMEOUT_SECONDS = 240
 _REVISION = re.compile(r"^[0-9a-f]{40}$")

backend/tests/test_appops_live_state_recovery.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import importlib.util
+import json
+import os
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+REPOSITORY = "jens/unraid-appops-gateway"
+CONTAINER = "Unraid-AppOps-Gateway"
+OBSERVED = "5" * 40
+RECORDED = "6" * 40
+ATTEMPT = "a" * 40
+IMAGE = "sha256:" + "5" * 64
+OLD_IMAGE = "sha256:" + "6" * 64
+
+
+def receipt(revision=RECORDED, image=OLD_IMAGE):
+    tree = "b" * 40
+    digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest()
+    built = "2026-10-02T00:00:00.000Z"
+    return {
+        "schema": "gitea-deploy-receipt.v1", "repository": REPOSITORY,
+        "revision": revision, "tree": tree, "treeDigest": digest,
+        "build": {"id": revision[:12], "repository": REPOSITORY,
+                  "revision": revision, "tree": tree, "treeDigest": digest,
+                  "builtAt": built},
+        "containers": {CONTAINER: {"image_id": image,
+            "image_revision": revision, "image_source": REPOSITORY,
+            "image_tree": tree, "image_version": revision[:12],
+            "image_created": built}}, "live": None,
+    }
+
+
+@pytest.fixture
+def recovery(monkeypatch, tmp_path):
+    if "fcntl" not in sys.modules:
+        shim = types.ModuleType("fcntl")
+        shim.LOCK_EX, shim.LOCK_UN = 2, 8
+        shim.flock = lambda *_args: None
+        monkeypatch.setitem(sys.modules, "fcntl", shim)
+    spec = importlib.util.spec_from_file_location(
+        "appops_recovery_test", ROOT / "ops/gitea-deploy-broker/deploy.py")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    state_root = tmp_path / "state"
+    state_root.mkdir(mode=0o700)
+    monkeypatch.setattr(module, "STATE_ROOT", state_root)
+    monkeypatch.setattr(module, "require_host_root", lambda: None)
+    monkeypatch.setattr(module, "_root_owned", lambda _path: True)
+    if os.name == "nt":
+        # Windows does not expose POSIX write bits; keep real inode/type checks.
+        real_lstat = Path.lstat
+        def posix_permissions(path):
+            observed = real_lstat(path)
+            return types.SimpleNamespace(st_mode=observed.st_mode & ~0o022,
+                st_dev=observed.st_dev, st_ino=observed.st_ino)
+        monkeypatch.setattr(Path, "lstat", posix_permissions)
+    fsyncs = []
+    monkeypatch.setattr(module, "_fsync_directory", lambda path: fsyncs.append(path))
+    project = {"slug": "unraid-appops-gateway", "branch": "main",
+        "gitea_repository_id": 9, "containers": [CONTAINER],
+        "compose_project": "unraid-appops-gateway",
+        "compose_files": ["docker-compose.yml"], "verify_oci_revision": True,
+        "environment": {"APPOPS_DATA_PATH": str(module.APPOPS_DATA_ROOT)},
+        "source_dir": str(tmp_path), "_repository_name": REPOSITORY}
+    (tmp_path / "docker-compose.yml").write_text("services: {}\n")
+    public = {key: value for key, value in project.items() if not key.startswith("_")}
+    monkeypatch.setattr(module, "load_projects", lambda: {REPOSITORY: copy.deepcopy(public)})
+    def validate(_repo, branch, revision):
+        assert branch == "main"
+        if revision not in {OBSERVED, ATTEMPT}:
+            raise module.DeployError("revision is not a protected ancestor")
+        return revision
+    monkeypatch.setattr(module, "validate_commit", validate)
+    source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway"
+    labels = {"org.opencontainers.image.revision": OBSERVED,
+              "org.opencontainers.image.source": source,
+              "com.docker.compose.project": "unraid-appops-gateway"}
+    container = {"Id": "c" * 64, "Image": IMAGE,
+        "State": {"Running": True, "Health": {"Status": "healthy"}},
+        "Config": {"Image": "appops:observed", "Labels": labels},
+        "NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}},
+        "Mounts": [{"Destination": "/data", "Type": "bind", "RW": True,
+                    "Source": str(module.APPOPS_DATA_ROOT)}]}
+    image = {"Id": IMAGE, "Config": {"Labels": copy.deepcopy(labels)}}
+    monkeypatch.setattr(module, "inspect_container", lambda name: copy.deepcopy(container))
+    monkeypatch.setattr(module, "inspect_image", lambda name: copy.deepcopy(image))
+    monkeypatch.setattr(module, "compose_project_container_names", lambda name: {CONTAINER})
+    def output(command, **_kwargs):
+        assert command == ["docker", "ps", "-a", "--format", "{{.Names}}"]
+        return CONTAINER + "\n"
+    monkeypatch.setattr(module, "output", output)
+    def forbidden(*_args, **_kwargs):
+        pytest.fail("recovery attempted a host mutation")
+    monkeypatch.setattr(module, "run", forbidden)
+    prior = {"schema_version": 3, "repository": REPOSITORY,
+        "slug": project["slug"], "compose_project": project["compose_project"],
+        "revision": RECORDED, "live_revision": RECORDED,
+        "status": "failed", "live_status": "unknown", "live_legacy_provenance": False,
+        "live_artifacts": {"containers": {CONTAINER: {"image_id": OLD_IMAGE}}},
+        "live_receipt": receipt(), "rollback_snapshot": {"revision": "7" * 40},
+        "last_attempt": {"revision": ATTEMPT, "status": "failed",
+                         "message": module._APPOPS_RECOVERY_FAILURE}}
+    path = state_root / "unraid-appops-gateway.json"
+    def save():
+        path.write_text(json.dumps(prior), encoding="utf-8")
+        path.chmod(0o600)
+    save()
+    return types.SimpleNamespace(m=module, project=project, repo=tmp_path,
+        path=path, prior=prior, container=container, image=image,
+        public=public, save=save, fsyncs=fsyncs)
+
+
+def test_reattest_preserves_evidence_and_observed_legacy_identity(recovery):
+    r = recovery
+    before = r.path.read_bytes()
+    compose = (r.repo / "docker-compose.yml").read_bytes()
+    result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
+    state = json.loads(r.path.read_bytes())
+    backup = r.path.parent / result["backupFile"]
+    assert backup.read_bytes() == before
+    assert state["recovery"]["prior_state_sha256"] == hashlib.sha256(before).hexdigest()
+    assert state["last_attempt"] == r.prior["last_attempt"]
+    assert state["live_revision"] == OBSERVED
+    assert state["live_legacy_provenance"] is True
+    assert state["live_status"] == "healthy"
+    assert state["live_receipt"] == state["rollback_snapshot"] == {}
+    assert state["live_artifacts"]["compose_sha256"] == ""
+    assert state["live_artifacts"]["containers"][CONTAINER]["image_id"] == IMAGE
+    assert state["recovery"]["retired_receipt_revision"] == RECORDED
+    assert (r.repo / "docker-compose.yml").read_bytes() == compose
+    assert r.fsyncs == [r.path.parent, r.path.parent]
+
+
+@pytest.mark.parametrize("field,value", [
+    ("schema_version", 2), ("status", "success"), ("live_status", "healthy"),
+    ("live_legacy_provenance", None), ("live_legacy_provenance", 0),
+    ("repository", "jens/other"), ("live_revision", OBSERVED),
+    ("rollback_snapshot", None),
+    ("live_receipt", None), ("live_receipt", {}), ("live_receipt", "invalid"),
+    ("live_artifacts", None), ("live_artifacts", {}),
+])
+def test_rejects_ineligible_recorded_state(recovery, field, value):
+    r = recovery
+    r.prior[field] = value
+    r.save()
+    before = r.path.read_bytes()
+    with pytest.raises(r.m.DeployError):
+        r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
+    assert r.path.read_bytes() == before
+    assert not list(r.path.parent.glob("*.bak"))
+
+
+@pytest.mark.parametrize("failure", ["wrong-failure", "non-ancestor", "same-image", "receipt-image", "receipt-revision"])
+def test_rejects_unproven_historical_drift(recovery, failure):
+    r = recovery
+    if failure == "wrong-failure":
+        r.prior["last_attempt"]["message"] = "Deployment failed"
+    elif failure == "non-ancestor":
+        r.prior["last_attempt"]["revision"] = "d" * 40
+    elif failure == "same-image":
+        r.prior["live_artifacts"]["containers"][CONTAINER]["image_id"] = IMAGE
+    elif failure == "receipt-image":
+        r.prior["live_receipt"]["containers"][CONTAINER]["image_id"] = IMAGE
+    else:
+        r.prior["live_receipt"]["revision"] = OBSERVED
+    r.save()
+    before = r.path.read_bytes()
+    with pytest.raises(r.m.DeployError):
+        r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
+    assert r.path.read_bytes() == before
+
+
+@pytest.mark.parametrize("failure", ["unhealthy", "volume", "readonly", "mount-path", "port", "image", "source", "revision", "extra-container", "config", "owner"])
+def test_rejects_wrong_actual_runtime_or_target(recovery, monkeypatch, failure):
+    r = recovery
+    if failure == "unhealthy":
+        r.container["State"]["Health"]["Status"] = "unhealthy"
+    elif failure in {"volume", "readonly", "mount-path"}:
+        key, value = {"volume": ("Type", "volume"), "readonly": ("RW", False),
+                      "mount-path": ("Source", "/other")}[failure]
+        r.container["Mounts"][0][key] = value
+    elif failure == "port":
+        r.container["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostPort"] = "9999"
+    elif failure == "image":
+        r.image["Id"] = OLD_IMAGE
+    elif failure in {"source", "revision"}:
+        r.image["Config"]["Labels"]["org.opencontainers.image." + failure] = "wrong"
+    elif failure == "extra-container":
+        monkeypatch.setattr(r.m, "compose_project_container_names", lambda name: {CONTAINER, "other"})
+    elif failure == "config":
+        r.public["branch"] = "other"
+    else:
+        monkeypatch.setattr(r.m, "_root_owned", lambda path: False)
+    before = r.path.read_bytes()
+    with pytest.raises(r.m.DeployError):
+        r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
+    assert r.path.read_bytes() == before
+
+
+@pytest.mark.parametrize("change", ["bytes", "inode", "runtime", "config"])
+def test_rechecks_after_durable_backup_before_replace(recovery, monkeypatch, change):
+    r = recovery
+    def after_backup(_path):
+        if change == "bytes":
+            r.path.write_bytes(b'{"external": true}')
+        elif change == "inode":
+            replacement = r.path.with_suffix(".replacement")
+            replacement.write_bytes(r.path.read_bytes())
+            replacement.chmod(0o600)
+            replacement.replace(r.path)
+        elif change == "runtime":
+            r.container["Id"] = "d" * 64
+        else:
+            r.public["branch"] = "changed"
+    monkeypatch.setattr(r.m, "_fsync_directory", after_backup)
+    with pytest.raises(r.m.DeployError):
+        r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
+    state = json.loads(r.path.read_bytes())
+    assert "recovery" not in state
+    assert len(list(r.path.parent.glob("*.bak"))) == 1
+    assert not list(r.path.parent.glob("*.tmp"))
+
+
+def test_next_normal_delivery_keeps_recovery_audit(recovery):
+    r = recovery
+    result = r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
+    audit = json.loads(r.path.read_bytes())["recovery"]
+    r.m.write_state(REPOSITORY, r.project, OBSERVED, "success", "Verified normal deployment",
+                    live_receipt=receipt(OBSERVED, IMAGE), live_legacy_provenance=False)
+    state = json.loads(r.path.read_bytes())
+    assert state["recovery"] == audit
+    assert (r.path.parent / result["backupFile"]).exists()
+    assert state["live_receipt"]["revision"] == OBSERVED
+    assert state["live_legacy_provenance"] is False
+    assert state["last_attempt"]["message"] == "Verified normal deployment"
+
+
+def test_post_replace_fsync_failure_reports_applied_unknown_and_retains_backup(recovery, monkeypatch):
+    r = recovery
+    calls = 0
+    def fail_after_replace(_path):
+        nonlocal calls
+        calls += 1
+        if calls == 2:
+            raise OSError("synthetic directory fsync failure")
+    monkeypatch.setattr(r.m, "_fsync_directory", fail_after_replace)
+    before = r.path.read_bytes()
+    with pytest.raises(r.m.DeployError, match="state was replaced but durability is unknown"):
+        r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)
+    state = json.loads(r.path.read_bytes())
+    assert state["live_revision"] == OBSERVED
+    assert (r.path.parent / state["recovery"]["backup_file"]).read_bytes() == before
+    with pytest.raises(r.m.DeployError, match="exact failed pre-mutation"):
+        r.m.reattest_appops_live_state(r.project, r.repo, OBSERVED)

backend/tests/test_runner_fleet_cutover_safety.py

@@ -0,0 +1,509 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import shlex
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+UPGRADE_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet_upgrade_v2.py"
+ACTIVE_JOBS_PATH = "/api/v1/admin/actions/jobs"
+UNSET = object()
+SCRIPT_PATH = ROOT / "scripts" / "upgrade_ci_runner_fleet_v2.sh"
+
+PRIMARY = "itworx-security-runner"
+SECONDARY = "itworx-validation-runner-2"
+LEGACY = "itworx-unraid-deploy"
+OTHER_FIXED_RUNNERS = (
+    "itworx-validation-runner-3",
+    "itworx-docker-runner",
+    "itworx-browser-runner",
+    "itworx-validation-runner-burst-1",
+)
+
+
+@pytest.fixture
+def upgrade_module():
+    spec = importlib.util.spec_from_file_location(
+        "appops_runner_fleet_upgrade_v2_cutover_test", UPGRADE_PATH
+    )
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _runner(
+    runner_id: int,
+    name: str,
+    *,
+    busy: object = False,
+    disabled: object = False,
+) -> dict[str, object]:
+    return {
+        "id": runner_id,
+        "name": name,
+        "busy": busy,
+        "disabled": disabled,
+    }
+
+
+def _safe_runners(*, include_legacy: bool = False) -> list[dict[str, object]]:
+    runners = [
+        _runner(1, PRIMARY, disabled=True),
+        _runner(2, SECONDARY, disabled=True),
+    ]
+    if include_legacy:
+        runners.append(_runner(3, LEGACY, disabled=True))
+    return runners
+
+
+class ServiceStub:
+    def __init__(
+        self,
+        runners: list[dict[str, object]] | None = None,
+        jobs: object = UNSET,
+        *,
+        inventory_error: Exception | None = None,
+        request_error: Exception | None = None,
+    ) -> None:
+        self.runners = list(runners if runners is not None else _safe_runners())
+        self.jobs = {"jobs": [], "total_count": 0} if jobs is UNSET else jobs
+        self.inventory_error = inventory_error
+        self.request_error = request_error
+        self.requests: list[tuple[str, dict[str, object]]] = []
+
+    def _gitea_runner_inventory(self) -> list[dict[str, object]]:
+        if self.inventory_error is not None:
+            raise self.inventory_error
+        return self.runners
+
+    def _gitea_request(
+        self, path: str, params: dict[str, object] | None = None
+    ) -> object:
+        self.requests.append((path, dict(params or {})))
+        if self.request_error is not None:
+            raise self.request_error
+        return self.jobs
+
+
+def _require_quiescent(module: Any, service: ServiceStub) -> dict[str, object]:
+    return module.require_cutover_quiescence(service)
+
+
+def _assert_rejected(module: Any, service: ServiceStub) -> None:
+    with pytest.raises(RuntimeError):
+        _require_quiescent(module, service)
+
+
+def test_active_gitea_job_blocks_cutover_even_when_runner_reports_idle_and_disabled(
+    upgrade_module,
+):
+    service = ServiceStub(
+        jobs={
+            "jobs": [
+                {"id": 101, "runner_id": 1, "status": "in_progress"},
+            ],
+            "total_count": 1,
+        }
+    )
+
+    _assert_rejected(upgrade_module, service)
+
+    assert len(service.requests) == 1
+    assert service.requests[0][0] == ACTIVE_JOBS_PATH
+    assert service.requests[0][1].get("status") == "in_progress"
+    assert service.requests[0][1].get("limit") == 100
+
+
+@pytest.mark.parametrize(
+    "runners",
+    [
+        [_runner(2, SECONDARY, disabled=True)],
+        [_runner(1, PRIMARY), _runner(2, SECONDARY, disabled=True)],
+        [
+            _runner(1, PRIMARY, busy=True, disabled=True),
+            _runner(2, SECONDARY, disabled=True),
+        ],
+    ],
+    ids=("primary-missing", "primary-not-fenced", "primary-busy"),
+)
+def test_primary_runner_must_be_unique_present_disabled_and_idle(
+    upgrade_module, runners
+):
+    _assert_rejected(upgrade_module, ServiceStub(runners=runners))
+
+
+@pytest.mark.parametrize(
+    "runners",
+    [
+        [_runner(1, PRIMARY, disabled=True), _runner(2, SECONDARY)],
+        [
+            _runner(1, PRIMARY, disabled=True),
+            _runner(2, SECONDARY, busy=True, disabled=True),
+        ],
+    ],
+    ids=("secondary-not-fenced", "secondary-busy"),
+)
+def test_secondary_runner_must_be_fenced_and_idle(upgrade_module, runners):
+    _assert_rejected(upgrade_module, ServiceStub(runners=runners))
+
+
+def test_duplicate_primary_registration_is_rejected(upgrade_module):
+    runners = _safe_runners()
+    runners.append(_runner(11, PRIMARY, disabled=True))
+
+    _assert_rejected(upgrade_module, ServiceStub(runners=runners))
+
+
+@pytest.mark.parametrize("bad_value", [None, "false", 0, 1])
+def test_primary_busy_and_disabled_fields_must_be_boolean(upgrade_module, bad_value):
+    busy_runners = _safe_runners()
+    busy_runners[0]["busy"] = bad_value
+    _assert_rejected(upgrade_module, ServiceStub(runners=busy_runners))
+
+    disabled_runners = _safe_runners()
+    disabled_runners[0]["disabled"] = bad_value
+    _assert_rejected(upgrade_module, ServiceStub(runners=disabled_runners))
+
+
+@pytest.mark.parametrize(
+    "legacy",
+    [
+        _runner(3, LEGACY, disabled=False),
+        _runner(3, LEGACY, busy=True, disabled=True),
+    ],
+    ids=("legacy-enabled", "legacy-busy"),
+)
+def test_optional_legacy_runner_must_be_fenced_and_idle(upgrade_module, legacy):
+    runners = _safe_runners()
+    runners.append(legacy)
+
+    _assert_rejected(upgrade_module, ServiceStub(runners=runners))
+
+
+@pytest.mark.parametrize("name", OTHER_FIXED_RUNNERS)
+def test_other_present_fixed_fleet_runner_must_be_idle(upgrade_module, name):
+    runners = _safe_runners()
+    runners.append(_runner(10, name, busy=True, disabled=False))
+
+    _assert_rejected(upgrade_module, ServiceStub(runners=runners))
+
+
+def test_unrelated_enabled_runner_cannot_accept_new_shared_dind_work(upgrade_module):
+    runners = [*_safe_runners(), _runner(90, "known-unrelated-runner")]
+    _assert_rejected(upgrade_module, ServiceStub(runners=runners))
+
+
+@pytest.mark.parametrize(
+    "jobs",
+    [
+        None,
+        [],
+        {},
+        {"jobs": []},
+        {"jobs": [], "total_count": 1},
+        {"jobs": [], "total_count": True},
+        {"jobs": [], "total_count": "0"},
+        {"jobs": None, "total_count": 0},
+        {"jobs": {}, "total_count": 0},
+        {"jobs": [None], "total_count": 1},
+        {"jobs": ["not-a-job"], "total_count": 1},
+        {"jobs": [{"id": 101, "runner_id": 1}], "total_count": 1},
+        {"jobs": [{"id": 101, "runner_id": 1, "status": "queued"}], "total_count": 1},
+    ],
+    ids=(
+        "response-null",
+        "response-list",
+        "jobs-field-missing",
+        "jobs-count-missing",
+        "jobs-count-mismatch",
+        "jobs-count-bool",
+        "jobs-count-not-integer",
+        "jobs-null",
+        "jobs-not-list",
+        "job-null",
+        "job-not-object",
+        "job-status-missing",
+        "job-status-mismatch",
+    ),
+)
+def test_malformed_or_unexpected_active_job_response_fails_closed(upgrade_module, jobs):
+    _assert_rejected(upgrade_module, ServiceStub(jobs=jobs))
+
+
+def test_full_100_job_page_is_rejected_as_truncated_or_ambiguous(upgrade_module):
+    jobs = {
+        "jobs": [
+            {"id": index + 1, "runner_id": 90, "status": "in_progress"}
+            for index in range(100)
+        ],
+        "total_count": 100,
+    }
+
+    _assert_rejected(
+        upgrade_module,
+        ServiceStub(
+            runners=[
+                *_safe_runners(),
+                _runner(90, "known-unrelated-runner", disabled=True),
+            ],
+            jobs=jobs,
+        ),
+    )
+
+
+@pytest.mark.parametrize(
+    "runner_id",
+    [None, True, False, 0, -1, "1", 999],
+    ids=("missing", "true", "false", "zero", "negative", "string", "unknown"),
+)
+def test_active_job_with_invalid_or_unknown_runner_id_fails_closed(
+    upgrade_module, runner_id
+):
+    jobs = {
+        "jobs": [{"id": 101, "runner_id": runner_id, "status": "in_progress"}],
+        "total_count": 1,
+    }
+
+    _assert_rejected(upgrade_module, ServiceStub(jobs=jobs))
+
+
+def test_active_job_on_any_fixed_runner_blocks_even_if_its_busy_flag_is_false(
+    upgrade_module,
+):
+    runners = [
+        *_safe_runners(),
+        _runner(10, "itworx-docker-runner", busy=False, disabled=True),
+    ]
+    service = ServiceStub(
+        runners=runners,
+        jobs={
+            "jobs": [{"id": 102, "runner_id": 10, "status": "in_progress"}],
+            "total_count": 1,
+        },
+    )
+
+    _assert_rejected(upgrade_module, service)
+
+
+@pytest.mark.parametrize("where", ["inventory", "jobs"])
+def test_gitea_api_errors_fail_closed(upgrade_module, where):
+    service = ServiceStub(
+        inventory_error=OSError("inventory unavailable")
+        if where == "inventory"
+        else None,
+        request_error=OSError("jobs unavailable") if where == "jobs" else None,
+    )
+
+    expected_message = (
+        "inventory unavailable" if where == "inventory" else "jobs unavailable"
+    )
+    with pytest.raises(OSError, match=expected_message):
+        _require_quiescent(upgrade_module, service)
+
+
+def test_active_job_on_unrelated_runner_also_blocks_cutover(upgrade_module):
+    service = ServiceStub(
+        runners=[*_safe_runners(), _runner(90, "known-unrelated-runner")],
+        jobs={
+            "jobs": [{"id": 201, "runner_id": 90, "status": "in_progress"}],
+            "total_count": 1,
+        },
+    )
+    _assert_rejected(upgrade_module, service)
+
+
+def test_quiescent_result_has_no_host_mutations(upgrade_module, monkeypatch):
+    def unexpected_shell(*_args, **_kwargs):
+        raise AssertionError("cutover guard must not execute Docker or shell commands")
+
+    monkeypatch.setattr(upgrade_module.subprocess, "run", unexpected_shell)
+    unrelated = _runner(90, "known-unrelated-runner", disabled=True)
+    service = ServiceStub(
+        runners=[*_safe_runners(include_legacy=True), unrelated],
+        jobs={"jobs": [], "total_count": 0},
+    )
+
+    result = _require_quiescent(upgrade_module, service)
+
+    assert result == {
+        "status": "quiescent",
+        "fencedRunners": [PRIMARY, SECONDARY, LEGACY],
+        "activeFleetJobs": 0,
+    }
+    assert service.requests == [
+        (
+            ACTIVE_JOBS_PATH,
+            {
+                "status": "in_progress",
+                "page": 1,
+                "limit": 100,
+                "sort": "id",
+                "order": "asc",
+            },
+        )
+    ]
+
+
+def _bash_function(name: str) -> str:
+    script = SCRIPT_PATH.read_text(encoding="utf-8")
+    start = script.index(f"{name}() {{")
+    end = script.index("\n}", start) + 2
+    return script[start:end]
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+@pytest.mark.parametrize(
+    ("proof", "executor_status", "allowed"),
+    [
+        ("", 0, False),
+        ("not-json", 0, False),
+        (
+            json.dumps(
+                {
+                    "status": "quiescent",
+                    "fencedRunners": [PRIMARY, SECONDARY],
+                    "activeFleetJobs": 1,
+                }
+            ),
+            0,
+            False,
+        ),
+        (
+            json.dumps(
+                {
+                    "status": "quiescent",
+                    "fencedRunners": [PRIMARY, SECONDARY],
+                    "activeFleetJobs": 0,
+                }
+            ),
+            1,
+            False,
+        ),
+        (
+            json.dumps(
+                {
+                    "status": "quiescent",
+                    "fencedRunners": [PRIMARY, SECONDARY],
+                    "activeFleetJobs": 0,
+                }
+            ),
+            0,
+            True,
+        ),
+    ],
+    ids=(
+        "old-executor-empty",
+        "malformed-proof",
+        "active-job",
+        "executor-error",
+        "quiescent",
+    ),
+)
+def test_cutover_shell_requires_real_successful_quiescence_proof(
+    proof, executor_status, allowed
+):
+    # The first python3 invocation stands in for the installed executor; the second
+    # invokes the actual JSON parser inside check_cutover.
+    body = "\n".join(
+        [
+            "ROOT=/synthetic",
+            f"MOCK_PROOF={shlex.quote(proof)}",
+            f"MOCK_EXECUTOR_STATUS={executor_status}",
+            "python3() {",
+            '  if [[ "$1" == /synthetic/controller/runner_fleet_upgrade_v2.py ]]; then',
+            '    printf "%s" "$MOCK_PROOF"',
+            '    return "$MOCK_EXECUTOR_STATUS"',
+            "  fi",
+            '  command python3 "$@"',
+            "}",
+            _bash_function("check_cutover"),
+            "check_cutover",
+        ]
+    )
+    result = subprocess.run(
+        ["bash"], input=body.encode(), capture_output=True, check=False
+    )
+    assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace")
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+@pytest.mark.parametrize(
+    ("inventory", "docker_status", "allowed"),
+    [("", 1, False), ("running-container-id", 0, False), ("", 0, True)],
+    ids=("docker-failed-with-empty-output", "busy-dind", "idle-dind"),
+)
+def test_dind_idle_shell_requires_successful_empty_docker_inventory(
+    inventory, docker_status, allowed
+):
+    body = "\n".join(
+        [
+            "DIND=synthetic-dind",
+            f"MOCK_DIND_INVENTORY={shlex.quote(inventory)}",
+            f"MOCK_DOCKER_STATUS={docker_status}",
+            'docker() { printf "%s" "$MOCK_DIND_INVENTORY"; return "$MOCK_DOCKER_STATUS"; }',
+            _bash_function("check_dind_idle"),
+            "check_dind_idle",
+        ]
+    )
+    result = subprocess.run(
+        ["bash"], input=body.encode(), capture_output=True, check=False
+    )
+    assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace")
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+@pytest.mark.parametrize(
+    ("cutover_started", "dispatch_started"),
+    [(0, 0), (1, 0), (1, 1)],
+    ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"),
+)
+def test_failure_restores_only_safe_runner_phase(cutover_started, dispatch_started):
+    body = "\n".join(
+        [
+            "backup=/synthetic-backup",
+            "COMPOSE_FILE=/synthetic-compose",
+            "PRIMARY_CONFIG=/synthetic-primary",
+            "SECONDARY_CONFIG=/synthetic-secondary",
+            "ROOT=/synthetic-root",
+            "guard_changed=0",
+            "host_guard_changed=0",
+            f"cutover_started={cutover_started}",
+            f"dispatch_started={dispatch_started}",
+            'new_runners=("new-container")',
+            'registered=("new-registration")',
+            'fenced=("itworx-security-runner:false")',
+            "created_paths=()",
+            "exec 3>&1",
+            'cp() { printf "cp:%s\\n" "$*"; }',
+            'docker() { printf "docker:%s\\n" "$*" >&3; }',
+            'delete_runner() { printf "delete:%s\\n" "$*"; }',
+            'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2"; }',
+            'stop_capacity_daemon() { printf "stop-daemon\\n"; }',
+            'rm() { printf "rm:%s\\n" "$*"; }',
+            _bash_function("restore"),
+            "false",
+            "restore",
+        ]
+    )
+    result = subprocess.run(
+        ["bash"], input=body.encode(), capture_output=True, check=False
+    )
+    output = result.stdout.decode(errors="replace")
+    errors = result.stderr.decode(errors="replace")
+    assert result.returncode == 1, errors
+    if dispatch_started:
+        assert output == ""
+        assert "runner_fleet_recovery_needed" in errors
+    else:
+        assert "cp:" in output
+        assert "delete:new-registration" in output
+        assert "restore-fence:itworx-security-runner:false" in output
+        assert ("stop-daemon" in output) is bool(cutover_started)
+        assert ("docker:compose " in output) is bool(cutover_started)