Publish curated Unraid-AppOps-Gateway source from 21729207a161d87393cfb73e79582bc373aa19cf

backend/tests/test_runner_fleet_cutover_safety.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import importlib.util
+import hashlib
 import json
 import shlex
 import shutil
@@ -19,6 +20,7 @@ SCRIPT_PATH = ROOT / "scripts" / "upgrade_ci_runner_fleet_v2.sh"
 PRIMARY = "itworx-security-runner"
 SECONDARY = "itworx-validation-runner-2"
 LEGACY = "itworx-unraid-deploy"
+WINDOWS_NATIVE = "itworx-windows11-native"
 OTHER_FIXED_RUNNERS = (
     "itworx-validation-runner-3",
     "itworx-docker-runner",
@@ -358,6 +360,280 @@ def _bash_function(name: str) -> str:
     return script[start:end]
 
 
+def _labels_digest(labels: list[object]) -> str:
+    canonical = json.dumps(
+        labels, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+    ).encode("utf-8")
+    return hashlib.sha256(canonical).hexdigest()
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+@pytest.mark.parametrize(
+    ("rows", "total_count", "expected_status", "expected_snapshot"),
+    [
+        (
+            [
+                {
+                    "id": 41,
+                    "name": WINDOWS_NATIVE,
+                    "busy": False,
+                    "disabled": False,
+                    "labels": ["windows-validation"],
+                },
+                {
+                    "id": 42,
+                    "name": "itworx-security-runner",
+                    "busy": False,
+                    "disabled": True,
+                    "labels": [],
+                },
+            ],
+            2,
+            0,
+            f"41:false:{_labels_digest(['windows-validation'])}",
+        ),
+        (
+            [
+                {
+                    "id": 42,
+                    "name": "itworx-security-runner",
+                    "busy": False,
+                    "disabled": True,
+                    "labels": [],
+                }
+            ],
+            1,
+            3,
+            "",
+        ),
+        (
+            [
+                {
+                    "id": 41,
+                    "name": WINDOWS_NATIVE,
+                    "busy": False,
+                    "disabled": False,
+                    "labels": [],
+                }
+            ],
+            2,
+            1,
+            "",
+        ),
+        (
+            [
+                {
+                    "id": 41,
+                    "name": WINDOWS_NATIVE,
+                    "busy": False,
+                    "disabled": False,
+                    "labels": [],
+                },
+                {
+                    "id": 41,
+                    "name": "other-runner",
+                    "busy": False,
+                    "disabled": False,
+                    "labels": [],
+                },
+            ],
+            2,
+            1,
+            "",
+        ),
+        (
+            [
+                {
+                    "id": 41,
+                    "name": WINDOWS_NATIVE,
+                    "busy": False,
+                    "disabled": False,
+                    "labels": [],
+                },
+                {
+                    "id": 42,
+                    "name": WINDOWS_NATIVE,
+                    "busy": False,
+                    "disabled": True,
+                    "labels": [],
+                },
+            ],
+            2,
+            1,
+            "",
+        ),
+        (
+            [
+                {
+                    "id": index + 1,
+                    "name": WINDOWS_NATIVE if index == 0 else f"other-{index}",
+                    "busy": False,
+                    "disabled": False,
+                    "labels": [],
+                }
+                for index in range(100)
+            ],
+            100,
+            1,
+            "",
+        ),
+        (
+            [
+                {
+                    "id": 41,
+                    "name": WINDOWS_NATIVE,
+                    "busy": "false",
+                    "disabled": False,
+                    "labels": [],
+                }
+            ],
+            1,
+            1,
+            "",
+        ),
+    ],
+    ids=(
+        "known-registration",
+        "optional-registration-absent",
+        "incomplete-inventory",
+        "duplicate-id",
+        "duplicate-name",
+        "bounded-inventory-limit",
+        "malformed-inventory-row",
+    ),
+)
+def test_windows_snapshot_requires_a_complete_unique_inventory(
+    tmp_path, rows, total_count, expected_status, expected_snapshot
+):
+    token_file = tmp_path / "token"
+    token_file.write_text("synthetic-token", encoding="utf-8")
+    inventory = json.dumps({"total_count": total_count, "runners": rows})
+    body = "\n".join(
+        [
+            f"ADMIN_TOKEN_FILE={shlex.quote(str(token_file))}",
+            f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}",
+            f"MOCK_INVENTORY={shlex.quote(inventory)}",
+            'curl() { printf "%s" "$MOCK_INVENTORY"; }',
+            _bash_function("windows_native_snapshot"),
+            "windows_native_snapshot",
+        ]
+    )
+    result = subprocess.run(
+        ["bash"], input=body.encode(), capture_output=True, check=False
+    )
+    assert result.returncode == expected_status, result.stderr.decode(errors="replace")
+    assert result.stdout.decode(errors="replace").strip() == expected_snapshot
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+def test_windows_snapshot_fails_closed_on_inventory_api_error(tmp_path):
+    token_file = tmp_path / "token"
+    token_file.write_text("synthetic-token", encoding="utf-8")
+    body = "\n".join(
+        [
+            f"ADMIN_TOKEN_FILE={shlex.quote(str(token_file))}",
+            f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}",
+            'curl() { return 22; }',
+            _bash_function("windows_native_snapshot"),
+            "windows_native_snapshot",
+        ]
+    )
+    result = subprocess.run(
+        ["bash"], input=body.encode(), capture_output=True, check=False
+    )
+    assert result.returncode == 1
+    assert result.stdout == b""
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+@pytest.mark.parametrize(
+    ("prior_disabled", "expected_patches"),
+    [("false", ["patch:41:true", "patch:41:false"]), ("true", [])],
+    ids=("restore-prior-enabled", "preserve-prior-paused"),
+)
+def test_windows_admission_capture_and_restore_preserve_prior_state(
+    prior_disabled, expected_patches
+):
+    labels = _labels_digest(["windows-validation"])
+    body = "\n".join(
+        [
+            f"WINDOWS_NATIVE={shlex.quote(WINDOWS_NATIVE)}",
+            'windows_native_saved=0; windows_native_id=""; windows_native_prior_disabled=""; windows_native_labels_digest=""',
+            'mock_id=41; mock_disabled="' + prior_disabled + '"; mock_labels="' + labels + '"',
+            'windows_native_snapshot() { printf "%s:%s:%s\\n" "$mock_id" "$mock_disabled" "$mock_labels"; }',
+            'set_runner_disabled_by_id() { printf "patch:%s:%s\\n" "$1" "$2"; mock_disabled="$2"; }',
+            _bash_function("capture_windows_native_admission"),
+            _bash_function("restore_windows_native_admission"),
+            "capture_windows_native_admission",
+            '[[ "$windows_native_saved" == 1 ]] || exit 91',
+            '[[ "$mock_disabled" == true ]] || exit 92',
+            "restore_windows_native_admission",
+            'printf "final:%s:saved:%s\\n" "$mock_disabled" "$windows_native_saved"',
+        ]
+    )
+    result = subprocess.run(
+        ["bash"], input=body.encode(), capture_output=True, check=False
+    )
+    assert result.returncode == 0, result.stderr.decode(errors="replace")
+    output = result.stdout.decode(errors="replace").splitlines()
+    assert output[:-1] == expected_patches
+    assert output[-1] == f"final:{prior_disabled}:saved:0"
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+@pytest.mark.parametrize(
+    "snapshot",
+    [
+        "99:true:{digest}",
+        "41:true:{other_digest}",
+    ],
+    ids=("registration-replaced", "labels-drifted"),
+)
+def test_windows_admission_restore_rejects_identity_or_label_drift(snapshot):
+    expected_digest = _labels_digest(["windows-validation"])
+    other_digest = _labels_digest(["changed-label"])
+    rendered = snapshot.format(digest=expected_digest, other_digest=other_digest)
+    body = "\n".join(
+        [
+            'windows_native_saved=1; windows_native_id=41; windows_native_prior_disabled=false',
+            f'windows_native_labels_digest="{expected_digest}"',
+            f'MOCK_SNAPSHOT="{rendered}"',
+            'windows_native_snapshot() { printf "%s\\n" "$MOCK_SNAPSHOT"; }',
+            'set_runner_disabled_by_id() { echo unexpected-patch; }',
+            _bash_function("restore_windows_native_admission"),
+            'if restore_windows_native_admission; then exit 93; else status=$?; fi',
+            'printf "status:%s:saved:%s\\n" "$status" "$windows_native_saved"',
+        ]
+    )
+    result = subprocess.run(
+        ["bash"], input=body.encode(), capture_output=True, check=False
+    )
+    assert result.returncode == 0, result.stderr.decode(errors="replace")
+    assert result.stdout.decode(errors="replace").strip() == "status:1:saved:1"
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+def test_windows_admission_restore_failure_keeps_recovery_intent():
+    labels = _labels_digest(["windows-validation"])
+    body = "\n".join(
+        [
+            'windows_native_saved=1; windows_native_id=41; windows_native_prior_disabled=false',
+            f'windows_native_labels_digest="{labels}"',
+            f'mock_snapshot="41:true:{labels}"',
+            'windows_native_snapshot() { printf "%s\\n" "$mock_snapshot"; }',
+            'set_runner_disabled_by_id() { return 1; }',
+            _bash_function("restore_windows_native_admission"),
+            'if restore_windows_native_admission; then exit 94; else status=$?; fi',
+            'printf "status:%s:saved:%s\\n" "$status" "$windows_native_saved"',
+        ]
+    )
+    result = subprocess.run(
+        ["bash"], input=body.encode(), capture_output=True, check=False
+    )
+    assert result.returncode == 0, result.stderr.decode(errors="replace")
+    assert result.stdout.decode(errors="replace").strip() == "status:1:saved:1"
+
+
 @pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
 @pytest.mark.parametrize(
     ("proof", "executor_status", "allowed"),
@@ -458,20 +734,25 @@ def test_dind_idle_shell_requires_successful_empty_docker_inventory(
     assert (result.returncode == 0) is allowed, result.stderr.decode(errors="replace")
 
 
-@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
-@pytest.mark.parametrize(
-    ("cutover_started", "dispatch_started"),
-    [(0, 0), (1, 0), (1, 1)],
-    ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"),
-)
-def test_failure_restores_only_safe_runner_phase(cutover_started, dispatch_started):
+def _restore_exit_harness(
+    *,
+    cutover_started: int,
+    dispatch_started: int,
+    exit_status: int,
+    fail_first_cp: bool = False,
+    fail_windows_patch: bool = False,
+):
+    labels = _labels_digest(["windows-validation"])
     body = "\n".join(
         [
+            "set -Eeuo pipefail",
             "backup=/synthetic-backup",
             "COMPOSE_FILE=/synthetic-compose",
             "PRIMARY_CONFIG=/synthetic-primary",
             "SECONDARY_CONFIG=/synthetic-secondary",
             "ROOT=/synthetic-root",
+            "RUNNER_GUARD_SCRIPT=/synthetic-runner-guard",
+            "HOST_GUARD_SCRIPT=/synthetic-host-guard",
             "guard_changed=0",
             "host_guard_changed=0",
             f"cutover_started={cutover_started}",
@@ -480,30 +761,132 @@ def test_failure_restores_only_safe_runner_phase(cutover_started, dispatch_start
             'registered=("new-registration")',
             'fenced=("itworx-security-runner:false")',
             "created_paths=()",
+            "windows_native_saved=1",
+            "windows_native_id=41",
+            "windows_native_prior_disabled=false",
+            f'windows_native_labels_digest="{labels}"',
+            "mock_windows_disabled=true",
+            f"fail_first_cp={int(fail_first_cp)}",
+            f"fail_windows_patch={int(fail_windows_patch)}",
+            "cp_calls=0",
             "exec 3>&1",
-            'cp() { printf "cp:%s\\n" "$*"; }',
+            'cp() { printf "cp:%s\\n" "$*" >&3; ((cp_calls += 1)); if ((fail_first_cp && cp_calls == 1)); then return 1; fi; }',
             'docker() { printf "docker:%s\\n" "$*" >&3; }',
-            'delete_runner() { printf "delete:%s\\n" "$*"; }',
-            'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2"; }',
-            'stop_capacity_daemon() { printf "stop-daemon\\n"; }',
-            'rm() { printf "rm:%s\\n" "$*"; }',
+            'delete_runner() { printf "delete:%s\\n" "$*" >&3; }',
+            'set_runner_disabled() { printf "restore-fence:%s:%s\\n" "$1" "$2" >&3; }',
+            'set_runner_disabled_by_id() { printf "windows-patch:%s:%s\\n" "$1" "$2" >&3; if ((fail_windows_patch)); then return 1; fi; mock_windows_disabled="$2"; }',
+            'windows_native_snapshot() { printf "windows-snapshot\\n" >&3; printf "41:%s:%s\\n" "$mock_windows_disabled" "$windows_native_labels_digest"; }',
+            'stop_capacity_daemon() { printf "stop-daemon\\n" >&3; }',
+            'rm() { printf "rm:%s\\n" "$*" >&3; }',
+            _bash_function("restore_managed_failure"),
+            _bash_function("restore_windows_native_admission"),
             _bash_function("restore"),
-            "false",
-            "restore",
+            "trap restore EXIT",
+            f"exit {exit_status}",
         ]
     )
-    result = subprocess.run(
+    return subprocess.run(
         ["bash"], input=body.encode(), capture_output=True, check=False
     )
-    output = result.stdout.decode(errors="replace")
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+@pytest.mark.parametrize(
+    ("cutover_started", "dispatch_started"),
+    [(0, 0), (1, 0), (1, 1)],
+    ids=("before-cutover", "cutover-before-dispatch", "after-dispatch"),
+)
+def test_exit_cleanup_restores_windows_after_managed_recovery(
+    cutover_started, dispatch_started
+):
+    result = _restore_exit_harness(
+        cutover_started=cutover_started,
+        dispatch_started=dispatch_started,
+        exit_status=1,
+    )
+    output = result.stdout.decode(errors="replace").splitlines()
     errors = result.stderr.decode(errors="replace")
     assert result.returncode == 1, errors
+    windows_patch_index = output.index("windows-patch:41:false")
+    assert not any(
+        line.startswith("docker:") for line in output[windows_patch_index + 1 :]
+    )
     if dispatch_started:
-        assert output == ""
+        assert output == [
+            "windows-snapshot",
+            "windows-patch:41:false",
+            "windows-snapshot",
+        ]
         assert "runner_fleet_recovery_needed" in errors
     else:
-        assert "cp:" in output
+        assert any(line.startswith("cp:") for line in output)
         assert "delete:new-registration" in output
         assert "restore-fence:itworx-security-runner:false" in output
         assert ("stop-daemon" in output) is bool(cutover_started)
-        assert ("docker:compose " in output) is bool(cutover_started)
+        compose_indexes = [
+            index
+            for index, line in enumerate(output)
+            if line.startswith("docker:compose ")
+        ]
+        assert bool(compose_indexes) is bool(cutover_started)
+        if compose_indexes:
+            assert compose_indexes[-1] < windows_patch_index
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+def test_copy_failure_still_restores_windows_after_managed_cleanup_aborts():
+    result = _restore_exit_harness(
+        cutover_started=1,
+        dispatch_started=0,
+        exit_status=1,
+        fail_first_cp=True,
+    )
+    output = result.stdout.decode(errors="replace").splitlines()
+    errors = result.stderr.decode(errors="replace")
+    assert result.returncode == 75
+    assert output == [
+        "cp:-a -- /synthetic-backup/docker-compose.yml /synthetic-compose",
+        "windows-snapshot",
+        "windows-patch:41:false",
+        "windows-snapshot",
+    ]
+    assert "managed rollback did not complete" in errors
+    assert not any(line.startswith("docker:") for line in output)
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+def test_exit_cleanup_restores_windows_on_success():
+    result = _restore_exit_harness(
+        cutover_started=0,
+        dispatch_started=1,
+        exit_status=0,
+    )
+    assert result.returncode == 0, result.stderr.decode(errors="replace")
+    assert result.stdout.decode(errors="replace").splitlines() == [
+        "windows-snapshot",
+        "windows-patch:41:false",
+        "windows-snapshot",
+    ]
+
+    script = SCRIPT_PATH.read_text(encoding="utf-8")
+    dispatch_index = script.index("dispatch_started=1")
+    restore_index = script.index("if ! restore_windows_native_admission; then", dispatch_index)
+    ready_index = script.index("printf 'ci_runner_fleet_v2_ready", restore_index)
+    trap_removed_index = script.index("trap - EXIT", ready_index)
+    assert restore_index < ready_index < trap_removed_index
+    assert "docker " not in script[restore_index:trap_removed_index]
+
+
+@pytest.mark.skipif(shutil.which("bash") is None, reason="bash is unavailable")
+def test_post_dispatch_windows_restore_failure_marks_recovery_needed():
+    result = _restore_exit_harness(
+        cutover_started=1,
+        dispatch_started=1,
+        exit_status=1,
+        fail_windows_patch=True,
+    )
+    assert result.returncode == 75
+    assert (
+        "runner_fleet_recovery_needed: Windows-native admission state could not be restored"
+        in result.stderr.decode(errors="replace")
+    )