From 7ee376bdbb98de1b11526b6a951e2b9c1022dc90 Mon Sep 17 00:00:00 2001 From: Public Source Publisher Date: Sat, 10 Oct 2026 05:25:48 +0000 Subject: [PATCH] Publish curated Unraid-AppOps-Gateway source from b2bb670dd165f2979e421ffc34de833438b2fd85 --- PUBLIC_SOURCE_EXPORT.md | 2 +- PUBLIC_SOURCE_MANIFEST.json | 11 +- backend/tests/test_runner_fleet_specialize.py | 448 +++++++++++++++++- .../tests/test_runner_guard_shell_syntax.py | 88 ++++ 4 files changed, 543 insertions(+), 6 deletions(-) create mode 100644 backend/tests/test_runner_guard_shell_syntax.py diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index b356317..f963ddb 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `1a5159c66078983299a1c2acb8837e4b5a8a607a`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `b2bb670dd165f2979e421ffc34de833438b2fd85`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index cb51218..f46eca6 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "1a5159c66078983299a1c2acb8837e4b5a8a607a", + "sourceRevision": "b2bb670dd165f2979e421ffc34de833438b2fd85", "files": [ { "path": ".dockerignore", @@ -1644,14 +1644,19 @@ }, { "path": "backend/tests/test_runner_fleet_specialize.py", - "sha256": "6771e7ae22ddb0d39e19272d311f9c77b898273c7106eef02ba0c76fc6e48873", - "bytes": 119270 + "sha256": "7cdb734b4596f2bdbd4703166792808a0183985c7165d9d8c51bdb54119f83d9", + "bytes": 145449 }, { "path": "backend/tests/test_runner_fleet_v2_execution_surface.py", "sha256": "62dcc692085ea9af004c4618e0c770701396be02254daf06bd5b55f56b5119fa", "bytes": 2337 }, + { + "path": "backend/tests/test_runner_guard_shell_syntax.py", + "sha256": "878fe58ff4dbb696ce6234b403f08f003a22881dba394cc2de1fbddc307c36ef", + "bytes": 3018 + }, { "path": "backend/tests/test_runner_profiles.py", "sha256": "45a8dba20d1e8d778dc90c20a997630037e2eaa1d462fa62f48671285aa82e0e", diff --git a/backend/tests/test_runner_fleet_specialize.py b/backend/tests/test_runner_fleet_specialize.py index c2952a7..f34b39c 100644 --- a/backend/tests/test_runner_fleet_specialize.py +++ b/backend/tests/test_runner_fleet_specialize.py @@ -1,15 +1,18 @@ from __future__ import annotations -import hashlib +import base64 import copy +import hashlib import importlib.util import io import json import os +import subprocess import sys import tarfile -from contextlib import nullcontext +from contextlib import contextmanager, nullcontext from pathlib import Path +from types import SimpleNamespace import pytest @@ -96,6 +99,7 @@ def test_legacy_live_source_does_not_open_promotion_backup(specialize, monkeypat def test_controller_compose_cas_blocks_rollback_before_any_lifecycle_action( specialize, controller_only_source, monkeypatch, separate_controller_hash, ): + monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance) model, compose, _archive, _archived = controller_only_source compose.write_text(json.dumps(model)) original_sha = hashlib.sha256(compose.read_bytes()).hexdigest() @@ -136,6 +140,11 @@ def specialize(monkeypatch): return module +@contextmanager +def _fixture_rollback_maintenance(): + yield lambda: None + + def _valid_task_image_manifest(specialize, revision="a" * 40, tree="b" * 40): source = "https://gitea.example.com/Jens/Unraid-AppOps-Gateway" return { @@ -1403,8 +1412,16 @@ def test_guard_scripts_patch_only_source_locked_files_and_reject_later_operator_ monkeypatch.setattr(specialize, "RUNNER_GUARD_SOURCE_SHA256", runner_sha) monkeypatch.setattr(specialize, "HOST_GUARD_SOURCE_SHA256", host_sha) monkeypatch.setattr(specialize, "_run", lambda *_args, **_kwargs: "") + monkeypatch.setattr(specialize, "_validate_guard_syntax", lambda _encoded: None) monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) + backup = tmp_path / "guard-backup" + backup.mkdir(mode=0o700) + for path, name in ((runner, "gitea-runner-guard.sh"), (host, "host-resource-guard.sh")): + target = backup / name + target.write_bytes(path.read_bytes()) + target.chmod(0o600) journal = { + "backupDirectory": str(backup), "guardFiles": [ {"path": str(runner), "originalSha256": runner_sha, "installedSha256": None}, {"path": str(host), "originalSha256": host_sha, "installedSha256": None}, @@ -1424,6 +1441,432 @@ def test_guard_scripts_patch_only_source_locked_files_and_reject_later_operator_ assert host.read_text(encoding="utf-8") == "operator-owned guard revision\n" +@pytest.fixture +def guard_transaction(specialize, tmp_path, monkeypatch): + monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance) + runner = tmp_path / "runner-guard" + host = tmp_path / "host-guard" + originals = ( + (b'#!/bin/bash\nstack_recovery=0\nset_managed_runners_disabled false\nunexpected=0\n' + b' [ "$name" = "$DEPLOY_RUNNER" ] && continue\n'), + (b'#!/bin/bash\n# ITWORX_RUNNER_CAPACITY_V2_BEGIN\nold capacity writer\n' + b'# ITWORX_RUNNER_CAPACITY_V2_END\nrunner_healthy=1\n' + b' if [ "$sandbox_healthy" -eq 1 ] && \\\n' + b' timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then\n' + b' recover legacy stack\n fi\n'), + ) + backup = tmp_path / "backups" + backup.mkdir(mode=0o700) + for path, original, name in zip((runner, host), originals, ("gitea-runner-guard.sh", "host-resource-guard.sh"), strict=True): + path.write_bytes(original) + target = backup / name + target.write_bytes(original) + target.chmod(0o600) + monkeypatch.setattr(specialize, "RUNNER_GUARD_PATH", runner) + monkeypatch.setattr(specialize, "HOST_GUARD_PATH", host) + for name, original in zip(("RUNNER_GUARD_SOURCE_SHA256", "HOST_GUARD_SOURCE_SHA256"), originals, strict=True): + monkeypatch.setattr(specialize, name, hashlib.sha256(original).hexdigest()) + journal = {"backupDirectory": str(backup), "guardFiles": [ + {"path": str(path), "originalSha256": hashlib.sha256(original).hexdigest(), "installedSha256": None} + for path, original in zip((runner, host), originals, strict=True) + ]} + monkeypatch.setattr(specialize, "_write_journal", lambda _journal: None) + monkeypatch.setattr(specialize, "_set_stage", lambda _journal, _stage: None) + monkeypatch.setattr(specialize, "_validate_guard_syntax", lambda _encoded: None) + return journal, runner, host, originals + + +@pytest.mark.parametrize("invalid", [0, 1]) +def test_both_candidate_syntax_checks_precede_any_intent_or_guard_write(specialize, guard_transaction, monkeypatch, invalid): + journal, runner, host, originals = guard_transaction + checked = [] + def syntax(encoded): + checked.append(encoded) + if len(checked) == invalid + 1: + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_guard_syntax_invalid") + monkeypatch.setattr(specialize, "_validate_guard_syntax", syntax) + monkeypatch.setattr(specialize, "_write_journal", lambda _journal: pytest.fail("invalid candidate published intent")) + monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("invalid candidate installed")) + with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_syntax_invalid"): + specialize._service_guard_scripts(journal) + assert (runner.read_bytes(), host.read_bytes()) == originals + assert all(row["installedSha256"] is None for row in journal["guardFiles"]) + + +@pytest.mark.parametrize("boundary", ["intent_failed", "after_intent", "after_runner", "after_host"]) +def test_guard_installation_durable_crash_boundaries_restore_only_owned_bytes(specialize, guard_transaction, monkeypatch, boundary): + journal, runner, host, originals = guard_transaction + persisted = [] + real_write = specialize._atomic_write_owned + writes = [] + def journal_write(value): + assert all(row["installedSha256"] for row in value["guardFiles"]) + assert (runner.read_bytes(), host.read_bytes()) == originals + if boundary == "intent_failed": + raise OSError("private canary write failure") + persisted.append(copy.deepcopy(value)) + if boundary == "after_intent": + raise OSError("private canary crash after intent") + def install(path, encoded, **kwargs): + assert persisted and persisted[0]["guardFiles"] == journal["guardFiles"] + real_write(path, encoded, **kwargs) + writes.append(path) + if boundary == "after_runner" or boundary == "after_host" and len(writes) == 2: + raise OSError("private canary crash after replace") + monkeypatch.setattr(specialize, "_write_journal", journal_write) + monkeypatch.setattr(specialize, "_atomic_write_owned", install) + with pytest.raises(OSError): + specialize._service_guard_scripts(journal) + monkeypatch.setattr(specialize, "_atomic_write_owned", real_write) + specialize._restore_guard_scripts(persisted[0] if persisted else journal) + assert (runner.read_bytes(), host.read_bytes()) == originals + specialize._restore_guard_scripts(journal) + assert (runner.read_bytes(), host.read_bytes()) == originals + + +@pytest.mark.parametrize("mutation", ["duplicate", "foreign_path", "original_hash", "planned_hash", "backup", "foreign_edit"]) +def test_guard_restore_rejects_foreign_journal_backup_or_live_bytes_before_any_write(specialize, guard_transaction, monkeypatch, mutation): + journal, runner, host, _originals = guard_transaction + specialize._service_guard_scripts(journal) + if mutation == "duplicate": + journal["guardFiles"][1] = copy.deepcopy(journal["guardFiles"][0]) + elif mutation == "foreign_path": + journal["guardFiles"][1]["path"] = str(host.with_name("foreign")) + elif mutation == "original_hash": + journal["guardFiles"][0]["originalSha256"] = "f" * 64 + elif mutation == "planned_hash": + journal["guardFiles"][0]["installedSha256"] = "f" * 64 + elif mutation == "backup": + (Path(journal["backupDirectory"]) / "host-resource-guard.sh").write_bytes(b"foreign private canary") + else: + host.write_bytes(b"foreign private canary") + before = runner.read_bytes(), host.read_bytes() + monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("foreign state restored")) + with pytest.raises(specialize.RunnerFleetSpecializeError): + specialize._restore_guard_scripts(journal) + assert (runner.read_bytes(), host.read_bytes()) == before + + +def test_guard_syntax_checker_never_exposes_bash_output(specialize, monkeypatch): + def parse(args, **kwargs): + assert args == ["bash", "-n"] and kwargs["input"] == b"candidate" + assert kwargs["timeout"] == 15 and kwargs["capture_output"] is True + return SimpleNamespace(returncode=2, stdout=b"private canary", stderr=b"private canary") + monkeypatch.setattr(specialize.subprocess, "run", parse) + with pytest.raises(specialize.RunnerFleetSpecializeError, match="^runner_fleet_specialization_guard_syntax_invalid$"): + specialize._validate_guard_syntax(b"candidate") + + +@pytest.mark.parametrize("mutation", [None, "foreign", "begin_only", "end_only", "repeated"]) +def test_host_guard_without_capacity_stanza_requires_exact_approved_bytes(specialize, guard_transaction, monkeypatch, mutation): + _journal, _runner, _host, originals = guard_transaction + text = originals[1].decode() + start = text.index("# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n") + end = text.index("# ITWORX_RUNNER_CAPACITY_V2_END\n") + len("# ITWORX_RUNNER_CAPACITY_V2_END\n") + approved = text[:start] + text[end:] + monkeypatch.setattr(specialize, "HOST_GUARD_SOURCE_SHA256", hashlib.sha256(approved.encode()).hexdigest()) + if mutation == "foreign": + approved += "# operator edit\n" + elif mutation == "begin_only": + approved += "# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n" + elif mutation == "end_only": + approved += "# ITWORX_RUNNER_CAPACITY_V2_END\n" + elif mutation == "repeated": + approved += "# ITWORX_RUNNER_CAPACITY_V2_BEGIN\n# ITWORX_RUNNER_CAPACITY_V2_END\n" * 2 + if mutation is None: + result = specialize._patch_host_guard(approved, "fixed --state") + assert 'specialization_state="$(fixed --state' in result + assert '"$specialization_state" != "legacy"' in result + else: + with pytest.raises(specialize.RunnerFleetSpecializeError, match="capacity_block_invalid"): + specialize._patch_host_guard(approved, "fixed --state") + + +@pytest.mark.parametrize("mutation", ["path", "backup", "foreign_edit"]) +def test_guard_rollback_refusal_preserves_marker_journal_before_lifecycle_effects(specialize, guard_transaction, monkeypatch, mutation): + journal, runner, host, _originals = guard_transaction + specialize._service_guard_scripts(journal) + live_compose = runner.with_name("compose") + live_compose.write_bytes(b"fixed compose") + monkeypatch.setattr(specialize, "COMPOSE_PATH", live_compose) + journal["baseComposeSha256"] = hashlib.sha256(live_compose.read_bytes()).hexdigest() + monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) + monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: pytest.fail("refusal reached lifecycle effects")) + marker, durable = runner.with_name("marker"), runner.with_name("journal") + marker.write_bytes(b"owned marker") + durable.write_bytes(b"owned journal") + monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker) + monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable) + if mutation == "path": + journal["guardFiles"][0]["path"] = str(runner.with_name("foreign")) + elif mutation == "backup": + (Path(journal["backupDirectory"]) / "host-resource-guard.sh").write_bytes(b"foreign canary") + elif mutation == "foreign_edit": + host.write_bytes(b"foreign canary") + assert specialize._rollback_before_activation(journal) is False + assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" + + +@pytest.mark.parametrize("failure", ["cleanup", "restore_cas", None]) +def test_rollback_retains_fences_until_coherent_lifecycle_recovery(specialize, guard_transaction, monkeypatch, failure): + journal, runner, host, originals = guard_transaction + specialize._service_guard_scripts(journal) + fenced = runner.read_bytes(), host.read_bytes() + compose, marker, durable = runner.with_name("compose"), runner.with_name("marker"), runner.with_name("journal") + compose.write_bytes(b"fixed compose") + marker.write_bytes(b"owned marker") + durable.write_bytes(b"owned journal") + monkeypatch.setattr(specialize, "COMPOSE_PATH", compose) + monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker) + monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable) + journal.update(baseComposeSha256=hashlib.sha256(compose.read_bytes()).hexdigest(), + originalDind={"present": False, "running": False, "restartPolicy": "no", "imageId": None}, + originalContainers=[], originalRegistrations=[], cgroupCreated=True, cgroupRootSubtree=[]) + monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) + monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose") + monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: []) + monkeypatch.setattr(specialize, "_inspect", lambda _name: None) + events = [] + def cleanup(_journal): + assert (runner.read_bytes(), host.read_bytes()) == fenced + events.append("cleanup") + if failure == "cleanup": + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_command_failed") + monkeypatch.setattr(specialize, "_cleanup_new_containers", cleanup) + def cgroup(**_kwargs): + assert (runner.read_bytes(), host.read_bytes()) == fenced + events.append("cgroup") + return {"familyExists": False} + monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup) + write = specialize._atomic_write_owned + def restore(*args, **kwargs): + assert events == ["cleanup", "cgroup"] + assert marker.exists() and durable.exists() + if failure == "restore_cas": + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_compare_and_swap_failed") + write(*args, **kwargs) + monkeypatch.setattr(specialize, "_atomic_write_owned", restore) + assert specialize._rollback_before_activation(journal) is (failure is None) + if failure: + assert (runner.read_bytes(), host.read_bytes()) == fenced + assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" + else: + assert (runner.read_bytes(), host.read_bytes()) == originals + assert not marker.exists() and not durable.exists() + + +def test_rollback_maintenance_refusal_keeps_all_state_without_any_restore(specialize, tmp_path, monkeypatch): + marker, journal = tmp_path / "marker", tmp_path / "journal" + marker.write_bytes(b"owned marker") + journal.write_bytes(b"owned journal") + @contextmanager + def busy(): + raise RuntimeError("private canary lock refusal") + yield + monkeypatch.setattr(specialize, "_rollback_maintenance", busy) + monkeypatch.setattr(specialize, "_rollback_before_activation_locked", lambda *_args: pytest.fail("busy barrier reached effects")) + assert specialize._rollback_before_activation({}) is False + assert marker.read_bytes() == b"owned marker" and journal.read_bytes() == b"owned journal" + + +def test_rollback_barrier_drift_refuses_before_guards_or_cleanup(specialize, guard_transaction, monkeypatch): + journal, _runner, _host, _originals = guard_transaction + @contextmanager + def changed(): + def refuse(): + raise RuntimeError("private canary changed inode") + yield refuse + monkeypatch.setattr(specialize, "_rollback_maintenance", changed) + monkeypatch.setattr(specialize, "_restore_guard_scripts", lambda *_args, **_kwargs: pytest.fail("drift restored guard")) + monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda *_args: pytest.fail("drift changed fleet")) + assert specialize._rollback_before_activation(journal) is False + + +@pytest.mark.skipif(os.name != "posix", reason="real common flock contract requires POSIX") +def test_real_common_maintenance_lock_excludes_process_until_whole_rollback_finishes(specialize, tmp_path, monkeypatch): + if os.geteuid() != 0: + pytest.skip("protected maintenance lock requires root") + import runner_acceptance_quiescence as quiescence + lock = tmp_path / "maintenance.lock" + lock.touch(mode=0o600) + monkeypatch.setattr(quiescence, "MAINTENANCE_LOCK", lock) + monkeypatch.setattr(quiescence, "_LOCK_PROBE", quiescence._LOCK_PROBE.replace( + "/srv/example/appdata/gitea-deploy-runners/state/runner-maintenance.lock", str(lock))) + monkeypatch.setitem(sys.modules, "deploy", SimpleNamespace(load_projects=dict)) + probe = ( + "import fcntl,os,sys\nfd=os.open(sys.argv[1],os.O_RDWR|os.O_NOFOLLOW)\n" + "try: fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)\n" + "except BlockingIOError: sys.exit(0)\n" + "sys.exit(1)\n" + ) + events = [] + def held(phase): + result = subprocess.run([sys.executable, "-c", probe, str(lock)], capture_output=True, timeout=5, check=False) + assert result.returncode == 0 and not result.stdout and not result.stderr + events.append(phase) + compose, marker, durable = tmp_path / "compose", tmp_path / "marker", tmp_path / "journal" + compose.write_bytes(b"captured compose") + marker.write_bytes(b"owned marker") + durable.write_bytes(b"owned journal") + monkeypatch.setattr(specialize, "COMPOSE_PATH", compose) + monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", marker) + monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable) + monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) + monkeypatch.setattr(specialize, "_guard_script_plan", lambda _journal: []) + monkeypatch.setattr(specialize, "_restore_guard_scripts", lambda *_args, **_kwargs: held("guards-restored")) + monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: held("cleanup")) + monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"captured compose") + monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: []) + monkeypatch.setattr(specialize, "_inspect", lambda _name: None) + def cgroup(**_kwargs): + held("cgroup") + assert marker.exists() and durable.exists() + return {"familyExists": False} + monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup) + journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(), "guardFiles": [{"owned": True}], + "originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None}, + "originalContainers": [], "originalRegistrations": [], "cgroupCreated": True, "cgroupRootSubtree": []} + assert specialize._rollback_before_activation(journal) is True + assert events == ["cleanup", "cgroup", "guards-restored"] + assert not marker.exists() and not durable.exists() + # The independent participant can acquire only after the wrapper returns. + result = subprocess.run([sys.executable, "-c", probe, str(lock)], capture_output=True, timeout=5, check=False) + assert result.returncode == 1 and not result.stdout and not result.stderr + + +@pytest.fixture +def owned_orphan(specialize, guard_transaction, tmp_path, monkeypatch): + import runner_acceptance_quiescence as quiescence + journal, runner, host, originals = guard_transaction + corrected = specialize._guard_script_plan(journal)[0][3] + closing = f'\nfi\n{specialize.SPECIALIZATION_GUARD_END}\n'.encode() + orphan = corrected.replace(closing, f'\n{specialize.SPECIALIZATION_GUARD_END}\n'.encode(), 1) + runner.write_bytes(orphan) + monkeypatch.setattr(specialize, "_OWNED_MALFORMED_RUNNER_GUARD_SHA256", hashlib.sha256(orphan).hexdigest()) + receipt = { + "schema": quiescence.SCHEMA, "revision": "0ea6f267c28d51ac7113e6d943c89857d3a189ab", + "manifestDigest": quiescence._digest(quiescence.OBSERVATION), "phase": "complete", + "progress": {row["id"]: "quiesced" for row in quiescence._fixed_rows()}, + "registrationProgress": {"9": "restored", "22": "restored"}, + "guardProgress": {"runner": "restored", "host": "restored"}, + "guards": {name: { + "original": base64.b64encode(original).decode(), "originalSha256": hashlib.sha256(original).hexdigest(), + "wrappedSha256": hashlib.sha256(quiescence.ControllerDriver._wrapped_guard(original)).hexdigest(), + "mode": path.stat().st_mode & 0o777, + } for name, path, original in zip(("runner", "host"), (runner, host), originals, strict=True)}, + "baseline": {"rows": [{"state": {"Running": False}} for _row in quiescence._fixed_rows()]}, + } + events = [] + for name in ("SPECIALIZED_STATE_PATH", "MIGRATION_JOURNAL_PATH"): + monkeypatch.setattr(specialize, name, tmp_path / name) + monkeypatch.setattr(specialize, "_specialized_storage_present", lambda: False) + monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False}) + monkeypatch.setattr(specialize, "_read_fleet_base_compose", lambda: (b"source", None)) + monkeypatch.setattr(specialize, "_validate_base_composition", lambda _source: "v1") + monkeypatch.setattr(specialize, "_runner_rows", lambda: [{"id": 9}]) + monkeypatch.setattr(specialize, "_validate_legacy_registrations", lambda *_args: [{"id": 9}]) + monkeypatch.setattr(specialize, "_validate_legacy_containers", lambda *_args: ("v1", [{"id": "fixed"}])) + monkeypatch.setitem(sys.modules, "deploy", SimpleNamespace(load_projects=lambda: {"fixed": "registry"})) + monkeypatch.setattr(quiescence, "_read_receipt", lambda: copy.deepcopy(receipt)) + monkeypatch.setattr(quiescence, "_idle", lambda *_args: events.append("idle")) + def attest(snapshot, *, baseline, progress): + assert baseline == receipt["baseline"] and progress == receipt["progress"] + assert snapshot["rows"] == baseline["rows"] + events.append("attested") + monkeypatch.setattr(quiescence, "attest_snapshot", attest) + class Driver: + _wrapped_guard = staticmethod(quiescence.ControllerDriver._wrapped_guard) + def __init__(self, projects): + assert projects == {"fixed": "registry"} + @contextmanager + def maintenance(self): + with (tmp_path / "fixture-lock").open("wb") as lock: + self._maintenance_fd = lock.fileno() + events.append("locked") + yield + events.append("unlocked") + def assert_barrier(self): + events.append("barrier") + def snapshot(self): + self._guard_binding() + events.append("snapshot") + return copy.deepcopy(receipt["baseline"]) + monkeypatch.setattr(quiescence, "ControllerDriver", Driver) + return quiescence, receipt, runner, host, originals, orphan, events, Driver + + +def test_owned_orphan_recovery_uses_full_repeated_admission_and_cas_then_is_idempotent(specialize, owned_orphan, monkeypatch): + _q, receipt, runner, host, originals, orphan, events, _driver = owned_orphan + before = copy.deepcopy(receipt) + write = specialize._atomic_write_owned + def restore(path, data, **kwargs): + assert events.count("attested") == 2 and events.count("idle") == 3 + assert path == runner and data == originals[0] + assert kwargs["expected_sha256"] == hashlib.sha256(orphan).hexdigest() + events.append("restore") + write(path, data, **kwargs) + monkeypatch.setattr(specialize, "_atomic_write_owned", restore) + specialize._recover_owned_orphan_guard() + assert (runner.read_bytes(), host.read_bytes()) == originals and receipt == before + assert events.index("locked") < events.index("restore") < events.index("unlocked") + specialize._recover_owned_orphan_guard() + assert events.count("restore") == 1 + + +@pytest.mark.parametrize("mutation", ["revision", "manifest", "phase", "progress", "registration", "guard_progress", + "original", "wrapped", "foreign_runner", "foreign_host", "marker", "journal", + "storage", "family", "idle", "lock", "origin", "changed_receipt", "container", "cas"]) +def test_orphan_recovery_refuses_unowned_or_changed_state_without_write(specialize, owned_orphan, monkeypatch, mutation): + q, receipt, runner, host, _originals, _orphan, _events, driver = owned_orphan + if mutation in {"revision", "manifest", "phase"}: + receipt[{"revision": "revision", "manifest": "manifestDigest", "phase": "phase"}[mutation]] = "foreign" + elif mutation == "progress": + receipt["progress"].pop(next(iter(receipt["progress"]))) + elif mutation == "registration": + receipt["registrationProgress"]["9"] = "disabled" + elif mutation == "guard_progress": + receipt["guardProgress"]["host"] = "installed" + elif mutation in {"original", "wrapped"}: + receipt["guards"]["runner"]["originalSha256" if mutation == "original" else "wrappedSha256"] = "f" * 64 + elif mutation in {"foreign_runner", "foreign_host"}: + (runner if mutation == "foreign_runner" else host).write_bytes(b"private foreign canary") + elif mutation in {"marker", "journal"}: + (specialize.SPECIALIZED_STATE_PATH if mutation == "marker" else specialize.MIGRATION_JOURNAL_PATH).touch() + elif mutation == "storage": + monkeypatch.setattr(specialize, "_specialized_storage_present", lambda: True) + elif mutation == "family": + monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": True}) + elif mutation == "idle": + monkeypatch.setattr(q, "_idle", lambda *_args: (_ for _ in ()).throw(q.QuiescenceError("global_ci_work_present"))) + elif mutation == "lock": + monkeypatch.setattr(driver, "assert_barrier", lambda _self: (_ for _ in ()).throw(q.QuiescenceError("dispatch_maintenance_busy"))) + elif mutation == "origin": + monkeypatch.setattr(q, "attest_snapshot", lambda *_args, **_kwargs: (_ for _ in ()).throw(q.QuiescenceError("origin_changed"))) + elif mutation == "changed_receipt": + reads = [] + def changed(): + reads.append(True) + result = copy.deepcopy(receipt) + if len(reads) > 1: + result["revision"] = "foreign" + return result + monkeypatch.setattr(q, "_read_receipt", changed) + elif mutation == "container": + reads = [] + monkeypatch.setattr(specialize, "_validate_legacy_containers", lambda *_args: reads.append(True) or ("v1", [{"id": str(len(reads))}])) + else: + monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: (_ for _ in ()).throw( + specialize.RunnerFleetSpecializeError("runner_fleet_specialization_compare_and_swap_failed"))) + before = runner.read_bytes(), host.read_bytes() + if mutation != "cas": + monkeypatch.setattr(specialize, "_atomic_write_owned", lambda *_args, **_kwargs: pytest.fail("unowned orphan restored")) + with pytest.raises(specialize.RunnerFleetSpecializeError): + specialize._recover_owned_orphan_guard() + assert (runner.read_bytes(), host.read_bytes()) == before + + def test_execute_runs_only_approved_shell_then_requires_verified_postcondition(specialize, monkeypatch): script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() digest = hashlib.sha256(script).hexdigest() @@ -2180,6 +2623,7 @@ def test_specialization_stage_fixture_quiesces_old_before_registering_and_never_ def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup( specialize, tmp_path, monkeypatch, original_running ): + monkeypatch.setattr(specialize, "_rollback_maintenance", _fixture_rollback_maintenance) image_id = specialize._DIND_IMAGE_ID base_model = _base_v1_compose_model(specialize) specialized_model = { diff --git a/backend/tests/test_runner_guard_shell_syntax.py b/backend/tests/test_runner_guard_shell_syntax.py new file mode 100644 index 0000000..67a05d5 --- /dev/null +++ b/backend/tests/test_runner_guard_shell_syntax.py @@ -0,0 +1,88 @@ +"""Validate generated guard programs with Bash's actual parser, without effects.""" +from __future__ import annotations + +import hashlib +import importlib.util +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +CONTROLLER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker" +RUNNER = '''#!/usr/bin/env bash +set -euo pipefail +readonly DEPLOY_RUNNER="deployment-runner" +stack_recovery=0 +if [ "$stack_recovery" -eq 0 ]; then + stack_recovery=1 +fi +unexpected=0 +for name in validation-runner; do + [ "$name" = "$DEPLOY_RUNNER" ] && continue + unexpected=1 +done +''' +HOST = '''#!/usr/bin/env bash +set -euo pipefail +# ITWORX_RUNNER_CAPACITY_V2_BEGIN +capacity_active=true +# ITWORX_RUNNER_CAPACITY_V2_END +runner_healthy=1 +if [ "$runner_healthy" -eq 1 ]; then + if [ "$sandbox_healthy" -eq 1 ] && \\ + timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then + : + fi +fi +''' + + +@pytest.fixture +def fleet(monkeypatch): + monkeypatch.syspath_prepend(str(CONTROLLER)) + spec = importlib.util.spec_from_file_location( + "runner_guard_shell_contract", CONTROLLER / "runner_fleet_specialize.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@pytest.mark.skipif(os.name != "posix", reason="Real Bash guard validation requires POSIX") +@pytest.mark.parametrize("kind", ["runner", "host", "host_legacy"]) +def test_complete_generated_guard_passes_real_bash_parser(fleet, kind, monkeypatch): + bash = shutil.which("bash") + assert bash is not None, "The deployment qualification requires Bash" + state_command = "timeout 15 /fixed/controller --state" + if kind == "runner": + original = RUNNER + generated = fleet._patch_runner_guard( + original, state_command, + 'timeout 10 /fixed/controller --guard-allows-container "$name"', + ) + else: + original = HOST + if kind == "host_legacy": + original = original.replace( + "# ITWORX_RUNNER_CAPACITY_V2_BEGIN\ncapacity_active=true\n" + "# ITWORX_RUNNER_CAPACITY_V2_END\n", "", + ) + # This isolated fixture represents a single approved legacy source. + # Live acceptance separately binds the actual unchanged guard hash. + monkeypatch.setattr( + fleet, "HOST_GUARD_SOURCE_SHA256", + hashlib.sha256(original.encode("utf-8")).hexdigest(), + ) + generated = fleet._patch_host_guard(original, state_command) + for program in (original, generated): + # Parse stdin only. No shell program, controller, Docker or guard runs. + result = subprocess.run( + [bash, "-n"], input=program.encode("utf-8"), + capture_output=True, timeout=5, check=False, + ) + assert result.returncode == 0, "Generated guard failed real Bash syntax validation"