from __future__ import annotations import copy import hashlib import importlib.util import json import os import stat import subprocess import sys from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace import pytest BROKER = Path(__file__).resolve().parents[2] / "ops/gitea-deploy-broker" def load(name, path): spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) sys.modules[name] = module spec.loader.exec_module(module) return module @pytest.fixture def recovery(monkeypatch): monkeypatch.syspath_prepend(str(BROKER)) return load("failed_cutover_recovery_test", BROKER / "runner_failed_cutover_recovery.py") @pytest.fixture def owned(recovery, monkeypatch, tmp_path): fixture = load("recovery_envelope_fixture", Path(__file__).with_name("test_runner_resource_envelope.py")) candidate = fixture._module() probe, image, code = fixture._host_probe_fixture(candidate) code = f"{candidate._HOST_VOLUME_ROOT}/controller-artifacts/{recovery.ORIGIN}" probe["Mounts"][1]["Source"] = code probe["HostConfig"]["Mounts"][1]["Source"] = code probe["Id"] = recovery.PROBE_ID probe["Mounts"][0].update(Source="/sys/fs/cgroup", RW=True) probe["HostConfig"]["Mounts"][0].update(Source="/sys/fs/cgroup", ReadOnly=False) probe["HostConfig"]["CapAdd"] = ["CAP_DAC_OVERRIDE"] probe["State"].update(Status="created", Pid=0, StartedAt="0001-01-01T00:00:00Z", FinishedAt="0001-01-01T00:00:00Z") monkeypatch.setattr(candidate, "_controller_runtime_identity", lambda: (image, code)) monkeypatch.setattr(candidate, "_inspect_host_probe_by_name", lambda: copy.deepcopy(probe)) monkeypatch.setattr(candidate, "_host_probe_image_labels", lambda _: {}) fleet = load("recovery_fleet_fixture", BROKER / "runner_fleet_specialize.py") q = load("recovery_quiescence_fixture", BROKER / "runner_acceptance_quiescence.py") root = tmp_path / "bundle" root.mkdir(mode=0o700) manifest_root = tmp_path / "task-images" manifest_root.mkdir(mode=0o700) manifest_path = manifest_root / "task-image-manifest.json" manifest = {"sourceRevision": recovery.ORIGIN, "sourceTree": recovery.TREE} manifest_raw = json.dumps(manifest).encode() manifest_path.write_bytes(manifest_raw) manifest_path.chmod(0o600) manifest_sha = hashlib.sha256(manifest_raw).hexdigest() marker = {"schemaVersion": 1, "status": "failed", "sourceScriptSha256": recovery.SCRIPT_SHA, "imageSourceRevision": recovery.ORIGIN, "imageSourceTree": recovery.TREE, "imageManifestSha256": manifest_sha, "resourceEnvelopeDigest": "a" * 64, "verifiedRunners": list(fleet._SPECIALIZED_NAMES), "updatedAt": "2026-10-10T00:00:00Z"} originals = [{"id": i, "name": name, "labels": [], "disabled": False} for i, name in zip((9, 22), fleet._legacy_names("v1"), strict=True)] rows = [{**row, "disabled": True, "busy": False} for row in originals] journal = {"status": "in_progress", "stage": "legacy_removed", "sourceRevision": recovery.ORIGIN, "sourceTree": recovery.TREE, "sourceScriptSha256": recovery.SCRIPT_SHA, "legacyTopology": "v1", "cgroupCreated": False, "cgroupEnsurePending": True, "cgroupFamilyInitiallyPresent": False, "newRegistrationIds": {}, "buildProbe": None, "originalRegistrations": originals, "baseComposeSha256": hashlib.sha256(b"compose").hexdigest(), "imageManifestSha256": manifest_sha} dind = {"Id": "b" * 64, "Image": image, "Created": "fixed", "Name": "/gitea-runner-dind", "Path": "entrypoint", "Args": [], "Config": {}, "HostConfig": {}, "Mounts": [], "NetworkSettings": {"Networks": {}}, "RestartCount": 0, "State": {"Running": True, "Pid": 123, "StartedAt": "fixed", "Health": {"Status": "healthy", "Log": []}}} events = [] for name in ("SPECIALIZED_STATE_PATH", "MIGRATION_JOURNAL_PATH", "COMPOSE_PATH", "SPECIALIZED_COMPOSE_PATH", "RUNNER_CONFIG_ROOT", "_SPECIALIZED_DATA_ROOT", "_SPECIALIZED_WORKSPACE_ROOT"): monkeypatch.setattr(fleet, name, tmp_path / name) monkeypatch.setattr(fleet, "TASK_IMAGE_ROOT", manifest_root) monkeypatch.setattr(fleet, "TASK_IMAGE_MANIFEST_PATH", manifest_path) monkeypatch.setattr(fleet, "_trusted_directory", lambda _: None) monkeypatch.setattr(fleet, "_valid_installed_manifest", lambda _: True) monkeypatch.setattr(fleet, "_read_journal", lambda: copy.deepcopy(journal)) monkeypatch.setattr(fleet, "_validate_legacy_registrations", lambda current, _: [{k: v for k, v in row.items() if k != "busy"} for row in current]) monkeypatch.setattr(fleet, "_inspect", lambda name: copy.deepcopy(dind) if name == fleet.DIND_NAME else None) monkeypatch.setattr(fleet, "_captured_base_source", lambda _: b"compose") monkeypatch.setattr(fleet, "_validate_base_composition", lambda _: "v1") monkeypatch.setattr(fleet, "_captured_compose_mapping", lambda *args, **kwargs: {}) monkeypatch.setattr(fleet, "_captured_compose_model", lambda *args, **kwargs: b"fixed") monkeypatch.setattr(fleet, "_read_source_file", lambda *args, **kwargs: b"compose") monkeypatch.setattr(fleet, "_guard_state_unchanged", lambda _: True) monkeypatch.setattr(fleet, "_guard_script_plan", lambda _: events.append("guard-cas-checked")) monkeypatch.setattr(fleet, "_is_captured_original_dind", lambda *args: True) def read(path, **kwargs): if path == fleet.SPECIALIZED_STATE_PATH: return json.dumps(marker).encode() if path == fleet.MIGRATION_JOURNAL_PATH: return json.dumps(journal).encode() if path == manifest_path: return manifest_path.read_bytes() return b"reviewed" monkeypatch.setattr(recovery, "_read", read) monkeypatch.setattr(recovery, "_installed_proof", lambda: b"reviewed") monkeypatch.setattr(recovery, "BUNDLE", root) monkeypatch.setattr(recovery, "__file__", str(root / "runner_failed_cutover_recovery.py")) monkeypatch.setattr(recovery, "CLAIM", tmp_path / "consumed.json") monkeypatch.setattr(recovery, "sys", SimpleNamespace(flags=SimpleNamespace(isolated=True), dont_write_bytecode=True)) monkeypatch.setattr(recovery, "os", SimpleNamespace(name="posix", geteuid=lambda: 0, path=os.path)) if os.name == "nt": # Windows chmod cannot materialize POSIX directory bits. The native # Linux qualification exercises the real root/mode checks separately. monkeypatch.setattr(recovery, "stat", SimpleNamespace( S_IMODE=lambda mode: 0o700 if stat.S_ISDIR(mode) else stat.S_IMODE(mode), S_ISDIR=stat.S_ISDIR, S_ISREG=stat.S_ISREG, )) pending = [] class Driver: def __init__(self, _projects): self.held = False @contextmanager def maintenance(self): events.append("maintenance-enter") self.held = True yield self.held = False events.append("maintenance-exit") def assert_barrier(self): assert self.held def pending_jobs(self): return pending def registrations(self): return copy.deepcopy(rows) def _run(self, args, **kwargs): assert args[6:] == ["ps", "-a", "--no-trunc", "-q"] return "" @contextmanager def migration(_fleet): events.append("migration-enter") yield lambda: None events.append("migration-exit") monkeypatch.setattr(recovery, "_migration_existing_lock", migration) monkeypatch.setattr(fleet, "_migration_lock", lambda **kwargs: pytest.fail("normal lock would create state")) monkeypatch.setattr(q, "ControllerDriver", Driver) monkeypatch.setattr(recovery, "_load", lambda: (fleet, candidate, q, {})) return fleet, candidate, q, Driver, marker, journal, probe, rows, dind, events, pending def test_readonly_preflight_has_no_claim_or_lifecycle_and_uses_both_locks(recovery, owned, monkeypatch): *_rest, events, _pending = owned monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("readonly claimed")) result = recovery.run() assert result["status"] == "ready" and result["effectsPerformed"] is False assert events == ["migration-enter", "maintenance-enter", "guard-cas-checked", "guard-cas-checked", "maintenance-exit", "migration-exit"] def test_execute_keeps_existing_wrapper_barrier_and_only_candidate_rollback_hook(recovery, owned, monkeypatch): fleet, candidate, _q, _driver, _marker, _journal, _probe, _rows, _dind, events, _pending = owned def claim(_proof): events.append("consumed") monkeypatch.setattr(recovery, "_claim", claim) def rollback(journal, assert_barrier): assert_barrier() assert events[-1] == "consumed" assert fleet.rollback_host_cgroup_family is candidate.rollback_host_cgroup_family events.append("existing-rollback-body") return True monkeypatch.setattr(fleet, "_rollback_before_activation_locked", rollback) result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) assert result["status"] == "rolled_back" and result["invocationConsumed"] is True assert events.index("consumed") < events.index("existing-rollback-body") < events.index("maintenance-exit") @pytest.mark.parametrize("change", ["source", "tree", "script", "active", "stage", "pending", "created", "initial", "queue", "busy", "registration", "probe_id", "probe_pid", "probe_started", "cap_extra", "probe_mount", "guard_cas", "privatebusy", "manifest", "extra_manifest_file"]) def test_preflight_rejects_drift_without_claim_or_lifecycle(recovery, owned, monkeypatch, change): fleet, _candidate, _q, driver, marker, journal, probe, rows, _dind, _events, pending = owned if change in {"source", "tree", "script"}: journal[{"source": "sourceRevision", "tree": "sourceTree", "script": "sourceScriptSha256"}[change]] = "foreign" elif change == "active": marker["status"] = "active" elif change == "stage": journal["stage"] = "dind_ready" elif change in {"pending", "created", "initial"}: journal[{"pending": "cgroupEnsurePending", "created": "cgroupCreated", "initial": "cgroupFamilyInitiallyPresent"}[change]] = change != "pending" elif change == "queue": pending.append({"id": 1}) elif change == "busy": rows[0]["busy"] = True elif change == "registration": rows[0]["labels"] = ["foreign"] elif change == "probe_id": probe["Id"] = "f" * 64 elif change in {"probe_pid", "probe_started"}: probe["State"]["Pid" if change == "probe_pid" else "StartedAt"] = 1 if change == "probe_pid" else "foreign" elif change == "cap_extra": probe["HostConfig"]["CapAdd"].append("SYS_ADMIN") elif change == "probe_mount": probe["Mounts"][0]["Source"] = "/foreign" elif change == "guard_cas": monkeypatch.setattr(fleet, "_guard_script_plan", lambda _: (_ for _ in ()).throw(RuntimeError("private-secret-canary"))) elif change == "privatebusy": monkeypatch.setattr(driver, "_run", lambda *args, **kwargs: "f" * 64) elif change == "manifest": fleet.TASK_IMAGE_MANIFEST_PATH.write_text('{"sourceRevision":"foreign"}') else: (fleet.TASK_IMAGE_ROOT / "foreign").touch() monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("refusal claimed")) monkeypatch.setattr(fleet, "_rollback_before_activation", lambda _: pytest.fail("refusal rolled back")) result = recovery.run() assert result["status"] == "blocked" and result["effectsPerformed"] is False assert "private-secret-canary" not in json.dumps(result) assert result["reason"] in recovery.REASONS | {"recovery_admission_rejected"} @pytest.mark.parametrize("field", ["health_log", "pid", "config", "mounts", "started"]) def test_repeated_admission_preserves_process_identity_except_health_log(recovery, owned, monkeypatch, field): fleet, _candidate, q, driver_type, _marker, _journal, _probe, _rows, dind, _events, _pending = owned driver = driver_type({}) with driver.maintenance(): _, before = recovery._admit(fleet, _candidate, driver, q) if field == "health_log": dind["State"]["Health"]["Log"] = [{"Start": "later", "End": "later"}] elif field in {"pid", "started"}: dind["State"]["Pid" if field == "pid" else "StartedAt"] = "foreign" elif field == "config": dind["Config"]["Labels"] = {"foreign": "value"} else: dind["Mounts"] = [{"Type": "bind", "Source": "/foreign", "Destination": "/foreign"}] _, after = recovery._admit(fleet, _candidate, driver, q) assert (before == after) is (field == "health_log") @pytest.mark.parametrize("change", ["order", "extra_field", "duplicate", "source", "rw", "propagation"]) def test_probe_mount_order_is_immaterial_but_complete_mount_identity_is_preserved(recovery, owned, change): fleet, candidate, q, driver_type, _marker, _journal, probe, *_rest = owned driver = driver_type({}) with driver.maintenance(): _, before = recovery._admit(fleet, candidate, driver, q) if change == "order": probe["Mounts"].reverse() elif change == "extra_field": probe["Mounts"][0]["unexpected_identity"] = "foreign" elif change == "duplicate": probe["Mounts"].append(copy.deepcopy(probe["Mounts"][0])) else: probe["Mounts"][0][{"source": "Source", "rw": "RW", "propagation": "Propagation"}[change]] = "foreign" try: _, after = recovery._admit(fleet, candidate, driver, q) except recovery.RecoveryRejected: assert change != "order" return assert (before == after) is (change == "order") def test_after_claim_failure_is_unknown_never_auto_retried_and_redacted(recovery, owned, monkeypatch): fleet, *_rest = owned calls = [] def claim(_proof): recovery.CLAIM.write_text("consumed") calls.append(True) monkeypatch.setattr(recovery, "_claim", claim) monkeypatch.setattr(fleet, "_rollback_before_activation_locked", lambda *args: False) result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) assert result["status"] == "unknown" and result["effectsPerformed"] is None and calls == [True] result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) assert result["status"] == "blocked" and calls == [True] and recovery.CLAIM.exists() @pytest.mark.parametrize("boundary", ["migration_busy", "maintenance_busy", "second_read", "queue_before_execute"]) def test_refused_barrier_or_changed_admission_never_claims_or_reaches_rollback_body(recovery, owned, monkeypatch, boundary): fleet, _candidate, _q, driver, _marker, _journal, probe, _rows, _dind, _events, pending = owned @contextmanager def refused(*args, **kwargs): raise RuntimeError("private-canary") yield if boundary == "migration_busy": monkeypatch.setattr(recovery, "_migration_existing_lock", refused) elif boundary == "maintenance_busy": monkeypatch.setattr(driver, "maintenance", refused) elif boundary == "queue_before_execute": pending.append({"id": 1}) else: reads = [] original = _candidate._inspect_host_probe_by_name def changed(): reads.append(True) value = original() if len(reads) == 2: value["Created"] = "changed" return value monkeypatch.setattr(_candidate, "_inspect_host_probe_by_name", changed) monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("refusal claimed")) monkeypatch.setattr(fleet, "_rollback_before_activation_locked", lambda *args: pytest.fail("refusal entered rollback body")) result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) assert result["status"] == "blocked" and result["effectsPerformed"] is False assert "private-canary" not in json.dumps(result) if boundary == "queue_before_execute": assert result["reason"] == "global_ci_work_present" elif boundary == "second_read": assert result["reason"] == "admission_changed" @pytest.mark.parametrize("confirmation", [None, "foreign"]) def test_execution_requires_exact_fixed_confirmation(recovery, owned, confirmation): result = recovery.run(execute=True, confirmation=confirmation) assert result["reason"] == "fixed_confirmation_required" and result["effectsPerformed"] is False @pytest.mark.skipif(os.name != "posix", reason="native POSIX root ownership and atomic publication") @pytest.mark.parametrize("failure", [None, "before_link", "after_link"]) def test_atomic_consumed_claim_survives_crash_boundaries_without_replay(recovery, monkeypatch, tmp_path, failure): marker = tmp_path / "once.json" tmp_path.chmod(0o700) monkeypatch.setattr(recovery, "CLAIM", marker) monkeypatch.setattr(recovery, "_parents", lambda _: None) original = recovery.os.fsync calls = [] def fsync(fd): calls.append(True) if failure == "before_link" and len(calls) == 1 or failure == "after_link" and len(calls) == 2: raise OSError("private-canary") original(fd) monkeypatch.setattr(recovery.os, "fsync", fsync) if failure: with pytest.raises(OSError): recovery._claim({"bound": True}) assert marker.exists() is (failure == "after_link") else: recovery._claim({"bound": True}) assert json.loads(marker.read_text())["phase"] == "consumed" assert marker.stat().st_mode & 0o777 == 0o600 with pytest.raises(FileExistsError): recovery._claim({"bound": True}) assert not list(tmp_path.glob(".ca3a-recovery-*")) @pytest.mark.parametrize("missing", ["directory", "lock"]) def test_existing_migration_lock_never_creates_missing_operation_state(recovery, monkeypatch, tmp_path, missing): parent = tmp_path / "state" if missing == "lock": parent.mkdir() monkeypatch.setattr(recovery, "_parents", lambda _: None) fleet = SimpleNamespace(MIGRATION_JOURNAL_PATH=parent / "specialization-journal.json") before = set(tmp_path.rglob("*")) with pytest.raises(FileNotFoundError): with recovery._migration_existing_lock(fleet): pytest.fail("missing lock acquired") assert set(tmp_path.rglob("*")) == before @pytest.mark.skipif(os.name != "posix", reason="native independent-process flock proof") def test_existing_migration_lock_excludes_other_process_and_rejects_leaf_replacement(recovery, monkeypatch, tmp_path): lock = tmp_path / "specialization.lock" lock.write_bytes(b"") lock.chmod(0o600) monkeypatch.setattr(recovery, "_parents", lambda _: None) fleet = SimpleNamespace(MIGRATION_JOURNAL_PATH=tmp_path / "specialization-journal.json") probe = ("import fcntl,os,sys; f=os.open(sys.argv[1],os.O_RDWR);\n" "try: fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB)\n" "except BlockingIOError: sys.exit(0)\n" "sys.exit(1)") with pytest.raises(recovery.RecoveryRejected, match="migration_lock_changed"): with recovery._migration_existing_lock(fleet) as assert_lock: result = subprocess.run([sys.executable, "-c", probe, str(lock)], capture_output=True, timeout=5, check=False) assert result.returncode == 0 and not result.stdout and not result.stderr assert_lock() replacement = tmp_path / "replacement" replacement.write_bytes(b"") replacement.chmod(0o600) replacement.replace(lock) assert_lock()