diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index 14492ad..8bf75d6 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `0ea6f267c28d51ac7113e6d943c89857d3a189ab`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `5f1a0fc2fdd07082637995b2c8b4514a1ecbbbb8`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index d42b96a..0988db8 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "0ea6f267c28d51ac7113e6d943c89857d3a189ab", + "sourceRevision": "5f1a0fc2fdd07082637995b2c8b4514a1ecbbbb8", "files": [ { "path": ".dockerignore", @@ -937,6 +937,16 @@ "sha256": "54b24a90e7238aaed22d0dfbae70d9772006d20cdb9dbd43d781af48892dcd94", "bytes": 33759 }, + { + "path": "backend/tests/test_ci_acceptance_root_recovery.py", + "sha256": "48bac4513e61610331195ccfef5aaf418eac8ba4ec3a4169351d293643900322", + "bytes": 14349 + }, + { + "path": "backend/tests/test_ci_api_term_root_recovery.py", + "sha256": "b59c4f65229a1a74850c512fe6144dec3ad9a2e41f39c28a57bab406c023fd9d", + "bytes": 12820 + }, { "path": "backend/tests/test_ci_job_docker.py", "sha256": "7292a3ad3e52e934e6a410fdea8adb6a00dc90fe8de28d8159205ec069951d73", @@ -1584,8 +1594,8 @@ }, { "path": "backend/tests/test_runner_acceptance_quiescence.py", - "sha256": "62ce270a8b3e9c9a613cd7bb91f0a7d8b6071bccf914def33f5672084b7a3841", - "bytes": 42074 + "sha256": "659948130bc1a1067ef849e32ba38078b05e407fe1fd80effb203f9af36cadaa", + "bytes": 48198 }, { "path": "backend/tests/test_runner_acceptance_surface.py", @@ -1649,18 +1659,23 @@ }, { "path": "backend/tests/test_runner_resource_envelope.py", - "sha256": "0181c36c9560c5833fb653b50d8739677b45df22b856ff514dd9cba69b732a3d", - "bytes": 51403 + "sha256": "42333a8bc138319e98e9756f2b3909d0c55dac1efa348cd1d6bd7c918098ae73", + "bytes": 53395 }, { "path": "backend/tests/test_runner_scheduler_liveness_recovery.py", "sha256": "40e69e3d0161fb0932a35889ccc7c51ab42bc74c1e0b39c0f5c31a7a76bfe1c3", "bytes": 19338 }, + { + "path": "backend/tests/test_runner_specialization_child_boundary.py", + "sha256": "820faac9350a3c30613cc3fe928857cff29f3df364563a4ceb916b4b8b5e0c51", + "bytes": 8474 + }, { "path": "backend/tests/test_runner_specialization_controller_surface.py", - "sha256": "a6caa41c68478e1ded3509e44ec319cfb3ca6a060f0129c0d6e090f8929dbf6a", - "bytes": 12349 + "sha256": "1885820b4f4306c863c104eb1a46409a93358acfccb95db2728386abbb78b369", + "bytes": 13370 }, { "path": "backend/tests/test_runner_statistics_binding.py", diff --git a/backend/tests/test_ci_acceptance_root_recovery.py b/backend/tests/test_ci_acceptance_root_recovery.py new file mode 100644 index 0000000..e877be0 --- /dev/null +++ b/backend/tests/test_ci_acceptance_root_recovery.py @@ -0,0 +1,310 @@ +from __future__ import annotations + +import ast +import copy +import hashlib +import importlib.util +import json +import os +import stat +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from test_runner_acceptance_quiescence import module, proven, smart_database # noqa: F401 + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def helper(): + spec = importlib.util.spec_from_file_location( + "root_recovery_review_test", ROOT / "scripts/recover_ci_acceptance_once.py") + helper = importlib.util.module_from_spec(spec) + spec.loader.exec_module(helper) + return helper + + +@pytest.fixture +def partial(helper, module, smart_database, monkeypatch, tmp_path): + driver, holder, pg, clients, waits = smart_database + baseline = driver.snapshot() + registrations = copy.deepcopy(driver.runners[:2]) + driver.disable_restart(pg["id"]) + driver.term(pg["id"]) + for row in driver.runners[:2]: + row["disabled"] = True + holder["value"] = { + "schema": module.SCHEMA, "revision": helper.ORIGIN, + "manifestDigest": module._digest(module.OBSERVATION), "phase": "quiescing", + "baseline": baseline, "registrations": registrations, + "registrationProgress": {"9": "disabled", "22": "disabled"}, + "progress": {pg["id"]: "term_intent"}, "guards": driver.capture_guards(), + "guardProgress": {"runner": "installed", "host": "installed"}, + } + + class Adapter: + def __init__(self, projects): + pass + + def __getattr__(self, name): + return getattr(driver, name) + + def maintenance(self): + return driver.maintenance() + + def snapshot(self): + return driver.snapshot() + + monkeypatch.setattr(module, "ControllerDriver", Adapter) + monkeypatch.setattr(helper, "_load", lambda: (module, {})) + monkeypatch.setattr(helper, "_installed_proof", lambda: None) + monkeypatch.setattr(helper, "_read", lambda *args, **kwargs: b"private reviewed helper") + bundle = tmp_path / "bundle" + bundle.mkdir(mode=0o700) + monkeypatch.setattr(helper, "BUNDLE", bundle) + monkeypatch.setattr(helper, "__file__", str(bundle / "recover_ci_acceptance_once.py")) + monkeypatch.setattr(helper, "sys", SimpleNamespace(flags=SimpleNamespace(isolated=1), + dont_write_bytecode=True)) + monkeypatch.setattr(helper, "os", SimpleNamespace(name="posix", geteuid=lambda: 0)) + monkeypatch.setattr(helper, "stat", SimpleNamespace( + S_IMODE=lambda mode: 0o700)) + claims = [] + monkeypatch.setattr(helper, "_claim", lambda digest: claims.append(digest)) + return driver, holder, pg, clients, claims + + +def test_default_preflight_is_readonly_and_preserves_receipt(helper, partial): + driver, holder, _, _, claims = partial + before = copy.deepcopy(holder["value"]) + effects = list(driver.effects) + result = helper.run() + assert result["status"] == "partial" and result["effectsPerformed"] is False + assert result["candidateRevision"] == helper.CANDIDATE + assert result["receiptOrigin"] == helper.ORIGIN and result["normalDeployment"] is False + assert not claims and driver.effects == effects and holder["value"] == before + + +def test_reviewed_execution_resumes_same_receipt_and_two_clients(helper, module, partial): + driver, holder, pg, clients, claims = partial + before = holder["value"] + identity = (before["revision"], before["manifestDigest"], module._digest(before["baseline"])) + result = helper.run(execute=True, confirmation=helper.CONFIRMATION) + assert result["status"] == "quiesced" and result["invocationClaimed"] is True + assert result["retryAllowed"] is False and result["normalDeployment"] is False + assert len(claims) == 1 + assert all(not row["state"]["Running"] for row in [pg, *clients]) + after = holder["value"] + assert identity == (after["revision"], after["manifestDigest"], module._digest(after["baseline"])) + assert after["phase"] == "complete" + assert all(effect[0] in {"TERM", "restart", "registration"} for effect in driver.effects) + + +@pytest.mark.parametrize("change", ["missing", "origin", "manifest", "complete", "guarding", + "restoring", "guards", "registrations", "cid", "stage"]) +def test_foreign_or_new_maintenance_never_executes(helper, partial, change): + driver, holder, _, _, claims = partial + if change == "missing": + holder["value"] = None + else: + receipt = holder["value"] + if change == "origin": + receipt["revision"] = "a" * 40 + elif change == "manifest": + receipt["manifestDigest"] = "a" * 64 + elif change in {"complete", "guarding", "restoring"}: + receipt["phase"] = change + elif change == "guards": + receipt["guardProgress"]["host"] = "restored" + elif change == "registrations": + receipt["registrationProgress"]["22"] = "restored" + elif change == "cid": + receipt["progress"]["a" * 64] = "term_intent" + else: + receipt["progress"][next(iter(receipt["progress"]))] = "foreign" + effects = list(driver.effects) + result = helper.run(execute=True, confirmation=helper.CONFIRMATION) + assert result["status"] == "blocked" and result["effectsPerformed"] is False + assert not claims and driver.effects == effects + + +@pytest.mark.parametrize("fault", ["queued", "changed-container", "busy-lock", "busy-fleet-lock", "claimed-marker"]) +def test_admission_races_before_claim_refuse(helper, partial, monkeypatch, fault): + from contextlib import contextmanager + driver, _, _, _, claims = partial + if fault == "queued": + driver.pending_jobs = lambda: [{"status": "queued"}] + elif fault == "changed-container": + driver.state["rows"][0]["imageId"] = "sha256:" + "0" * 64 + elif fault == "busy-lock": + @contextmanager + def busy(): + raise RuntimeError("sensitive unexpected detail") + yield + driver.maintenance = busy + elif fault == "busy-fleet-lock": + @contextmanager + def busy_fleet(**kwargs): + raise RuntimeError("fleet lock busy") + yield + module = driver.module + monkeypatch.setattr(module.fleet, "_migration_lock", busy_fleet) + else: + def claimed(digest): + raise helper.RecoveryRejected("invocation_already_claimed_inspect_only") + monkeypatch.setattr(helper, "_claim", claimed) + effects = list(driver.effects) + result = helper.run(execute=True, confirmation=helper.CONFIRMATION) + assert result["status"] == "blocked" and not claims and driver.effects == effects + assert "sensitive" not in json.dumps(result) + + +def test_failure_after_claim_is_unknown_never_blind_retry(helper, partial, monkeypatch): + driver, _, _, _, claims = partial + driver.term = lambda cid: (_ for _ in ()).throw(RuntimeError("token-secret-value")) + result = helper.run(execute=True, confirmation=helper.CONFIRMATION) + assert len(claims) == 1 and result["status"] == "unknown" + assert result["effectsPerformed"] is None and result["retryAllowed"] is False + assert "token-secret-value" not in json.dumps(result) + + +@pytest.mark.parametrize("execute,confirmation", [(True, None), (True, "other"), (False, "other")]) +def test_confirmation_cannot_be_selected(helper, partial, execute, confirmation): + with pytest.raises(helper.RecoveryRejected, match="explicit_fixed_confirmation_required"): + helper.run(execute=execute, confirmation=confirmation) + assert not partial[-1] + + +def test_fixed_artifact_inventory_and_candidate_hash_match_immutable_code(helper): + source = ROOT / "ops/gitea-deploy-broker" + tree = ast.parse((source / "status_server.py").read_text()) + files = next(ast.literal_eval(node.value) for node in tree.body + if isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) + and target.id == "_CONTROLLER_ARTIFACT_FILES" for target in node.targets)) + assert helper.FILES == files + # The private bundle must contain Git's LF bytes, not a Windows CRLF checkout. + canonical = (source / helper.MODULE_NAME).read_bytes().replace(b"\r\n", b"\n") + assert hashlib.sha256(canonical).hexdigest() == helper.CANDIDATE_SHA + assert helper.ORIGIN != helper.CANDIDATE + + +@pytest.mark.parametrize("change", ["module", "artifact", "revision", "tree", "version"]) +def test_installed_identity_pin_rejects_drift(helper, monkeypatch, change): + raws = {name: name.encode() for name in helper.FILES} + original_hash = hashlib.sha256(raws[helper.MODULE_NAME]).hexdigest() + digest = hashlib.sha256() + for name, raw in raws.items(): + digest.update(name.encode() + b"\0" + raw + b"\0") + monkeypatch.setattr(helper, "ORIGINAL_MODULE_SHA", original_hash) + monkeypatch.setattr(helper, "ARTIFACT_SHA", digest.hexdigest()) + release = {"apiVersion": "itworx.controller-artifact/v1", "controllerVersion": "2.1.21", + "sourceRepository": "jens/unraid-appops-gateway", "sourceRevision": helper.ORIGIN, + "sourceTree": helper.ORIGIN_TREE, "artifactSha256": helper.ARTIFACT_SHA, + "builtAt": "2026-10-10T00:00:00Z"} + if change in {"module", "artifact"}: + raws[helper.MODULE_NAME if change == "module" else helper.FILES[0]] += b"altered" + else: + release[{"revision": "sourceRevision", "tree": "sourceTree", + "version": "controllerVersion"}[change]] = "different" + monkeypatch.setattr(helper, "_read", lambda path, **kwargs: + json.dumps(release).encode() if path.name == "controller-release.json" + else raws[path.name]) + with pytest.raises(helper.RecoveryRejected): + helper._installed_proof() + + +def test_candidate_hash_refuses_before_any_import(helper, monkeypatch): + monkeypatch.setattr(helper, "_installed_proof", lambda: None) + monkeypatch.setattr(helper, "_read", lambda *args, **kwargs: b"foreign candidate") + with pytest.raises(helper.RecoveryRejected, match="candidate_bytes_changed"): + helper._load() + + +@pytest.mark.parametrize("field", ["root", "isolated", "bytecode"]) +def test_untrusted_interpreter_refuses(helper, partial, monkeypatch, field): + if field == "root": + helper.os.geteuid = lambda: 1000 + elif field == "isolated": + helper.sys.flags.isolated = 0 + else: + helper.sys.dont_write_bytecode = False + with pytest.raises(helper.RecoveryRejected, match="isolated_root_python_required"): + helper.run() + assert not partial[-1] + + +@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="native root POSIX file safety") +@pytest.mark.parametrize("boundary", ["before-link", "after-link"]) +def test_claim_publication_failure_is_atomic_and_never_replayed(helper, monkeypatch, tmp_path, boundary): + bundle = tmp_path / "bundle" + bundle.mkdir(mode=0o700) + monkeypatch.setattr(helper, "BUNDLE", bundle) + monkeypatch.setattr(helper, "MARKER", bundle / "invoked.json") + monkeypatch.setattr(helper, "_trusted_parents", lambda path: None) + calls = 0 + + def fsync(fd): + nonlocal calls + calls += 1 + if calls == (1 if boundary == "before-link" else 2): + raise OSError("simulated actual durability failure") + os.fsync(fd) + + proxy = SimpleNamespace(**{name: getattr(os, name) for name in dir(os)}) + proxy.fsync = fsync + monkeypatch.setattr(helper, "os", proxy) + with pytest.raises(OSError): + helper._claim("a" * 64) + marker = bundle / "invoked.json" + assert marker.exists() is (boundary == "after-link") + if marker.exists(): + assert json.loads(helper._read(marker, private=True))["candidateRevision"] == helper.CANDIDATE + with pytest.raises(helper.RecoveryRejected, match="invocation_already_claimed"): + helper._claim("a" * 64) + assert not list(bundle.glob(".acceptance-recovery-invocation-*")) + + +@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="native root POSIX file safety") +def test_private_marker_atomic_once_and_reader_metadata(helper, monkeypatch, tmp_path): + bundle = tmp_path / "bundle" + bundle.mkdir(mode=0o700) + monkeypatch.setattr(helper, "BUNDLE", bundle) + monkeypatch.setattr(helper, "MARKER", bundle / "invoked.json") + # pytest's sandbox ancestors are not the deployment bundle; test leaf metadata here. + monkeypatch.setattr(helper, "_trusted_parents", lambda path: None) + helper._claim("a" * 64) + marker = bundle / "invoked.json" + assert stat.S_IMODE(marker.stat().st_mode) == 0o600 + assert json.loads(helper._read(marker, private=True))["receiptOrigin"] == helper.ORIGIN + with pytest.raises(helper.RecoveryRejected, match="invocation_already_claimed"): + helper._claim("a" * 64) + with pytest.raises(helper.RecoveryRejected, match="invocation_already_claimed"): + helper._claim("b" * 64) + for mode in (0o644, 0o640, 0o664, 0o777): + marker.chmod(mode) + with pytest.raises(helper.RecoveryRejected, match="recovery_file_untrusted"): + helper._read(marker, private=True) + marker.chmod(0o600) + os.chown(marker, 1000, 0) + with pytest.raises(helper.RecoveryRejected, match="recovery_file_untrusted"): + helper._read(marker, private=True) + os.chown(marker, 0, 0) + link = bundle / "extra-link" + os.link(marker, link) + with pytest.raises(helper.RecoveryRejected, match="recovery_file_untrusted"): + helper._read(marker, private=True) + link.unlink() + proxy = SimpleNamespace(**{name: getattr(os, name) for name in dir(os)}) + proxy.fstat = lambda fd: SimpleNamespace(**{ + key: getattr(os.fstat(fd), key) + (1 if key == "st_ino" else 0) + for key in ("st_dev", "st_ino", "st_uid", "st_gid", "st_mode", "st_nlink", "st_size", + "st_mtime_ns", "st_ctime_ns")}) + with monkeypatch.context() as scope: + scope.setattr(helper, "os", proxy) + with pytest.raises(helper.RecoveryRejected, match="recovery_file_changed"): + helper._read(marker, private=True) + marker.unlink() + marker.symlink_to(bundle / "absent") + with pytest.raises(helper.RecoveryRejected, match="recovery_file_untrusted"): + helper._read(marker, private=True) diff --git a/backend/tests/test_ci_api_term_root_recovery.py b/backend/tests/test_ci_api_term_root_recovery.py new file mode 100644 index 0000000..d7107cb --- /dev/null +++ b/backend/tests/test_ci_api_term_root_recovery.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import builtins +import copy +import hashlib +import importlib.util +import json +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from test_runner_acceptance_quiescence import module, proven # noqa: F401 + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def helper(): + spec = importlib.util.spec_from_file_location("api_child_recovery_test", + ROOT / "scripts/recover_ci_api_term_once.py") + result = importlib.util.module_from_spec(spec) + spec.loader.exec_module(result) + return result + + +@pytest.fixture +def prepared(helper, module, proven, monkeypatch, tmp_path): + from contextlib import nullcontext + driver, holder = proven + baseline = driver.snapshot() + holder["value"] = { + "schema": module.SCHEMA, "revision": module.REVIEWED_RECEIPT_PREDECESSOR, + "manifestDigest": module._digest(module.OBSERVATION), "phase": "quiescing", + "baseline": baseline, "registrations": copy.deepcopy(driver.runners[:2]), + "registrationProgress": {"9": "disabled", "22": "disabled"}, + "progress": {row["id"]: "term_intent" for row in baseline["rows"]}, + "guards": driver.capture_guards(), "guardProgress": {"runner": "installed", "host": "installed"}, + } + for row in driver.state["rows"]: + row["restartPolicy"] = {"Name": "no", "MaximumRetryCount": 0} + for row in driver.runners[:2]: + row["disabled"] = True + driver._effect_admission = module.ControllerDriver._effect_admission.__get__(driver) + base = SimpleNamespace(ORIGIN=module.REVIEWED_RECEIPT_PREDECESSOR, + CANDIDATE="07c75a3aae4d0881f93e4bef2aa0275235c4253a", + CANDIDATE_SHA="236af955566bd2713684977008045c7b9f1390b83fff4b160da83c6473bf4246", + MARKER=tmp_path / "original-marker.json") + def original_scope(module, receipt): + assert receipt is not None and receipt["revision"] == base.ORIGIN + assert receipt["phase"] == "quiescing" + assert receipt["manifestDigest"] == module._digest(module.OBSERVATION) + base._scope = original_scope + base._load = lambda: (module, {}) + base._installed_proof = lambda: None + base._read = lambda *args, **kwargs: json.dumps({"candidateRevision": base.CANDIDATE, + "candidateSha256": base.CANDIDATE_SHA, "receiptOrigin": base.ORIGIN, + "receiptDigestBefore": "a" * 64}).encode() + monkeypatch.setattr(helper, "_load_base", lambda: base) + monkeypatch.setattr(module, "ControllerDriver", lambda projects: driver) + bundle = tmp_path / "bundle" + bundle.mkdir() + monkeypatch.setattr(helper, "BUNDLE", bundle) + monkeypatch.setattr(helper, "__file__", str(bundle / "recover_ci_api_term_once.py")) + monkeypatch.setattr(helper, "os", SimpleNamespace(name="posix", geteuid=lambda: 0)) + monkeypatch.setattr(helper, "sys", SimpleNamespace(flags=SimpleNamespace(isolated=1), dont_write_bytecode=True)) + monkeypatch.setattr(helper, "stat", SimpleNamespace(S_IMODE=lambda mode: 0o700)) + claims, effects = [], [] + monkeypatch.setattr(helper, "_claim", lambda *args: claims.append(args)) + def probe(driver, module, cid, ticket=None): + if ticket is None: + return {"pid": 11, "parent": 1, "start": 123, "initStart": 10, "namespace": "pid:[111]"} + effects.append(cid) + monkeypatch.setattr(helper, "_probe", probe) + monkeypatch.setattr(module.fleet, "_migration_lock", lambda **kwargs: nullcontext()) + return driver, holder, base, claims, effects + + +def test_readonly_default_never_claims_or_signals(helper, prepared): + driver, holder, _, claims, effects = prepared + receipt = copy.deepcopy(holder["value"]) + result = helper.run() + assert result["status"] == "ready" and result["effectsPerformed"] is False + assert result["existingMaintenanceEffects"] is True and result["normalDeployment"] is False + assert not claims and not effects and holder["value"] == receipt + assert not driver.effects + + +def test_three_exact_children_only_and_original_receipt_unchanged(helper, prepared): + _, holder, _, claims, effects = prepared + before = copy.deepcopy(holder["value"]) + result = helper.run(execute=True, confirmation=helper.CONFIRMATION) + assert result["status"] == "term_sent_inspect_exits" and result["resolvedChildren"] == 3 + assert result["confirmedSignalAcks"] == 3 + assert len(claims) == 1 and effects == list(helper.API_IDS) + assert holder["value"] == before and result["retryAllowed"] is False + + +@pytest.mark.parametrize("fault", ["missing", "foreign", "complete", "missing-intent", "foreign-intent", + "queue", "busy-fleet", "claimed", "container", "previous-marker"]) +def test_failed_admission_never_signals(helper, prepared, module, monkeypatch, fault): + from contextlib import contextmanager + driver, holder, base, claims, effects = prepared + if fault == "missing": + holder["value"] = None + elif fault == "foreign": + holder["value"]["revision"] = "f" * 40 + elif fault == "complete": + holder["value"]["phase"] = "complete" + elif fault == "missing-intent": + holder["value"]["progress"].pop(helper.API_IDS[0]) + elif fault == "foreign-intent": + holder["value"]["progress"][helper.API_IDS[0]] = "restart_intent" + elif fault == "queue": + driver.pending_jobs = lambda: [{"status": "queued"}] + elif fault == "busy-fleet": + @contextmanager + def busy(**kwargs): + raise RuntimeError("busy") + yield + monkeypatch.setattr(module.fleet, "_migration_lock", busy) + elif fault == "claimed": + monkeypatch.setattr(helper, "_claim", lambda *args: (_ for _ in ()).throw(RuntimeError("exists"))) + elif fault == "container": + driver.state["rows"][0]["imageId"] = "foreign" + else: + base._read = lambda *args, **kwargs: b"{}" + result = helper.run(execute=True, confirmation=helper.CONFIRMATION) + assert result["status"] == "blocked" and not effects and not claims + + +@pytest.mark.parametrize("exit_attested", [True, False]) +def test_missing_ack_advances_only_on_exact_owned_exit(helper, prepared, monkeypatch, exit_attested): + driver, holder, _, claims, effects = prepared + before = copy.deepcopy(holder["value"]) + normal = helper._probe + def probe(driver, module, cid, ticket=None): + if ticket is not None and cid == helper.API_IDS[0]: + effects.append(cid) + if exit_attested: + row = next(row for row in driver.state["rows"] if row["id"] == cid) + row["state"].update(Running=False, Status="exited", Pid=0, FinishedAt="2026-10-10T00:00:00Z") + raise RuntimeError("namespace teardown, token-secret-not-output") + return normal(driver, module, cid, ticket) + monkeypatch.setattr(helper, "_probe", probe) + result = helper.run(execute=True, confirmation=helper.CONFIRMATION) + assert len(claims) == 1 and effects.count(helper.API_IDS[0]) == 1 + assert result["status"] == ("term_sent_inspect_exits" if exit_attested else "unknown") + assert len(effects) == (3 if exit_attested else 1) + assert "token-secret" not in json.dumps(result) and holder["value"] == before + + +@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="native root durable marker") +@pytest.mark.parametrize("fault", [None, "before-link", "after-link"]) +def test_actual_atomic_marker_once_under_write_failures(helper, monkeypatch, tmp_path, fault): + marker = tmp_path / "private" / "once.json" + marker.parent.mkdir(mode=0o700) + monkeypatch.setattr(helper, "MARKER", marker) + base = SimpleNamespace(_trusted_parents=lambda path: None, + ORIGIN="0ea6f267c28d51ac7113e6d943c89857d3a189ab", + CANDIDATE_SHA="a" * 64, + _read=lambda *args, **kwargs: b"reviewed source") + calls = 0 + def fsync(fd): + nonlocal calls + calls += 1 + if (fault == "before-link" and calls == 1) or (fault == "after-link" and calls == 2): + raise OSError("injected actual fsync failure") + os.fsync(fd) + proxy = SimpleNamespace(**{name: getattr(os, name) for name in dir(os)}) + proxy.fsync = fsync + monkeypatch.setattr(helper, "os", proxy) + if fault: + with pytest.raises(OSError): + helper._claim(base, "b" * 64, {}) + else: + helper._claim(base, "b" * 64, {}) + assert marker.exists() is (fault != "before-link") + assert not list(marker.parent.glob(".api-term-invocation-*")) + if marker.exists(): + payload = json.loads(marker.read_bytes()) + assert payload["receiptOrigin"] == base.ORIGIN and payload["kind"] == "ci_api_child_term_once" + for digest in ("b" * 64, "c" * 64): + with pytest.raises(RuntimeError, match="already_invoked"): + helper._claim(base, digest, {}) + + +def test_explicit_confirmation_required(helper, prepared): + with pytest.raises(RuntimeError, match="separate_fixed_confirmation_required"): + helper.run(execute=True) + assert not prepared[-1] and not prepared[-2] + + +@pytest.mark.parametrize("fault", [None, "argv", "parent", "extra-child", "namespace", "exe", "pid-reuse", "esrch", "shell-term"]) +def test_actual_child_script_pidfd_and_process_races(helper, monkeypatch, capsys, fault): + signals, closes = [], [] + start = 123 + def procstat(parent, tick): + return "11 (python) S " + str(parent) + " " + " ".join(["0"] * 17) + " " + str(tick) + class FakePath: + def __init__(self, path): + self.path = path + self.name = path.rsplit("/", 1)[-1] + def iterdir(self): + return [FakePath("/proc/" + str(pid)) for pid in ([1, 11, 99, 12] if fault == "extra-child" else [1, 11, 99])] + def read_bytes(self): + args = (["sh", "-c", helper.SHELL_COMMAND] if self.path == "/proc/1/cmdline" else + ["foreign"] if fault == "argv" else helper.CHILD_ARGV) + return ("\0".join(args) + "\0").encode() + def read_text(self): + if self.path.endswith("/status"): + return "SigCgt:\t00014002" if fault == "shell-term" else "SigCgt:\t00010002" + if self.path == "/proc/1/stat": + return procstat(0, 10) + return procstat(0 if self.path == "/proc/99/stat" else 2 if fault == "parent" else 1, start) + def pidfd_open(pid, flags): + nonlocal start + assert pid == 11 and flags == 0 + if fault == "pid-reuse": + start += 1 + if fault == "esrch": + raise ProcessLookupError() + return 77 + def readlink(path): + if path.endswith("/exe"): + return "/foreign" if fault == "exe" else "/usr/local/bin/python3.12" + return "pid:[222]" if fault == "namespace" and "/11/" in path else "pid:[111]" + fake_os = SimpleNamespace(geteuid=lambda: 0, getpid=lambda: 99, pidfd_open=pidfd_open, + readlink=readlink, close=lambda fd: closes.append(fd)) + fake_signal = SimpleNamespace(SIGTERM=15, pidfd_send_signal=lambda fd, sig, info, flags: + signals.append((fd, sig, info, flags))) + ticket = {"pid": 11, "parent": 1, "start": 123, "initStart": 10, "namespace": "pid:[111]"} + fake_sys = SimpleNamespace(flags=SimpleNamespace(isolated=1), dont_write_bytecode=True, + argv=["-c", json.dumps(ticket)], exit=lambda code: (_ for _ in ()).throw(SystemExit(code))) + actual_import = builtins.__import__ + modules = {"os": fake_os, "signal": fake_signal, "sys": fake_sys, + "pathlib": SimpleNamespace(Path=FakePath)} + namespace = {"__builtins__": {**vars(builtins), "__import__": lambda name, *args, **kwargs: + modules.get(name) or actual_import(name, *args, **kwargs)}} + if fault is None: + exec(helper.CHILD_SOURCE, namespace) + assert signals == [(77, 15, None, 0)] and closes == [77] + assert json.loads(capsys.readouterr().out) == {"status": "term_sent"} + else: + with pytest.raises(SystemExit): + exec(helper.CHILD_SOURCE, namespace) + assert not signals + assert json.loads(capsys.readouterr().out) == {"status": "rejected"} + + +def test_original_c185_pin_and_no_stronger_signal_or_host_pid(helper): + original = (ROOT / "scripts/recover_ci_acceptance_once.py").read_bytes().replace(b"\r\n", b"\n") + assert hashlib.sha256(original).hexdigest() == helper.BASE_SHA + assert len(set(helper.API_IDS)) == 3 + assert "os.kill" not in helper.CHILD_SOURCE and "SIGKILL" not in helper.CHILD_SOURCE + assert "pidfd_send_signal(fd,signal.SIGTERM,None,0)" in helper.CHILD_SOURCE diff --git a/backend/tests/test_runner_acceptance_quiescence.py b/backend/tests/test_runner_acceptance_quiescence.py index 4b5cea2..de93795 100644 --- a/backend/tests/test_runner_acceptance_quiescence.py +++ b/backend/tests/test_runner_acceptance_quiescence.py @@ -228,6 +228,124 @@ def test_positive_preserves_resources_and_windows_then_idempotent(module, proven module.quiesce(revision="e" * 40, projects={}, driver=driver) +@pytest.fixture +def smart_database(proven): + # Synthetic dependency graph over two existing running identities; this + # models retained sessions, not a claim that mock-ats uses this production DB. + driver, receipt = proven + project = "workmesh-runtime-19421" + pg = next(row for row in driver.state["rows"] + if row["composeLabels"]["com.docker.compose.project"] == project + and row["composeLabels"]["com.docker.compose.service"] == "postgres") + clients = [row for row in driver.state["rows"] + if row["composeLabels"]["com.docker.compose.project"] == project + and row["composeLabels"]["com.docker.compose.service"] in {"api", "mock-ats"}] + assert len(clients) == 2 and all(row["state"]["Running"] for row in clients) + driver.state["rows"] = [pg] + [row for row in driver.state["rows"] if row is not pg] + original_term = driver.term + pending = False + waits = [] + + def term(cid): + nonlocal pending + if cid == pg["id"] and pg["state"]["Running"]: + pending = True + driver.effects.append(("TERM", cid)) + else: + original_term(cid) + if pending and all(not row["state"]["Running"] for row in clients): + pg["state"].update(Running=False, Status="exited", Pid=0, + FinishedAt="2026-10-10T00:00:00Z") + + def wait(cid): + waits.append(sum(effect[0] == "TERM" for effect in driver.effects)) + return not next(row for row in driver.state["rows"] if row["id"] == cid)["state"]["Running"] + + driver.term, driver.wait_exited = term, wait + return driver, receipt, pg, clients, waits + + +def test_two_clients_receive_term_before_waiting_for_smart_database_shutdown(module, smart_database): + driver, receipt, pg, clients, waits = smart_database + result = module.quiesce(revision="d" * 40, projects={}, driver=driver) + assert result["status"] == "quiesced" + assert all(not row["state"]["Running"] for row in [pg, *clients]) + assert waits and all(count == 12 for count in waits) + assert receipt["value"]["phase"] == "complete" + assert sum(effect[0] == "TERM" for effect in driver.effects) == 12 + + +def test_old_partial_receipt_resumes_two_passes_without_rewriting_origin(module, smart_database, monkeypatch): + from types import SimpleNamespace + + driver, holder, pg, clients, _ = smart_database + baseline = copy.deepcopy(driver.state) + original = { + "schema": module.SCHEMA, "revision": module.REVIEWED_RECEIPT_PREDECESSOR, + "manifestDigest": module._digest(module.OBSERVATION), "phase": "quiescing", + "baseline": baseline, "registrations": copy.deepcopy(driver.runners[:2]), + "registrationProgress": {"9": "disabled", "22": "disabled"}, + "progress": {pg["id"]: "term_intent"}, "guards": driver.capture_guards(), + "guardProgress": {"runner": "installed", "host": "installed"}, + } + driver.disable_restart(pg["id"]) + driver.term(pg["id"]) + assert pg["state"]["Running"] and all(row["state"]["Running"] for row in clients) + for runner in driver.runners[:2]: + runner["disabled"] = True + holder["value"] = original + frozen = (original["revision"], original["manifestDigest"], module._digest(baseline)) + release = {"controllerVersion": "2.1.21", "sourceRepository": module.fleet.SOURCE_REPOSITORY, + "sourceRevision": "e" * 40, "sourceTree": "f" * 40} + monkeypatch.setitem(sys.modules, "status_server", SimpleNamespace(_require_controller_release=lambda: release)) + driver.accepts_reviewed_receipt_predecessor = module.ControllerDriver({}).accepts_reviewed_receipt_predecessor + assert module.preflight(revision="e" * 40, projects={}, driver=driver)["status"] == "partial" + assert module.quiesce(revision="e" * 40, projects={}, driver=driver)["status"] == "quiesced" + final = holder["value"] + assert (final["revision"], final["manifestDigest"], module._digest(final["baseline"])) == frozen + assert all(not row["state"]["Running"] for row in [pg, *clients]) + assert module.quiesce(revision="e" * 40, projects={}, driver=driver)["status"] == "already_quiesced" + + +@pytest.mark.parametrize("change", ["origin", "manifest", "revision", "tree", "repository", "version", "missing-reader"]) +def test_predecessor_resume_rejects_foreign_receipt_or_current_source(module, proven, monkeypatch, change): + from types import SimpleNamespace + + driver, holder = proven + driver.timeout = True + module.quiesce(revision=module.REVIEWED_RECEIPT_PREDECESSOR, projects={}, driver=driver) + release = {"controllerVersion": "2.1.21", "sourceRepository": module.fleet.SOURCE_REPOSITORY, + "sourceRevision": "e" * 40, "sourceTree": "f" * 40} + monkeypatch.setitem(sys.modules, "status_server", SimpleNamespace(_require_controller_release=lambda: release)) + if change == "origin": + holder["value"]["revision"] = "a" * 40 + elif change == "manifest": + holder["value"]["manifestDigest"] = "a" * 64 + elif change == "revision": + release["sourceRevision"] = "a" * 40 + elif change == "tree": + release["sourceTree"] = "invalid" + elif change == "repository": + release["sourceRepository"] = "jens/foreign" + elif change == "version": + release["controllerVersion"] = "2.1.22" + if change != "missing-reader": + driver.accepts_reviewed_receipt_predecessor = module.ControllerDriver({}).accepts_reviewed_receipt_predecessor + before = len(driver.effects) + with pytest.raises(module.QuiescenceError, match="maintenance_owner_changed"): + module.quiesce(revision="e" * 40, projects={}, driver=driver) + assert len(driver.effects) == before + + +def test_predecessor_failure_projection_preserves_effect_uncertainty(module, proven): + driver, _ = proven + driver.timeout = True + module.quiesce(revision=module.REVIEWED_RECEIPT_PREDECESSOR, projects={}, driver=driver) + result = module.failure_projection("e" * 40) + assert result["status"] == "partial" and result["guardHeld"] is True + assert result["effectsPerformed"] is True + + def test_mount_order_is_canonical_for_fixed_current_and_resume_baseline(module, proven): driver, _ = proven baseline = copy.deepcopy(driver.state) diff --git a/backend/tests/test_runner_resource_envelope.py b/backend/tests/test_runner_resource_envelope.py index cf8a68f..9e55535 100644 --- a/backend/tests/test_runner_resource_envelope.py +++ b/backend/tests/test_runner_resource_envelope.py @@ -906,6 +906,38 @@ def test_host_family_inspection_only_allows_dind_start_from_exact_empty_family(m assert occupied["safeToStartDindReason"] == "resource_cgroup_family_not_empty" +def test_rollback_preserves_unraid_existing_hugetlb_controller(monkeypatch): + envelope = _module() + initial = ["cpuset", "cpu", "io", "memory", "hugetlb", "pids"] + calls = [] + monkeypatch.setattr(envelope, "_run_host_probe", lambda mode, payload: + calls.append((mode, payload)) or {"removed": True, "familyExists": False}) + envelope.rollback_host_cgroup_family(created=True, initial_controllers=initial) + assert calls == [("rollback", {"created": True, "initialControllers": initial})] + + monkeypatch.setattr(envelope, "_probe_open_family", lambda _fd: 123) + monkeypatch.setattr(envelope, "_probe_walk", lambda _fd: [{ + "cpuMax": "800000 100000", "memoryMax": str(16 * 1024**3), + }]) + monkeypatch.setattr(envelope, "_probe_read", lambda fd, name: + "" if name == "cgroup.procs" else + "cpu memory pids" if fd == 123 else " ".join(initial)) + monkeypatch.setattr(envelope.os, "close", lambda _fd: None) + monkeypatch.setattr(envelope.os, "rmdir", lambda *args, **kwargs: None) + writes = [] + monkeypatch.setattr(envelope, "_probe_write", lambda *args: writes.append(args)) + result = envelope._probe_rollback_family(1, {"op": "rollback", "created": True, + "initialControllers": initial}) + assert result == {"removed": True, "familyExists": False} + assert writes == [(123, "cgroup.subtree_control", "-cpu -memory -pids")] + for controllers in (["unknown"], initial + ["unknown"]): + with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"): + envelope.rollback_host_cgroup_family(created=True, initial_controllers=controllers) + with pytest.raises(RuntimeError, match="resource_cgroup_family_request_invalid"): + envelope._probe_rollback_family(1, {"op": "rollback", "created": True, + "initialControllers": controllers}) + + def test_host_probe_rw_modes_are_fixed_and_limited_to_ensure_or_rollback(): envelope = _module() image_id = "sha256:" + "a" * 64 diff --git a/backend/tests/test_runner_specialization_child_boundary.py b/backend/tests/test_runner_specialization_child_boundary.py new file mode 100644 index 0000000..0fc12f6 --- /dev/null +++ b/backend/tests/test_runner_specialization_child_boundary.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import hashlib +import os +import shutil +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from test_runner_fleet_specialize import specialize # noqa: F401 + + +def test_child_fixed_failure_is_finite_and_does_not_echo_exception(specialize, monkeypatch, capsys): + def fail(): + raise specialize.RunnerFleetSpecializeError("runner_fleet_specialization_entrypoint_untrusted") + monkeypatch.setattr(specialize, "_apply_from_fixed_script", fail) + assert specialize._main(["--apply-fixed", "--confirm", specialize.CONFIRMATION]) == 1 + captured = capsys.readouterr() + assert captured.out == "" + assert captured.err == "APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_entrypoint_untrusted\n" + + +@pytest.mark.parametrize("private", ["token-secret", "runner_fleet_specialization_arbitrary_secret", + "runner_fleet_specialization_entrypoint_untrusted\nsecret"]) +def test_unknown_child_details_are_redacted(specialize, monkeypatch, capsys, private): + def fail(): + raise specialize.RunnerFleetSpecializeError(private) + monkeypatch.setattr(specialize, "_apply_from_fixed_script", fail) + assert specialize._main(["--apply-fixed", "--confirm", specialize.CONFIRMATION]) == 1 + assert capsys.readouterr().err == "APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_script_execution_failed\n" + assert specialize.public_specialization_failure(RuntimeError(private)) == "runner_fleet_specialization_script_execution_failed" + + +@pytest.mark.parametrize("fault", [None, "stdout", "multiline", "secret", "code-prefix", "exit", "too-large", "unknown"]) +def test_parent_adopts_only_exact_single_finite_envelope(specialize, fault): + code = "runner_fleet_specialization_entrypoint_untrusted" + result = SimpleNamespace(returncode=1, stdout="", stderr=f"APPOPS_SPECIALIZATION_FAILURE:{code}\n") + if fault == "stdout": + result.stdout = "sensitive log" + elif fault == "multiline": + result.stderr += "secret\n" + elif fault == "secret": + result.stderr = "secret-value" + elif fault == "code-prefix": + result.stderr = f"{code}\n" + elif fault == "exit": + result.returncode = 137 + elif fault == "too-large": + result.stderr = "x" * 257 + elif fault == "unknown": + result.stderr = "APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_not_reviewed\n" + assert specialize._script_failure_code(result) == (code if fault is None else "runner_fleet_specialization_script_execution_failed") + + +@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0 or shutil.which("bash") is None, + reason="real POSIX root Bash parent proof in isolated Linux qualification") +@pytest.mark.parametrize("deny", [False, True]) +def test_real_bash_parent_identity_and_child_error_propagation_with_stub_only( + specialize, monkeypatch, tmp_path, deny, +): + # Source-identical Bash structure with only its fixed executor redirected to + # this sandbox's stub. Never call apply, deploy, Docker, native APIs or journals. + source_root = Path(__file__).resolve().parents[2] + state = tmp_path / "state" + state.mkdir(mode=0o700) + stub = tmp_path / "readonly_stub.py" + shell = (source_root / "scripts/upgrade_ci_runner_fleet_specialized.sh").read_text() + shell = shell.replace('/opt/gitea-deploy/runner_fleet_specialize.py', str(stub)) + approved = hashlib.sha256(shell.encode()).hexdigest() + stub.write_text( + "import sys\nfrom pathlib import Path\n" + + "sys.path.insert(0," + repr(str(source_root / "ops/gitea-deploy-broker")) + ")\n" + + "import runner_fleet_specialize as f\n" + + "f.SPECIALIZED_STATE_ROOT=Path(" + repr(str(state)) + ")\n" + + "f._APPROVED_SCRIPT_SHA256=" + repr("0" * 64 if deny else approved) + "\n" + + "if not f._authorized_script_parent():\n" + + " print('APPOPS_SPECIALIZATION_FAILURE:runner_fleet_specialization_entrypoint_untrusted',file=sys.stderr)\n" + + " sys.exit(1)\n", + encoding="utf-8", + ) + monkeypatch.setattr(specialize, "ROOT", tmp_path) + monkeypatch.setattr(specialize, "SPECIALIZED_STATE_ROOT", state) + monkeypatch.setattr(specialize, "SPECIALIZED_STATE_PATH", state / "specialized.json") + monkeypatch.setattr(specialize, "_ensure_state_root", lambda: None) + if deny: + with pytest.raises(specialize.RunnerFleetSpecializeError, + match="^runner_fleet_specialization_entrypoint_untrusted$"): + specialize._run_approved_shell_script(shell.encode()) + else: + specialize._run_approved_shell_script(shell.encode()) + assert list(state.iterdir()) == [] + + +@pytest.mark.parametrize("controllers,accepted", [ + (["cpu", "memory", "pids"], True), + (["cpuset", "cpu", "io", "memory", "hugetlb", "pids"], True), + (["cpuset", "cpu", "io", "memory", "hugetlb", "pids", "foreign"], False), +]) +@pytest.mark.parametrize("boundary", ["snapshot", "ensure", "journal"]) +def test_observed_hugetlb_inventory_and_legacy_records_remain_finite( + specialize, monkeypatch, tmp_path, controllers, accepted, boundary, +): + if boundary == "snapshot": + snapshot = {"exists": False, "rootSubtreeControllers": controllers, + "safeToStartDind": True} + monkeypatch.setattr(specialize, "inspect_host_cgroup_family", lambda: snapshot) + read = specialize._cgroup_family_snapshot + expected = snapshot + error = "cgroup_inventory_invalid" + elif boundary == "ensure": + result = {"created": True, "rootSubtreeBefore": controllers} + monkeypatch.setattr(specialize, "ensure_host_cgroup_family", lambda **kwargs: result) + def read(): + return specialize._ensure_cgroup_family_details(allow_existing=False) + expected = result + error = "cgroup_family_result_invalid" + else: + # Actual schema-v3 parser, including absent legacy-DinD state and its + # unchanged provenance, validates both historical and current inventories. + backup = specialize._SPECIALIZED_BACKUP_ROOT / "specialized-finite-fixture" + journal = { + "schemaVersion": 3, "status": "in_progress", "stage": "prepared", + "sourceScriptSha256": specialize._APPROVED_SCRIPT_SHA256, + "sourceRevision": "a" * 40, "sourceTree": "b" * 40, + "createdAt": "2026-10-10T00:00:00Z", "updatedAt": "2026-10-10T00:00:00Z", + "backupDirectory": str(backup), "baseComposeSha256": "c" * 64, + "baseComposeModelSha256": "d" * 64, "specializedComposeSha256": "e" * 64, + "legacyTopology": "v1", "originalRegistrations": [], "originalContainers": [], + "originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None}, + "guardFiles": [], "guardMarkers": {}, "cgroupCreated": False, + "cgroupEnsurePending": False, "cgroupFamilyInitiallyPresent": False, + "cgroupRootSubtree": controllers, "newRegistrationIds": {}, + "imageManifestSha256": None, "buildProbe": None, + } + monkeypatch.setattr(specialize, "_read_json_file", lambda *args, **kwargs: journal) + monkeypatch.setattr(specialize, "_trusted_directory", lambda path: None) + read = specialize._read_journal + expected = journal + error = "journal_invalid" + if accepted: + assert read() == expected + else: + with pytest.raises(specialize.RunnerFleetSpecializeError, match=error): + read() + + +@pytest.mark.parametrize("suffix", ["cgroup_inventory_invalid", "cgroup_inventory_unavailable", + "identity_adoption_rejected", "legacy_config_missing"]) +def test_new_finite_prejournal_failure_codes_cross_both_boundaries(specialize, monkeypatch, capsys, suffix): + code = "runner_fleet_specialization_" + suffix + def fail(): + raise specialize.RunnerFleetSpecializeError(code) + monkeypatch.setattr(specialize, "_apply_from_fixed_script", fail) + assert specialize._main(["--apply-fixed", "--confirm", specialize.CONFIRMATION]) == 1 + captured = capsys.readouterr() + assert captured.out == "" + result = SimpleNamespace(returncode=1, stdout=captured.out, stderr=captured.err) + assert specialize._script_failure_code(result) == code + assert specialize.public_specialization_failure(specialize.RunnerFleetSpecializeError(code)) == code diff --git a/backend/tests/test_runner_specialization_controller_surface.py b/backend/tests/test_runner_specialization_controller_surface.py index aed3c1f..e6c3155 100644 --- a/backend/tests/test_runner_specialization_controller_surface.py +++ b/backend/tests/test_runner_specialization_controller_surface.py @@ -87,6 +87,25 @@ def test_pending_protected_validation_blocks_specialization(controller, monkeypa assert responses[0][1]["status"] == "preflight_rejected" +@pytest.mark.parametrize("private, expected", [ + ("runner_fleet_specialization_entrypoint_untrusted", "runner_fleet_specialization_entrypoint_untrusted"), + ("private-token-value", "runner_fleet_specialization_script_execution_failed"), + ("runner_fleet_specialization_entrypoint_untrusted\nprivate-token-value", + "runner_fleet_specialization_script_execution_failed"), +]) +def test_specialization_failure_response_contains_only_finite_safe_reason(controller, monkeypatch, private, expected): + instance, responses, _ = handler(controller, monkeypatch) + monkeypatch.setattr(controller.subprocess, "run", lambda *args, **kwargs: + SimpleNamespace(returncode=0, stdout=b"exact source script")) + + def fail(_script): + raise controller.RunnerFleetSpecializeError(private) + + monkeypatch.setattr(controller, "execute_runner_fleet_specialize", fail) + instance.do_POST() + assert responses == [(409, {"status": "upgrade_failed", "revision": "a" * 40, "reason": expected})] + + def test_caller_cannot_choose_script_image_or_runner(controller, monkeypatch): instance, responses, _ = handler(controller, monkeypatch, {"reason": "Specialize CI profiles", "script": "untrusted.sh"}) instance.do_POST()