From ad3a9736cc191b44845ba4542a8843bf159be4b9 Mon Sep 17 00:00:00 2001 From: Public Source Publisher Date: Sat, 10 Oct 2026 14:24:16 +0000 Subject: [PATCH] Publish curated Unraid-AppOps-Gateway source from 87aa7dda6889fd951d202754d76daaaa1e98e26b --- PUBLIC_SOURCE_EXPORT.md | 2 +- PUBLIC_SOURCE_MANIFEST.json | 16 +- ...est_runner_dind_health_cutover_recovery.py | 245 ++++++++++++ ...test_runner_dind_rollback_tail_recovery.py | 278 ++++++++++++++ backend/tests/test_runner_fleet_specialize.py | 356 +++++++++++++++++- 5 files changed, 891 insertions(+), 6 deletions(-) create mode 100644 backend/tests/test_runner_dind_health_cutover_recovery.py create mode 100644 backend/tests/test_runner_dind_rollback_tail_recovery.py diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index ce54d88..51919e9 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `5572962b172902d61bdd220ceb8fb69426f17b5f`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `87aa7dda6889fd951d202754d76daaaa1e98e26b`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index 0f974ae..d047281 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "5572962b172902d61bdd220ceb8fb69426f17b5f", + "sourceRevision": "87aa7dda6889fd951d202754d76daaaa1e98e26b", "files": [ { "path": ".dockerignore", @@ -1627,6 +1627,16 @@ "sha256": "ec3a53cebab1fae3a68817798d72b2831f35eda4c46fbb3d07ad84fc5eb55ab3", "bytes": 2830 }, + { + "path": "backend/tests/test_runner_dind_health_cutover_recovery.py", + "sha256": "ae087c30252ba9c864d8ddd3a3063feb0d2870fb549f747eaba956d95b3304eb", + "bytes": 13128 + }, + { + "path": "backend/tests/test_runner_dind_rollback_tail_recovery.py", + "sha256": "62538fc576fc8cae56fd2c939892bcbc1360800f3276ccdc78a493697c87380e", + "bytes": 14095 + }, { "path": "backend/tests/test_runner_failed_cutover_recovery.py", "sha256": "18ca28b3e218fc4b4d8069b7b51fb2195b95ee96386bdc5fa075ed610f168528", @@ -1649,8 +1659,8 @@ }, { "path": "backend/tests/test_runner_fleet_specialize.py", - "sha256": "a79cf1f123284ef501fbe03363d0cc1b046df519442af1e63702e6cf8201ccb9", - "bytes": 167920 + "sha256": "8d75435eb4f7b4adb52ffaff7bbb234941c382c68a4e71bdbbc3abc177aa279a", + "bytes": 186243 }, { "path": "backend/tests/test_runner_fleet_v2_execution_surface.py", diff --git a/backend/tests/test_runner_dind_health_cutover_recovery.py b/backend/tests/test_runner_dind_health_cutover_recovery.py new file mode 100644 index 0000000..be00f49 --- /dev/null +++ b/backend/tests/test_runner_dind_health_cutover_recovery.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +import os +import sys +from pathlib import Path + +import pytest + +BROKER = Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker" + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def recovery(monkeypatch): + monkeypatch.syspath_prepend(str(BROKER)) + return load("dind_health_recovery_test", BROKER / "runner_dind_health_cutover_recovery.py") + + +@pytest.fixture +def owned(recovery, monkeypatch, tmp_path): + shared = load("dind_recovery_fixture", Path(__file__).with_name("test_runner_failed_cutover_recovery.py")) + monkeypatch.setattr(recovery, "PROBE_ID", "a" * 64, raising=False) + values = shared.owned.__wrapped__(recovery, monkeypatch, tmp_path) + fleet, _, q, driver, marker, journal, _, rows, dind, events, pending = values + journal.update(stage="registrations_created", cgroupCreated=True, cgroupEnsurePending=False, + newRegistrationIds=dict(recovery.REGISTRATIONS), backupDirectory=recovery.BACKUP) + for key, profile in fleet.RUNNER_PROFILES.items(): + rows.append({"id": recovery.REGISTRATIONS[key], "name": profile.container_name, + "displayName": profile.display_name, "labels": sorted({profile.runner_label, *profile.legacy_labels}), + "disabled": True, "busy": False}) + monkeypatch.setattr(fleet, "_validate_legacy_registrations", lambda current, _: [ + {k: v for k, v in row.items() if k != "busy"} for row in current if row["id"] in (9, 22)]) + manifest = fleet.TASK_IMAGE_MANIFEST_PATH.read_bytes() + sha = hashlib.sha256(manifest).hexdigest() + monkeypatch.setattr(recovery, "MANIFEST_SHA", sha) + marker["imageManifestSha256"] = journal["imageManifestSha256"] = sha + monkeypatch.setattr(recovery, "MARKER_SHA", hashlib.sha256(json.dumps(marker).encode()).hexdigest()) + monkeypatch.setattr(recovery, "JOURNAL_SHA", hashlib.sha256(json.dumps(journal).encode()).hexdigest()) + health = ["CMD-SHELL", "docker -H tcp://127.0.0.1:2375 info >/dev/null 2>&1"] + dind["Id"] = recovery.DIND_ID + dind["HostConfig"]["RestartPolicy"] = {"Name": "unless-stopped", "MaximumRetryCount": 0} + dind["Config"]["Healthcheck"] = {"Test": health} + dind["State"]["Health"]["Status"] = "unhealthy" + model = {"services": {"runner-dind": {"healthcheck": {"test": health}}}} + monkeypatch.setattr(fleet, "_captured_compose_mapping", lambda *a, **kw: copy.deepcopy(model)) + expected = copy.deepcopy(dind) + expected["HostConfig"]["RestartPolicy"]["Name"] = "no" + def validate(item, _model): + return all(item.get(key) == expected[key] for key in ("Id", "Image", "Config", "HostConfig", "Mounts", "NetworkSettings")) + monkeypatch.setattr(fleet, "_validate_dind_outer", validate) + monkeypatch.setattr(fleet, "ADMISSION_BOOTSTRAP_RECEIPT_PATH", tmp_path / "absent") + monkeypatch.setattr(recovery, "_workspace_proof", lambda _: {"owned": "six-empty"}) + monkeypatch.setattr(recovery, "_profile_state_proof", lambda _: {"private": "hash-only"}) + monkeypatch.setattr(recovery, "_probe_absent", lambda: True) + monkeypatch.setattr(recovery, "__file__", str(recovery.BUNDLE / "runner_dind_health_cutover_recovery.py")) + def native(args, **kw): + assert args[3].startswith("DOCKER_HOST=unix://") + if args[6:] == ["info", "--format", "{{json .}}"]: + return json.dumps({"ID": "fixed-inner-engine", "ServerVersion": "28.5.2"}) + assert args[6:] == ["ps", "-a", "--no-trunc", "-q"] + return "private-secret-canary" if pending else "" + monkeypatch.setattr(driver, "_run", lambda self, *a, **kw: native(*a, **kw)) + def effect(args, **kw): + assert args == ["docker", "update", "--restart=no", recovery.DIND_ID] + assert kw == {"timeout": 20, "max_output": 16 * 1024} + events.append("exact-policy-update") + dind["HostConfig"]["RestartPolicy"]["Name"] = "no" + monkeypatch.setattr(fleet, "_run", effect) + monkeypatch.setattr(recovery, "_load", lambda: (fleet, fleet, q, {})) + return values + + +def test_readonly_double_admission_never_claims_or_updates(recovery, owned, monkeypatch): + fleet, *rest = owned + monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("read-only claim")) + monkeypatch.setattr(fleet, "_run", lambda *a, **kw: pytest.fail("read-only effect")) + assert recovery.run()["status"] == "ready" + assert rest[-2].count("guard-cas-checked") == 2 + + +def test_claim_precedes_exact_update_and_existing_locked_rollback(recovery, owned, monkeypatch): + fleet, _, _, _, _, _, _, _, dind, events, _ = owned + monkeypatch.setattr(recovery, "_claim", lambda _: events.append("claim")) + def rollback(journal, barrier): + barrier() + assert dind["HostConfig"]["RestartPolicy"]["Name"] == "no" + events.append("existing-rollback") + return True + monkeypatch.setattr(fleet, "_rollback_before_activation_locked", rollback) + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result["status"] == "rolled_back" and result["invocationConsumed"] is True + assert events.index("claim") < events.index("exact-policy-update") < events.index("existing-rollback") < events.index("maintenance-exit") + + +@pytest.mark.parametrize("change", ["cid", "image", "mount", "config", "health", "policy", "retry", "queue", "busy", + "registration", "guard", "claim", "marker", "journal", "backup", "manifest", "workspace", "profile"]) +def test_changed_incident_refuses_before_claim_or_effect(recovery, owned, monkeypatch, change): + fleet, _, _, _, marker, journal, _, rows, dind, _, pending = owned + if change in {"cid", "image"}: + dind["Id" if change == "cid" else "Image"] = "f" * 64 + elif change == "mount": + dind["Mounts"].append({"Source": "foreign"}) + elif change == "config": + dind["Config"]["foreign"] = True + elif change == "health": + dind["Config"]["Healthcheck"]["Test"] = ["NONE"] + elif change == "policy": + dind["HostConfig"]["RestartPolicy"]["Name"] = "always" + elif change == "retry": + dind["HostConfig"]["RestartPolicy"]["MaximumRetryCount"] = 1 + elif change == "queue": + pending.append({"id": 1}) + elif change == "busy": + rows[-1]["busy"] = True + elif change == "registration": + rows[-1]["id"] = 999 + elif change == "guard": + monkeypatch.setattr(fleet, "_guard_script_plan", lambda _: (_ for _ in ()).throw(RuntimeError("private-secret-canary"))) + elif change == "claim": + recovery.CLAIM.write_text("consumed") + elif change == "marker": + marker["updatedAt"] = "foreign" + elif change == "journal": + journal["stage"] = "receipt_pending" + elif change == "backup": + journal["backupDirectory"] += "-foreign" + monkeypatch.setattr(recovery, "JOURNAL_SHA", hashlib.sha256(json.dumps(journal).encode()).hexdigest()) + elif change == "manifest": + fleet.TASK_IMAGE_MANIFEST_PATH.write_text("{}") + else: + monkeypatch.setattr(recovery, "_workspace_proof" if change == "workspace" else "_profile_state_proof", + lambda _: (_ for _ in ()).throw(recovery.RecoveryRejected("profile_state_changed"))) + monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("refusal must not claim")) + monkeypatch.setattr(fleet, "_run", lambda *a, **kw: pytest.fail("refusal must not update")) + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result["status"] == "blocked" and result["effectsPerformed"] is False + assert "private-secret-canary" not in str(result) + + +def test_postclaim_policy_failure_is_unknown_and_claim_not_replayed(recovery, owned, monkeypatch): + fleet = owned[0] + monkeypatch.setattr(recovery, "_claim", lambda _: recovery.CLAIM.write_text("consumed")) + monkeypatch.setattr(fleet, "_run", lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("private-secret-canary"))) + first = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert first["status"] == "unknown" and first["invocationConsumed"] is True and first["effectsPerformed"] is None + second = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert second["status"] == "blocked" and second["reason"] == "invocation_consumed_inspect_only" + assert "private-secret-canary" not in str(first) + str(second) + + +@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="native root metadata") +@pytest.mark.parametrize("change", [None, "extra_file", "config", "id", "labels", "ephemeral", "lock_content", "lock_mode", "lock_symlink"]) +def test_native_profile_state_hashes_secret_without_exposing_it(recovery, monkeypatch, tmp_path, change): + from types import SimpleNamespace + root = tmp_path / "state" + root.mkdir(mode=0o700) + config, data = root / "config", root / "data" + config.mkdir(mode=0o700) + data.mkdir(mode=0o700) + for key in ("checks", "build"): + file = config / f"{key}.yaml" + file.write_text("rendered-config") + file.chmod(0o600) + directory = data / key + directory.mkdir(mode=0o700) + file = directory / ".runner" + file.write_text(json.dumps({"WARNING": "private metadata", "ephemeral": False, + "id": recovery.REGISTRATIONS[key], "uuid": "private-uuid", + "name": key, "token": "private-canary", "address": "https://gitea.example.com", + "labels": [key + ":docker://fixed"]})) + file.chmod(0o644) + (directory / ".runner.lock").touch(mode=0o600) + compose = root / "compose" + compose.write_bytes(b"captured") + compose.chmod(0o600) + fleet = SimpleNamespace(RUNNER_PROFILES={key: SimpleNamespace(display_name=key, runner_label=key, legacy_labels=()) for key in ("checks", "build")}, + _render_runner_config=lambda *a: b"rendered-config", + _task_image_reference=lambda *a: "fixed", RUNNER_CONFIG_ROOT=config, + _SPECIALIZED_DATA_ROOT=data, SPECIALIZED_COMPOSE_PATH=compose, + _captured_compose_model=lambda *a, **kw: b"captured", _read_journal=lambda: {}) + monkeypatch.setattr(recovery, "_parents", lambda _: None) + if change == "extra_file": + (data / "build" / "foreign").write_text("private-canary") + elif change == "config": + (config / "build.yaml").write_text("foreign") + elif change in {"id", "labels", "ephemeral"}: + path = data / "build" / ".runner" + metadata = json.loads(path.read_text()) + metadata[change] = {"id": 999, "labels": ["foreign"], "ephemeral": True}[change] + path.write_text(json.dumps(metadata)) + elif change == "lock_content": + (data / "build" / ".runner.lock").write_text("foreign") + elif change == "lock_mode": + (data / "build" / ".runner.lock").chmod(0o644) + elif change == "lock_symlink": + path = data / "build" / ".runner.lock" + path.unlink() + path.symlink_to(compose) + if change is not None: + with pytest.raises(recovery.RecoveryRejected): + recovery._profile_state_proof(fleet) + else: + proof = recovery._profile_state_proof(fleet) + assert "secret-canary" not in str(proof) and "private-canary" not in str(proof) + + +@pytest.mark.parametrize("change", ["cid", "registration", "journal", "private_work", "workspace"]) +def test_postclaim_drift_never_reaches_policy_update(recovery, owned, monkeypatch, change): + fleet, _, _, _, _, journal, _, rows, dind, _, pending = owned + def claim(_): + if change == "cid": + dind["Id"] = "f" * 64 + elif change == "registration": + rows[-1]["disabled"] = False + elif change == "journal": + journal["foreign"] = True + elif change == "private_work": + pending.append({"id": 1}) + else: + monkeypatch.setattr(recovery, "_workspace_proof", lambda _: {"changed": True}) + monkeypatch.setattr(recovery, "_claim", claim) + monkeypatch.setattr(fleet, "_run", lambda *a, **kw: pytest.fail("changed admission must not update")) + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result["status"] == "unknown" and result["effectsPerformed"] is None + assert result["invocationConsumed"] is True + + +def test_inherited_health_counter_changes_only_do_not_change_process_identity(recovery, owned): + fleet, _, q, _, _, _, _, _, dind, *_ = owned + before = recovery._dind_identity(fleet, dind, q) + dind["State"]["Health"].update(FailingStreak=99, Log=[{"Output": "secret-canary"}]) + assert recovery._dind_identity(fleet, dind, q) == before + dind["State"]["Pid"] += 1 + assert recovery._dind_identity(fleet, dind, q) != before diff --git a/backend/tests/test_runner_dind_rollback_tail_recovery.py b/backend/tests/test_runner_dind_rollback_tail_recovery.py new file mode 100644 index 0000000..ccbadd0 --- /dev/null +++ b/backend/tests/test_runner_dind_rollback_tail_recovery.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import copy +import hashlib +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +BROKER = Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker" + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def recovery(monkeypatch): + monkeypatch.syspath_prepend(str(BROKER)) + return load("dind_tail_test", BROKER / "runner_dind_rollback_tail_recovery.py") + + +@pytest.fixture +def owned(recovery, monkeypatch, tmp_path): + shared = load("tail_shared", Path(__file__).with_name("test_runner_failed_cutover_recovery.py")) + monkeypatch.setattr(recovery, "PROBE_ID", "a" * 64, raising=False) + values = shared.owned.__wrapped__(recovery, monkeypatch, tmp_path) + fleet, _, q, driver, marker, journal, _, rows, dind, events, pending = values + journal.update(stage="registrations_created", cgroupCreated=True, cgroupEnsurePending=False, + newRegistrationIds=dict(recovery.REGISTRATIONS), backupDirectory=recovery.BACKUP, + cgroupRootSubtree=list(recovery.CONTROLLERS), originalContainers=[]) + for row in rows: + row.update(displayName=None, status="online") + rows.extend([{"id": rid, "name": name, "labels": [label], "disabled": False, "busy": False, + "displayName": None, "status": "online"} + for rid, name, label in ((13, "itworx-unraid-deploy", "unraid-deploy"), + (14, "itworx-windows11-native", "windows-native"))]) + fleet.TASK_IMAGE_MANIFEST_PATH.unlink() + fleet.TASK_IMAGE_ROOT.rmdir() + monkeypatch.setattr(fleet, "ADMISSION_BOOTSTRAP_RECEIPT_PATH", tmp_path / "absent") + dind["Id"] = recovery.RUNTIME_IDS[fleet.DIND_NAME] + legacy = {name: {**copy.deepcopy(dind), "Id": cid} for name, cid in recovery.RUNTIME_IDS.items() + if name != fleet.DIND_NAME} + monkeypatch.setattr(fleet, "_validate_legacy_containers", lambda _: (copy.deepcopy(legacy), [])) + monkeypatch.setattr(fleet, "_inspect", lambda name: copy.deepcopy(dind) if name == fleet.DIND_NAME else copy.deepcopy(legacy.get(name))) + family = {"exists": True, "controllers": list(recovery.CONTROLLERS), + "rootSubtreeControllers": list(recovery.CONTROLLERS), "familySubtreeControllers": list(recovery.CONTROLLERS), + "groupCount": 1, "hasProcesses": False, "familyCpuMax": "max 100000", "familyMemoryMax": "max"} + monkeypatch.setattr(fleet, "_cgroup_family_snapshot", lambda: copy.deepcopy(family)) + monkeypatch.setattr(recovery, "_probe_absent", lambda: True) + monkeypatch.setattr(recovery, "_prior_receipt", lambda: b"pinned") + monkeypatch.setattr(recovery, "MARKER_SHA", hashlib.sha256(json.dumps(marker).encode()).hexdigest()) + monkeypatch.setattr(recovery, "JOURNAL_SHA", hashlib.sha256(json.dumps(journal).encode()).hexdigest()) + monkeypatch.setattr(recovery, "__file__", str(recovery.BUNDLE / "runner_dind_rollback_tail_recovery.py")) + def legacy_private_inventory(self, args, **kwargs): + assert args == ["docker", "exec", "-e", "DOCKER_HOST=tcp://127.0.0.1:2375", + fleet.DIND_NAME, "docker", "ps", "-a", "--no-trunc", "-q"] + assert kwargs == {"timeout": 20, "max_output": 128 * 1024} + return "" + monkeypatch.setattr(driver, "_run", legacy_private_inventory) + monkeypatch.setattr(recovery, "_load", lambda: (fleet, fleet, q, {})) + def ensure(*, allow_existing): + assert allow_existing is True + events.append("ensure") + family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3)) + return False + monkeypatch.setattr(fleet, "_ensure_cgroup_family", ensure) + return values, family, legacy + + +def test_readonly_discloses_probe_and_never_claims_or_ensures(recovery, owned, monkeypatch): + values, _, _ = owned + monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("claim")) + monkeypatch.setattr(values[0], "_ensure_cgroup_family", lambda **kw: pytest.fail("ensure")) + result = recovery.run() + assert result["status"] == "ready" + assert result["diagnosticProbeLifecycle"] is True + assert result["recoveryEffectsPerformed"] is False + assert values[9].count("guard-cas-checked") == 2 + + +def test_claim_ensure_native_rollback_all_under_both_locks(recovery, owned, monkeypatch): + values, family, _ = owned + fleet, _, _, _, _, _, _, _, _, events, _ = values + monkeypatch.setattr(recovery, "_claim", lambda _: events.append("claim")) + def tail(journal, barrier): + barrier() + assert events.index("claim") < events.index("ensure") + assert "maintenance-exit" not in events and "migration-exit" not in events + assert family["familyCpuMax"] == "800000 100000" + events.append("native-tail") + return True + monkeypatch.setattr(fleet, "_rollback_before_activation_locked", tail) + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result["status"] == "rolled_back" and result["invocationConsumed"] is True + assert events.index("native-tail") < events.index("maintenance-exit") < events.index("migration-exit") + + +@pytest.mark.parametrize("mutation", ["cid", "unhealthy", "busy", "extra-registration", "specialized-file", "guards", "private-work", + "family-process", "family-child", "family-cpu", "family-memory", "family-subtree", "journal", "claim"]) +def test_mutations_rejected_before_claim_and_effect(recovery, owned, monkeypatch, mutation): + values, family, legacy = owned + fleet, _, _, _, _, journal, _, rows, dind, events, pending = values + if mutation == "cid": + dind["Id"] = "f" * 64 + elif mutation == "unhealthy": + dind["State"]["Health"]["Status"] = "unhealthy" + elif mutation == "busy": + rows[0]["busy"] = True + elif mutation == "extra-registration": + rows.append({"id": 23, "busy": False}) + elif mutation == "specialized-file": + fleet.RUNNER_CONFIG_ROOT.write_text("private-canary") + elif mutation == "guards": + monkeypatch.setattr(fleet, "_guard_state_unchanged", lambda _: False) + elif mutation == "private-work": + monkeypatch.setattr(values[3], "_run", lambda *a, **kw: "private-canary") + elif mutation == "family-process": + family["hasProcesses"] = True + elif mutation == "family-child": + family["groupCount"] = 2 + elif mutation == "family-cpu": + family["familyCpuMax"] = "800000 100000" + elif mutation == "family-memory": + family["familyMemoryMax"] = "1" + elif mutation == "family-subtree": + family["familySubtreeControllers"].pop() + elif mutation == "journal": + journal["backupDirectory"] += "foreign" + elif mutation == "claim": + recovery.CLAIM.write_text("consumed") + monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("claim")) + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result["status"] == "blocked" + assert result["recoveryEffectsPerformed"] is False + assert "ensure" not in events and "private-canary" not in json.dumps(result) + + +def test_admission_drift_refuses_before_claim(recovery, owned, monkeypatch): + original = recovery._admit + calls = [] + def drift(*args, **kwargs): + journal, proof = original(*args, **kwargs) + calls.append(1) + return journal, {**proof, "runtimeDigest": str(len(calls))} + monkeypatch.setattr(recovery, "_admit", drift) + monkeypatch.setattr(recovery, "_claim", lambda _: pytest.fail("claim")) + assert recovery.run(execute=True, confirmation=recovery.CONFIRMATION)["reason"] == "admission_changed" + + +def test_ensure_failure_consumes_and_reports_unknown_redacted(recovery, owned, monkeypatch): + monkeypatch.setattr(recovery, "_claim", lambda _: None) + def fail(**kwargs): + raise RuntimeError("token=private-canary") + monkeypatch.setattr(owned[0][0], "_ensure_cgroup_family", fail) + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result["status"] == "unknown" and result["invocationConsumed"] is True + assert result["recoveryEffectsPerformed"] is None and "private-canary" not in json.dumps(result) + + +def test_prior_receipt_closed_schema_and_raw_hash(recovery, monkeypatch): + raw = json.dumps({"schema": "appops.failed-cutover-recovery/v1", "phase": "consumed", + "incidentJobId": recovery.INCIDENT_JOB, "incidentBackupDirectory": recovery.BACKUP, + "sourceRevision": recovery.ORIGIN, "sourceTree": recovery.TREE, "artifactSha256": recovery.ARTIFACT_SHA, + "candidateSha256": recovery.FLEET_SHA, "proof": recovery.PRIOR_PROOF}).encode() + monkeypatch.setattr(recovery, "_read", lambda *a, **kw: raw) + monkeypatch.setattr(recovery, "PRIOR_SHA", hashlib.sha256(raw).hexdigest()) + assert recovery._prior_receipt() == raw + for changed in ({"extra": "private-canary"}, {"incidentJobId": "other"}, {"proof": {}}): + altered = json.dumps({**json.loads(raw), **changed}).encode() + monkeypatch.setattr(recovery, "_read", lambda *a, **kw: altered) + monkeypatch.setattr(recovery, "PRIOR_SHA", hashlib.sha256(altered).hexdigest()) + with pytest.raises(recovery.RecoveryRejected, match="prior_receipt_changed"): + recovery._prior_receipt() + + +def test_postclaim_runtime_change_refuses_before_ensure(recovery, owned, monkeypatch): + values, _, _ = owned + def consume(_): + values[8]["Id"] = "f" * 64 + monkeypatch.setattr(recovery, "_claim", consume) + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result["status"] == "unknown" and result["invocationConsumed"] is True + assert "ensure" not in values[9] + + +def test_once_claim_cannot_be_replayed(recovery, owned, monkeypatch): + values, _, _ = owned + def consume(_): + recovery.CLAIM.write_text("consumed") + monkeypatch.setattr(recovery, "_claim", consume) + monkeypatch.setattr(values[0], "_rollback_before_activation_locked", lambda journal, barrier: False) + assert recovery.run(execute=True, confirmation=recovery.CONFIRMATION)["status"] == "unknown" + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result["status"] == "blocked" and result["reason"] == "invocation_consumed_inspect_only" + + +def test_claim_write_uncertainty_is_unknown(recovery, owned, monkeypatch): + def uncertain(_): + raise OSError("secret-canary") + monkeypatch.setattr(recovery, "_claim", uncertain) + result = recovery.run(execute=True, confirmation=recovery.CONFIRMATION) + assert result["status"] == "unknown" and result["invocationConsumed"] is None + assert result["recoveryEffectsPerformed"] is None and "secret-canary" not in json.dumps(result) + assert "ensure" not in owned[0][9] + + +def test_all_runtime_ids_are_full_immutable_ids(recovery): + assert all(len(cid) == 64 and set(cid) <= set("0123456789abcdef") for cid in recovery.RUNTIME_IDS.values()) + assert recovery.CANDIDATE_SHA == recovery.FLEET_SHA + assert recovery.CLAIM != recovery.PRIOR_CLAIM + + +@pytest.mark.skipif(sys.platform == "win32", reason="native POSIX ownership required") +def test_native_existing_migration_lock_rejects_missing_and_retains_lock(recovery): + import os + import tempfile + from types import SimpleNamespace + if os.geteuid() != 0: + pytest.skip("native root ownership required") + parent = Path("/data") if Path("/data").is_dir() else Path("/root") + recovery._parents(parent / "native-test-root") + with tempfile.TemporaryDirectory(prefix="appops-tail-lock-test-", dir=parent) as temporary: + root = Path(temporary) + root.chmod(0o700) + fleet = SimpleNamespace(MIGRATION_JOURNAL_PATH=root / "journal.json") + with pytest.raises(FileNotFoundError): + with recovery._migration_existing_lock(fleet): + pytest.fail("missing lock admitted") + lock = root / "specialization.lock" + lock.write_bytes(b"") + lock.chmod(0o600) + with recovery._migration_existing_lock(fleet) as barrier: + barrier() + with pytest.raises(recovery.RecoveryRejected, match="migration_lock_busy"): + with recovery._migration_existing_lock(fleet): + pytest.fail("second lock acquired") + assert lock.exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="native POSIX ownership required") +def test_native_untrusted_parent_refused_before_missing_lock(recovery): + import os + import tempfile + from types import SimpleNamespace + if os.geteuid() != 0: + pytest.skip("native root ownership required") + parent = Path("/data") if Path("/data").is_dir() else Path("/root") + recovery._parents(parent / "native-test-root") + with tempfile.TemporaryDirectory(prefix="appops-tail-untrusted-test-", dir=parent) as temporary: + root = Path(temporary) + root.chmod(0o777) + fleet = SimpleNamespace(MIGRATION_JOURNAL_PATH=root / "journal.json") + with pytest.raises(recovery.RecoveryRejected, match="root_path_untrusted"): + with recovery._migration_existing_lock(fleet): + pytest.fail("untrusted parent admitted") + + + +def test_tail_uses_captured_legacy_tcp_not_specialized_unix(recovery, owned, monkeypatch): + values, _, _ = owned + fleet, _, _, driver, *_ = values + seen = [] + def inventory(self, args, **kwargs): + assert args[3] == "DOCKER_HOST=tcp://127.0.0.1:2375" + assert "unix://" not in " ".join(args) + seen.append(args[3]) + return "" + monkeypatch.setattr(driver, "_run", inventory) + monkeypatch.setattr(fleet, "_DIND_DOCKER_HOST", "unix:///run/user/1000/docker.sock") + assert recovery.run()["status"] == "ready" + assert seen == ["DOCKER_HOST=tcp://127.0.0.1:2375"] * 2 diff --git a/backend/tests/test_runner_fleet_specialize.py b/backend/tests/test_runner_fleet_specialize.py index 05541b1..ce42dfe 100644 --- a/backend/tests/test_runner_fleet_specialize.py +++ b/backend/tests/test_runner_fleet_specialize.py @@ -1181,6 +1181,66 @@ def test_specialized_compose_validation_accepts_real_compose_normalization( assert specialize._validate_composition(b"normalized candidate") +def test_specialized_dind_healthcheck_replaces_captured_tcp_endpoint(specialize, monkeypatch): + base = _base_v1_compose_model(specialize) + base["services"]["runner-dind"]["healthcheck"] = { + "test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"], + "interval": "15s", "timeout": "5s", "retries": 4, "start_period": "45s", + } + rendered = json.loads(specialize._render_specialized_compose(base)) + check = rendered["services"]["runner-dind"]["healthcheck"] + assert check == { + "test": ["CMD-SHELL", "DOCKER_HOST=unix:///run/itworx-ci-control/docker.sock docker info >/dev/null 2>&1"], + "interval": "15s", "timeout": "5s", "retries": 4, "start_period": "45s", + } + assert base["services"]["runner-dind"]["healthcheck"]["test"][1].startswith("DOCKER_HOST=tcp:") + monkeypatch.setattr(specialize, "_compose_model", lambda _raw: rendered) + assert specialize._validate_composition(b"reviewed model") + + +@pytest.mark.parametrize("check", [None, {"disable": True}, {"test": ["NONE"]}, + {"disable": True, "test": ["CMD-SHELL", "DOCKER_HOST=unix:///run/itworx-ci-control/docker.sock docker info >/dev/null 2>&1"]}, + {"test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"]}, + {"test": ["CMD-SHELL", "docker info >/dev/null 2>&1"]}, + {"test": ["CMD-SHELL", "DOCKER_HOST=unix:///var/run/docker.sock docker info >/dev/null 2>&1"]}, +]) +def test_specialized_compose_rejects_healthcheck_endpoint_drift(specialize, monkeypatch, check): + model = json.loads(specialize._render_specialized_compose(_base_v1_compose_model(specialize))) + model["services"]["runner-dind"]["healthcheck"] = check + monkeypatch.setattr(specialize, "_compose_model", lambda _raw: model) + with pytest.raises(specialize.RunnerFleetSpecializeError, match="dind_compose_invalid"): + specialize._validate_composition(b"foreign healthcheck") + + +@pytest.mark.parametrize("check", [None, {"Test": ["NONE"]}, + {"Test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"]}, + {"Test": ["CMD-SHELL", "docker info >/dev/null 2>&1"]}, + {"Test": ["CMD-SHELL", "DOCKER_HOST=unix:///var/run/docker.sock docker info >/dev/null 2>&1"]}, +]) +def test_specialized_dind_runtime_rejects_healthcheck_drift(specialize, check): + base = _base_v1_compose_model(specialize) + model = json.loads(specialize._render_specialized_compose(base)) + service = model["services"]["runner-dind"] + item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID, + running=True, restart_policy="no") + item["Config"].update(User="0", Cmd=service["command"], + Env=[f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"], + Healthcheck={"Test": service["healthcheck"]["test"]}) + item["HostConfig"].update(CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={}) + item["Mounts"] = [{"Type": "bind", "Source": mount["source"], + "Destination": mount["target"], "RW": not mount["read_only"]} + for mount in service["volumes"]] + assert specialize._validate_dind_outer(item, model) + changed = copy.deepcopy(item) + changed["Config"]["Healthcheck"] = check + assert specialize._validate_dind_outer(changed, model) is False + changed_model = copy.deepcopy(model) + changed_model["services"]["runner-dind"]["healthcheck"] = { + "test": ["CMD-SHELL", "DOCKER_HOST=tcp://127.0.0.1:2375 docker info >/dev/null 2>&1"], + } + assert specialize._validate_dind_outer(item, changed_model) is False + + def test_fixed_legacy_dind_runtime_preserves_rootless_storage_and_anonymous_volume(specialize): base = _base_v1_compose_model(specialize) item = _legacy_dind_fixture(specialize, base, image_id=specialize._DIND_IMAGE_ID, @@ -1208,6 +1268,7 @@ def test_new_dind_requires_actual_resource_support_and_fixed_shared_workspaces( image_id=specialize._DIND_IMAGE_ID, running=True, restart_policy="no") item["Config"].update(User="0", Cmd=service["command"]) item["Config"]["Env"] = [f"DOCKER_HOST={specialize._DIND_DOCKER_HOST}"] + item["Config"]["Healthcheck"] = {"Test": service["healthcheck"]["test"]} item["HostConfig"].update( CgroupParent="itworx-ci", PidsLimit=2048, PortBindings={} ) @@ -1358,6 +1419,123 @@ def test_runner_guard_patch_fences_legacy_and_preserves_exact_old_shape(speciali specialize._patch_runner_guard(source.replace("unexpected=0\n", ""), "state", "allow") +def _resource_guard_source(kind): + prefix = '''#!/bin/bash +set -eu +RUNNER="legacy-primary" +SECONDARY_RUNNER="legacy-secondary" +RUNNER_DIND="private-dind" +DEPLOY_RUNNER="deploy-runner" +RUNNER_PAUSE_FILE="/nonexistent-test-pause" +RUNNER_MARKER="/nonexistent-test-marker" +controller() { printf "controller:%s\\n" "$*" >> "$EVENTS"; if [ "$1" = --state ]; then printf "%s" "$GUARD_STATE"; fi; } +timeout() { shift; "$@"; } +docker() { printf "docker:%s\\n" "$*" >> "$EVENTS"; if [ "$1" = inspect ]; then printf legacy; fi; } +enforce_limits() { printf "enforce:%s\\n" "$*" >> "$EVENTS"; } +''' + if kind == "runner": + return prefix + '''# The sandbox is the aggregate resource envelope for all jobs and services. +if [ -n "$(timeout 10 docker inspect -f '{{.Id}}' "$RUNNER_DIND" 2>/dev/null || true)" ]; then + dind_limits="$(timeout 10 docker inspect -f '{{.HostConfig.NanoCpus}}' "$RUNNER_DIND")" + if [ "$dind_limits" != "expected" ]; then + timeout 20 docker update --restart unless-stopped "$RUNNER_DIND" + fi + if [ "$dind_limits" = "legacy" ]; then + timeout 20 docker update --cpuset-cpus 0-5 "$RUNNER_DIND" + fi +fi +enforce_limits "$DEPLOY_RUNNER" +stack_recovery=0 +: +unexpected=0 +for name in "$DEPLOY_RUNNER"; do + [ "$name" = "$DEPLOY_RUNNER" ] && continue +done +''' + return prefix + '''# ITWORX_RUNNER_CAPACITY_V2_BEGIN +: +# ITWORX_RUNNER_CAPACITY_V2_END +if [ -e "$RUNNER_PAUSE_FILE" ] || [ -e "$RUNNER_MARKER" ]; then + enforce_limits "$RUNNER" no + enforce_limits "$SECONDARY_RUNNER" no + enforce_limits "$RUNNER_DIND" no +else + enforce_limits "$RUNNER" unless-stopped + enforce_limits "$SECONDARY_RUNNER" unless-stopped + enforce_limits "$RUNNER_DIND" unless-stopped +fi +# The deploy runner is deliberately tiny and is required for deployment and +enforce_limits "$DEPLOY_RUNNER" +enforce_limits tdarr +enforce_limits controller +enforce_limits containment-pressure +runner_healthy=1 +if [ "$runner_healthy" -eq 0 ]; then + if [ "$sandbox_healthy" -eq 1 ] && \\ + timeout 20 docker start "$RUNNER" "$SECONDARY_RUNNER" >/dev/null 2>&1; then + : + fi +fi +''' + + +@pytest.mark.parametrize("kind", ["runner", "host"]) +def test_resource_writer_is_gated_before_any_legacy_mutation(specialize, kind): + source = _resource_guard_source(kind) + if kind == "runner": + patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow") + writer = "# The sandbox is the aggregate resource envelope" + else: + patched = specialize._patch_host_guard(source, "controller --state") + writer = 'if [ -e "$RUNNER_PAUSE_FILE" ]' + assert patched.index('specialization_state="$(controller --state') < patched.index(writer) + assert 'if [ "$specialization_state" = "legacy" ]; then\n ' + writer in patched + assert 'enforce_limits "$DEPLOY_RUNNER"' in patched + + +@pytest.mark.skipif(os.name != "posix", reason="requires actual Bash stub execution") +@pytest.mark.parametrize("kind", ["runner", "host"]) +@pytest.mark.parametrize("state", ["legacy", "blocked", "active"]) +def test_guard_bash_skips_only_legacy_resource_writer_in_cutover(specialize, tmp_path, kind, state): + source = _resource_guard_source(kind) + if kind == "runner": + patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow") + else: + patched = specialize._patch_host_guard(source, "controller --state") + events = tmp_path / "events" + result = subprocess.run(["bash", "-c", patched], capture_output=True, check=False, timeout=5, + env={"PATH": "/usr/bin:/bin", "EVENTS": str(events), "GUARD_STATE": state}) + assert result.returncode == 0 + rows = events.read_text().splitlines() + assert rows[0] == "controller:--state" + legacy_writes = [row for row in rows if row.startswith("docker:update") + or row.startswith("enforce:legacy-") or row.startswith("enforce:private-dind")] + assert bool(legacy_writes) is (state == "legacy") + assert "enforce:deploy-runner" in rows + if kind == "host": + assert {"enforce:tdarr", "enforce:controller", "enforce:containment-pressure"}.issubset(rows) + + +@pytest.mark.parametrize("kind", ["runner", "host"]) +def test_resource_writer_shape_drift_is_rejected(specialize, kind): + source = _resource_guard_source(kind) + anchor = ("# The sandbox is the aggregate resource envelope for all jobs and services.\n" + if kind == "runner" else 'if [ -e "$RUNNER_PAUSE_FILE" ] || [ -e "$RUNNER_MARKER" ]; then\n') + patch = (lambda text: specialize._patch_runner_guard(text, "controller --state", "controller --allow")) if kind == "runner" else ( + lambda text: specialize._patch_host_guard(text, "controller --state")) + with pytest.raises(specialize.RunnerFleetSpecializeError, match="guard_shape_invalid"): + patch(source.replace(anchor, anchor + anchor, 1)) + + +def test_historical_orphan_transform_does_not_acquire_new_resource_gate(specialize): + source = _resource_guard_source("runner") + historical = specialize._patch_runner_guard_historical(source, "controller --state", "controller --allow") + patched = specialize._patch_runner_guard(source, "controller --state", "controller --allow") + writer = historical.index("# The sandbox is the aggregate resource envelope") + assert 'specialization_state="$(controller --state' not in historical[:writer] + assert 'specialization_state="$(controller --state' in patched[:patched.index("# The sandbox is the aggregate resource envelope")] + + def test_execute_rejects_modified_script_before_starting_shell(specialize, monkeypatch): script = (ROOT / "scripts" / "upgrade_ci_runner_fleet_specialized.sh").read_bytes() monkeypatch.setattr(specialize, "_APPROVED_SCRIPT_SHA256", hashlib.sha256(script).hexdigest()) @@ -1621,7 +1799,9 @@ def test_rollback_retains_fences_until_coherent_lifecycle_recovery(specialize, g monkeypatch.setattr(specialize, "MIGRATION_JOURNAL_PATH", durable) journal.update(baseComposeSha256=hashlib.sha256(compose.read_bytes()).hexdigest(), originalDind={"present": False, "running": False, "restartPolicy": "no", "imageId": None}, - originalContainers=[], originalRegistrations=[], cgroupCreated=True, cgroupRootSubtree=[]) + originalContainers=[], originalRegistrations=[], cgroupCreated=True, + cgroupFamilyInitiallyPresent=False, cgroupRootSubtree=[]) + monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False}) monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose") monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) @@ -1656,6 +1836,174 @@ def test_rollback_retains_fences_until_coherent_lifecycle_recovery(specialize, g assert not marker.exists() and not durable.exists() +@pytest.fixture +def rollback_empty_family(specialize, tmp_path, monkeypatch): + compose, marker, durable = tmp_path / "compose", tmp_path / "marker", tmp_path / "journal" + compose.write_bytes(b"fixed compose") + marker.write_bytes(b"owned marker") + durable.write_bytes(b"owned journal") + for name, value in (("COMPOSE_PATH", compose), ("SPECIALIZED_STATE_PATH", marker), + ("MIGRATION_JOURNAL_PATH", durable)): + monkeypatch.setattr(specialize, name, value) + journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(), + "originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None}, + "originalContainers": [], "originalRegistrations": [], "guardFiles": [], + "cgroupCreated": True, "cgroupEnsurePending": False, "cgroupFamilyInitiallyPresent": False, + "cgroupRootSubtree": ["cpu", "cpuset", "hugetlb", "io", "memory", "pids"]} + family = {"exists": True, "groupCount": 1, "hasProcesses": False, + "familyCpuMax": "max 100000", "familyMemoryMax": "max", + "familySubtreeControllers": list(journal["cgroupRootSubtree"]), + "rootSubtreeControllers": list(journal["cgroupRootSubtree"])} + rebound = {"created": False, "rootSubtreeBefore": list(journal["cgroupRootSubtree"])} + events = [] + @contextmanager + def maintenance(): + yield lambda: events.append("barrier") + monkeypatch.setattr(specialize, "_rollback_maintenance", maintenance) + monkeypatch.setattr(specialize, "_guard_state_unchanged", lambda _journal: True) + monkeypatch.setattr(specialize, "_cleanup_new_containers", lambda _journal: None) + monkeypatch.setattr(specialize, "_captured_compose_model", lambda *_args, **_kwargs: b"fixed compose") + monkeypatch.setattr(specialize, "_captured_compose_mapping", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(specialize, "_legacy_services_for_restore", lambda _journal: []) + monkeypatch.setattr(specialize, "_inspect", lambda _name: None) + monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: events.append("snapshot") or family) + def ensure(**kwargs): + assert kwargs == {"allow_existing": True} + events.append("ensure") + return rebound + def rollback(**kwargs): + assert kwargs == {"created": True, "initial_controllers": tuple(journal["cgroupRootSubtree"])} + events.append("rollback") + return {"removed": True, "familyExists": False} + monkeypatch.setattr(specialize, "_ensure_cgroup_family_details", ensure) + monkeypatch.setattr(specialize, "rollback_host_cgroup_family", rollback) + return journal, family, rebound, events, marker, durable + + +@pytest.mark.parametrize("pending", [False, True]) +@pytest.mark.parametrize("cpu", ["max", "max 100000"]) +def test_rollback_rebinds_only_owned_empty_default_family_before_strict_removal( + specialize, rollback_empty_family, pending, cpu, +): + journal, family, _rebound, events, marker, durable = rollback_empty_family + journal.update(cgroupCreated=not pending, cgroupEnsurePending=pending) + family["familyCpuMax"] = cpu + original = copy.deepcopy(journal) + assert specialize._rollback_before_activation(journal) is True + assert journal == original + snapshot = events.index("snapshot") + assert events[snapshot:snapshot + 6] == ["snapshot", "barrier", "ensure", "barrier", "barrier", "rollback"] + assert not marker.exists() and not durable.exists() + + +@pytest.mark.parametrize("absent", [False, True]) +def test_rollback_never_ensures_absent_or_already_bounded_family(specialize, rollback_empty_family, absent): + journal, family, _rebound, events, _marker, _durable = rollback_empty_family + if absent: + family.clear() + family["exists"] = False + else: + family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3)) + assert specialize._rollback_before_activation(journal) is True + assert "ensure" not in events and "rollback" in events + + +@pytest.mark.parametrize("mutation", ["initially_present", "initially_unknown", "foreign_cpu", "foreign_memory", + "partial_default", "processes", "unknown_processes", "children", "bool_count", "newly_created", "root_changed"]) +def test_empty_family_rebind_refusal_retains_marker_and_journal(specialize, rollback_empty_family, mutation): + journal, family, rebound, events, marker, durable = rollback_empty_family + if mutation == "initially_present": + journal["cgroupFamilyInitiallyPresent"] = True + elif mutation == "initially_unknown": + journal.pop("cgroupFamilyInitiallyPresent") + elif mutation == "foreign_cpu": + family["familyCpuMax"] = "400000 100000" + elif mutation == "foreign_memory": + family["familyMemoryMax"] = str(8 * 1024**3) + elif mutation == "partial_default": + family["familyCpuMax"] = "800000 100000" + elif mutation == "processes": + family["hasProcesses"] = True + elif mutation == "unknown_processes": + family.pop("hasProcesses") + elif mutation == "children": + family["groupCount"] = 2 + elif mutation == "bool_count": + family["groupCount"] = True + elif mutation == "newly_created": + rebound["created"] = True + else: + rebound["rootSubtreeBefore"] = ["cpu", "memory", "pids"] + assert specialize._rollback_before_activation(journal) is False + assert "rollback" not in events + assert ("ensure" in events) is (mutation in {"newly_created", "root_changed"}) + assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" + + +def test_rebind_barrier_loss_prevents_cgroup_removal_and_keeps_durable_state( + specialize, rollback_empty_family, monkeypatch, +): + journal, _family, _rebound, events, marker, durable = rollback_empty_family + @contextmanager + def maintenance(): + def barrier(): + if events and events[-1] == "ensure": + raise RuntimeError("maintenance identity changed") + events.append("barrier") + yield barrier + monkeypatch.setattr(specialize, "_rollback_maintenance", maintenance) + assert specialize._rollback_before_activation(journal) is False + assert "ensure" in events and "rollback" not in events + assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" + + +@pytest.mark.parametrize("bounded", [False, True]) +def test_family_rollback_accepts_only_required_controllers_for_minimal_initial_root( + specialize, rollback_empty_family, bounded, +): + journal, family, rebound, events, _marker, _durable = rollback_empty_family + initial = ["cpu", "memory", "pids"] + journal["cgroupRootSubtree"] = list(initial) + family["rootSubtreeControllers"] = list(initial) + family["familySubtreeControllers"] = list(initial) + rebound["rootSubtreeBefore"] = list(initial) + if bounded: + family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3)) + assert specialize._rollback_before_activation(journal) is True + assert ("ensure" in events) is not bounded + assert "rollback" in events + + +@pytest.mark.parametrize("bounded", [False, True]) +@pytest.mark.parametrize("mutation", ["missing", "null", "string", "duplicate", "unknown", "nonstring", + "required_missing", "unowned_extra"]) +def test_family_subtree_drift_refuses_before_ensure_or_removal( + specialize, rollback_empty_family, bounded, mutation, +): + journal, family, _rebound, events, marker, durable = rollback_empty_family + if bounded: + family.update(familyCpuMax="800000 100000", familyMemoryMax=str(16 * 1024**3)) + if mutation == "missing": + family.pop("familySubtreeControllers") + elif mutation == "null": + family["familySubtreeControllers"] = None + elif mutation == "string": + family["familySubtreeControllers"] = "cpu memory pids" + elif mutation == "duplicate": + family["familySubtreeControllers"].append("cpu") + elif mutation == "unknown": + family["familySubtreeControllers"].append("foreign-controller") + elif mutation == "nonstring": + family["familySubtreeControllers"].append(["cpu"]) + elif mutation == "required_missing": + family["familySubtreeControllers"].remove("pids") + else: + journal["cgroupRootSubtree"] = ["cpu", "memory", "pids"] + assert specialize._rollback_before_activation(journal) is False + assert "ensure" not in events and "rollback" not in events + assert marker.read_bytes() == b"owned marker" and durable.read_bytes() == b"owned journal" + + def test_rollback_maintenance_refusal_keeps_all_state_without_any_restore(specialize, tmp_path, monkeypatch): marker, journal = tmp_path / "marker", tmp_path / "journal" marker.write_bytes(b"owned marker") @@ -1727,7 +2075,9 @@ def test_real_common_maintenance_lock_excludes_process_until_whole_rollback_fini monkeypatch.setattr(specialize, "rollback_host_cgroup_family", cgroup) journal = {"baseComposeSha256": hashlib.sha256(compose.read_bytes()).hexdigest(), "guardFiles": [{"owned": True}], "originalDind": {"present": False, "running": False, "restartPolicy": "no", "imageId": None}, - "originalContainers": [], "originalRegistrations": [], "cgroupCreated": True, "cgroupRootSubtree": []} + "originalContainers": [], "originalRegistrations": [], "cgroupCreated": True, + "cgroupFamilyInitiallyPresent": False, "cgroupRootSubtree": []} + monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False}) assert specialize._rollback_before_activation(journal) is True assert events == ["cleanup", "cgroup", "guards-restored"] assert not marker.exists() and not durable.exists() @@ -3059,6 +3409,7 @@ def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup( "guardFiles": [], "cgroupCreated": True, "cgroupEnsurePending": False, + "cgroupFamilyInitiallyPresent": False, "cgroupRootSubtree": ["cpu", "memory", "pids"], "backupDirectory": str(tmp_path / "specialized-test"), } @@ -3118,6 +3469,7 @@ def test_migration_failure_restores_captured_dind_state_before_cgroup_cleanup( monkeypatch.setattr(specialize, "_run", run) monkeypatch.setattr(specialize, "_run_compose", run_compose) monkeypatch.setattr(specialize, "_runner_rows", lambda: []) + monkeypatch.setattr(specialize, "_cgroup_family_snapshot", lambda: {"exists": False}) monkeypatch.setattr( specialize, "rollback_host_cgroup_family",