diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index 41b1de2..ebeafe9 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `2ff7105f7d3066d6c38825540340505233836a91`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `af7a7bd69966cb9939bf331cab169e2ec61e8f26`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index d8c9213..cc1e719 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "2ff7105f7d3066d6c38825540340505233836a91", + "sourceRevision": "af7a7bd69966cb9939bf331cab169e2ec61e8f26", "files": [ { "path": ".dockerignore", @@ -1659,8 +1659,8 @@ }, { "path": "backend/tests/test_runner_fleet_specialize.py", - "sha256": "6b4da6ed984a8c6741b1232f54f977521631ab6a3cc030284c1ea5c42ff99c47", - "bytes": 201815 + "sha256": "e3134a8ad84ab23fa2fef3d377d25b891cc4d854b8bdad7f271048404fbfc249", + "bytes": 205429 }, { "path": "backend/tests/test_runner_fleet_v2_execution_surface.py", @@ -1684,8 +1684,8 @@ }, { "path": "backend/tests/test_runner_resource_envelope.py", - "sha256": "6c3405c25d965a6c8e492385c03c465204f4723d85e9b81ed7320ffc2ecc0c1a", - "bytes": 98090 + "sha256": "1af414118af3be0dd4466fffe6a707249a84fc6cbeaf45e1b8aba9c739746241", + "bytes": 99527 }, { "path": "backend/tests/test_runner_scheduler_liveness_recovery.py", diff --git a/backend/tests/test_runner_fleet_specialize.py b/backend/tests/test_runner_fleet_specialize.py index bc40d3a..cd8ae56 100644 --- a/backend/tests/test_runner_fleet_specialize.py +++ b/backend/tests/test_runner_fleet_specialize.py @@ -18,6 +18,87 @@ from types import SimpleNamespace import pytest +@pytest.mark.parametrize("reason", [ + "resource_cgroup_process_mapping_invalid", "resource_cgroup_probe_identity_invalid", + "resource_cgroup_attestation_failed", "resource_cgroup_v2_unavailable", + "Bearer secret-value", None, +]) +def test_build_resource_refusal_keeps_only_allowlisted_private_cause(specialize, monkeypatch, reason): + monkeypatch.setattr(specialize, "_docker_outer_inventory", lambda: ([], [])) + monkeypatch.setattr(specialize, "attest_build_probe_workload", lambda *_a, **_k: { + "ready": False, "reason": reason, + }) + with pytest.raises(specialize.RunnerFleetSpecializeError) as error: + specialize._build_probe_resource_attestation({"containerId": "a" * 64}, {"State": {"Pid": 77}}) + assert str(error.value) == "runner_fleet_specialization_build_probe_cgroup_untrusted" + if reason in specialize._RESOURCE_FAILURE_CODES: + assert str(error.value.__cause__) == reason + else: + assert error.value.__cause__ is None + records = [] + monkeypatch.setattr(specialize, "_trusted_directory", lambda _path: None) + monkeypatch.setattr(specialize, "_write_private_atomic", lambda _path, raw: records.append(json.loads(raw))) + specialize._record_migration_failure({ + "backupDirectory": str(specialize._SPECIALIZED_BACKUP_ROOT / "specialized-bound"), + "sourceRevision": "a" * 40, "sourceTree": "b" * 40, + "stage": "build_probe_pending", + }, error.value) + assert len(records) == 1 + assert records[0]["causeCode"] == (reason if reason in specialize._RESOURCE_FAILURE_CODES else str(error.value)) + assert "secret-value" not in json.dumps(records) + + +@pytest.mark.parametrize("case", ["released", "busy", "untrusted"]) +def test_rollback_waits_only_for_natural_maintenance_release(specialize, monkeypatch, case): + deploy = SimpleNamespace(load_projects=lambda: {}) + monkeypatch.setitem(sys.modules, "deploy", deploy) + import runner_acceptance_quiescence as q + + events = [] + now = [0.0] + + class Driver: + @contextmanager + def maintenance(self): + events.append("acquire") + if case == "untrusted": + raise q.QuiescenceError("dispatch_lock_untrusted") + if case == "busy" or events.count("acquire") < 3: + raise q.QuiescenceError("dispatch_maintenance_busy") + events.append("owned") + try: + yield + finally: + events.append("release") + + def remaining(self): + return 240 - now[0] + + def assert_barrier(self): + assert "owned" in events + + def sleep(seconds): + assert 0 < seconds <= 1 + events.append("wait") + now[0] += seconds + + monkeypatch.setattr(deploy, "load_projects", lambda: {}) + monkeypatch.setattr(q, "ControllerDriver", lambda _projects: Driver()) + monkeypatch.setattr(specialize.time, "monotonic", lambda: now[0]) + monkeypatch.setattr(specialize.time, "sleep", sleep) + if case == "released": + with specialize._rollback_maintenance() as barrier: + barrier() + events.append("rollback-effect") + assert events == ["acquire", "wait", "acquire", "wait", "acquire", "owned", "rollback-effect", "release"] + else: + with pytest.raises(q.QuiescenceError, match="dispatch_maintenance_busy" if case == "busy" else "dispatch_lock_untrusted"): + with specialize._rollback_maintenance(): + pytest.fail("No effect before exclusive acquisition") + assert "owned" not in events + assert now[0] == (90 if case == "busy" else 0) + + @pytest.fixture def controller_only_source(specialize, tmp_path, monkeypatch): image = "sha256:" + "a" * 64 diff --git a/backend/tests/test_runner_resource_envelope.py b/backend/tests/test_runner_resource_envelope.py index 0b0373d..c18bb23 100644 --- a/backend/tests/test_runner_resource_envelope.py +++ b/backend/tests/test_runner_resource_envelope.py @@ -1921,7 +1921,8 @@ def test_attestor_rejects_unowned_descendants_below_dind(): assert result["reason"] == "resource_cgroup_unmapped_descendant" -def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkeypatch): +@pytest.mark.parametrize("race_count", [0, 1, 2, 3]) +def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkeypatch, race_count): envelope = _module() snapshot = _snapshot() family_path = str(snapshot["family"]["path"]) @@ -1949,10 +1950,28 @@ def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkey "cpuMilli": 4000, "memoryBytes": 6 * 1024**3, }] - monkeypatch.setattr(envelope, "_collect_snapshot", lambda _expected: snapshot) + attempts = [] + waits = [] + + def collect(_expected): + attempts.append(True) + if len(attempts) <= race_count: + raise RuntimeError("resource_cgroup_process_mapping_invalid") + return snapshot + + monkeypatch.setattr(envelope, "_collect_snapshot", collect) + monkeypatch.setattr(envelope.time, "sleep", waits.append) result = envelope.attest_build_probe_workload(probe_id, 77) + assert len(attempts) == min(race_count + 1, 3) + assert waits == [0.1] * min(race_count, 2) + if race_count == 3: + assert result["ready"] is False + assert result["reason"] == "resource_cgroup_process_mapping_invalid" + assert "probeWorkload" not in result + return + assert result["ready"] is True, result assert result["runningRunnerContainers"] == 0 assert result["probeWorkload"] == { @@ -1967,6 +1986,25 @@ def test_build_probe_attestor_binds_exact_cid_nested_pid_and_build_cgroup(monkey assert envelope.attest_build_probe_workload(probe_id, 78)["ready"] is False +@pytest.mark.parametrize("reason", [ + "resource_cgroup_probe_namespace_invalid", "resource_cgroup_probe_identity_invalid", + "resource_cgroup_probe_lock_busy", "resource_cgroup_workload_unmapped", +]) +def test_build_probe_attestor_does_not_retry_other_resource_refusals(monkeypatch, reason): + envelope = _module() + attempts = [] + + def collect(_expected): + attempts.append(True) + raise RuntimeError(reason) + + monkeypatch.setattr(envelope, "_collect_snapshot", collect) + monkeypatch.setattr(envelope.time, "sleep", lambda _seconds: pytest.fail("Refusal cannot be retried")) + proof = envelope.attest_build_probe_workload("f" * 64, 77) + assert proof["ready"] is False and proof["reason"] == reason + assert len(attempts) == 1 and "probeWorkload" not in proof + + def test_collect_snapshot_uses_host_helper_and_exports_real_family_counters(monkeypatch): envelope = _module() snapshot = _snapshot()