diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index ccb0391..a74207f 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `0f89e1506065c5365c48968b1d7845d666a18be8`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `1350d8bf6527e085bbfa950ec0d1ee3e973d618e`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index 9f5cfb1..f530c75 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "0f89e1506065c5365c48968b1d7845d666a18be8", + "sourceRevision": "1350d8bf6527e085bbfa950ec0d1ee3e973d618e", "files": [ { "path": ".dockerignore", @@ -524,7 +524,7 @@ }, { "path": "backend/app/services/controller_artifact_helper_spec.py", - "sha256": "e345e51a81eb483ffe8532592deb197296bd28db99e1e5bc358c590a59f1b94d", + "sha256": "573a1a79ff75940a1a0a49cab5718ad62ddd20703369d47167f1ab796f2e5cae", "bytes": 6029 }, { @@ -949,7 +949,7 @@ }, { "path": "backend/tests/test_controller_artifact_bootstrap.py", - "sha256": "6774b7fdf9fd91c463661970512b93027af93979fd4b6d61d89476f0465cc48b", + "sha256": "179c50723fc29524088b9991af9f7d924337fe11585f4fe6456917c1932efefa", "bytes": 82644 }, { @@ -964,7 +964,7 @@ }, { "path": "backend/tests/test_controller_artifact_promoter.py", - "sha256": "1a274fb06503ce5568e2096898e1a98ded10dee333f49bd45060bbc82adf00cb", + "sha256": "41b10414669b592f6055488f2dc51f1a0034e2aaf8506153c180ab93c4121560", "bytes": 7570 }, { @@ -989,8 +989,8 @@ }, { "path": "backend/tests/test_controller_hub_release_volume.py", - "sha256": "c3de3b47673a5fa365bf20aedc020a2ca733804adc9cd05d15b26aca2a1a2b05", - "bytes": 161657 + "sha256": "07a28688018b027d0590e68e65ca619fe223f21d4f8f58196d95eb344b087d55", + "bytes": 165301 }, { "path": "backend/tests/test_controller_root_procedure_contract.py", @@ -1639,8 +1639,8 @@ }, { "path": "backend/tests/test_runner_resource_envelope.py", - "sha256": "9bb731508ac3cc4d36ba5eae7839c0f73b411880b92a45118dfaa4b62c4e019f", - "bytes": 40972 + "sha256": "4970625e2fb1d95750f7a0acc1ce328865711abf3450e089a8a2304b21c20d9f", + "bytes": 46791 }, { "path": "backend/tests/test_runner_scheduler_liveness_recovery.py", diff --git a/backend/app/services/controller_artifact_helper_spec.py b/backend/app/services/controller_artifact_helper_spec.py index b3599f7..c75e8f3 100644 --- a/backend/app/services/controller_artifact_helper_spec.py +++ b/backend/app/services/controller_artifact_helper_spec.py @@ -5,7 +5,7 @@ from typing import Any SOURCE_REPOSITORY = "jens/unraid-appops-gateway" HELPER_NAME = "itworx-controller-artifact-promoter-modern" -CONTROLLER_VERSION = "2.1.19" +CONTROLLER_VERSION = "2.1.20" HELPER_LABEL = "itworx.controller.promoter" _REVISION = re.compile(r"^[0-9a-f]{40}$") _IMAGE_ID = re.compile(r"^sha256:[0-9a-f]{64}$") diff --git a/backend/tests/test_controller_artifact_bootstrap.py b/backend/tests/test_controller_artifact_bootstrap.py index 35d8ef1..113436e 100644 --- a/backend/tests/test_controller_artifact_bootstrap.py +++ b/backend/tests/test_controller_artifact_bootstrap.py @@ -589,7 +589,7 @@ def _health( *, revision: str = REVISION, tree: str = TREE, - version: str = "2.1.19", + version: str = "2.1.20", artifact_sha256: str = "1" * 64, ) -> dict[str, Any]: return { @@ -1347,7 +1347,7 @@ def test_current_214_baseline_is_verified_and_bound_to_new_helper(): "itworx.controller.promoter": "true", "itworx.source.repository": "jens/unraid-appops-gateway", "itworx.source.revision": REVISION, - "itworx.controller.version": "2.1.19", + "itworx.controller.version": "2.1.20", "itworx.controller.bootstrap.target.id": MODERN_CONTROLLER_CONTAINER_ID, "itworx.controller.bootstrap.target.image": CONTROLLER_IMAGE_ID, "itworx.controller.bootstrap.target.revision": MODERN_CONTROLLER_REVISION, @@ -1826,7 +1826,7 @@ def test_successful_child_requires_exact_source_tree_version_and_health(): assert result["status"] == "promoted" assert result["revision"] == REVISION - assert result["controllerVersion"] == "2.1.19" + assert result["controllerVersion"] == "2.1.20" assert result["artifactSha256"] == "1" * 64 assert docker.started == [MODERN_HELPER_ID] assert docker.created[0]["Image"] == CONTROLLER_IMAGE_ID @@ -2086,32 +2086,32 @@ def test_modern_upgrade_path_still_requires_private_archive_and_image_id( def test_current_helper_version_is_distinct_from_fixed_controller_baselines(): source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_bytes() namespace = bootstrap._safe_modern_spec_namespace(source) - assert namespace["CONTROLLER_VERSION"] == "2.1.19" - assert bootstrap._verified_historical_helper_version(source) == "2.1.19" + assert namespace["CONTROLLER_VERSION"] == "2.1.20" + assert bootstrap._verified_historical_helper_version(source) == "2.1.20" assert [baseline.version for baseline in bootstrap.CONTROLLER_BASELINES] == ["2.1.12", "2.1.14"] with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"): bootstrap._safe_modern_spec_namespace(RETIRED_MODERN_HELPER_SOURCE.encode("utf-8")) assert bootstrap._verified_historical_helper_version(RETIRED_MODERN_HELPER_SOURCE.encode("utf-8")) == "2.1.14" -@pytest.mark.parametrize("version", ["2.1.16", "2.1.20", "9.9.9"]) +@pytest.mark.parametrize("version", ["2.1.16", "2.1.21", "9.9.9"]) def test_current_and_historical_helper_reject_unreviewed_target_versions(version): source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_bytes() - foreign = source.replace(b'CONTROLLER_VERSION = "2.1.19"', f'CONTROLLER_VERSION = "{version}"'.encode()) + foreign = source.replace(b'CONTROLLER_VERSION = "2.1.20"', f'CONTROLLER_VERSION = "{version}"'.encode()) with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"): bootstrap._safe_modern_spec_namespace(foreign) with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"): bootstrap._verified_historical_helper_version(foreign) -@pytest.mark.parametrize("version", ["2.1.14", "2.1.16", "2.1.20"]) +@pytest.mark.parametrize("version", ["2.1.14", "2.1.16", "2.1.21"]) def test_promoted_health_cannot_substitute_a_baseline_or_unknown_version(version): assert not bootstrap._health_matches(_health(version=version), REVISION, TREE) def test_previous_reviewed_helper_is_historical_only(): source = (ROOT / "backend/app/services/controller_artifact_helper_spec.py").read_bytes() - prior = source.replace(b'CONTROLLER_VERSION = "2.1.19"', b'CONTROLLER_VERSION = "2.1.18"') + prior = source.replace(b'CONTROLLER_VERSION = "2.1.20"', b'CONTROLLER_VERSION = "2.1.19"') with pytest.raises(bootstrap.BootstrapError, match="controller_promoter_modern_spec_identity_invalid"): bootstrap._safe_modern_spec_namespace(prior) - assert bootstrap._verified_historical_helper_version(prior) == "2.1.18" + assert bootstrap._verified_historical_helper_version(prior) == "2.1.19" diff --git a/backend/tests/test_controller_artifact_promoter.py b/backend/tests/test_controller_artifact_promoter.py index 4d4a05b..9f77f37 100644 --- a/backend/tests/test_controller_artifact_promoter.py +++ b/backend/tests/test_controller_artifact_promoter.py @@ -46,7 +46,7 @@ def test_controller_promoter_helper_spec_is_fixed_and_bounded(): spec = helper_container_spec(revision=revision, image_id=image_id) assert HELPER_NAME == "itworx-controller-artifact-promoter-modern" - assert CONTROLLER_VERSION == "2.1.19" + assert CONTROLLER_VERSION == "2.1.20" assert re.fullmatch(r"2\.1\.\d+", CONTROLLER_VERSION) assert spec["Image"] == image_id assert spec["HostConfig"]["Privileged"] is False @@ -79,7 +79,7 @@ def test_controller_promoter_helper_spec_is_fixed_and_bounded(): assert revision in command assert "--image-id" in command assert CONTROLLER_VERSION in command - assert "version='2.1.19'" in command + assert "version='2.1.20'" in command assert "/gitea-repositories/jens/unraid-appops-gateway.git" in command archive_start = command.index('git --git-dir="$repo" archive') archive_end = command.index('> "$work/controller.tar"', archive_start) @@ -110,7 +110,7 @@ def test_controller_promoter_bootstrap_child_spec_binds_legacy_target_identity() "itworx.controller.promoter": "true", "itworx.source.repository": "jens/unraid-appops-gateway", "itworx.source.revision": revision, - "itworx.controller.version": "2.1.19", + "itworx.controller.version": "2.1.20", "itworx.controller.bootstrap.target.id": target_id, "itworx.controller.bootstrap.target.image": image_id, "itworx.controller.bootstrap.target.revision": target_revision, diff --git a/backend/tests/test_controller_hub_release_volume.py b/backend/tests/test_controller_hub_release_volume.py index ccea065..5826043 100644 --- a/backend/tests/test_controller_hub_release_volume.py +++ b/backend/tests/test_controller_hub_release_volume.py @@ -2461,6 +2461,7 @@ def test_stop_rejects_failed_storage_before_recording_intent_or_stopping(monkeyp @pytest.mark.parametrize('native_read_only_graph', [False, True]) def test_stop_allows_healthy_storage_and_preserves_intent_before_stop(monkeypatch, native_read_only_graph): + monkeypatch.setattr(module, '_cleanup_promotion_host_probe', lambda *a: None) receipt = {'phase': 'prepared'} events = [] monkeypatch.setattr(module, '_load_transaction', lambda _: (42, receipt, {})) @@ -3540,6 +3541,7 @@ def test_legacy_receipt_projection_rejects_unsafe_mount_before_docker(compose, d @pytest.mark.parametrize("phase", ["prepared", "stopped", "activated", "verified"]) @pytest.mark.parametrize("failure", ["read_only", "full"]) def test_forward_stop_rechecks_storage_after_intent_or_on_resume(monkeypatch, phase, failure): + monkeypatch.setattr(module, "_cleanup_promotion_host_probe", lambda *a: None) receipt = {"phase": phase, "imageId": IMAGE_ID, "previousRevision": OLD_REVISION, "sourceRevision": REVISION} persisted = phase != "prepared" @@ -3568,6 +3570,69 @@ def test_forward_stop_rechecks_storage_after_intent_or_on_resume(monkeypatch, ph assert events == [] +def test_probe_cleanup_failure_precedes_stop_intent_and_controller_stop(monkeypatch): + receipt = {"phase": "prepared"} + monkeypatch.setattr(module, "_load_transaction", lambda _: (42, receipt, {})) + monkeypatch.setattr(module.os, "close", lambda _: None) + monkeypatch.setattr(module, "_require_writable_promotion_storage", lambda: None) + def reject(*args): + raise module.VolumeTransitionError("controller_volume_command_failed") + monkeypatch.setattr(module, "_cleanup_promotion_host_probe", reject) + monkeypatch.setattr(module, "_set_promotion_phase", lambda *a: pytest.fail("No stop intent")) + monkeypatch.setattr(module, "_stop_current_controller", lambda *a, **kw: pytest.fail("No stop")) + with pytest.raises(module.VolumeTransitionError, match="controller_volume_command_failed"): + module.stop_promotion(Path("/unused-backup")) + assert receipt["phase"] == "prepared" + + +def test_promotion_cleanup_uses_incoming_code_but_only_attested_previous_source(monkeypatch, tmp_path): + receipt = {"imageId": IMAGE_ID, "previousRevision": OLD_REVISION, + "sourceRevision": REVISION, "sourceTree": "d" * 40} + old_source = module._expected_revision_path(OLD_REVISION).as_posix() + events = [] + def inspect(image, **kw): + assert image == IMAGE_ID + assert kw in ({"require_running": True}, {"expected_source": old_source, "require_running": True}) + events.append("attest") + return {"Id": "c" * 64, "Mounts": [{"Destination": module.EXECUTABLE_TARGET, + "Source": old_source}]} + monkeypatch.setattr(module, "_controller_image_inspect", inspect) + monkeypatch.setattr(module, "_read_tar_tree", lambda _: ({}, {"": 0o755})) + monkeypatch.setattr(module, "_release_identity", lambda _: { + "sourceRevision": REVISION, "sourceTree": "d" * 40}) + monkeypatch.setattr(module, "_write_private_tree", lambda *a: None) + def run(args, **kw): + assert args[0] == "python3" + assert Path(args[1]).name == "runner_resource_envelope.py" + assert args[2:] == ["--cleanup-stopped-host-probe", IMAGE_ID, old_source] + assert kw == {"max_output": 4096} + events.append("cleanup") + return b"" + monkeypatch.setattr(module, "_run", run) + module._cleanup_promotion_host_probe(receipt, {"controller-after.tar": b"qualified"}) + assert events == ["attest", "cleanup", "attest"] + + +@pytest.mark.parametrize("present", [False, True]) +def test_legacy_promotion_allows_only_no_probe_without_adopting_legacy_source(monkeypatch, present): + receipt = {"imageId": IMAGE_ID, "previousRevision": OLD_REVISION} + monkeypatch.setattr(module, "_controller_image_inspect", lambda *a, **kw: { + "Id": "c" * 64, "Mounts": [{"Destination": module.EXECUTABLE_TARGET, + "Source": module.CONTROLLER_DIR.as_posix()}]}) + calls = [] + def run(args, **kw): + calls.append(args) + return b"d" * 64 if present else b"" + monkeypatch.setattr(module, "_run", run) + if present: + with pytest.raises(module.VolumeTransitionError, match="controller_volume_probe_cleanup_identity_invalid"): + module._cleanup_promotion_host_probe(receipt, {}) + else: + module._cleanup_promotion_host_probe(receipt, {}) + assert calls == [["docker", "ps", "-a", "--no-trunc", "--filter", + "name=^itworx-runner-cgroup-probe$", "--format", "{{.ID}}"]] + + def test_resumed_forward_stop_of_stopped_controller_needs_no_storage(monkeypatch): receipt = {"phase": "stopped", "imageId": IMAGE_ID, "previousRevision": OLD_REVISION, "sourceRevision": REVISION} diff --git a/backend/tests/test_runner_resource_envelope.py b/backend/tests/test_runner_resource_envelope.py index 87d2e11..403d264 100644 --- a/backend/tests/test_runner_resource_envelope.py +++ b/backend/tests/test_runner_resource_envelope.py @@ -484,8 +484,7 @@ def test_host_probe_contract_is_bounded_and_never_mounts_host_proc_or_docker_soc assert args[-3:] == [image_id, "/opt/gitea-deploy/runner_resource_envelope.py", "--host-probe"] -def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift(): - envelope = _module() +def _host_probe_fixture(envelope): image_id = "sha256:" + "a" * 64 code_source = f"{envelope._HOST_VOLUME_ROOT}/controller-artifacts/{'c' * 40}" base = { @@ -552,6 +551,12 @@ def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift(): ], } + return base, image_id, code_source + + +def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift(): + envelope = _module() + base, image_id, code_source = _host_probe_fixture(envelope) assert envelope._validate_host_probe_inspect( base, mode="snapshot", image_id=image_id, code_source=code_source ) @@ -614,6 +619,115 @@ def test_host_probe_inspection_rejects_namespace_mount_and_resource_drift(): ) +@pytest.mark.parametrize("mode", ["snapshot", "family", "ensure", "rollback"]) +def test_inherited_image_labels_are_exact_and_owned_probe_cleanup_is_bounded(monkeypatch, mode): + envelope = _module() + probe, image_id, code_source = _host_probe_fixture(envelope) + inherited = {"io.buildah.version": "1.35.0", "org.opencontainers.image.title": "Official runner image"} + probe["Config"]["Labels"].update(inherited) + rw = mode in {"ensure", "rollback"} + source = "/sys/fs/cgroup/itworx-ci" if mode == "snapshot" else "/sys/fs/cgroup" + probe["Mounts"][0].update(Source=source, RW=rw) + probe["HostConfig"]["Mounts"][0].update(Source=source, ReadOnly=not rw) + probe["HostConfig"]["CapAdd"] = ["DAC_OVERRIDE"] if rw else [] + assert envelope._validate_host_probe_inspect( + probe, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited + ) + for mutate in ( + lambda item: item["Config"]["Labels"].update(unknown="extra"), + lambda item: item["Config"]["Labels"].update({"org.opencontainers.image.title": "changed"}), + lambda item: item["Config"]["Labels"].pop("io.buildah.version"), + lambda item: item["Config"]["Labels"].update({"tech.itworx.owner": "foreign"}), + lambda item: item.update(Image="sha256:" + "f" * 64), + ): + candidate = copy.deepcopy(probe) + mutate(candidate) + assert not envelope._validate_host_probe_inspect( + candidate, mode=mode, image_id=image_id, code_source=code_source, image_labels=inherited + ) + monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: inherited) + inspected = iter([probe, copy.deepcopy(probe), None]) + monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected)) + calls = [] + monkeypatch.setattr(envelope, "_run", lambda args, **kw: calls.append(args)) + assert envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source) + assert calls == [["docker", "rm", probe["Id"]]] + + +@pytest.mark.parametrize("drift", ["running", "foreign_source", "race", "missing_state"]) +def test_probe_cleanup_refuses_foreign_running_or_changed_identity(monkeypatch, drift): + envelope = _module() + probe, image_id, code_source = _host_probe_fixture(envelope) + fresh = copy.deepcopy(probe) + if drift == "running": + probe["State"]["Running"] = True + elif drift == "foreign_source": + probe["Mounts"][1]["Source"] = code_source.replace("c" * 40, "d" * 40) + elif drift == "race": + fresh["Id"] = "f" * 64 + else: + probe.pop("State") + monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: {}) + inspected = iter([probe, fresh]) + monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: next(inspected)) + monkeypatch.setattr(envelope, "_run", lambda *a, **kw: pytest.fail("No removal or probe execution")) + with pytest.raises(RuntimeError, match="resource_cgroup_probe_cleanup_identity_"): + envelope.cleanup_stopped_host_probe(image_id=image_id, code_source=code_source) + + +@pytest.mark.parametrize("labels", [None, {}, {"org.opencontainers.image.title": "runner"}]) +def test_probe_label_metadata_comes_from_exact_image(monkeypatch, labels): + envelope = _module() + image_id = "sha256:" + "a" * 64 + calls = [] + def run(args, **kw): + calls.append(args) + return json.dumps([{"Id": image_id, "Config": {"Labels": labels}}]) + monkeypatch.setattr(envelope, "_run", run) + assert envelope._host_probe_image_labels(image_id) == (labels or {}) + assert calls == [["docker", "image", "inspect", image_id]] + + +@pytest.mark.parametrize("row", [ + {"Id": "sha256:" + "b" * 64, "Config": {"Labels": {}}}, + {"Id": "sha256:" + "a" * 64, "Config": {"Labels": ["foreign"]}}, + {"Id": "sha256:" + "a" * 64, "Config": {"Labels": {"secret-canary": 123}}}, + {"Id": "sha256:" + "a" * 64, "Config": {"Labels": {"tech.itworx.owner": "foreign"}}}, +]) +def test_probe_image_metadata_rejects_drift_without_echo(monkeypatch, row): + envelope = _module() + monkeypatch.setattr(envelope, "_run", lambda *a, **kw: json.dumps([row])) + with pytest.raises(RuntimeError, match="^resource_cgroup_probe_image_labels_invalid$"): + envelope._host_probe_image_labels("sha256:" + "a" * 64) + + +def test_normal_probe_executes_and_removes_its_image_label_bound_container(monkeypatch): + envelope = _module() + probe, image_id, code_source = _host_probe_fixture(envelope) + inherited = {"org.opencontainers.image.title": "Official runner image"} + probe["Config"]["Labels"].update(inherited) + monkeypatch.setattr(envelope, "_controller_runtime_identity", lambda: (image_id, code_source)) + monkeypatch.setattr(envelope, "_host_probe_image_labels", lambda _: inherited) + monkeypatch.setattr(envelope, "_inspect_host_probe_by_name", lambda: None) + calls = [] + def run(args, **kw): + calls.append(args) + if args[:2] == ["docker", "create"]: + return probe["Id"] + if args[:2] == ["docker", "inspect"]: + return json.dumps([probe]) + assert args == ["docker", "rm", probe["Id"]] + return "" + def start(args, **kw): + assert args == ["docker", "start", "--attach", "--interactive", probe["Id"]] + assert json.loads(kw["input"]) == {"op": "snapshot"} + return SimpleNamespace(returncode=0, stdout=b'{"ready":true,"result":{"verified":true}}') + monkeypatch.setattr(envelope, "_run", run) + monkeypatch.setattr(envelope.subprocess, "run", start) + assert envelope._run_host_probe("snapshot") == {"verified": True} + assert calls[-1] == ["docker", "rm", probe["Id"]] + + def test_host_probe_image_and_source_come_from_verified_running_controller(monkeypatch): envelope = _module() revision = "d" * 40