from __future__ import annotations import contextlib import copy import importlib.util import io import json import tarfile import types from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] spec = importlib.util.spec_from_file_location( "immutable_promotion_test", ROOT / "ops/gitea-deploy-broker/immutable_service_promotion.py", ) m = importlib.util.module_from_spec(spec) spec.loader.exec_module(m) KEY = b"test-receipt-key" * 3 REVISION = "b" * 40 class Runtime: def __init__(self, path): self.receipt_root = path / "receipts" self.state = { "target": { "containerId": "a" * 64, "imageId": "sha256:" + "a" * 64, "revision": m.OLD_REVISION, "spec": { "image": "sha256:" + "a" * 64, "environment": {"GITEA_CONNECTOR_BUILD_SHA": m.OLD_REVISION}, }, }, "nonTargets": { "postgres": {"containerId": "c" * 64, "imageId": "sha256:" + "c" * 64} }, } self.calls = [] self.fail_health = False self.fail_rollback = False self.artifact = { "revision": REVISION, "imageId": "sha256:" + "b" * 64, "candidateSha256": "d" * 64, } def lock(self): return contextlib.nullcontext() def snapshot(self): return copy.deepcopy(self.state) def candidate(self, revision, **kwargs): self.calls.append("candidate") assert revision == REVISION return copy.deepcopy(self.artifact) desired = m.ControllerRuntime.desired def promote(self, desired): self.calls.append("promote") self.state["target"] = {**copy.deepcopy(desired), "containerId": "e" * 64} def verify(self, desired, baseline, *, rollback=False): if self.fail_health and not rollback: raise ValueError("private diagnostic must not escape") assert self.state["nonTargets"] == baseline["nonTargets"] return {"containerId": self.state["target"]["containerId"]} def rollback(self, baseline, desired): self.calls.append("rollback") if self.fail_rollback: raise ValueError("secret rollback diagnostic") self.state = copy.deepcopy(baseline) def verify_record(self, record): self.calls.append("verify_record") @pytest.fixture def fixture(tmp_path): runtime = Runtime(tmp_path) request = { "repository": "Jens/ITWorx_MCP_Hub", "service": m.SERVICE, "revision": REVISION, "expectedBaseline": m.digest(runtime.state), "reason": "Repair verified Gitea compatibility", "requestId": "d6fe0ba3-81f0-4ed6-b36d-10b12dbf55e4", } return runtime, request def plan(runtime, request): return m.preflight(request, api_json=None, receipt_key=KEY, runtime=runtime) def arguments(request, result): return {**request, "planId": result["planId"], "planDigest": result["planDigest"]} @pytest.mark.parametrize( "change", [ {"service": "postgres"}, {"repository": "Jens/other"}, {"revision": "main"}, {"revision": "B" * 40}, {"requestId": "bad"}, {"image": "evil"}, {"command": "sh"}, {"expectedBaseline": "bad"}, {"reason": "line\nbreak"}, ], ) def test_closed_request_rejects_untrusted_target_or_extra_fields(fixture, change): runtime, request = fixture with pytest.raises(m.PromotionRejected): plan(runtime, {**request, **change}) assert not runtime.calls def test_preflight_is_readonly_and_receipts_do_not_expose_configuration(fixture): runtime, request = fixture result = plan(runtime, request) assert result["status"] == "prepared" assert result["expiresAt"] - result["createdAt"] == 300 assert runtime.calls == ["candidate"] assert "spec" not in json.dumps(result) and "environment" not in json.dumps(result) assert result["planId"] == m.digest(m.request(request)) def test_queue_is_durable_idempotent_and_no_runtime_operation(fixture): runtime, request = fixture result = plan(runtime, request) runtime.calls.clear() args = arguments(request, result) queued = m.accept_apply(args, receipt_key=KEY, runtime=runtime) assert queued["status"] == "queued" assert m.accept_apply(args, receipt_key=KEY, runtime=runtime) == queued assert runtime.calls == [] assert len(list(runtime.receipt_root.rglob("*.json"))) == 2 def test_apply_rechecks_artifact_and_baseline_then_only_promotes_once(fixture): runtime, request = fixture result = plan(runtime, request) args = arguments(request, result) m.accept_apply(args, receipt_key=KEY, runtime=runtime) applied = m.apply( args, api_json=None, download_artifact=None, receipt_key=KEY, runtime=runtime ) assert applied["status"] == "applied" assert runtime.calls == ["candidate", "candidate", "promote"] assert ( m.apply( args, api_json=None, download_artifact=None, receipt_key=KEY, runtime=runtime, ) == applied ) assert runtime.calls.count("promote") == 1 assert runtime.state["nonTargets"]["postgres"]["containerId"] == "c" * 64 assert ( m.status({"planId": result["planId"]}, receipt_key=KEY, runtime=runtime) == applied ) def test_unrelated_baseline_drift_blocks_before_any_import_or_write(fixture): runtime, request = fixture result = plan(runtime, request) runtime.calls.clear() runtime.state["nonTargets"]["postgres"]["containerId"] = "f" * 64 with pytest.raises(m.PromotionRejected, match="baseline_changed"): m.apply( arguments(request, result), api_json=None, download_artifact=None, receipt_key=KEY, runtime=runtime, ) assert not runtime.calls def test_artifact_change_after_preflight_blocks_without_promoting(fixture): runtime, request = fixture result = plan(runtime, request) runtime.artifact["candidateSha256"] = "f" * 64 with pytest.raises(m.PromotionRejected, match="artifact_changed"): m.apply( arguments(request, result), api_json=None, download_artifact=None, receipt_key=KEY, runtime=runtime, ) assert "promote" not in runtime.calls def test_health_failure_restores_exact_baseline_and_records_rollback(fixture): runtime, request = fixture before = runtime.snapshot() result = plan(runtime, request) runtime.fail_health = True outcome = m.apply( arguments(request, result), api_json=None, download_artifact=None, receipt_key=KEY, runtime=runtime, ) assert outcome["status"] == "rolled_back" assert runtime.snapshot() == before assert "private" not in json.dumps(outcome) def test_rollback_failure_is_durable_and_cannot_blindly_retry(fixture): runtime, request = fixture result = plan(runtime, request) runtime.fail_health = runtime.fail_rollback = True args = arguments(request, result) with pytest.raises(m.PromotionRejected, match="rollback_recovery_required"): m.apply( args, api_json=None, download_artifact=None, receipt_key=KEY, runtime=runtime, ) assert ( m.status({"planId": result["planId"]}, receipt_key=KEY, runtime=runtime)[ "status" ] == "recovery_required" ) with pytest.raises(m.PromotionRejected, match="recovery_required"): m.apply( args, api_json=None, download_artifact=None, receipt_key=KEY, runtime=runtime, ) assert runtime.calls.count("promote") == 1 def test_expired_plan_and_different_intent_are_rejected(fixture, monkeypatch): runtime, request = fixture result = plan(runtime, request) args = arguments(request, result) with pytest.raises(m.PromotionRejected, match="plan_mismatch"): m.accept_apply( {**args, "reason": "Changed reason"}, receipt_key=KEY, runtime=runtime ) monkeypatch.setattr(m.time, "time", lambda: result["expiresAt"] + 1) with pytest.raises(m.PromotionRejected, match="plan_expired"): m.accept_apply(args, receipt_key=KEY, runtime=runtime) def test_signature_tampering_chain_gaps_and_weak_key_fail_closed(fixture): runtime, request = fixture result = plan(runtime, request) path = next(runtime.receipt_root.rglob("*.json")) data = json.loads(path.read_text()) data["record"]["revision"] = "f" * 40 path.write_text(json.dumps(data)) with pytest.raises(m.PromotionRejected, match="signature_invalid"): m.status({"planId": result["planId"]}, receipt_key=KEY, runtime=runtime) with pytest.raises(m.PromotionRejected, match="key_invalid"): m.ReceiptStore(runtime.receipt_root, b"weak") def test_async_rejection_records_only_safe_reason_and_never_overwrites_recovery( fixture, ): runtime, request = fixture result = plan(runtime, request) m.accept_apply(arguments(request, result), receipt_key=KEY, runtime=runtime) rejected = m.record_worker_failure( result["planId"], "secret=raw error!", receipt_key=KEY, runtime=runtime ) assert rejected["status"] == "rejected" assert rejected["reasonCode"] == "component_worker_failed" assert ( m.record_worker_failure( result["planId"], "later", receipt_key=KEY, runtime=runtime ) == rejected ) @pytest.fixture def adapter(): runtime = m.ControllerRuntime.__new__(m.ControllerRuntime) runtime.docker = lambda *_args: [ { "Id": "sha256:" + "a" * 64, "Config": { "Labels": { "org.opencontainers.image.revision": m.OLD_REVISION, "org.opencontainers.image.source": "https://gitea.example.com/jens/itworx-mcp-hub", } }, } ] runtime.runtime_fingerprint = lambda: { "blob": m.ABI_BLOBS["services/runtime/connector_gitea.py"][0] } env = { "SERVICE_NAME": "connector-gitea", "GITEA_CONNECTOR_BUILD_SHA": m.OLD_REVISION, "GITEA_ENDPOINT": "https://gitea.example.com", "GITEA_TOKEN_FILE": "/run/secrets/gitea_mcp_token", "GITEA_WRITE_TOKEN_FILE": "/run/secrets/gitea_mcp_write_token", "SECRET_EXPECTED_UID": "99", "SECRET_EXPECTED_GID": "100", "CONNECTOR_IDENTITY_KEY_FILE": "/run/secrets/connector_identity_signing_key", "GPG_KEY": "public image fingerprint", } mounts = [ { "Type": "bind", "RW": False, "Source": "/srv/example/appdata/itworx-mcp-hub/secrets/" + name, "Destination": "/run/secrets/" + name, } for name in ( "gitea_mcp_token", "gitea_mcp_write_token", "connector_identity_signing_key", ) ] mounts.append( { "Type": "bind", "RW": False, "Source": f"/srv/example/appdata/itworx-mcp-hub/deployment/releases/{m.OLD_REVISION}/services/runtime/connector_gitea.py", "Destination": "/opt/hub/services/runtime/connector_gitea.py", } ) row = { "Id": "a" * 64, "Image": "sha256:" + "a" * 64, "Config": { "Entrypoint": ["python", "-m", "services.runtime"], "Cmd": None, "User": "99:100", "WorkingDir": "/opt/hub", "Tty": False, "OpenStdin": False, "ExposedPorts": {"8080/tcp": {}}, "Env": [k + "=" + v for k, v in env.items()], "Labels": {}, "Healthcheck": { "Interval": 60000000000, "Retries": 5, "StartPeriod": 10000000000, "Timeout": 3000000000, "Test": [ "CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz', timeout=2)", ], }, }, "HostConfig": { "ReadonlyRootfs": True, "Privileged": False, "CapDrop": ["ALL"], "SecurityOpt": ["no-new-privileges:true"], "Memory": 536870912, "MemorySwap": -1, "MemoryReservation": 0, "NetworkMode": "itworx-mcp-hub-connectors", "LogConfig": { "Type": "json-file", "Config": {"max-file": "3", "max-size": "10m"}, }, "ShmSize": 67108864, "AutoRemove": False, "CgroupnsMode": "private", "ReadonlyPaths": [ "/proc/bus", "/proc/fs", "/proc/irq", "/proc/sys", "/proc/sysrq-trigger", ], "MaskedPaths": [ "/proc/asound", "/proc/acpi", "/proc/kcore", "/proc/keys", "/proc/latency_stats", "/proc/timer_list", "/proc/timer_stats", "/proc/sched_debug", "/proc/scsi", "/sys/firmware", "/sys/devices/virtual/powercap", ], "NanoCpus": 1000000000, "PidsLimit": 128, "Tmpfs": {"/tmp": "size=512m,mode=1777"}, "IpcMode": "private", "RestartPolicy": {"Name": "unless-stopped", "MaximumRetryCount": 0}, }, "Mounts": mounts, "NetworkSettings": { "Networks": { n: {"Aliases": [m.CONTAINER, m.SERVICE]} for n in ( "itworx-mcp-hub-connectors", "itworx-mcp-hub-observe", "itworx-mcp-hub-egress", ) } }, } return runtime, row def test_known_live_target_preserves_public_metadata_and_only_image_build_sha_changes( adapter, ): runtime, row = adapter target = runtime.target(row) desired = runtime.desired( {"target": target}, {"imageId": "sha256:" + "b" * 64, "revision": REVISION} ) changed = copy.deepcopy(desired) changed.update(imageId=target["imageId"], revision=target["revision"]) changed["spec"]["image"] = target["imageId"] changed["spec"]["environment"]["GITEA_CONNECTOR_BUILD_SHA"] = m.OLD_REVISION assert changed == target assert target["spec"]["environment"]["SECRET_EXPECTED_UID"] == "99" assert target["spec"]["environment"]["GPG_KEY"] == "public image fingerprint" @pytest.mark.parametrize( "field,value", [ ("Privileged", True), ("CapAdd", ["SYS_ADMIN"]), ("Devices", [{"PathOnHost": "/dev/sda"}]), ("ExtraHosts", ["attacker:127.0.0.1"]), ("PidMode", "host"), ("Memory", 0), ("ReadonlyRootfs", False), ], ) def test_unexpected_security_runtime_composition_is_rejected(adapter, field, value): runtime, row = adapter row["HostConfig"][field] = value with pytest.raises(m.PromotionRejected, match="configuration_unexpected"): runtime.target(row) @pytest.mark.parametrize( "key,value", [ ("GITEA_TOKEN", "secret"), ("GITEA_ENDPOINT", "https://attacker.invalid"), ("PYTHONPATH", "/tmp/evil"), ("SECRET_EXPECTED_UID", "0"), ], ) def test_env_drift_inline_secret_or_alternate_upstream_is_rejected(adapter, key, value): runtime, row = adapter row["Config"]["Env"] = [ v for v in row["Config"]["Env"] if not v.startswith(key + "=") ] row["Config"]["Env"].append(key + "=" + value) with pytest.raises(m.PromotionRejected): runtime.target(row) def test_fixed_promote_contains_only_one_service_no_database_or_build_commands( adapter, tmp_path, monkeypatch ): runtime, row = adapter runtime.receipt_root = tmp_path target = runtime.target(row) calls = [] def run(argv, **kwargs): calls.append(argv) config = json.loads(Path(argv[5]).read_text()) assert set(config["services"]) == {m.SERVICE} assert ( config["services"][m.SERVICE]["environment"] == target["spec"]["environment"] ) assert ( kwargs["stdout"] == m.subprocess.DEVNULL and kwargs["stderr"] == m.subprocess.DEVNULL ) return types.SimpleNamespace(returncode=0) monkeypatch.setattr(m.subprocess, "run", run) runtime.docker = lambda *_args: [{"Id": "d" * 64}] runtime.promote(target) assert calls[0][:5] == [ "docker", "compose", "--project-name", "itworx-mcp-hub", "--file", ] assert calls[0][6:] == [ "up", "--detach", "--no-deps", "--no-build", "--pull", "never", "connector-gitea", ] assert not list(tmp_path.glob(".component-*")) def source_archive( *, uid=0, regular=True, mode=0o664, content=b"reviewed", name="connector_gitea.py" ): buffer = io.BytesIO() with tarfile.open(fileobj=buffer, mode="w") as archive: member = tarfile.TarInfo(name) member.uid = uid member.gid = 100 member.mode = mode member.mtime = 1234 member.type = tarfile.REGTYPE if regular else tarfile.SYMTYPE member.size = len(content) if regular else 0 archive.addfile(member, io.BytesIO(content) if regular else None) return buffer.getvalue() @pytest.mark.parametrize( "value", [ {"uid": 99}, {"uid": 0, "regular": False}, {"uid": 0, "regular": True, "mode": 0o666}, {"uid": 0, "regular": True, "mode": 0o664}, ], ) def test_runtime_source_change_symlink_unowned_or_writable_file_rejected( value, monkeypatch ): runtime = m.ControllerRuntime.__new__(m.ControllerRuntime) raw = source_archive( uid=value.get("uid", 0), regular=value.get("regular", True), mode=value.get("mode", 0o664), content=b"changed", ) monkeypatch.setattr( m.subprocess, "run", lambda *_args, **_kwargs: types.SimpleNamespace(returncode=0, stdout=raw), ) with pytest.raises(m.PromotionRejected, match="runtime_source_unreviewed"): runtime.runtime_fingerprint() def test_source_fingerprint_uses_readonly_archive_even_when_target_is_stopped( monkeypatch, ): runtime = m.ControllerRuntime.__new__(m.ControllerRuntime) data = b"reviewed" blob = m.hashlib.sha1(b"blob " + str(len(data)).encode() + b"\0" + data).hexdigest() monkeypatch.setitem( m.ABI_BLOBS, "services/runtime/connector_gitea.py", (blob, "new") ) def run(argv, **kwargs): assert argv == [ "docker", "cp", m.CONTAINER + ":/opt/hub/services/runtime/connector_gitea.py", "-", ] assert kwargs["stderr"] == m.subprocess.DEVNULL return types.SimpleNamespace(returncode=0, stdout=source_archive(content=data)) monkeypatch.setattr(m.subprocess, "run", run) assert runtime.runtime_fingerprint() == { "uid": 0, "gid": 100, "mode": 0o664, "size": 8, "mtime": "1234", "blob": blob, } def test_nonterminal_status_never_waits_for_the_deployment_lock(fixture): runtime, request = fixture result = plan(runtime, request) m.accept_apply(arguments(request, result), receipt_key=KEY, runtime=runtime) runtime.lock = lambda: pytest.fail("status must not block behind runtime promotion") assert ( m.status({"planId": result["planId"]}, receipt_key=KEY, runtime=runtime)[ "status" ] == "queued" ) def test_component_requires_explicit_auto_apply_hold(): runtime = m.ControllerRuntime.__new__(m.ControllerRuntime) runtime.controller = types.SimpleNamespace( load_projects=lambda: {m.REPOSITORY: {"auto_apply": True}} ) with pytest.raises(m.PromotionRejected, match="auto_apply_hold_required"): runtime.snapshot() def test_rollback_never_overwrites_a_recreated_or_edited_operator_target(adapter): runtime, row = adapter before = runtime.target(row) baseline = {"target": before, "nonTargets": {"postgres": "unchanged"}} desired = runtime.desired( baseline, {"revision": REVISION, "imageId": "sha256:" + "b" * 64} ) runtime.promoted_id = "d" * 64 observed = { "target": {**desired, "containerId": "e" * 64}, "nonTargets": baseline["nonTargets"], } runtime.snapshot = lambda: observed runtime.promote = lambda *_: pytest.fail("must never overwrite operator container") with pytest.raises(m.PromotionRejected, match="rollback_cas_failed"): runtime.rollback(baseline, desired) observed["target"]["containerId"] = runtime.promoted_id observed["target"]["runtimeFingerprint"] = {"blob": "changed"} with pytest.raises(m.PromotionRejected, match="rollback_cas_failed"): runtime.rollback(baseline, desired) def test_rollback_loaded_image_provenance_must_match_bound_revision(adapter): runtime, row = adapter runtime.docker = lambda *_: [ { "Id": row["Image"], "Config": {"Labels": {"org.opencontainers.image.revision": "f" * 40}}, } ] with pytest.raises(m.PromotionRejected, match="rollback_image_unverified"): runtime.target(row) def test_baseline_digest_changes_when_bound_source_stat_or_blob_changes(fixture): runtime, _request = fixture runtime.state["target"]["runtimeFingerprint"] = {"blob": "old", "inode": 5} before = m.digest(runtime.state) runtime.state["target"]["runtimeFingerprint"]["inode"] = 6 assert m.digest(runtime.state) != before