Files
Unraid-AppOps-Gateway-Public/backend/tests/test_immutable_service_promotion.py
T

668 lines
22 KiB
Python

from __future__ import annotations
import contextlib
import copy
import importlib.util
import io
import json
import tarfile
import types
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
spec = importlib.util.spec_from_file_location(
"immutable_promotion_test",
ROOT / "ops/gitea-deploy-broker/immutable_service_promotion.py",
)
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
KEY = b"test-receipt-key" * 3
REVISION = "b" * 40
class Runtime:
def __init__(self, path):
self.receipt_root = path / "receipts"
self.state = {
"target": {
"containerId": "a" * 64,
"imageId": "sha256:" + "a" * 64,
"revision": m.OLD_REVISION,
"spec": {
"image": "sha256:" + "a" * 64,
"environment": {"GITEA_CONNECTOR_BUILD_SHA": m.OLD_REVISION},
},
},
"nonTargets": {
"postgres": {"containerId": "c" * 64, "imageId": "sha256:" + "c" * 64}
},
}
self.calls = []
self.fail_health = False
self.fail_rollback = False
self.artifact = {
"revision": REVISION,
"imageId": "sha256:" + "b" * 64,
"candidateSha256": "d" * 64,
}
def lock(self):
return contextlib.nullcontext()
def snapshot(self):
return copy.deepcopy(self.state)
def candidate(self, revision, **kwargs):
self.calls.append("candidate")
assert revision == REVISION
return copy.deepcopy(self.artifact)
desired = m.ControllerRuntime.desired
def promote(self, desired):
self.calls.append("promote")
self.state["target"] = {**copy.deepcopy(desired), "containerId": "e" * 64}
def verify(self, desired, baseline, *, rollback=False):
if self.fail_health and not rollback:
raise ValueError("private diagnostic must not escape")
assert self.state["nonTargets"] == baseline["nonTargets"]
return {"containerId": self.state["target"]["containerId"]}
def rollback(self, baseline, desired):
self.calls.append("rollback")
if self.fail_rollback:
raise ValueError("secret rollback diagnostic")
self.state = copy.deepcopy(baseline)
def verify_record(self, record):
self.calls.append("verify_record")
@pytest.fixture
def fixture(tmp_path):
runtime = Runtime(tmp_path)
request = {
"repository": "Jens/ITWorx_MCP_Hub",
"service": m.SERVICE,
"revision": REVISION,
"expectedBaseline": m.digest(runtime.state),
"reason": "Repair verified Gitea compatibility",
"requestId": "d6fe0ba3-81f0-4ed6-b36d-10b12dbf55e4",
}
return runtime, request
def plan(runtime, request):
return m.preflight(request, api_json=None, receipt_key=KEY, runtime=runtime)
def arguments(request, result):
return {**request, "planId": result["planId"], "planDigest": result["planDigest"]}
@pytest.mark.parametrize(
"change",
[
{"service": "postgres"},
{"repository": "Jens/other"},
{"revision": "main"},
{"revision": "B" * 40},
{"requestId": "bad"},
{"image": "evil"},
{"command": "sh"},
{"expectedBaseline": "bad"},
{"reason": "line\nbreak"},
],
)
def test_closed_request_rejects_untrusted_target_or_extra_fields(fixture, change):
runtime, request = fixture
with pytest.raises(m.PromotionRejected):
plan(runtime, {**request, **change})
assert not runtime.calls
def test_preflight_is_readonly_and_receipts_do_not_expose_configuration(fixture):
runtime, request = fixture
result = plan(runtime, request)
assert result["status"] == "prepared"
assert result["expiresAt"] - result["createdAt"] == 300
assert runtime.calls == ["candidate"]
assert "spec" not in json.dumps(result) and "environment" not in json.dumps(result)
assert result["planId"] == m.digest(m.request(request))
def test_queue_is_durable_idempotent_and_no_runtime_operation(fixture):
runtime, request = fixture
result = plan(runtime, request)
runtime.calls.clear()
args = arguments(request, result)
queued = m.accept_apply(args, receipt_key=KEY, runtime=runtime)
assert queued["status"] == "queued"
assert m.accept_apply(args, receipt_key=KEY, runtime=runtime) == queued
assert runtime.calls == []
assert len(list(runtime.receipt_root.rglob("*.json"))) == 2
def test_apply_rechecks_artifact_and_baseline_then_only_promotes_once(fixture):
runtime, request = fixture
result = plan(runtime, request)
args = arguments(request, result)
m.accept_apply(args, receipt_key=KEY, runtime=runtime)
applied = m.apply(
args, api_json=None, download_artifact=None, receipt_key=KEY, runtime=runtime
)
assert applied["status"] == "applied"
assert runtime.calls == ["candidate", "candidate", "promote"]
assert (
m.apply(
args,
api_json=None,
download_artifact=None,
receipt_key=KEY,
runtime=runtime,
)
== applied
)
assert runtime.calls.count("promote") == 1
assert runtime.state["nonTargets"]["postgres"]["containerId"] == "c" * 64
assert (
m.status({"planId": result["planId"]}, receipt_key=KEY, runtime=runtime)
== applied
)
def test_unrelated_baseline_drift_blocks_before_any_import_or_write(fixture):
runtime, request = fixture
result = plan(runtime, request)
runtime.calls.clear()
runtime.state["nonTargets"]["postgres"]["containerId"] = "f" * 64
with pytest.raises(m.PromotionRejected, match="baseline_changed"):
m.apply(
arguments(request, result),
api_json=None,
download_artifact=None,
receipt_key=KEY,
runtime=runtime,
)
assert not runtime.calls
def test_artifact_change_after_preflight_blocks_without_promoting(fixture):
runtime, request = fixture
result = plan(runtime, request)
runtime.artifact["candidateSha256"] = "f" * 64
with pytest.raises(m.PromotionRejected, match="artifact_changed"):
m.apply(
arguments(request, result),
api_json=None,
download_artifact=None,
receipt_key=KEY,
runtime=runtime,
)
assert "promote" not in runtime.calls
def test_health_failure_restores_exact_baseline_and_records_rollback(fixture):
runtime, request = fixture
before = runtime.snapshot()
result = plan(runtime, request)
runtime.fail_health = True
outcome = m.apply(
arguments(request, result),
api_json=None,
download_artifact=None,
receipt_key=KEY,
runtime=runtime,
)
assert outcome["status"] == "rolled_back"
assert runtime.snapshot() == before
assert "private" not in json.dumps(outcome)
def test_rollback_failure_is_durable_and_cannot_blindly_retry(fixture):
runtime, request = fixture
result = plan(runtime, request)
runtime.fail_health = runtime.fail_rollback = True
args = arguments(request, result)
with pytest.raises(m.PromotionRejected, match="rollback_recovery_required"):
m.apply(
args,
api_json=None,
download_artifact=None,
receipt_key=KEY,
runtime=runtime,
)
assert (
m.status({"planId": result["planId"]}, receipt_key=KEY, runtime=runtime)[
"status"
]
== "recovery_required"
)
with pytest.raises(m.PromotionRejected, match="recovery_required"):
m.apply(
args,
api_json=None,
download_artifact=None,
receipt_key=KEY,
runtime=runtime,
)
assert runtime.calls.count("promote") == 1
def test_expired_plan_and_different_intent_are_rejected(fixture, monkeypatch):
runtime, request = fixture
result = plan(runtime, request)
args = arguments(request, result)
with pytest.raises(m.PromotionRejected, match="plan_mismatch"):
m.accept_apply(
{**args, "reason": "Changed reason"}, receipt_key=KEY, runtime=runtime
)
monkeypatch.setattr(m.time, "time", lambda: result["expiresAt"] + 1)
with pytest.raises(m.PromotionRejected, match="plan_expired"):
m.accept_apply(args, receipt_key=KEY, runtime=runtime)
def test_signature_tampering_chain_gaps_and_weak_key_fail_closed(fixture):
runtime, request = fixture
result = plan(runtime, request)
path = next(runtime.receipt_root.rglob("*.json"))
data = json.loads(path.read_text())
data["record"]["revision"] = "f" * 40
path.write_text(json.dumps(data))
with pytest.raises(m.PromotionRejected, match="signature_invalid"):
m.status({"planId": result["planId"]}, receipt_key=KEY, runtime=runtime)
with pytest.raises(m.PromotionRejected, match="key_invalid"):
m.ReceiptStore(runtime.receipt_root, b"weak")
def test_async_rejection_records_only_safe_reason_and_never_overwrites_recovery(
fixture,
):
runtime, request = fixture
result = plan(runtime, request)
m.accept_apply(arguments(request, result), receipt_key=KEY, runtime=runtime)
rejected = m.record_worker_failure(
result["planId"], "secret=raw error!", receipt_key=KEY, runtime=runtime
)
assert rejected["status"] == "rejected"
assert rejected["reasonCode"] == "component_worker_failed"
assert (
m.record_worker_failure(
result["planId"], "later", receipt_key=KEY, runtime=runtime
)
== rejected
)
@pytest.fixture
def adapter():
runtime = m.ControllerRuntime.__new__(m.ControllerRuntime)
runtime.docker = lambda *_args: [
{
"Id": "sha256:" + "a" * 64,
"Config": {
"Labels": {
"org.opencontainers.image.revision": m.OLD_REVISION,
"org.opencontainers.image.source": "https://gitea.example.com/jens/itworx-mcp-hub",
}
},
}
]
runtime.runtime_fingerprint = lambda: {
"blob": m.ABI_BLOBS["services/runtime/connector_gitea.py"][0]
}
env = {
"SERVICE_NAME": "connector-gitea",
"GITEA_CONNECTOR_BUILD_SHA": m.OLD_REVISION,
"GITEA_ENDPOINT": "https://gitea.example.com",
"GITEA_TOKEN_FILE": "/run/secrets/gitea_mcp_token",
"GITEA_WRITE_TOKEN_FILE": "/run/secrets/gitea_mcp_write_token",
"SECRET_EXPECTED_UID": "99",
"SECRET_EXPECTED_GID": "100",
"CONNECTOR_IDENTITY_KEY_FILE": "/run/secrets/connector_identity_signing_key",
"GPG_KEY": "public image fingerprint",
}
mounts = [
{
"Type": "bind",
"RW": False,
"Source": "/srv/example/appdata/itworx-mcp-hub/secrets/" + name,
"Destination": "/run/secrets/" + name,
}
for name in (
"gitea_mcp_token",
"gitea_mcp_write_token",
"connector_identity_signing_key",
)
]
mounts.append(
{
"Type": "bind",
"RW": False,
"Source": f"/srv/example/appdata/itworx-mcp-hub/deployment/releases/{m.OLD_REVISION}/services/runtime/connector_gitea.py",
"Destination": "/opt/hub/services/runtime/connector_gitea.py",
}
)
row = {
"Id": "a" * 64,
"Image": "sha256:" + "a" * 64,
"Config": {
"Entrypoint": ["python", "-m", "services.runtime"],
"Cmd": None,
"User": "99:100",
"WorkingDir": "/opt/hub",
"Tty": False,
"OpenStdin": False,
"ExposedPorts": {"8080/tcp": {}},
"Env": [k + "=" + v for k, v in env.items()],
"Labels": {},
"Healthcheck": {
"Interval": 60000000000,
"Retries": 5,
"StartPeriod": 10000000000,
"Timeout": 3000000000,
"Test": [
"CMD",
"python",
"-c",
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/healthz', timeout=2)",
],
},
},
"HostConfig": {
"ReadonlyRootfs": True,
"Privileged": False,
"CapDrop": ["ALL"],
"SecurityOpt": ["no-new-privileges:true"],
"Memory": 536870912,
"MemorySwap": -1,
"MemoryReservation": 0,
"NetworkMode": "itworx-mcp-hub-connectors",
"LogConfig": {
"Type": "json-file",
"Config": {"max-file": "3", "max-size": "10m"},
},
"ShmSize": 67108864,
"AutoRemove": False,
"CgroupnsMode": "private",
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger",
],
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware",
"/sys/devices/virtual/powercap",
],
"NanoCpus": 1000000000,
"PidsLimit": 128,
"Tmpfs": {"/tmp": "size=512m,mode=1777"},
"IpcMode": "private",
"RestartPolicy": {"Name": "unless-stopped", "MaximumRetryCount": 0},
},
"Mounts": mounts,
"NetworkSettings": {
"Networks": {
n: {"Aliases": [m.CONTAINER, m.SERVICE]}
for n in (
"itworx-mcp-hub-connectors",
"itworx-mcp-hub-observe",
"itworx-mcp-hub-egress",
)
}
},
}
return runtime, row
def test_known_live_target_preserves_public_metadata_and_only_image_build_sha_changes(
adapter,
):
runtime, row = adapter
target = runtime.target(row)
desired = runtime.desired(
{"target": target}, {"imageId": "sha256:" + "b" * 64, "revision": REVISION}
)
changed = copy.deepcopy(desired)
changed.update(imageId=target["imageId"], revision=target["revision"])
changed["spec"]["image"] = target["imageId"]
changed["spec"]["environment"]["GITEA_CONNECTOR_BUILD_SHA"] = m.OLD_REVISION
assert changed == target
assert target["spec"]["environment"]["SECRET_EXPECTED_UID"] == "99"
assert target["spec"]["environment"]["GPG_KEY"] == "public image fingerprint"
@pytest.mark.parametrize(
"field,value",
[
("Privileged", True),
("CapAdd", ["SYS_ADMIN"]),
("Devices", [{"PathOnHost": "/dev/sda"}]),
("ExtraHosts", ["attacker:127.0.0.1"]),
("PidMode", "host"),
("Memory", 0),
("ReadonlyRootfs", False),
],
)
def test_unexpected_security_runtime_composition_is_rejected(adapter, field, value):
runtime, row = adapter
row["HostConfig"][field] = value
with pytest.raises(m.PromotionRejected, match="configuration_unexpected"):
runtime.target(row)
@pytest.mark.parametrize(
"key,value",
[
("GITEA_TOKEN", "secret"),
("GITEA_ENDPOINT", "https://attacker.invalid"),
("PYTHONPATH", "/tmp/evil"),
("SECRET_EXPECTED_UID", "0"),
],
)
def test_env_drift_inline_secret_or_alternate_upstream_is_rejected(adapter, key, value):
runtime, row = adapter
row["Config"]["Env"] = [
v for v in row["Config"]["Env"] if not v.startswith(key + "=")
]
row["Config"]["Env"].append(key + "=" + value)
with pytest.raises(m.PromotionRejected):
runtime.target(row)
def test_fixed_promote_contains_only_one_service_no_database_or_build_commands(
adapter, tmp_path, monkeypatch
):
runtime, row = adapter
runtime.receipt_root = tmp_path
target = runtime.target(row)
calls = []
def run(argv, **kwargs):
calls.append(argv)
config = json.loads(Path(argv[5]).read_text())
assert set(config["services"]) == {m.SERVICE}
assert (
config["services"][m.SERVICE]["environment"]
== target["spec"]["environment"]
)
assert (
kwargs["stdout"] == m.subprocess.DEVNULL
and kwargs["stderr"] == m.subprocess.DEVNULL
)
return types.SimpleNamespace(returncode=0)
monkeypatch.setattr(m.subprocess, "run", run)
runtime.docker = lambda *_args: [{"Id": "d" * 64}]
runtime.promote(target)
assert calls[0][:5] == [
"docker",
"compose",
"--project-name",
"itworx-mcp-hub",
"--file",
]
assert calls[0][6:] == [
"up",
"--detach",
"--no-deps",
"--no-build",
"--pull",
"never",
"connector-gitea",
]
assert not list(tmp_path.glob(".component-*"))
def source_archive(
*, uid=0, regular=True, mode=0o664, content=b"reviewed", name="connector_gitea.py"
):
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w") as archive:
member = tarfile.TarInfo(name)
member.uid = uid
member.gid = 100
member.mode = mode
member.mtime = 1234
member.type = tarfile.REGTYPE if regular else tarfile.SYMTYPE
member.size = len(content) if regular else 0
archive.addfile(member, io.BytesIO(content) if regular else None)
return buffer.getvalue()
@pytest.mark.parametrize(
"value",
[
{"uid": 99},
{"uid": 0, "regular": False},
{"uid": 0, "regular": True, "mode": 0o666},
{"uid": 0, "regular": True, "mode": 0o664},
],
)
def test_runtime_source_change_symlink_unowned_or_writable_file_rejected(
value, monkeypatch
):
runtime = m.ControllerRuntime.__new__(m.ControllerRuntime)
raw = source_archive(
uid=value.get("uid", 0),
regular=value.get("regular", True),
mode=value.get("mode", 0o664),
content=b"changed",
)
monkeypatch.setattr(
m.subprocess,
"run",
lambda *_args, **_kwargs: types.SimpleNamespace(returncode=0, stdout=raw),
)
with pytest.raises(m.PromotionRejected, match="runtime_source_unreviewed"):
runtime.runtime_fingerprint()
def test_source_fingerprint_uses_readonly_archive_even_when_target_is_stopped(
monkeypatch,
):
runtime = m.ControllerRuntime.__new__(m.ControllerRuntime)
data = b"reviewed"
blob = m.hashlib.sha1(b"blob " + str(len(data)).encode() + b"\0" + data).hexdigest()
monkeypatch.setitem(
m.ABI_BLOBS, "services/runtime/connector_gitea.py", (blob, "new")
)
def run(argv, **kwargs):
assert argv == [
"docker",
"cp",
m.CONTAINER + ":/opt/hub/services/runtime/connector_gitea.py",
"-",
]
assert kwargs["stderr"] == m.subprocess.DEVNULL
return types.SimpleNamespace(returncode=0, stdout=source_archive(content=data))
monkeypatch.setattr(m.subprocess, "run", run)
assert runtime.runtime_fingerprint() == {
"uid": 0,
"gid": 100,
"mode": 0o664,
"size": 8,
"mtime": "1234",
"blob": blob,
}
def test_nonterminal_status_never_waits_for_the_deployment_lock(fixture):
runtime, request = fixture
result = plan(runtime, request)
m.accept_apply(arguments(request, result), receipt_key=KEY, runtime=runtime)
runtime.lock = lambda: pytest.fail("status must not block behind runtime promotion")
assert (
m.status({"planId": result["planId"]}, receipt_key=KEY, runtime=runtime)[
"status"
]
== "queued"
)
def test_component_requires_explicit_auto_apply_hold():
runtime = m.ControllerRuntime.__new__(m.ControllerRuntime)
runtime.controller = types.SimpleNamespace(
load_projects=lambda: {m.REPOSITORY: {"auto_apply": True}}
)
with pytest.raises(m.PromotionRejected, match="auto_apply_hold_required"):
runtime.snapshot()
def test_rollback_never_overwrites_a_recreated_or_edited_operator_target(adapter):
runtime, row = adapter
before = runtime.target(row)
baseline = {"target": before, "nonTargets": {"postgres": "unchanged"}}
desired = runtime.desired(
baseline, {"revision": REVISION, "imageId": "sha256:" + "b" * 64}
)
runtime.promoted_id = "d" * 64
observed = {
"target": {**desired, "containerId": "e" * 64},
"nonTargets": baseline["nonTargets"],
}
runtime.snapshot = lambda: observed
runtime.promote = lambda *_: pytest.fail("must never overwrite operator container")
with pytest.raises(m.PromotionRejected, match="rollback_cas_failed"):
runtime.rollback(baseline, desired)
observed["target"]["containerId"] = runtime.promoted_id
observed["target"]["runtimeFingerprint"] = {"blob": "changed"}
with pytest.raises(m.PromotionRejected, match="rollback_cas_failed"):
runtime.rollback(baseline, desired)
def test_rollback_loaded_image_provenance_must_match_bound_revision(adapter):
runtime, row = adapter
runtime.docker = lambda *_: [
{
"Id": row["Image"],
"Config": {"Labels": {"org.opencontainers.image.revision": "f" * 40}},
}
]
with pytest.raises(m.PromotionRejected, match="rollback_image_unverified"):
runtime.target(row)
def test_baseline_digest_changes_when_bound_source_stat_or_blob_changes(fixture):
runtime, _request = fixture
runtime.state["target"]["runtimeFingerprint"] = {"blob": "old", "inode": 5}
before = m.digest(runtime.state)
runtime.state["target"]["runtimeFingerprint"]["inode"] = 6
assert m.digest(runtime.state) != before