891 lines
38 KiB
Python
891 lines
38 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import stat
|
|
import sys
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
CONTROLLER = ROOT / "ops" / "gitea-deploy-broker"
|
|
RECOVERY_ID = "12345678-1234-4234-8234-123456789abc"
|
|
REVISION = "c" * 40
|
|
TREE = "d" * 40
|
|
IMAGE_ID = "sha256:" + "e" * 64
|
|
CONTAINER_ID = "f" * 64
|
|
BUILT_AT = "2026-10-07T00:00:00.000Z"
|
|
SECRET_SENTINEL = "model-private-secret$literal$$value"
|
|
|
|
|
|
@pytest.fixture
|
|
def controller(monkeypatch, tmp_path):
|
|
monkeypatch.syspath_prepend(str(CONTROLLER))
|
|
spec = importlib.util.spec_from_file_location(
|
|
"appops_self_upgrade_runtime_model_test",
|
|
CONTROLLER / "status_server.py",
|
|
)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
|
|
recovery_root = tmp_path / "recovery"
|
|
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root)
|
|
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_REQUEST_ROOT", recovery_root / "requests")
|
|
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_MODEL_ROOT", recovery_root / "models")
|
|
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_TRANSACTION_ROOT", recovery_root / "transactions")
|
|
secret = b"runtime-model-test-secret-key-0123456789"
|
|
monkeypatch.setattr(module, "_self_upgrade_secret", lambda: secret)
|
|
return module, recovery_root, secret
|
|
|
|
|
|
def _environment(revision: str = REVISION, tree: str = TREE) -> dict[str, str]:
|
|
return {
|
|
"APPOPS_SOURCE_REPOSITORY": "jens/unraid-appops-gateway",
|
|
"APPOPS_SOURCE_REVISION": revision,
|
|
"APPOPS_SOURCE_TREE": tree,
|
|
"APPOPS_BUILD_ID": revision[:12],
|
|
"APPOPS_BUILD_DATE": BUILT_AT,
|
|
"KNOWLEDGE_OPTIMIZER_MODEL": "gpt-6-luna",
|
|
"OPERATOR_TOKEN": SECRET_SENTINEL,
|
|
}
|
|
|
|
|
|
def _model(revision: str = REVISION, tree: str = TREE) -> dict[str, object]:
|
|
return {
|
|
"name": "unraid-appops-gateway",
|
|
"services": {
|
|
"appops-gateway": {
|
|
"image": "unraid-appops-gateway:production",
|
|
"container_name": "Unraid-AppOps-Gateway",
|
|
"environment": _environment(revision, tree),
|
|
"ports": [{"target": 1216, "published": "1216", "host_ip": "198.18.10.150"}],
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _legacy_environment() -> dict[str, str]:
|
|
return {
|
|
**{f"LEGACY_SETTING_{index:02d}": f"fixture-value-{index:02d}" for index in range(43)},
|
|
"KNOWLEDGE_OPTIMIZER_MODEL": "gpt-6-luna",
|
|
"APPOPS_FRONTEND_ORIGIN": "https://apps.example.test",
|
|
"OPERATOR_TOKEN": SECRET_SENTINEL,
|
|
}
|
|
|
|
|
|
def _legacy_model(module, image_ref: str | None = None) -> dict[str, object]:
|
|
return {
|
|
"name": "unraid-appops-gateway",
|
|
"services": {
|
|
"appops-gateway": {
|
|
"image": image_ref or module.SELF_UPGRADE_OLD_IMAGE_REF,
|
|
"container_name": "Unraid-AppOps-Gateway",
|
|
"environment": _legacy_environment(),
|
|
"ports": [{
|
|
"target": 1216, "published": "1216", "host_ip": "198.18.10.150",
|
|
}],
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _mock_legacy_docker(monkeypatch, module, *, candidate=False):
|
|
environment = _environment() if candidate else _legacy_environment()
|
|
raw_env = [f"{key}={value}" for key, value in environment.items()]
|
|
if not candidate:
|
|
raw_env.append(f"APPOPS_FRONTEND_ORIGIN={environment['APPOPS_FRONTEND_ORIGIN']}")
|
|
revision = REVISION if candidate else module.SELF_UPGRADE_OLD_REVISION
|
|
tree = TREE if candidate else module.SELF_UPGRADE_OLD_TREE
|
|
image_id = IMAGE_ID if candidate else module.SELF_UPGRADE_IMAGE_ID
|
|
image_labels = {
|
|
"org.opencontainers.image.source": module.SELF_UPGRADE_REPOSITORY,
|
|
"org.opencontainers.image.revision": revision,
|
|
"org.opencontainers.image.source-tree": tree,
|
|
"org.opencontainers.image.version": revision[:12],
|
|
"org.opencontainers.image.created": BUILT_AT if candidate else "2026-09-01T00:00:00.000Z",
|
|
}
|
|
container = {
|
|
"Id": CONTAINER_ID,
|
|
"Image": image_id,
|
|
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
|
"Mounts": [
|
|
{"Type": "bind", "Source": module.SELF_UPGRADE_DATA_PATH,
|
|
"Destination": "/data", "RW": True},
|
|
{"Type": "bind", "Source": module.SELF_UPGRADE_MODELS_PATH,
|
|
"Destination": module.SELF_UPGRADE_MODELS_TARGET, "RW": True},
|
|
],
|
|
"Config": {
|
|
"Image": module.SELF_UPGRADE_PRODUCTION_IMAGE_REF if candidate else module.SELF_UPGRADE_OLD_IMAGE_REF,
|
|
"Hostname": "unraid-appops-gateway",
|
|
"Env": raw_env,
|
|
"Labels": {
|
|
**image_labels,
|
|
"com.docker.compose.project": "unraid-appops-gateway",
|
|
"com.docker.compose.service": "appops-gateway",
|
|
"com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source",
|
|
"com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml",
|
|
},
|
|
},
|
|
"HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}},
|
|
"NetworkSettings": {
|
|
"Ports": {"1216/tcp": [{"HostIp": module.SELF_UPGRADE_PUBLISH_HOST, "HostPort": "1216"}]},
|
|
"Networks": {
|
|
"itworx-mcp-hub-npm": {},
|
|
"unraid-appops-gateway_default": {},
|
|
},
|
|
},
|
|
}
|
|
image = {"Id": image_id, "Config": {"Labels": image_labels}}
|
|
|
|
def fake_run(args, **_kwargs):
|
|
if args[:3] == ["docker", "ps", "-a"]:
|
|
return SimpleNamespace(stdout="Unraid-AppOps-Gateway\n")
|
|
if args[:2] == ["docker", "inspect"]:
|
|
return SimpleNamespace(stdout=json.dumps([container]))
|
|
if args[:3] == ["docker", "image", "inspect"]:
|
|
return SimpleNamespace(stdout=json.dumps([image]))
|
|
raise AssertionError("unexpected Docker command category")
|
|
|
|
monkeypatch.setattr(module.subprocess, "run", fake_run)
|
|
monkeypatch.setattr(module, "_load_projects", lambda: {
|
|
module.SELF_UPGRADE_REPOSITORY: {
|
|
"containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True,
|
|
},
|
|
})
|
|
monkeypatch.setattr(
|
|
module, "_self_upgrade_read_legacy_cache_json",
|
|
lambda *_args, **_kwargs: (b"{}", {"live_artifacts": {"containers": {}}}),
|
|
)
|
|
monkeypatch.setattr(module, "_self_upgrade_source_is_valid", lambda _source: True)
|
|
monkeypatch.setattr(module, "_self_upgrade_source_fingerprint", lambda *_args: "2" * 64)
|
|
return container
|
|
|
|
|
|
def _receipt(module, revision: str = REVISION, tree: str = TREE) -> dict[str, object]:
|
|
digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest()
|
|
return {
|
|
"schema": "gitea-deploy-receipt.v1",
|
|
"repository": module.SELF_UPGRADE_REPOSITORY,
|
|
"revision": revision,
|
|
"tree": tree,
|
|
"treeDigest": digest,
|
|
"build": {
|
|
"id": revision[:12],
|
|
"repository": module.SELF_UPGRADE_REPOSITORY,
|
|
"revision": revision,
|
|
"tree": tree,
|
|
"treeDigest": digest,
|
|
"builtAt": BUILT_AT,
|
|
},
|
|
"containers": {
|
|
"Unraid-AppOps-Gateway": {
|
|
"image_id": IMAGE_ID,
|
|
"image_revision": revision,
|
|
"image_source": module.SELF_UPGRADE_REPOSITORY,
|
|
"image_tree": tree,
|
|
"image_version": revision[:12],
|
|
"image_created": BUILT_AT,
|
|
},
|
|
},
|
|
"live": None,
|
|
}
|
|
|
|
|
|
def _binding(module, receipt, *, container_id: str = CONTAINER_ID) -> dict[str, object]:
|
|
return {
|
|
"containerId": container_id,
|
|
"imageId": IMAGE_ID,
|
|
"runtimeFingerprint": "1" * 64,
|
|
"sourceFingerprint": "2" * 64,
|
|
"receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json(receipt)).hexdigest(),
|
|
}
|
|
|
|
|
|
def test_plain_compose_file_preserves_literal_dollars_under_private_attestation(
|
|
controller,
|
|
):
|
|
module, recovery_root, _secret = controller
|
|
if os.name == "nt" or os.geteuid() != 0:
|
|
pytest.skip("The controller model store requires root-owned POSIX files")
|
|
|
|
pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
|
assert isinstance(pointer, dict)
|
|
loaded = module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION)
|
|
assert loaded is not None
|
|
assert loaded["model"] == _model()
|
|
assert loaded["pointer"] == pointer
|
|
assert Path(loaded["path"]).name == f"{RECOVERY_ID}-{REVISION}.compose.json"
|
|
|
|
compose_path = Path(loaded["path"])
|
|
attestation_path = compose_path.with_name(
|
|
f"{RECOVERY_ID}-{REVISION}.model.json"
|
|
)
|
|
compose_raw = compose_path.read_bytes()
|
|
attestation_raw = attestation_path.read_bytes()
|
|
assert b"model-private-secret$$literal$$$$value" in compose_raw
|
|
assert SECRET_SENTINEL.encode() not in compose_raw
|
|
assert SECRET_SENTINEL.encode() not in attestation_raw
|
|
assert b"OPERATOR_TOKEN" not in attestation_raw
|
|
assert stat.S_IMODE(compose_path.stat().st_mode) == 0o600
|
|
assert stat.S_IMODE(attestation_path.stat().st_mode) == 0o600
|
|
assert stat.S_IMODE(recovery_root.stat().st_mode) == 0o700
|
|
assert stat.S_IMODE((recovery_root / "models").stat().st_mode) == 0o700
|
|
assert module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) == pointer
|
|
assert module._self_upgrade_store_compose_model(
|
|
RECOVERY_ID, REVISION, _model(tree="a" * 40),
|
|
) is None
|
|
|
|
compose_path.write_text(compose_path.read_text(encoding="utf-8").replace("gpt-6-luna", "gpt-5.5"))
|
|
compose_path.chmod(0o600)
|
|
assert module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION) is None
|
|
|
|
|
|
def test_live_model_records_are_content_addressed_and_activate_only_from_state(
|
|
controller, monkeypatch,
|
|
):
|
|
module, _recovery_root, _secret = controller
|
|
if os.name == "nt" or os.geteuid() != 0:
|
|
pytest.skip("The controller model store requires root-owned POSIX files")
|
|
pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
|
assert isinstance(pointer, dict)
|
|
receipt = _receipt(module)
|
|
live_pointer = module._self_upgrade_store_live_model(pointer, _binding(module, receipt))
|
|
assert isinstance(live_pointer, dict)
|
|
first_live = module._self_upgrade_read_live_model_reference(live_pointer)
|
|
assert first_live is not None
|
|
assert first_live["model"] == _model()
|
|
|
|
fresh_container_id = "a" * 64
|
|
restored_pointer = module._self_upgrade_store_live_model(
|
|
pointer, _binding(module, receipt, container_id=fresh_container_id),
|
|
)
|
|
assert isinstance(restored_pointer, dict)
|
|
assert restored_pointer["liveRecordSha256"] != live_pointer["liveRecordSha256"]
|
|
assert module._self_upgrade_read_live_model_reference(live_pointer) == first_live
|
|
restored = module._self_upgrade_read_live_model_reference(restored_pointer)
|
|
assert restored is not None
|
|
assert restored["live"]["containerId"] == fresh_container_id
|
|
|
|
state = {
|
|
"schema_version": 3,
|
|
"repository": module.SELF_UPGRADE_REPOSITORY,
|
|
"status": "failed",
|
|
"live_revision": REVISION,
|
|
"live_status": "healthy",
|
|
"live_legacy_provenance": False,
|
|
"live_receipt": receipt,
|
|
"live_artifacts": {
|
|
"containers": {
|
|
"Unraid-AppOps-Gateway": {
|
|
"container_id": fresh_container_id,
|
|
"image_id": IMAGE_ID,
|
|
},
|
|
},
|
|
"appops_runtime_model": restored_pointer,
|
|
},
|
|
}
|
|
project = {
|
|
"containers": ["Unraid-AppOps-Gateway"],
|
|
"verify_oci_revision": True,
|
|
}
|
|
active = module._self_upgrade_active_compose_model(state, project=project)
|
|
assert active is not None
|
|
assert active["model"] == _model()
|
|
runtime = {
|
|
"revision": REVISION,
|
|
"containerId": fresh_container_id,
|
|
"imageId": IMAGE_ID,
|
|
"runtimeFingerprint": "1" * 64,
|
|
"sourceFingerprint": "2" * 64,
|
|
"runtimeEnvironment": _environment(),
|
|
}
|
|
assert module._self_upgrade_active_compose_model(state, runtime, project=project) is not None
|
|
runtime["runtimeEnvironment"] = {**_environment(), "KNOWLEDGE_OPTIMIZER_MODEL": "gpt-5.5"}
|
|
assert module._self_upgrade_active_compose_model(state, runtime, project=project) is None
|
|
state["live_status"] = "unhealthy"
|
|
assert module._self_upgrade_active_compose_model(state, project=project) is None
|
|
|
|
|
|
@pytest.mark.parametrize("drift", [None, "host", "port", "extra-port", "environment", "image", "unsigned", "stopped", "not-restarting"])
|
|
def test_signed_crashloop_candidate_can_be_identified_only_for_rollback(controller, monkeypatch, drift):
|
|
module, _root, _secret = controller
|
|
if os.name == "nt" or os.geteuid() != 0:
|
|
pytest.skip("The signed model store requires root-owned POSIX files")
|
|
pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
|
assert isinstance(pointer, dict)
|
|
loaded = module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION)
|
|
container = _mock_legacy_docker(monkeypatch, module, candidate=True)
|
|
container["Config"]["Labels"]["com.docker.compose.project.config_files"] = loaded["path"]
|
|
container["State"].update({"Restarting": True, "Status": "restarting", "Health": {"Status": "unhealthy"}})
|
|
container["NetworkSettings"]["Ports"] = {}
|
|
container["HostConfig"]["PortBindings"] = {
|
|
"1216/tcp": [{"HostIp": module.SELF_UPGRADE_PUBLISH_HOST, "HostPort": "1216"}],
|
|
}
|
|
if drift == "host":
|
|
container["HostConfig"]["PortBindings"]["1216/tcp"][0]["HostIp"] = "0.0.0.0"
|
|
elif drift == "port":
|
|
container["HostConfig"]["PortBindings"]["1216/tcp"][0]["HostPort"] = "9999"
|
|
elif drift == "extra-port":
|
|
container["HostConfig"]["PortBindings"]["22/tcp"] = [{"HostIp": "0.0.0.0", "HostPort": "22"}]
|
|
elif drift == "environment":
|
|
container["Config"]["Env"].append("UNREVIEWED_SETTING=drift")
|
|
elif drift == "image":
|
|
container["Image"] = "sha256:" + "0" * 64
|
|
elif drift == "unsigned":
|
|
pointer = {**pointer, "modelRecordSha256": "0" * 64}
|
|
elif drift == "stopped":
|
|
container["State"]["Running"] = False
|
|
elif drift == "not-restarting":
|
|
container["State"]["Restarting"] = False
|
|
assert module._self_upgrade_runtime() is None
|
|
assert module._self_upgrade_model_runtime_identity(pointer, IMAGE_ID) is None
|
|
identity = module._self_upgrade_model_runtime_identity(pointer, IMAGE_ID, require_healthy=False)
|
|
assert (identity is not None) == (drift is None)
|
|
|
|
|
|
def test_candidate_binding_rechecks_exact_model_receipt_and_runtime(
|
|
controller, monkeypatch,
|
|
):
|
|
module, _recovery_root, _secret = controller
|
|
if os.name == "nt" or os.geteuid() != 0:
|
|
pytest.skip("The controller model store requires root-owned POSIX files")
|
|
pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
|
assert isinstance(pointer, dict)
|
|
receipt = _receipt(module)
|
|
project = {
|
|
"containers": ["Unraid-AppOps-Gateway"],
|
|
"verify_oci_revision": True,
|
|
}
|
|
monkeypatch.setattr(module, "_load_projects", lambda: {module.SELF_UPGRADE_REPOSITORY: project})
|
|
monkeypatch.setattr(module, "_self_upgrade_runtime", lambda **_kwargs: {
|
|
"revision": REVISION,
|
|
"sourceRepository": module.SELF_UPGRADE_REPOSITORY,
|
|
"sourceTree": TREE,
|
|
"imageId": IMAGE_ID,
|
|
"containerId": CONTAINER_ID,
|
|
"runtimeFingerprint": "1" * 64,
|
|
"sourceFingerprint": "2" * 64,
|
|
"runtimeEnvironment": _environment(),
|
|
})
|
|
binding = module._self_upgrade_candidate_model_binding(pointer, receipt)
|
|
assert binding == _binding(module, receipt)
|
|
wrong_receipt = _receipt(module, tree="b" * 40)
|
|
assert module._self_upgrade_candidate_model_binding(pointer, wrong_receipt) is None
|
|
bad_pointer = {**pointer, "modelSha256": "0" * 64}
|
|
assert module._self_upgrade_candidate_model_binding(bad_pointer, receipt) is None
|
|
|
|
|
|
def test_fixed_555_model_preserves_exact_environment_and_duplicate_origin(
|
|
controller, monkeypatch,
|
|
):
|
|
module, _recovery_root, _secret = controller
|
|
if os.name == "nt" or os.geteuid() != 0:
|
|
pytest.skip("The controller model store requires root-owned POSIX files")
|
|
journal = {
|
|
"expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION,
|
|
"currentImageId": module.SELF_UPGRADE_IMAGE_ID,
|
|
"runtimeEnvironment": _legacy_environment(),
|
|
}
|
|
monkeypatch.setattr(module, "_self_upgrade_recovery_journal", lambda _rid: journal)
|
|
pointer = module._self_upgrade_store_compose_model(
|
|
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, _legacy_model(module),
|
|
)
|
|
assert isinstance(pointer, dict)
|
|
loaded = module._self_upgrade_load_compose_model(
|
|
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION,
|
|
)
|
|
assert loaded is not None
|
|
assert loaded["model"] == _legacy_model(module)
|
|
changed = _legacy_model(module)
|
|
changed["services"]["appops-gateway"]["environment"]["APPOPS_BUILD_ID"] = "untrusted"
|
|
assert not module._self_upgrade_compose_model_valid(
|
|
changed, module.SELF_UPGRADE_OLD_REVISION,
|
|
)
|
|
|
|
container = _mock_legacy_docker(monkeypatch, module)
|
|
initial_runtime = module._self_upgrade_runtime()
|
|
assert initial_runtime is not None
|
|
assert initial_runtime["runtimeEnvironment"] == _legacy_environment()
|
|
assert len(container["Config"]["Env"]) == 47
|
|
assert len(initial_runtime["runtimeEnvironment"]) == 46
|
|
runtime = module._self_upgrade_runtime(
|
|
authorized_model_pointer=pointer,
|
|
allow_legacy_canonical_config=True,
|
|
)
|
|
assert runtime is not None
|
|
assert runtime["runtimeEnvironment"] == _legacy_environment()
|
|
assert len(runtime["runtimeEnvironment"]) == 46
|
|
assert len(container["Config"]["Env"]) == 47
|
|
assert runtime["imageId"] == module.SELF_UPGRADE_IMAGE_ID
|
|
assert runtime["containerId"] == container["Id"]
|
|
binding = module._self_upgrade_legacy_model_binding(pointer)
|
|
assert binding == {
|
|
"containerId": CONTAINER_ID,
|
|
"imageId": module.SELF_UPGRADE_IMAGE_ID,
|
|
"runtimeFingerprint": runtime["runtimeFingerprint"],
|
|
"sourceFingerprint": "2" * 64,
|
|
"receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json({})).hexdigest(),
|
|
}
|
|
|
|
live_pointer = module._self_upgrade_store_live_model(pointer, binding)
|
|
assert isinstance(live_pointer, dict)
|
|
state = {
|
|
"schema_version": 3,
|
|
"repository": module.SELF_UPGRADE_REPOSITORY,
|
|
"status": "rolled-back",
|
|
"live_revision": module.SELF_UPGRADE_OLD_REVISION,
|
|
"live_status": "healthy",
|
|
"live_legacy_provenance": True,
|
|
"live_receipt": {},
|
|
"live_artifacts": {
|
|
"containers": {
|
|
"Unraid-AppOps-Gateway": {
|
|
"container_id": CONTAINER_ID,
|
|
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
|
},
|
|
},
|
|
"appops_runtime_model": live_pointer,
|
|
},
|
|
}
|
|
project = {
|
|
"containers": ["Unraid-AppOps-Gateway"],
|
|
"verify_oci_revision": True,
|
|
}
|
|
active = module._self_upgrade_active_compose_model(state, project=project)
|
|
assert active is not None
|
|
assert active["legacy"] is True
|
|
assert active["model"] == _legacy_model(module)
|
|
assert module._self_upgrade_active_compose_model(state, runtime, project=project) is not None
|
|
|
|
container["State"]["Health"]["Status"] = "unhealthy"
|
|
assert module._self_upgrade_runtime() is None
|
|
assert module._self_upgrade_model_runtime_identity(pointer, module.SELF_UPGRADE_IMAGE_ID) is None
|
|
recovery_identity = module._self_upgrade_model_runtime_identity(
|
|
pointer, module.SELF_UPGRADE_IMAGE_ID, require_healthy=False,
|
|
)
|
|
assert isinstance(recovery_identity, dict)
|
|
assert set(recovery_identity) == {
|
|
"containerId", "imageId", "revision", "sourceRepository", "sourceTree",
|
|
"runtimeFingerprint", "sourceFingerprint", "runtimeCasSha256",
|
|
}
|
|
assert module._self_upgrade_model_runtime_identity(
|
|
pointer, module.SELF_UPGRADE_IMAGE_ID, require_healthy="false",
|
|
) is None
|
|
container["State"]["Health"]["Status"] = "healthy"
|
|
|
|
# The identical duplicate is accepted only for the exact immutable old
|
|
# image; conflicting duplicates remain fail-closed.
|
|
raw_env = container["Config"]["Env"]
|
|
raw_env[-1] = "APPOPS_FRONTEND_ORIGIN=https://attacker.invalid"
|
|
assert module._self_upgrade_runtime(
|
|
authorized_model_pointer=pointer,
|
|
allow_legacy_canonical_config=True,
|
|
) is None
|
|
|
|
|
|
def test_fixed_555_production_reference_is_valid_only_under_signed_legacy_model(
|
|
controller, monkeypatch,
|
|
):
|
|
module, _recovery_root, _secret = controller
|
|
if os.name == "nt" or os.geteuid() != 0:
|
|
pytest.skip("The controller model store requires root-owned POSIX files")
|
|
journal = {
|
|
"expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION,
|
|
"currentImageId": module.SELF_UPGRADE_IMAGE_ID,
|
|
"runtimeEnvironment": _legacy_environment(),
|
|
}
|
|
monkeypatch.setattr(module, "_self_upgrade_recovery_journal", lambda _rid: journal)
|
|
model = _legacy_model(module, module.SELF_UPGRADE_PRODUCTION_IMAGE_REF)
|
|
pointer = module._self_upgrade_store_compose_model(
|
|
"42345678-1234-4234-8234-123456789abc",
|
|
module.SELF_UPGRADE_OLD_REVISION,
|
|
model,
|
|
)
|
|
assert isinstance(pointer, dict)
|
|
container = _mock_legacy_docker(monkeypatch, module)
|
|
# The pre-capture container can retain the old platform tag while the
|
|
# signed effective rollback model uses the closed production tag.
|
|
container["Config"]["Image"] = module.SELF_UPGRADE_OLD_IMAGE_REF
|
|
initial_runtime = module._self_upgrade_runtime()
|
|
assert initial_runtime is not None
|
|
assert initial_runtime["imageRef"] == module.SELF_UPGRADE_OLD_IMAGE_REF
|
|
|
|
runtime = module._self_upgrade_runtime(
|
|
authorized_model_pointer=pointer,
|
|
allow_legacy_canonical_config=True,
|
|
)
|
|
assert isinstance(runtime, dict)
|
|
assert runtime["imageId"] == module.SELF_UPGRADE_IMAGE_ID
|
|
assert runtime["imageRef"] == module.SELF_UPGRADE_OLD_IMAGE_REF
|
|
binding = module._self_upgrade_legacy_model_binding(pointer)
|
|
assert isinstance(binding, dict)
|
|
reference = module._self_upgrade_store_live_model(pointer, binding)
|
|
assert isinstance(reference, dict)
|
|
|
|
project = {
|
|
"containers": ["Unraid-AppOps-Gateway"],
|
|
"verify_oci_revision": True,
|
|
}
|
|
monkeypatch.setattr(module, "_load_projects", lambda: {
|
|
module.SELF_UPGRADE_REPOSITORY: project,
|
|
})
|
|
snapshot = {
|
|
"revision": module.SELF_UPGRADE_OLD_REVISION,
|
|
"legacy_provenance": True,
|
|
"receipt": {},
|
|
"appops_runtime_model": reference,
|
|
"containers": [{
|
|
"name": "Unraid-AppOps-Gateway",
|
|
"container_id": CONTAINER_ID,
|
|
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
|
"image_ref": module.SELF_UPGRADE_OLD_IMAGE_REF,
|
|
"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497",
|
|
"restore_image_ref": module.SELF_UPGRADE_PRODUCTION_IMAGE_REF,
|
|
"was_running": True,
|
|
}],
|
|
}
|
|
active_state = {
|
|
"schema_version": 3,
|
|
"repository": module.SELF_UPGRADE_REPOSITORY,
|
|
"status": "rolled-back",
|
|
"live_revision": module.SELF_UPGRADE_OLD_REVISION,
|
|
"live_status": "healthy",
|
|
"live_legacy_provenance": True,
|
|
"live_receipt": {},
|
|
"live_artifacts": {
|
|
"appops_runtime_model": reference,
|
|
"containers": {
|
|
"Unraid-AppOps-Gateway": {
|
|
"container_id": CONTAINER_ID,
|
|
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
monkeypatch.setattr(
|
|
module,
|
|
"_self_upgrade_read_legacy_cache_json",
|
|
lambda *_args, **_kwargs: (b"{}", active_state),
|
|
)
|
|
assert module._self_upgrade_runtime() is not None
|
|
verified = module._self_upgrade_verify_rollback_model(reference, snapshot)
|
|
assert verified is not None
|
|
assert verified["model"] == model
|
|
assert verified["legacy"] is True
|
|
|
|
container["Config"]["Image"] = module.SELF_UPGRADE_PRODUCTION_IMAGE_REF
|
|
production_model = module._self_upgrade_load_compose_model(
|
|
pointer["recoveryId"], pointer["revision"],
|
|
)
|
|
assert production_model is not None
|
|
container["Config"]["Labels"]["com.docker.compose.project.config_files"] = str(
|
|
production_model["path"],
|
|
)
|
|
restored_runtime = module._self_upgrade_runtime(
|
|
authorized_model_pointer=pointer,
|
|
allow_legacy_canonical_config=True,
|
|
)
|
|
assert isinstance(restored_runtime, dict)
|
|
assert restored_runtime["imageRef"] == module.SELF_UPGRADE_PRODUCTION_IMAGE_REF
|
|
|
|
|
|
@pytest.mark.parametrize("cache", ["registered-builds", "registered-builds-v2"])
|
|
def test_recovery_transaction_binds_authorized_555_preimage_and_holds_pending_state(
|
|
controller, monkeypatch, cache,
|
|
):
|
|
module, recovery_root, secret = controller
|
|
if os.name == "nt" or os.geteuid() != 0:
|
|
pytest.skip("The controller transaction store requires root-owned POSIX files")
|
|
recovery_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
semantic = module._webhook_semantic_digest(9, "refs/heads/main", REVISION)
|
|
original_record = {
|
|
"schema": module._WEBHOOK_SCHEMA,
|
|
"semantic_sha256": semantic,
|
|
"revision": REVISION,
|
|
"status": "failed",
|
|
"reason": "deployment_failed",
|
|
}
|
|
original_raw = module._self_upgrade_canonical_json(original_record) + b"\n"
|
|
original_sha = hashlib.sha256(original_raw).hexdigest()
|
|
journal = {
|
|
"schema": module._SELF_UPGRADE_RECOVERY_SCHEMA,
|
|
"recoveryId": RECOVERY_ID,
|
|
"semanticSha256": semantic,
|
|
"repository": module.SELF_UPGRADE_REPOSITORY,
|
|
"revision": REVISION,
|
|
"originalRawSha256": original_sha,
|
|
"idempotencyKeySha256": "3" * 64,
|
|
"preflightSha256": "7" * 64,
|
|
"expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION,
|
|
"runtimeEnvironment": _legacy_environment(),
|
|
"sourceFingerprint": "4" * 64,
|
|
"runtimeFingerprint": "5" * 64,
|
|
"registryFingerprint": "b" * 64,
|
|
"currentContainerId": CONTAINER_ID,
|
|
"currentImageId": module.SELF_UPGRADE_IMAGE_ID,
|
|
"stateSha256": "a" * 64,
|
|
"currentSourceTree": module.SELF_UPGRADE_OLD_TREE,
|
|
"createdAt": 1,
|
|
}
|
|
monkeypatch.setattr(module, "_self_upgrade_model_transaction_origin_valid", lambda _payload: True)
|
|
module._self_upgrade_write_private_journal(RECOVERY_ID, journal, secret)
|
|
request_root = recovery_root / "requests"
|
|
request_root.mkdir(mode=0o700)
|
|
request_payload = {
|
|
"schema": module._SELF_UPGRADE_RECOVERY_REQUEST_SCHEMA,
|
|
"revision": REVISION,
|
|
"preflightSha256": "7" * 64,
|
|
"operationId": semantic,
|
|
"reasonSha256": "8" * 64,
|
|
"idempotencyKeySha256": "3" * 64,
|
|
"recoveryId": RECOVERY_ID,
|
|
"recoveryOfSha256": original_sha,
|
|
"createdAt": 1,
|
|
}
|
|
request_record = {
|
|
**request_payload,
|
|
"hmac": module._self_upgrade_mac(
|
|
secret, "recovery-request", module._self_upgrade_canonical_json(request_payload),
|
|
),
|
|
}
|
|
module._self_upgrade_write_private_record(
|
|
request_root / f"{'3' * 64}.json",
|
|
module._self_upgrade_canonical_json(request_record) + b"\n",
|
|
)
|
|
inbox_root = recovery_root / "inbox"
|
|
inbox_root.mkdir(mode=0o700)
|
|
monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", inbox_root)
|
|
original_path = inbox_root / f"{semantic}.json"
|
|
original_path.write_bytes(original_raw)
|
|
original_path.chmod(0o600)
|
|
recovery_delivery = {
|
|
"status": "deploying",
|
|
"recovery_id": RECOVERY_ID,
|
|
"validation": {"last_result": "pass"},
|
|
"audit": [
|
|
{"stage": "validation_passed"},
|
|
{"stage": "deployment_started"},
|
|
],
|
|
}
|
|
module._self_upgrade_write_private_record(
|
|
recovery_root / f"{semantic}.delivery.json",
|
|
module._self_upgrade_canonical_json(recovery_delivery) + b"\n",
|
|
)
|
|
project = {"branch": "main", "gitea_repository_id": 9}
|
|
monkeypatch.setattr(module, "_load_projects", lambda: {module.SELF_UPGRADE_REPOSITORY: project})
|
|
monkeypatch.setattr(module, "_validation_policy", lambda *_args: {"repository": module.SELF_UPGRADE_REPOSITORY})
|
|
monkeypatch.setattr(module, "_self_upgrade_original_failed_delivery", lambda *_args: (
|
|
original_path, original_raw, original_record,
|
|
))
|
|
monkeypatch.setattr(module, "_self_upgrade_recovery_record_valid", lambda *_args, **_kwargs: True)
|
|
monkeypatch.setattr(module, "_self_upgrade_recovery_preimage_matches", lambda _record: True)
|
|
pointer = module._self_upgrade_store_compose_model(
|
|
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, _legacy_model(module),
|
|
)
|
|
assert isinstance(pointer, dict)
|
|
monkeypatch.setattr(module, "_self_upgrade_runtime", lambda **_kwargs: {
|
|
"runtimeCasSha256": "6" * 64,
|
|
"runtimeFingerprint": "5" * 64,
|
|
"sourceFingerprint": "4" * 64,
|
|
"containerId": CONTAINER_ID,
|
|
"imageId": module.SELF_UPGRADE_IMAGE_ID,
|
|
"revision": module.SELF_UPGRADE_OLD_REVISION,
|
|
"runtimeEnvironment": _legacy_environment(),
|
|
})
|
|
|
|
metadata = {
|
|
"id": REVISION[:12], "repository": module.SELF_UPGRADE_REPOSITORY,
|
|
"revision": REVISION, "tree": TREE,
|
|
"treeDigest": hashlib.sha256(f"git-tree:{TREE}".encode()).hexdigest(),
|
|
"builtAt": BUILT_AT,
|
|
}
|
|
snapshot = {
|
|
"revision": module.SELF_UPGRADE_OLD_REVISION,
|
|
"legacy_provenance": True,
|
|
"receipt": {},
|
|
"compose_containers": ["Unraid-AppOps-Gateway"],
|
|
"containers": [{
|
|
"name": "Unraid-AppOps-Gateway", "container_id": CONTAINER_ID,
|
|
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
|
"image_ref": module.SELF_UPGRADE_OLD_IMAGE_REF,
|
|
"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497",
|
|
"was_running": True,
|
|
}],
|
|
"captured_at_epoch": 1,
|
|
"appops_runtime_model": pointer,
|
|
}
|
|
now = 100
|
|
transaction_id = "22345678-1234-4234-8234-123456789abc"
|
|
request = {
|
|
"schema": module._SELF_UPGRADE_MODEL_TRANSACTION_SCHEMA,
|
|
"transactionId": transaction_id,
|
|
"recoveryId": RECOVERY_ID,
|
|
"repository": module.SELF_UPGRADE_REPOSITORY,
|
|
"kind": "recovery",
|
|
"revision": REVISION,
|
|
"metadata": metadata,
|
|
"expectedStateSha256": "a" * 64,
|
|
"registryFingerprint": "b" * 64,
|
|
"preMutationRuntimeSha256": "6" * 64,
|
|
"sourceFingerprint": "4" * 64,
|
|
"priorModelPointer": pointer,
|
|
"priorSnapshot": snapshot,
|
|
"candidateModelPointer": None,
|
|
"buildContexts": {},
|
|
"candidateImageId": None,
|
|
"verifiedLiveReference": None,
|
|
"verifiedReceipt": None,
|
|
"phase": "prepared",
|
|
"createdAt": now,
|
|
"updatedAt": now,
|
|
}
|
|
assert module._self_upgrade_recovery_journal(RECOVERY_ID) == journal
|
|
assert module._self_upgrade_recovery_request_payload(request_record) == request_payload
|
|
assert module._self_upgrade_original_failed_delivery(semantic, REVISION, {"repository": module.SELF_UPGRADE_REPOSITORY}) == (
|
|
original_path, original_raw, original_record,
|
|
)
|
|
assert module._self_upgrade_recovery_preimage_matches(recovery_delivery) is True
|
|
read_legacy = module._self_upgrade_read_legacy_cache_json
|
|
state_root = recovery_root / "state"
|
|
state_root.mkdir(mode=0o700)
|
|
state_path = state_root / "unraid-appops-gateway.json"
|
|
restored_state_raw: bytes | None = None
|
|
restored_state: dict[str, object] | None = None
|
|
|
|
def read_test_inbox(path, **kwargs):
|
|
if path == original_path:
|
|
assert path.read_bytes() == original_raw
|
|
return original_raw, original_record
|
|
if path == state_path and restored_state_raw is not None and restored_state is not None:
|
|
assert path.read_bytes() == restored_state_raw
|
|
return restored_state_raw, restored_state
|
|
return read_legacy(path, **kwargs)
|
|
|
|
# The production reader pins the real legacy cache directory profile. For
|
|
# this temp-root test, stub only that fixed-path adapter while preserving
|
|
# the actual HMAC request, journal, authorization, and transaction checks.
|
|
monkeypatch.setattr(module, "_self_upgrade_read_legacy_cache_json", read_test_inbox)
|
|
actual_authorization = module._self_upgrade_recovery_authorization(RECOVERY_ID)
|
|
assert actual_authorization is not None
|
|
preimage = module._self_upgrade_transaction_live_preimage(RECOVERY_ID, "recovery")
|
|
assert preimage is not None
|
|
assert preimage["currentRevision"] == module.SELF_UPGRADE_OLD_REVISION
|
|
assert preimage["runtimeEnvironment"] == _legacy_environment()
|
|
assert module._self_upgrade_load_compose_model(
|
|
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION,
|
|
)["model"]["services"]["appops-gateway"]["environment"] == journal["runtimeEnvironment"]
|
|
created = module._self_upgrade_begin_model_transaction(request)
|
|
assert isinstance(created, dict)
|
|
assert created["payload"]["priorSnapshot"]["appops_runtime_model"] == pointer
|
|
assert stat.S_IMODE((recovery_root / "transactions" / f"{transaction_id}.json").stat().st_mode) == 0o600
|
|
assert module._self_upgrade_load_model_transaction(transaction_id) == created
|
|
|
|
assert module._self_upgrade_update_model_transaction(
|
|
transaction_id, "0" * 64, "recovery_required",
|
|
) is None
|
|
candidate_pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
|
assert isinstance(candidate_pointer, dict)
|
|
context = f"{module.EXACT_SOURCE_ROOT.as_posix()}/{cache}/unraid-appops-gateway/{REVISION}/tree"
|
|
for bad_context in (
|
|
context + "/../outside", context.replace(REVISION, "f" * 40),
|
|
context.replace(cache, "registered-builds-v3"), context + "-outside",
|
|
):
|
|
assert module._self_upgrade_update_model_transaction(
|
|
transaction_id, created["sha256"], "source_staged",
|
|
{"candidateModelPointer": candidate_pointer, "buildContexts": {"appops-gateway": bad_context}},
|
|
) is None
|
|
staged = module._self_upgrade_update_model_transaction(
|
|
transaction_id, created["sha256"], "source_staged",
|
|
{"candidateModelPointer": candidate_pointer, "buildContexts": {"appops-gateway": context}},
|
|
)
|
|
assert isinstance(staged, dict)
|
|
assert module._self_upgrade_load_model_transaction(transaction_id) == staged
|
|
created = staged
|
|
held = module._self_upgrade_update_model_transaction(
|
|
transaction_id, created["sha256"], "recovery_required",
|
|
)
|
|
assert isinstance(held, dict)
|
|
assert held["payload"]["phase"] == "recovery_required"
|
|
assert module._self_upgrade_pending_model_transaction() == held
|
|
|
|
rollback_binding = {
|
|
"containerId": CONTAINER_ID,
|
|
"imageId": module.SELF_UPGRADE_IMAGE_ID,
|
|
"runtimeFingerprint": "5" * 64,
|
|
"sourceFingerprint": "4" * 64,
|
|
"receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json({})).hexdigest(),
|
|
}
|
|
restored_reference = module._self_upgrade_store_live_model(pointer, rollback_binding)
|
|
assert isinstance(restored_reference, dict)
|
|
restored_state = {
|
|
"repository": module.SELF_UPGRADE_REPOSITORY,
|
|
"status": "failed",
|
|
"live_revision": module.SELF_UPGRADE_OLD_REVISION,
|
|
"live_status": "healthy",
|
|
"live_legacy_provenance": True,
|
|
"live_receipt": {},
|
|
"live_artifacts": {
|
|
"appops_runtime_model": restored_reference,
|
|
"containers": {
|
|
"Unraid-AppOps-Gateway": {
|
|
"container_id": CONTAINER_ID,
|
|
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
restored_state_raw = module._self_upgrade_canonical_json(restored_state) + b"\n"
|
|
state_path.write_bytes(restored_state_raw)
|
|
state_path.chmod(0o600)
|
|
monkeypatch.setattr(module, "STATE_ROOT", state_root)
|
|
|
|
rollback_transaction_id = "32345678-1234-4234-8234-123456789abc"
|
|
rollback_payload = dict(created["payload"])
|
|
rollback_payload.update({
|
|
"transactionId": rollback_transaction_id,
|
|
"phase": "rolling_back",
|
|
"failureCode": "rollback_started",
|
|
"updatedAt": now + 1,
|
|
})
|
|
transaction_path = module._self_upgrade_transaction_path(rollback_transaction_id)
|
|
module._self_upgrade_write_private_record(
|
|
transaction_path,
|
|
module._self_upgrade_canonical_json(
|
|
module._self_upgrade_model_transaction_envelope(rollback_payload, secret),
|
|
) + b"\n",
|
|
)
|
|
rollback_before = module._self_upgrade_load_model_transaction(rollback_transaction_id)
|
|
assert isinstance(rollback_before, dict)
|
|
intent = {
|
|
"schema": "appops-fixed-555-rollback-resume.v1", "stateSha256": "a" * 64,
|
|
"candidateRuntimeSha256": "b" * 64, "candidateStableRuntimeSha256": "d" * 64,
|
|
"snapshotSha256": hashlib.sha256(module._self_upgrade_canonical_json(rollback_payload["priorSnapshot"])).hexdigest(),
|
|
}
|
|
resumed = module._self_upgrade_update_model_transaction(
|
|
rollback_transaction_id, rollback_before["sha256"], "rolling_back", {"rollbackResumeIntent": intent},
|
|
)
|
|
assert isinstance(resumed, dict)
|
|
assert module._self_upgrade_load_model_transaction(rollback_transaction_id) == resumed
|
|
assert module._self_upgrade_update_model_transaction(
|
|
rollback_transaction_id, resumed["sha256"], "rolling_back",
|
|
{"rollbackResumeIntent": {**intent, "stateSha256": "c" * 64}},
|
|
) is None
|
|
rollback_before = resumed
|
|
rolled_back = module._self_upgrade_update_model_transaction(
|
|
rollback_transaction_id,
|
|
rollback_before["sha256"],
|
|
"rolled_back",
|
|
{"restoredLiveReference": restored_reference},
|
|
)
|
|
assert isinstance(rolled_back, dict)
|
|
assert rolled_back["payload"]["phase"] == "rolled_back"
|
|
assert rolled_back["payload"]["restoredLiveReference"] == restored_reference
|
|
assert rolled_back["payload"]["restoredStateSha256"] == hashlib.sha256(restored_state_raw).hexdigest()
|
|
assert module._self_upgrade_pending_model_transaction() == held
|