Publish curated Unraid-AppOps-Gateway source from 2e6cf771e11a9cdc8b2e2453586a16d518adf40b
This commit is contained in:
Public Source Publisher committed
1 parent
5d598db0df
commit
631b846397
10 files changed
+4709
-8
No files matched your search
@@ -0,0 +1,815 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CONTROLLER = ROOT / "ops" / "gitea-deploy-broker"
|
||||
RECOVERY_ID = "12345678-1234-4234-8234-123456789abc"
|
||||
REVISION = "c" * 40
|
||||
TREE = "d" * 40
|
||||
IMAGE_ID = "sha256:" + "e" * 64
|
||||
CONTAINER_ID = "f" * 64
|
||||
BUILT_AT = "2026-10-07T00:00:00.000Z"
|
||||
SECRET_SENTINEL = "model-private-secret$literal$$value"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def controller(monkeypatch, tmp_path):
|
||||
monkeypatch.syspath_prepend(str(CONTROLLER))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"appops_self_upgrade_runtime_model_test",
|
||||
CONTROLLER / "status_server.py",
|
||||
)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
recovery_root = tmp_path / "recovery"
|
||||
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root)
|
||||
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_REQUEST_ROOT", recovery_root / "requests")
|
||||
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_MODEL_ROOT", recovery_root / "models")
|
||||
monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_TRANSACTION_ROOT", recovery_root / "transactions")
|
||||
secret = b"runtime-model-test-secret-key-0123456789"
|
||||
monkeypatch.setattr(module, "_self_upgrade_secret", lambda: secret)
|
||||
return module, recovery_root, secret
|
||||
|
||||
|
||||
def _environment(revision: str = REVISION, tree: str = TREE) -> dict[str, str]:
|
||||
return {
|
||||
"APPOPS_SOURCE_REPOSITORY": "jens/unraid-appops-gateway",
|
||||
"APPOPS_SOURCE_REVISION": revision,
|
||||
"APPOPS_SOURCE_TREE": tree,
|
||||
"APPOPS_BUILD_ID": revision[:12],
|
||||
"APPOPS_BUILD_DATE": BUILT_AT,
|
||||
"KNOWLEDGE_OPTIMIZER_MODEL": "gpt-6-luna",
|
||||
"OPERATOR_TOKEN": SECRET_SENTINEL,
|
||||
}
|
||||
|
||||
|
||||
def _model(revision: str = REVISION, tree: str = TREE) -> dict[str, object]:
|
||||
return {
|
||||
"name": "unraid-appops-gateway",
|
||||
"services": {
|
||||
"appops-gateway": {
|
||||
"image": "unraid-appops-gateway:production",
|
||||
"container_name": "Unraid-AppOps-Gateway",
|
||||
"environment": _environment(revision, tree),
|
||||
"ports": [{"target": 1216, "published": "1216", "host_ip": "198.18.10.150"}],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _legacy_environment() -> dict[str, str]:
|
||||
return {
|
||||
**{f"LEGACY_SETTING_{index:02d}": f"fixture-value-{index:02d}" for index in range(43)},
|
||||
"KNOWLEDGE_OPTIMIZER_MODEL": "gpt-6-luna",
|
||||
"APPOPS_FRONTEND_ORIGIN": "https://apps.example.test",
|
||||
"OPERATOR_TOKEN": SECRET_SENTINEL,
|
||||
}
|
||||
|
||||
|
||||
def _legacy_model(module, image_ref: str | None = None) -> dict[str, object]:
|
||||
return {
|
||||
"name": "unraid-appops-gateway",
|
||||
"services": {
|
||||
"appops-gateway": {
|
||||
"image": image_ref or module.SELF_UPGRADE_OLD_IMAGE_REF,
|
||||
"container_name": "Unraid-AppOps-Gateway",
|
||||
"environment": _legacy_environment(),
|
||||
"ports": [{
|
||||
"target": 1216, "published": "1216", "host_ip": "198.18.10.150",
|
||||
}],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _mock_legacy_docker(monkeypatch, module):
|
||||
environment = _legacy_environment()
|
||||
raw_env = [f"{key}={value}" for key, value in environment.items()]
|
||||
raw_env.append(f"APPOPS_FRONTEND_ORIGIN={environment['APPOPS_FRONTEND_ORIGIN']}")
|
||||
image_labels = {
|
||||
"org.opencontainers.image.source": module.SELF_UPGRADE_REPOSITORY,
|
||||
"org.opencontainers.image.revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"org.opencontainers.image.source-tree": module.SELF_UPGRADE_OLD_TREE,
|
||||
"org.opencontainers.image.version": module.SELF_UPGRADE_OLD_REVISION[:12],
|
||||
"org.opencontainers.image.created": "2026-09-01T00:00:00.000Z",
|
||||
}
|
||||
container = {
|
||||
"Id": CONTAINER_ID,
|
||||
"Image": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
||||
"Mounts": [
|
||||
{"Type": "bind", "Source": module.SELF_UPGRADE_DATA_PATH,
|
||||
"Destination": "/data", "RW": True},
|
||||
{"Type": "bind", "Source": module.SELF_UPGRADE_MODELS_PATH,
|
||||
"Destination": module.SELF_UPGRADE_MODELS_TARGET, "RW": True},
|
||||
],
|
||||
"Config": {
|
||||
"Image": module.SELF_UPGRADE_OLD_IMAGE_REF,
|
||||
"Hostname": "unraid-appops-gateway",
|
||||
"Env": raw_env,
|
||||
"Labels": {
|
||||
**image_labels,
|
||||
"com.docker.compose.project": "unraid-appops-gateway",
|
||||
"com.docker.compose.service": "appops-gateway",
|
||||
"com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source",
|
||||
"com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml",
|
||||
},
|
||||
},
|
||||
"HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}},
|
||||
"NetworkSettings": {
|
||||
"Ports": {"1216/tcp": [{"HostIp": module.SELF_UPGRADE_PUBLISH_HOST, "HostPort": "1216"}]},
|
||||
"Networks": {
|
||||
"itworx-mcp-hub-npm": {},
|
||||
"unraid-appops-gateway_default": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
image = {"Id": module.SELF_UPGRADE_IMAGE_ID, "Config": {"Labels": image_labels}}
|
||||
|
||||
def fake_run(args, **_kwargs):
|
||||
if args[:3] == ["docker", "ps", "-a"]:
|
||||
return SimpleNamespace(stdout="Unraid-AppOps-Gateway\n")
|
||||
if args[:2] == ["docker", "inspect"]:
|
||||
return SimpleNamespace(stdout=json.dumps([container]))
|
||||
if args[:3] == ["docker", "image", "inspect"]:
|
||||
return SimpleNamespace(stdout=json.dumps([image]))
|
||||
raise AssertionError("unexpected Docker command category")
|
||||
|
||||
monkeypatch.setattr(module.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(module, "_load_projects", lambda: {
|
||||
module.SELF_UPGRADE_REPOSITORY: {
|
||||
"containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True,
|
||||
},
|
||||
})
|
||||
monkeypatch.setattr(
|
||||
module, "_self_upgrade_read_legacy_cache_json",
|
||||
lambda *_args, **_kwargs: (b"{}", {"live_artifacts": {"containers": {}}}),
|
||||
)
|
||||
monkeypatch.setattr(module, "_self_upgrade_source_is_valid", lambda _source: True)
|
||||
monkeypatch.setattr(module, "_self_upgrade_source_fingerprint", lambda *_args: "2" * 64)
|
||||
return container
|
||||
|
||||
|
||||
def _receipt(module, revision: str = REVISION, tree: str = TREE) -> dict[str, object]:
|
||||
digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest()
|
||||
return {
|
||||
"schema": "gitea-deploy-receipt.v1",
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": revision,
|
||||
"tree": tree,
|
||||
"treeDigest": digest,
|
||||
"build": {
|
||||
"id": revision[:12],
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": revision,
|
||||
"tree": tree,
|
||||
"treeDigest": digest,
|
||||
"builtAt": BUILT_AT,
|
||||
},
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"image_id": IMAGE_ID,
|
||||
"image_revision": revision,
|
||||
"image_source": module.SELF_UPGRADE_REPOSITORY,
|
||||
"image_tree": tree,
|
||||
"image_version": revision[:12],
|
||||
"image_created": BUILT_AT,
|
||||
},
|
||||
},
|
||||
"live": None,
|
||||
}
|
||||
|
||||
|
||||
def _binding(module, receipt, *, container_id: str = CONTAINER_ID) -> dict[str, object]:
|
||||
return {
|
||||
"containerId": container_id,
|
||||
"imageId": IMAGE_ID,
|
||||
"runtimeFingerprint": "1" * 64,
|
||||
"sourceFingerprint": "2" * 64,
|
||||
"receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json(receipt)).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def test_plain_compose_file_preserves_literal_dollars_under_private_attestation(
|
||||
controller,
|
||||
):
|
||||
module, recovery_root, _secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller model store requires root-owned POSIX files")
|
||||
|
||||
pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
||||
assert isinstance(pointer, dict)
|
||||
loaded = module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION)
|
||||
assert loaded is not None
|
||||
assert loaded["model"] == _model()
|
||||
assert loaded["pointer"] == pointer
|
||||
assert Path(loaded["path"]).name == f"{RECOVERY_ID}-{REVISION}.compose.json"
|
||||
|
||||
compose_path = Path(loaded["path"])
|
||||
attestation_path = compose_path.with_name(
|
||||
f"{RECOVERY_ID}-{REVISION}.model.json"
|
||||
)
|
||||
compose_raw = compose_path.read_bytes()
|
||||
attestation_raw = attestation_path.read_bytes()
|
||||
assert b"model-private-secret$$literal$$$$value" in compose_raw
|
||||
assert SECRET_SENTINEL.encode() not in compose_raw
|
||||
assert SECRET_SENTINEL.encode() not in attestation_raw
|
||||
assert b"OPERATOR_TOKEN" not in attestation_raw
|
||||
assert stat.S_IMODE(compose_path.stat().st_mode) == 0o600
|
||||
assert stat.S_IMODE(attestation_path.stat().st_mode) == 0o600
|
||||
assert stat.S_IMODE(recovery_root.stat().st_mode) == 0o700
|
||||
assert stat.S_IMODE((recovery_root / "models").stat().st_mode) == 0o700
|
||||
assert module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) == pointer
|
||||
assert module._self_upgrade_store_compose_model(
|
||||
RECOVERY_ID, REVISION, _model(tree="a" * 40),
|
||||
) is None
|
||||
|
||||
compose_path.write_text(compose_path.read_text(encoding="utf-8").replace("gpt-6-luna", "gpt-5.5"))
|
||||
compose_path.chmod(0o600)
|
||||
assert module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION) is None
|
||||
|
||||
|
||||
def test_live_model_records_are_content_addressed_and_activate_only_from_state(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
module, _recovery_root, _secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller model store requires root-owned POSIX files")
|
||||
pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
||||
assert isinstance(pointer, dict)
|
||||
receipt = _receipt(module)
|
||||
live_pointer = module._self_upgrade_store_live_model(pointer, _binding(module, receipt))
|
||||
assert isinstance(live_pointer, dict)
|
||||
first_live = module._self_upgrade_read_live_model_reference(live_pointer)
|
||||
assert first_live is not None
|
||||
assert first_live["model"] == _model()
|
||||
|
||||
fresh_container_id = "a" * 64
|
||||
restored_pointer = module._self_upgrade_store_live_model(
|
||||
pointer, _binding(module, receipt, container_id=fresh_container_id),
|
||||
)
|
||||
assert isinstance(restored_pointer, dict)
|
||||
assert restored_pointer["liveRecordSha256"] != live_pointer["liveRecordSha256"]
|
||||
assert module._self_upgrade_read_live_model_reference(live_pointer) == first_live
|
||||
restored = module._self_upgrade_read_live_model_reference(restored_pointer)
|
||||
assert restored is not None
|
||||
assert restored["live"]["containerId"] == fresh_container_id
|
||||
|
||||
state = {
|
||||
"schema_version": 3,
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"status": "failed",
|
||||
"live_revision": REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": False,
|
||||
"live_receipt": receipt,
|
||||
"live_artifacts": {
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"container_id": fresh_container_id,
|
||||
"image_id": IMAGE_ID,
|
||||
},
|
||||
},
|
||||
"appops_runtime_model": restored_pointer,
|
||||
},
|
||||
}
|
||||
project = {
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"verify_oci_revision": True,
|
||||
}
|
||||
active = module._self_upgrade_active_compose_model(state, project=project)
|
||||
assert active is not None
|
||||
assert active["model"] == _model()
|
||||
runtime = {
|
||||
"revision": REVISION,
|
||||
"containerId": fresh_container_id,
|
||||
"imageId": IMAGE_ID,
|
||||
"runtimeFingerprint": "1" * 64,
|
||||
"sourceFingerprint": "2" * 64,
|
||||
"runtimeEnvironment": _environment(),
|
||||
}
|
||||
assert module._self_upgrade_active_compose_model(state, runtime, project=project) is not None
|
||||
runtime["runtimeEnvironment"] = {**_environment(), "KNOWLEDGE_OPTIMIZER_MODEL": "gpt-5.5"}
|
||||
assert module._self_upgrade_active_compose_model(state, runtime, project=project) is None
|
||||
state["live_status"] = "unhealthy"
|
||||
assert module._self_upgrade_active_compose_model(state, project=project) is None
|
||||
|
||||
|
||||
def test_candidate_binding_rechecks_exact_model_receipt_and_runtime(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
module, _recovery_root, _secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller model store requires root-owned POSIX files")
|
||||
pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model())
|
||||
assert isinstance(pointer, dict)
|
||||
receipt = _receipt(module)
|
||||
project = {
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"verify_oci_revision": True,
|
||||
}
|
||||
monkeypatch.setattr(module, "_load_projects", lambda: {module.SELF_UPGRADE_REPOSITORY: project})
|
||||
monkeypatch.setattr(module, "_self_upgrade_runtime", lambda **_kwargs: {
|
||||
"revision": REVISION,
|
||||
"sourceRepository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"sourceTree": TREE,
|
||||
"imageId": IMAGE_ID,
|
||||
"containerId": CONTAINER_ID,
|
||||
"runtimeFingerprint": "1" * 64,
|
||||
"sourceFingerprint": "2" * 64,
|
||||
"runtimeEnvironment": _environment(),
|
||||
})
|
||||
binding = module._self_upgrade_candidate_model_binding(pointer, receipt)
|
||||
assert binding == _binding(module, receipt)
|
||||
wrong_receipt = _receipt(module, tree="b" * 40)
|
||||
assert module._self_upgrade_candidate_model_binding(pointer, wrong_receipt) is None
|
||||
bad_pointer = {**pointer, "modelSha256": "0" * 64}
|
||||
assert module._self_upgrade_candidate_model_binding(bad_pointer, receipt) is None
|
||||
|
||||
|
||||
def test_fixed_555_model_preserves_exact_environment_and_duplicate_origin(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
module, _recovery_root, _secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller model store requires root-owned POSIX files")
|
||||
journal = {
|
||||
"expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"currentImageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"runtimeEnvironment": _legacy_environment(),
|
||||
}
|
||||
monkeypatch.setattr(module, "_self_upgrade_recovery_journal", lambda _rid: journal)
|
||||
pointer = module._self_upgrade_store_compose_model(
|
||||
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, _legacy_model(module),
|
||||
)
|
||||
assert isinstance(pointer, dict)
|
||||
loaded = module._self_upgrade_load_compose_model(
|
||||
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION,
|
||||
)
|
||||
assert loaded is not None
|
||||
assert loaded["model"] == _legacy_model(module)
|
||||
changed = _legacy_model(module)
|
||||
changed["services"]["appops-gateway"]["environment"]["APPOPS_BUILD_ID"] = "untrusted"
|
||||
assert not module._self_upgrade_compose_model_valid(
|
||||
changed, module.SELF_UPGRADE_OLD_REVISION,
|
||||
)
|
||||
|
||||
container = _mock_legacy_docker(monkeypatch, module)
|
||||
initial_runtime = module._self_upgrade_runtime()
|
||||
assert initial_runtime is not None
|
||||
assert initial_runtime["runtimeEnvironment"] == _legacy_environment()
|
||||
assert len(container["Config"]["Env"]) == 47
|
||||
assert len(initial_runtime["runtimeEnvironment"]) == 46
|
||||
runtime = module._self_upgrade_runtime(
|
||||
authorized_model_pointer=pointer,
|
||||
allow_legacy_canonical_config=True,
|
||||
)
|
||||
assert runtime is not None
|
||||
assert runtime["runtimeEnvironment"] == _legacy_environment()
|
||||
assert len(runtime["runtimeEnvironment"]) == 46
|
||||
assert len(container["Config"]["Env"]) == 47
|
||||
assert runtime["imageId"] == module.SELF_UPGRADE_IMAGE_ID
|
||||
assert runtime["containerId"] == container["Id"]
|
||||
binding = module._self_upgrade_legacy_model_binding(pointer)
|
||||
assert binding == {
|
||||
"containerId": CONTAINER_ID,
|
||||
"imageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"runtimeFingerprint": runtime["runtimeFingerprint"],
|
||||
"sourceFingerprint": "2" * 64,
|
||||
"receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json({})).hexdigest(),
|
||||
}
|
||||
|
||||
live_pointer = module._self_upgrade_store_live_model(pointer, binding)
|
||||
assert isinstance(live_pointer, dict)
|
||||
state = {
|
||||
"schema_version": 3,
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"status": "rolled-back",
|
||||
"live_revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": True,
|
||||
"live_receipt": {},
|
||||
"live_artifacts": {
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"container_id": CONTAINER_ID,
|
||||
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
||||
},
|
||||
},
|
||||
"appops_runtime_model": live_pointer,
|
||||
},
|
||||
}
|
||||
project = {
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"verify_oci_revision": True,
|
||||
}
|
||||
active = module._self_upgrade_active_compose_model(state, project=project)
|
||||
assert active is not None
|
||||
assert active["legacy"] is True
|
||||
assert active["model"] == _legacy_model(module)
|
||||
assert module._self_upgrade_active_compose_model(state, runtime, project=project) is not None
|
||||
|
||||
container["State"]["Health"]["Status"] = "unhealthy"
|
||||
assert module._self_upgrade_runtime() is None
|
||||
assert module._self_upgrade_model_runtime_identity(pointer, module.SELF_UPGRADE_IMAGE_ID) is None
|
||||
recovery_identity = module._self_upgrade_model_runtime_identity(
|
||||
pointer, module.SELF_UPGRADE_IMAGE_ID, require_healthy=False,
|
||||
)
|
||||
assert isinstance(recovery_identity, dict)
|
||||
assert set(recovery_identity) == {
|
||||
"containerId", "imageId", "revision", "sourceRepository", "sourceTree",
|
||||
"runtimeFingerprint", "sourceFingerprint", "runtimeCasSha256",
|
||||
}
|
||||
assert module._self_upgrade_model_runtime_identity(
|
||||
pointer, module.SELF_UPGRADE_IMAGE_ID, require_healthy="false",
|
||||
) is None
|
||||
container["State"]["Health"]["Status"] = "healthy"
|
||||
|
||||
# The identical duplicate is accepted only for the exact immutable old
|
||||
# image; conflicting duplicates remain fail-closed.
|
||||
raw_env = container["Config"]["Env"]
|
||||
raw_env[-1] = "APPOPS_FRONTEND_ORIGIN=https://attacker.invalid"
|
||||
assert module._self_upgrade_runtime(
|
||||
authorized_model_pointer=pointer,
|
||||
allow_legacy_canonical_config=True,
|
||||
) is None
|
||||
|
||||
|
||||
def test_fixed_555_production_reference_is_valid_only_under_signed_legacy_model(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
module, _recovery_root, _secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller model store requires root-owned POSIX files")
|
||||
journal = {
|
||||
"expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"currentImageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"runtimeEnvironment": _legacy_environment(),
|
||||
}
|
||||
monkeypatch.setattr(module, "_self_upgrade_recovery_journal", lambda _rid: journal)
|
||||
model = _legacy_model(module, module.SELF_UPGRADE_PRODUCTION_IMAGE_REF)
|
||||
pointer = module._self_upgrade_store_compose_model(
|
||||
"42345678-1234-4234-8234-123456789abc",
|
||||
module.SELF_UPGRADE_OLD_REVISION,
|
||||
model,
|
||||
)
|
||||
assert isinstance(pointer, dict)
|
||||
container = _mock_legacy_docker(monkeypatch, module)
|
||||
# The pre-capture container can retain the old platform tag while the
|
||||
# signed effective rollback model uses the closed production tag.
|
||||
container["Config"]["Image"] = module.SELF_UPGRADE_OLD_IMAGE_REF
|
||||
initial_runtime = module._self_upgrade_runtime()
|
||||
assert initial_runtime is not None
|
||||
assert initial_runtime["imageRef"] == module.SELF_UPGRADE_OLD_IMAGE_REF
|
||||
|
||||
runtime = module._self_upgrade_runtime(
|
||||
authorized_model_pointer=pointer,
|
||||
allow_legacy_canonical_config=True,
|
||||
)
|
||||
assert isinstance(runtime, dict)
|
||||
assert runtime["imageId"] == module.SELF_UPGRADE_IMAGE_ID
|
||||
assert runtime["imageRef"] == module.SELF_UPGRADE_OLD_IMAGE_REF
|
||||
binding = module._self_upgrade_legacy_model_binding(pointer)
|
||||
assert isinstance(binding, dict)
|
||||
reference = module._self_upgrade_store_live_model(pointer, binding)
|
||||
assert isinstance(reference, dict)
|
||||
|
||||
project = {
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"verify_oci_revision": True,
|
||||
}
|
||||
monkeypatch.setattr(module, "_load_projects", lambda: {
|
||||
module.SELF_UPGRADE_REPOSITORY: project,
|
||||
})
|
||||
snapshot = {
|
||||
"revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"legacy_provenance": True,
|
||||
"receipt": {},
|
||||
"appops_runtime_model": reference,
|
||||
"containers": [{
|
||||
"name": "Unraid-AppOps-Gateway",
|
||||
"container_id": CONTAINER_ID,
|
||||
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"image_ref": module.SELF_UPGRADE_OLD_IMAGE_REF,
|
||||
"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497",
|
||||
"restore_image_ref": module.SELF_UPGRADE_PRODUCTION_IMAGE_REF,
|
||||
"was_running": True,
|
||||
}],
|
||||
}
|
||||
active_state = {
|
||||
"schema_version": 3,
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"status": "rolled-back",
|
||||
"live_revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": True,
|
||||
"live_receipt": {},
|
||||
"live_artifacts": {
|
||||
"appops_runtime_model": reference,
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"container_id": CONTAINER_ID,
|
||||
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"_self_upgrade_read_legacy_cache_json",
|
||||
lambda *_args, **_kwargs: (b"{}", active_state),
|
||||
)
|
||||
assert module._self_upgrade_runtime() is not None
|
||||
verified = module._self_upgrade_verify_rollback_model(reference, snapshot)
|
||||
assert verified is not None
|
||||
assert verified["model"] == model
|
||||
assert verified["legacy"] is True
|
||||
|
||||
container["Config"]["Image"] = module.SELF_UPGRADE_PRODUCTION_IMAGE_REF
|
||||
production_model = module._self_upgrade_load_compose_model(
|
||||
pointer["recoveryId"], pointer["revision"],
|
||||
)
|
||||
assert production_model is not None
|
||||
container["Config"]["Labels"]["com.docker.compose.project.config_files"] = str(
|
||||
production_model["path"],
|
||||
)
|
||||
restored_runtime = module._self_upgrade_runtime(
|
||||
authorized_model_pointer=pointer,
|
||||
allow_legacy_canonical_config=True,
|
||||
)
|
||||
assert isinstance(restored_runtime, dict)
|
||||
assert restored_runtime["imageRef"] == module.SELF_UPGRADE_PRODUCTION_IMAGE_REF
|
||||
|
||||
|
||||
def test_recovery_transaction_binds_authorized_555_preimage_and_holds_pending_state(
|
||||
controller, monkeypatch,
|
||||
):
|
||||
module, recovery_root, secret = controller
|
||||
if os.name == "nt" or os.geteuid() != 0:
|
||||
pytest.skip("The controller transaction store requires root-owned POSIX files")
|
||||
recovery_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
semantic = module._webhook_semantic_digest(9, "refs/heads/main", REVISION)
|
||||
original_record = {
|
||||
"schema": module._WEBHOOK_SCHEMA,
|
||||
"semantic_sha256": semantic,
|
||||
"revision": REVISION,
|
||||
"status": "failed",
|
||||
"reason": "deployment_failed",
|
||||
}
|
||||
original_raw = module._self_upgrade_canonical_json(original_record) + b"\n"
|
||||
original_sha = hashlib.sha256(original_raw).hexdigest()
|
||||
journal = {
|
||||
"schema": module._SELF_UPGRADE_RECOVERY_SCHEMA,
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"semanticSha256": semantic,
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": REVISION,
|
||||
"originalRawSha256": original_sha,
|
||||
"idempotencyKeySha256": "3" * 64,
|
||||
"preflightSha256": "7" * 64,
|
||||
"expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"runtimeEnvironment": _legacy_environment(),
|
||||
"sourceFingerprint": "4" * 64,
|
||||
"runtimeFingerprint": "5" * 64,
|
||||
"registryFingerprint": "b" * 64,
|
||||
"currentContainerId": CONTAINER_ID,
|
||||
"currentImageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"stateSha256": "a" * 64,
|
||||
"currentSourceTree": module.SELF_UPGRADE_OLD_TREE,
|
||||
"createdAt": 1,
|
||||
}
|
||||
monkeypatch.setattr(module, "_self_upgrade_model_transaction_origin_valid", lambda _payload: True)
|
||||
module._self_upgrade_write_private_journal(RECOVERY_ID, journal, secret)
|
||||
request_root = recovery_root / "requests"
|
||||
request_root.mkdir(mode=0o700)
|
||||
request_payload = {
|
||||
"schema": module._SELF_UPGRADE_RECOVERY_REQUEST_SCHEMA,
|
||||
"revision": REVISION,
|
||||
"preflightSha256": "7" * 64,
|
||||
"operationId": semantic,
|
||||
"reasonSha256": "8" * 64,
|
||||
"idempotencyKeySha256": "3" * 64,
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"recoveryOfSha256": original_sha,
|
||||
"createdAt": 1,
|
||||
}
|
||||
request_record = {
|
||||
**request_payload,
|
||||
"hmac": module._self_upgrade_mac(
|
||||
secret, "recovery-request", module._self_upgrade_canonical_json(request_payload),
|
||||
),
|
||||
}
|
||||
module._self_upgrade_write_private_record(
|
||||
request_root / f"{'3' * 64}.json",
|
||||
module._self_upgrade_canonical_json(request_record) + b"\n",
|
||||
)
|
||||
inbox_root = recovery_root / "inbox"
|
||||
inbox_root.mkdir(mode=0o700)
|
||||
monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", inbox_root)
|
||||
original_path = inbox_root / f"{semantic}.json"
|
||||
original_path.write_bytes(original_raw)
|
||||
original_path.chmod(0o600)
|
||||
recovery_delivery = {
|
||||
"status": "deploying",
|
||||
"recovery_id": RECOVERY_ID,
|
||||
"validation": {"last_result": "pass"},
|
||||
"audit": [
|
||||
{"stage": "validation_passed"},
|
||||
{"stage": "deployment_started"},
|
||||
],
|
||||
}
|
||||
module._self_upgrade_write_private_record(
|
||||
recovery_root / f"{semantic}.delivery.json",
|
||||
module._self_upgrade_canonical_json(recovery_delivery) + b"\n",
|
||||
)
|
||||
project = {"branch": "main", "gitea_repository_id": 9}
|
||||
monkeypatch.setattr(module, "_load_projects", lambda: {module.SELF_UPGRADE_REPOSITORY: project})
|
||||
monkeypatch.setattr(module, "_validation_policy", lambda *_args: {"repository": module.SELF_UPGRADE_REPOSITORY})
|
||||
monkeypatch.setattr(module, "_self_upgrade_original_failed_delivery", lambda *_args: (
|
||||
original_path, original_raw, original_record,
|
||||
))
|
||||
monkeypatch.setattr(module, "_self_upgrade_recovery_record_valid", lambda *_args, **_kwargs: True)
|
||||
monkeypatch.setattr(module, "_self_upgrade_recovery_preimage_matches", lambda _record: True)
|
||||
pointer = module._self_upgrade_store_compose_model(
|
||||
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, _legacy_model(module),
|
||||
)
|
||||
assert isinstance(pointer, dict)
|
||||
monkeypatch.setattr(module, "_self_upgrade_runtime", lambda **_kwargs: {
|
||||
"runtimeCasSha256": "6" * 64,
|
||||
"runtimeFingerprint": "5" * 64,
|
||||
"sourceFingerprint": "4" * 64,
|
||||
"containerId": CONTAINER_ID,
|
||||
"imageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"runtimeEnvironment": _legacy_environment(),
|
||||
})
|
||||
|
||||
metadata = {
|
||||
"id": REVISION[:12], "repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": REVISION, "tree": TREE,
|
||||
"treeDigest": hashlib.sha256(f"git-tree:{TREE}".encode()).hexdigest(),
|
||||
"builtAt": BUILT_AT,
|
||||
}
|
||||
snapshot = {
|
||||
"revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"legacy_provenance": True,
|
||||
"receipt": {},
|
||||
"compose_containers": ["Unraid-AppOps-Gateway"],
|
||||
"containers": [{
|
||||
"name": "Unraid-AppOps-Gateway", "container_id": CONTAINER_ID,
|
||||
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"image_ref": module.SELF_UPGRADE_OLD_IMAGE_REF,
|
||||
"rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497",
|
||||
"was_running": True,
|
||||
}],
|
||||
"captured_at_epoch": 1,
|
||||
"appops_runtime_model": pointer,
|
||||
}
|
||||
now = 100
|
||||
transaction_id = "22345678-1234-4234-8234-123456789abc"
|
||||
request = {
|
||||
"schema": module._SELF_UPGRADE_MODEL_TRANSACTION_SCHEMA,
|
||||
"transactionId": transaction_id,
|
||||
"recoveryId": RECOVERY_ID,
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"kind": "recovery",
|
||||
"revision": REVISION,
|
||||
"metadata": metadata,
|
||||
"expectedStateSha256": "a" * 64,
|
||||
"registryFingerprint": "b" * 64,
|
||||
"preMutationRuntimeSha256": "6" * 64,
|
||||
"sourceFingerprint": "4" * 64,
|
||||
"priorModelPointer": pointer,
|
||||
"priorSnapshot": snapshot,
|
||||
"candidateModelPointer": None,
|
||||
"buildContexts": {},
|
||||
"candidateImageId": None,
|
||||
"verifiedLiveReference": None,
|
||||
"verifiedReceipt": None,
|
||||
"phase": "prepared",
|
||||
"createdAt": now,
|
||||
"updatedAt": now,
|
||||
}
|
||||
assert module._self_upgrade_recovery_journal(RECOVERY_ID) == journal
|
||||
assert module._self_upgrade_recovery_request_payload(request_record) == request_payload
|
||||
assert module._self_upgrade_original_failed_delivery(semantic, REVISION, {"repository": module.SELF_UPGRADE_REPOSITORY}) == (
|
||||
original_path, original_raw, original_record,
|
||||
)
|
||||
assert module._self_upgrade_recovery_preimage_matches(recovery_delivery) is True
|
||||
read_legacy = module._self_upgrade_read_legacy_cache_json
|
||||
state_root = recovery_root / "state"
|
||||
state_root.mkdir(mode=0o700)
|
||||
state_path = state_root / "unraid-appops-gateway.json"
|
||||
restored_state_raw: bytes | None = None
|
||||
restored_state: dict[str, object] | None = None
|
||||
|
||||
def read_test_inbox(path, **kwargs):
|
||||
if path == original_path:
|
||||
assert path.read_bytes() == original_raw
|
||||
return original_raw, original_record
|
||||
if path == state_path and restored_state_raw is not None and restored_state is not None:
|
||||
assert path.read_bytes() == restored_state_raw
|
||||
return restored_state_raw, restored_state
|
||||
return read_legacy(path, **kwargs)
|
||||
|
||||
# The production reader pins the real legacy cache directory profile. For
|
||||
# this temp-root test, stub only that fixed-path adapter while preserving
|
||||
# the actual HMAC request, journal, authorization, and transaction checks.
|
||||
monkeypatch.setattr(module, "_self_upgrade_read_legacy_cache_json", read_test_inbox)
|
||||
actual_authorization = module._self_upgrade_recovery_authorization(RECOVERY_ID)
|
||||
assert actual_authorization is not None
|
||||
preimage = module._self_upgrade_transaction_live_preimage(RECOVERY_ID, "recovery")
|
||||
assert preimage is not None
|
||||
assert preimage["currentRevision"] == module.SELF_UPGRADE_OLD_REVISION
|
||||
assert preimage["runtimeEnvironment"] == _legacy_environment()
|
||||
assert module._self_upgrade_load_compose_model(
|
||||
RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION,
|
||||
)["model"]["services"]["appops-gateway"]["environment"] == journal["runtimeEnvironment"]
|
||||
created = module._self_upgrade_begin_model_transaction(request)
|
||||
assert isinstance(created, dict)
|
||||
assert created["payload"]["priorSnapshot"]["appops_runtime_model"] == pointer
|
||||
assert stat.S_IMODE((recovery_root / "transactions" / f"{transaction_id}.json").stat().st_mode) == 0o600
|
||||
assert module._self_upgrade_load_model_transaction(transaction_id) == created
|
||||
|
||||
assert module._self_upgrade_update_model_transaction(
|
||||
transaction_id, "0" * 64, "recovery_required",
|
||||
) is None
|
||||
held = module._self_upgrade_update_model_transaction(
|
||||
transaction_id, created["sha256"], "recovery_required",
|
||||
)
|
||||
assert isinstance(held, dict)
|
||||
assert held["payload"]["phase"] == "recovery_required"
|
||||
assert module._self_upgrade_pending_model_transaction() == held
|
||||
|
||||
rollback_binding = {
|
||||
"containerId": CONTAINER_ID,
|
||||
"imageId": module.SELF_UPGRADE_IMAGE_ID,
|
||||
"runtimeFingerprint": "5" * 64,
|
||||
"sourceFingerprint": "4" * 64,
|
||||
"receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json({})).hexdigest(),
|
||||
}
|
||||
restored_reference = module._self_upgrade_store_live_model(pointer, rollback_binding)
|
||||
assert isinstance(restored_reference, dict)
|
||||
restored_state = {
|
||||
"repository": module.SELF_UPGRADE_REPOSITORY,
|
||||
"status": "failed",
|
||||
"live_revision": module.SELF_UPGRADE_OLD_REVISION,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": True,
|
||||
"live_receipt": {},
|
||||
"live_artifacts": {
|
||||
"appops_runtime_model": restored_reference,
|
||||
"containers": {
|
||||
"Unraid-AppOps-Gateway": {
|
||||
"container_id": CONTAINER_ID,
|
||||
"image_id": module.SELF_UPGRADE_IMAGE_ID,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
restored_state_raw = module._self_upgrade_canonical_json(restored_state) + b"\n"
|
||||
state_path.write_bytes(restored_state_raw)
|
||||
state_path.chmod(0o600)
|
||||
monkeypatch.setattr(module, "STATE_ROOT", state_root)
|
||||
|
||||
rollback_transaction_id = "32345678-1234-4234-8234-123456789abc"
|
||||
rollback_payload = dict(created["payload"])
|
||||
rollback_payload.update({
|
||||
"transactionId": rollback_transaction_id,
|
||||
"phase": "rolling_back",
|
||||
"failureCode": "rollback_started",
|
||||
"updatedAt": now + 1,
|
||||
})
|
||||
transaction_path = module._self_upgrade_transaction_path(rollback_transaction_id)
|
||||
module._self_upgrade_write_private_record(
|
||||
transaction_path,
|
||||
module._self_upgrade_canonical_json(
|
||||
module._self_upgrade_model_transaction_envelope(rollback_payload, secret),
|
||||
) + b"\n",
|
||||
)
|
||||
rollback_before = module._self_upgrade_load_model_transaction(rollback_transaction_id)
|
||||
assert isinstance(rollback_before, dict)
|
||||
rolled_back = module._self_upgrade_update_model_transaction(
|
||||
rollback_transaction_id,
|
||||
rollback_before["sha256"],
|
||||
"rolled_back",
|
||||
{"restoredLiveReference": restored_reference},
|
||||
)
|
||||
assert isinstance(rolled_back, dict)
|
||||
assert rolled_back["payload"]["phase"] == "rolled_back"
|
||||
assert rolled_back["payload"]["restoredLiveReference"] == restored_reference
|
||||
assert rolled_back["payload"]["restoredStateSha256"] == hashlib.sha256(restored_state_raw).hexdigest()
|
||||
assert module._self_upgrade_pending_model_transaction() == held
|
||||
Reference in new issue
Block a user