from __future__ import annotations import hashlib import importlib.util import json import os import stat import sys from pathlib import Path from types import SimpleNamespace import pytest ROOT = Path(__file__).resolve().parents[2] CONTROLLER = ROOT / "ops" / "gitea-deploy-broker" RECOVERY_ID = "12345678-1234-4234-8234-123456789abc" REVISION = "c" * 40 TREE = "d" * 40 IMAGE_ID = "sha256:" + "e" * 64 CONTAINER_ID = "f" * 64 BUILT_AT = "2026-10-07T00:00:00.000Z" SECRET_SENTINEL = "model-private-secret$literal$$value" @pytest.fixture def controller(monkeypatch, tmp_path): monkeypatch.syspath_prepend(str(CONTROLLER)) spec = importlib.util.spec_from_file_location( "appops_self_upgrade_runtime_model_test", CONTROLLER / "status_server.py", ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) recovery_root = tmp_path / "recovery" monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root) monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_REQUEST_ROOT", recovery_root / "requests") monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_MODEL_ROOT", recovery_root / "models") monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_TRANSACTION_ROOT", recovery_root / "transactions") secret = b"runtime-model-test-secret-key-0123456789" monkeypatch.setattr(module, "_self_upgrade_secret", lambda: secret) return module, recovery_root, secret def _environment(revision: str = REVISION, tree: str = TREE) -> dict[str, str]: return { "APPOPS_SOURCE_REPOSITORY": "jens/unraid-appops-gateway", "APPOPS_SOURCE_REVISION": revision, "APPOPS_SOURCE_TREE": tree, "APPOPS_BUILD_ID": revision[:12], "APPOPS_BUILD_DATE": BUILT_AT, "KNOWLEDGE_OPTIMIZER_MODEL": "gpt-6-luna", "OPERATOR_TOKEN": SECRET_SENTINEL, } def _model(revision: str = REVISION, tree: str = TREE) -> dict[str, object]: return { "name": "unraid-appops-gateway", "services": { "appops-gateway": { "image": "unraid-appops-gateway:production", "container_name": "Unraid-AppOps-Gateway", "environment": _environment(revision, tree), "ports": [{"target": 1216, "published": "1216", "host_ip": "198.18.10.150"}], }, }, } def _legacy_environment() -> dict[str, str]: return { **{f"LEGACY_SETTING_{index:02d}": f"fixture-value-{index:02d}" for index in range(43)}, "KNOWLEDGE_OPTIMIZER_MODEL": "gpt-6-luna", "APPOPS_FRONTEND_ORIGIN": "https://apps.example.test", "OPERATOR_TOKEN": SECRET_SENTINEL, } def _legacy_model(module, image_ref: str | None = None) -> dict[str, object]: return { "name": "unraid-appops-gateway", "services": { "appops-gateway": { "image": image_ref or module.SELF_UPGRADE_OLD_IMAGE_REF, "container_name": "Unraid-AppOps-Gateway", "environment": _legacy_environment(), "ports": [{ "target": 1216, "published": "1216", "host_ip": "198.18.10.150", }], }, }, } def _mock_legacy_docker(monkeypatch, module, *, candidate=False): environment = _environment() if candidate else _legacy_environment() raw_env = [f"{key}={value}" for key, value in environment.items()] if not candidate: raw_env.append(f"APPOPS_FRONTEND_ORIGIN={environment['APPOPS_FRONTEND_ORIGIN']}") revision = REVISION if candidate else module.SELF_UPGRADE_OLD_REVISION tree = TREE if candidate else module.SELF_UPGRADE_OLD_TREE image_id = IMAGE_ID if candidate else module.SELF_UPGRADE_IMAGE_ID image_labels = { "org.opencontainers.image.source": module.SELF_UPGRADE_REPOSITORY, "org.opencontainers.image.revision": revision, "org.opencontainers.image.source-tree": tree, "org.opencontainers.image.version": revision[:12], "org.opencontainers.image.created": BUILT_AT if candidate else "2026-09-01T00:00:00.000Z", } container = { "Id": CONTAINER_ID, "Image": image_id, "State": {"Running": True, "Health": {"Status": "healthy"}}, "Mounts": [ {"Type": "bind", "Source": module.SELF_UPGRADE_DATA_PATH, "Destination": "/data", "RW": True}, {"Type": "bind", "Source": module.SELF_UPGRADE_MODELS_PATH, "Destination": module.SELF_UPGRADE_MODELS_TARGET, "RW": True}, ], "Config": { "Image": module.SELF_UPGRADE_PRODUCTION_IMAGE_REF if candidate else module.SELF_UPGRADE_OLD_IMAGE_REF, "Hostname": "unraid-appops-gateway", "Env": raw_env, "Labels": { **image_labels, "com.docker.compose.project": "unraid-appops-gateway", "com.docker.compose.service": "appops-gateway", "com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source", "com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml", }, }, "HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}}, "NetworkSettings": { "Ports": {"1216/tcp": [{"HostIp": module.SELF_UPGRADE_PUBLISH_HOST, "HostPort": "1216"}]}, "Networks": { "itworx-mcp-hub-npm": {}, "unraid-appops-gateway_default": {}, }, }, } image = {"Id": image_id, "Config": {"Labels": image_labels}} def fake_run(args, **_kwargs): if args[:3] == ["docker", "ps", "-a"]: return SimpleNamespace(stdout="Unraid-AppOps-Gateway\n") if args[:2] == ["docker", "inspect"]: return SimpleNamespace(stdout=json.dumps([container])) if args[:3] == ["docker", "image", "inspect"]: return SimpleNamespace(stdout=json.dumps([image])) raise AssertionError("unexpected Docker command category") monkeypatch.setattr(module.subprocess, "run", fake_run) monkeypatch.setattr(module, "_load_projects", lambda: { module.SELF_UPGRADE_REPOSITORY: { "containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True, }, }) monkeypatch.setattr( module, "_self_upgrade_read_legacy_cache_json", lambda *_args, **_kwargs: (b"{}", {"live_artifacts": {"containers": {}}}), ) monkeypatch.setattr(module, "_self_upgrade_source_is_valid", lambda _source: True) monkeypatch.setattr(module, "_self_upgrade_source_fingerprint", lambda *_args: "2" * 64) return container def _receipt(module, revision: str = REVISION, tree: str = TREE) -> dict[str, object]: digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest() return { "schema": "gitea-deploy-receipt.v1", "repository": module.SELF_UPGRADE_REPOSITORY, "revision": revision, "tree": tree, "treeDigest": digest, "build": { "id": revision[:12], "repository": module.SELF_UPGRADE_REPOSITORY, "revision": revision, "tree": tree, "treeDigest": digest, "builtAt": BUILT_AT, }, "containers": { "Unraid-AppOps-Gateway": { "image_id": IMAGE_ID, "image_revision": revision, "image_source": module.SELF_UPGRADE_REPOSITORY, "image_tree": tree, "image_version": revision[:12], "image_created": BUILT_AT, }, }, "live": None, } def _binding(module, receipt, *, container_id: str = CONTAINER_ID) -> dict[str, object]: return { "containerId": container_id, "imageId": IMAGE_ID, "runtimeFingerprint": "1" * 64, "sourceFingerprint": "2" * 64, "receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json(receipt)).hexdigest(), } def test_plain_compose_file_preserves_literal_dollars_under_private_attestation( controller, ): module, recovery_root, _secret = controller if os.name == "nt" or os.geteuid() != 0: pytest.skip("The controller model store requires root-owned POSIX files") pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) assert isinstance(pointer, dict) loaded = module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION) assert loaded is not None assert loaded["model"] == _model() assert loaded["pointer"] == pointer assert Path(loaded["path"]).name == f"{RECOVERY_ID}-{REVISION}.compose.json" compose_path = Path(loaded["path"]) attestation_path = compose_path.with_name( f"{RECOVERY_ID}-{REVISION}.model.json" ) compose_raw = compose_path.read_bytes() attestation_raw = attestation_path.read_bytes() assert b"model-private-secret$$literal$$$$value" in compose_raw assert SECRET_SENTINEL.encode() not in compose_raw assert SECRET_SENTINEL.encode() not in attestation_raw assert b"OPERATOR_TOKEN" not in attestation_raw assert stat.S_IMODE(compose_path.stat().st_mode) == 0o600 assert stat.S_IMODE(attestation_path.stat().st_mode) == 0o600 assert stat.S_IMODE(recovery_root.stat().st_mode) == 0o700 assert stat.S_IMODE((recovery_root / "models").stat().st_mode) == 0o700 assert module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) == pointer assert module._self_upgrade_store_compose_model( RECOVERY_ID, REVISION, _model(tree="a" * 40), ) is None compose_path.write_text(compose_path.read_text(encoding="utf-8").replace("gpt-6-luna", "gpt-5.5")) compose_path.chmod(0o600) assert module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION) is None def test_live_model_records_are_content_addressed_and_activate_only_from_state( controller, monkeypatch, ): module, _recovery_root, _secret = controller if os.name == "nt" or os.geteuid() != 0: pytest.skip("The controller model store requires root-owned POSIX files") pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) assert isinstance(pointer, dict) receipt = _receipt(module) live_pointer = module._self_upgrade_store_live_model(pointer, _binding(module, receipt)) assert isinstance(live_pointer, dict) first_live = module._self_upgrade_read_live_model_reference(live_pointer) assert first_live is not None assert first_live["model"] == _model() fresh_container_id = "a" * 64 restored_pointer = module._self_upgrade_store_live_model( pointer, _binding(module, receipt, container_id=fresh_container_id), ) assert isinstance(restored_pointer, dict) assert restored_pointer["liveRecordSha256"] != live_pointer["liveRecordSha256"] assert module._self_upgrade_read_live_model_reference(live_pointer) == first_live restored = module._self_upgrade_read_live_model_reference(restored_pointer) assert restored is not None assert restored["live"]["containerId"] == fresh_container_id state = { "schema_version": 3, "repository": module.SELF_UPGRADE_REPOSITORY, "status": "failed", "live_revision": REVISION, "live_status": "healthy", "live_legacy_provenance": False, "live_receipt": receipt, "live_artifacts": { "containers": { "Unraid-AppOps-Gateway": { "container_id": fresh_container_id, "image_id": IMAGE_ID, }, }, "appops_runtime_model": restored_pointer, }, } project = { "containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True, } active = module._self_upgrade_active_compose_model(state, project=project) assert active is not None assert active["model"] == _model() runtime = { "revision": REVISION, "containerId": fresh_container_id, "imageId": IMAGE_ID, "runtimeFingerprint": "1" * 64, "sourceFingerprint": "2" * 64, "runtimeEnvironment": _environment(), } assert module._self_upgrade_active_compose_model(state, runtime, project=project) is not None runtime["runtimeEnvironment"] = {**_environment(), "KNOWLEDGE_OPTIMIZER_MODEL": "gpt-5.5"} assert module._self_upgrade_active_compose_model(state, runtime, project=project) is None state["live_status"] = "unhealthy" assert module._self_upgrade_active_compose_model(state, project=project) is None @pytest.mark.parametrize("drift", [None, "host", "port", "extra-port", "environment", "image", "unsigned", "stopped", "not-restarting"]) def test_signed_crashloop_candidate_can_be_identified_only_for_rollback(controller, monkeypatch, drift): module, _root, _secret = controller if os.name == "nt" or os.geteuid() != 0: pytest.skip("The signed model store requires root-owned POSIX files") pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) assert isinstance(pointer, dict) loaded = module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION) container = _mock_legacy_docker(monkeypatch, module, candidate=True) container["Config"]["Labels"]["com.docker.compose.project.config_files"] = loaded["path"] container["State"].update({"Restarting": True, "Status": "restarting", "Health": {"Status": "unhealthy"}}) container["NetworkSettings"]["Ports"] = {} container["HostConfig"]["PortBindings"] = { "1216/tcp": [{"HostIp": module.SELF_UPGRADE_PUBLISH_HOST, "HostPort": "1216"}], } if drift == "host": container["HostConfig"]["PortBindings"]["1216/tcp"][0]["HostIp"] = "0.0.0.0" elif drift == "port": container["HostConfig"]["PortBindings"]["1216/tcp"][0]["HostPort"] = "9999" elif drift == "extra-port": container["HostConfig"]["PortBindings"]["22/tcp"] = [{"HostIp": "0.0.0.0", "HostPort": "22"}] elif drift == "environment": container["Config"]["Env"].append("UNREVIEWED_SETTING=drift") elif drift == "image": container["Image"] = "sha256:" + "0" * 64 elif drift == "unsigned": pointer = {**pointer, "modelRecordSha256": "0" * 64} elif drift == "stopped": container["State"]["Running"] = False elif drift == "not-restarting": container["State"]["Restarting"] = False assert module._self_upgrade_runtime() is None assert module._self_upgrade_model_runtime_identity(pointer, IMAGE_ID) is None identity = module._self_upgrade_model_runtime_identity(pointer, IMAGE_ID, require_healthy=False) assert (identity is not None) == (drift is None) def test_candidate_binding_rechecks_exact_model_receipt_and_runtime( controller, monkeypatch, ): module, _recovery_root, _secret = controller if os.name == "nt" or os.geteuid() != 0: pytest.skip("The controller model store requires root-owned POSIX files") pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) assert isinstance(pointer, dict) receipt = _receipt(module) project = { "containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True, } monkeypatch.setattr(module, "_load_projects", lambda: {module.SELF_UPGRADE_REPOSITORY: project}) monkeypatch.setattr(module, "_self_upgrade_runtime", lambda **_kwargs: { "revision": REVISION, "sourceRepository": module.SELF_UPGRADE_REPOSITORY, "sourceTree": TREE, "imageId": IMAGE_ID, "containerId": CONTAINER_ID, "runtimeFingerprint": "1" * 64, "sourceFingerprint": "2" * 64, "runtimeEnvironment": _environment(), }) binding = module._self_upgrade_candidate_model_binding(pointer, receipt) assert binding == _binding(module, receipt) wrong_receipt = _receipt(module, tree="b" * 40) assert module._self_upgrade_candidate_model_binding(pointer, wrong_receipt) is None bad_pointer = {**pointer, "modelSha256": "0" * 64} assert module._self_upgrade_candidate_model_binding(bad_pointer, receipt) is None def test_fixed_555_model_preserves_exact_environment_and_duplicate_origin( controller, monkeypatch, ): module, _recovery_root, _secret = controller if os.name == "nt" or os.geteuid() != 0: pytest.skip("The controller model store requires root-owned POSIX files") journal = { "expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION, "currentImageId": module.SELF_UPGRADE_IMAGE_ID, "runtimeEnvironment": _legacy_environment(), } monkeypatch.setattr(module, "_self_upgrade_recovery_journal", lambda _rid: journal) pointer = module._self_upgrade_store_compose_model( RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, _legacy_model(module), ) assert isinstance(pointer, dict) loaded = module._self_upgrade_load_compose_model( RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, ) assert loaded is not None assert loaded["model"] == _legacy_model(module) changed = _legacy_model(module) changed["services"]["appops-gateway"]["environment"]["APPOPS_BUILD_ID"] = "untrusted" assert not module._self_upgrade_compose_model_valid( changed, module.SELF_UPGRADE_OLD_REVISION, ) container = _mock_legacy_docker(monkeypatch, module) initial_runtime = module._self_upgrade_runtime() assert initial_runtime is not None assert initial_runtime["runtimeEnvironment"] == _legacy_environment() assert len(container["Config"]["Env"]) == 47 assert len(initial_runtime["runtimeEnvironment"]) == 46 runtime = module._self_upgrade_runtime( authorized_model_pointer=pointer, allow_legacy_canonical_config=True, ) assert runtime is not None assert runtime["runtimeEnvironment"] == _legacy_environment() assert len(runtime["runtimeEnvironment"]) == 46 assert len(container["Config"]["Env"]) == 47 assert runtime["imageId"] == module.SELF_UPGRADE_IMAGE_ID assert runtime["containerId"] == container["Id"] binding = module._self_upgrade_legacy_model_binding(pointer) assert binding == { "containerId": CONTAINER_ID, "imageId": module.SELF_UPGRADE_IMAGE_ID, "runtimeFingerprint": runtime["runtimeFingerprint"], "sourceFingerprint": "2" * 64, "receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json({})).hexdigest(), } live_pointer = module._self_upgrade_store_live_model(pointer, binding) assert isinstance(live_pointer, dict) state = { "schema_version": 3, "repository": module.SELF_UPGRADE_REPOSITORY, "status": "rolled-back", "live_revision": module.SELF_UPGRADE_OLD_REVISION, "live_status": "healthy", "live_legacy_provenance": True, "live_receipt": {}, "live_artifacts": { "containers": { "Unraid-AppOps-Gateway": { "container_id": CONTAINER_ID, "image_id": module.SELF_UPGRADE_IMAGE_ID, }, }, "appops_runtime_model": live_pointer, }, } project = { "containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True, } active = module._self_upgrade_active_compose_model(state, project=project) assert active is not None assert active["legacy"] is True assert active["model"] == _legacy_model(module) assert module._self_upgrade_active_compose_model(state, runtime, project=project) is not None container["State"]["Health"]["Status"] = "unhealthy" assert module._self_upgrade_runtime() is None assert module._self_upgrade_model_runtime_identity(pointer, module.SELF_UPGRADE_IMAGE_ID) is None recovery_identity = module._self_upgrade_model_runtime_identity( pointer, module.SELF_UPGRADE_IMAGE_ID, require_healthy=False, ) assert isinstance(recovery_identity, dict) assert set(recovery_identity) == { "containerId", "imageId", "revision", "sourceRepository", "sourceTree", "runtimeFingerprint", "sourceFingerprint", "runtimeCasSha256", } assert module._self_upgrade_model_runtime_identity( pointer, module.SELF_UPGRADE_IMAGE_ID, require_healthy="false", ) is None container["State"]["Health"]["Status"] = "healthy" # The identical duplicate is accepted only for the exact immutable old # image; conflicting duplicates remain fail-closed. raw_env = container["Config"]["Env"] raw_env[-1] = "APPOPS_FRONTEND_ORIGIN=https://attacker.invalid" assert module._self_upgrade_runtime( authorized_model_pointer=pointer, allow_legacy_canonical_config=True, ) is None def test_fixed_555_production_reference_is_valid_only_under_signed_legacy_model( controller, monkeypatch, ): module, _recovery_root, _secret = controller if os.name == "nt" or os.geteuid() != 0: pytest.skip("The controller model store requires root-owned POSIX files") journal = { "expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION, "currentImageId": module.SELF_UPGRADE_IMAGE_ID, "runtimeEnvironment": _legacy_environment(), } monkeypatch.setattr(module, "_self_upgrade_recovery_journal", lambda _rid: journal) model = _legacy_model(module, module.SELF_UPGRADE_PRODUCTION_IMAGE_REF) pointer = module._self_upgrade_store_compose_model( "42345678-1234-4234-8234-123456789abc", module.SELF_UPGRADE_OLD_REVISION, model, ) assert isinstance(pointer, dict) container = _mock_legacy_docker(monkeypatch, module) # The pre-capture container can retain the old platform tag while the # signed effective rollback model uses the closed production tag. container["Config"]["Image"] = module.SELF_UPGRADE_OLD_IMAGE_REF initial_runtime = module._self_upgrade_runtime() assert initial_runtime is not None assert initial_runtime["imageRef"] == module.SELF_UPGRADE_OLD_IMAGE_REF runtime = module._self_upgrade_runtime( authorized_model_pointer=pointer, allow_legacy_canonical_config=True, ) assert isinstance(runtime, dict) assert runtime["imageId"] == module.SELF_UPGRADE_IMAGE_ID assert runtime["imageRef"] == module.SELF_UPGRADE_OLD_IMAGE_REF binding = module._self_upgrade_legacy_model_binding(pointer) assert isinstance(binding, dict) reference = module._self_upgrade_store_live_model(pointer, binding) assert isinstance(reference, dict) project = { "containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True, } monkeypatch.setattr(module, "_load_projects", lambda: { module.SELF_UPGRADE_REPOSITORY: project, }) snapshot = { "revision": module.SELF_UPGRADE_OLD_REVISION, "legacy_provenance": True, "receipt": {}, "appops_runtime_model": reference, "containers": [{ "name": "Unraid-AppOps-Gateway", "container_id": CONTAINER_ID, "image_id": module.SELF_UPGRADE_IMAGE_ID, "image_ref": module.SELF_UPGRADE_OLD_IMAGE_REF, "rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497", "restore_image_ref": module.SELF_UPGRADE_PRODUCTION_IMAGE_REF, "was_running": True, }], } active_state = { "schema_version": 3, "repository": module.SELF_UPGRADE_REPOSITORY, "status": "rolled-back", "live_revision": module.SELF_UPGRADE_OLD_REVISION, "live_status": "healthy", "live_legacy_provenance": True, "live_receipt": {}, "live_artifacts": { "appops_runtime_model": reference, "containers": { "Unraid-AppOps-Gateway": { "container_id": CONTAINER_ID, "image_id": module.SELF_UPGRADE_IMAGE_ID, }, }, }, } monkeypatch.setattr( module, "_self_upgrade_read_legacy_cache_json", lambda *_args, **_kwargs: (b"{}", active_state), ) assert module._self_upgrade_runtime() is not None verified = module._self_upgrade_verify_rollback_model(reference, snapshot) assert verified is not None assert verified["model"] == model assert verified["legacy"] is True container["Config"]["Image"] = module.SELF_UPGRADE_PRODUCTION_IMAGE_REF production_model = module._self_upgrade_load_compose_model( pointer["recoveryId"], pointer["revision"], ) assert production_model is not None container["Config"]["Labels"]["com.docker.compose.project.config_files"] = str( production_model["path"], ) restored_runtime = module._self_upgrade_runtime( authorized_model_pointer=pointer, allow_legacy_canonical_config=True, ) assert isinstance(restored_runtime, dict) assert restored_runtime["imageRef"] == module.SELF_UPGRADE_PRODUCTION_IMAGE_REF @pytest.mark.parametrize("cache", ["registered-builds", "registered-builds-v2"]) def test_recovery_transaction_binds_authorized_555_preimage_and_holds_pending_state( controller, monkeypatch, cache, ): module, recovery_root, secret = controller if os.name == "nt" or os.geteuid() != 0: pytest.skip("The controller transaction store requires root-owned POSIX files") recovery_root.mkdir(mode=0o700, parents=True, exist_ok=True) semantic = module._webhook_semantic_digest(9, "refs/heads/main", REVISION) original_record = { "schema": module._WEBHOOK_SCHEMA, "semantic_sha256": semantic, "revision": REVISION, "status": "failed", "reason": "deployment_failed", } original_raw = module._self_upgrade_canonical_json(original_record) + b"\n" original_sha = hashlib.sha256(original_raw).hexdigest() journal = { "schema": module._SELF_UPGRADE_RECOVERY_SCHEMA, "recoveryId": RECOVERY_ID, "semanticSha256": semantic, "repository": module.SELF_UPGRADE_REPOSITORY, "revision": REVISION, "originalRawSha256": original_sha, "idempotencyKeySha256": "3" * 64, "preflightSha256": "7" * 64, "expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION, "runtimeEnvironment": _legacy_environment(), "sourceFingerprint": "4" * 64, "runtimeFingerprint": "5" * 64, "registryFingerprint": "b" * 64, "currentContainerId": CONTAINER_ID, "currentImageId": module.SELF_UPGRADE_IMAGE_ID, "stateSha256": "a" * 64, "currentSourceTree": module.SELF_UPGRADE_OLD_TREE, "createdAt": 1, } monkeypatch.setattr(module, "_self_upgrade_model_transaction_origin_valid", lambda _payload: True) module._self_upgrade_write_private_journal(RECOVERY_ID, journal, secret) request_root = recovery_root / "requests" request_root.mkdir(mode=0o700) request_payload = { "schema": module._SELF_UPGRADE_RECOVERY_REQUEST_SCHEMA, "revision": REVISION, "preflightSha256": "7" * 64, "operationId": semantic, "reasonSha256": "8" * 64, "idempotencyKeySha256": "3" * 64, "recoveryId": RECOVERY_ID, "recoveryOfSha256": original_sha, "createdAt": 1, } request_record = { **request_payload, "hmac": module._self_upgrade_mac( secret, "recovery-request", module._self_upgrade_canonical_json(request_payload), ), } module._self_upgrade_write_private_record( request_root / f"{'3' * 64}.json", module._self_upgrade_canonical_json(request_record) + b"\n", ) inbox_root = recovery_root / "inbox" inbox_root.mkdir(mode=0o700) monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", inbox_root) original_path = inbox_root / f"{semantic}.json" original_path.write_bytes(original_raw) original_path.chmod(0o600) recovery_delivery = { "status": "deploying", "recovery_id": RECOVERY_ID, "validation": {"last_result": "pass"}, "audit": [ {"stage": "validation_passed"}, {"stage": "deployment_started"}, ], } module._self_upgrade_write_private_record( recovery_root / f"{semantic}.delivery.json", module._self_upgrade_canonical_json(recovery_delivery) + b"\n", ) project = {"branch": "main", "gitea_repository_id": 9} monkeypatch.setattr(module, "_load_projects", lambda: {module.SELF_UPGRADE_REPOSITORY: project}) monkeypatch.setattr(module, "_validation_policy", lambda *_args: {"repository": module.SELF_UPGRADE_REPOSITORY}) monkeypatch.setattr(module, "_self_upgrade_original_failed_delivery", lambda *_args: ( original_path, original_raw, original_record, )) monkeypatch.setattr(module, "_self_upgrade_recovery_record_valid", lambda *_args, **_kwargs: True) monkeypatch.setattr(module, "_self_upgrade_recovery_preimage_matches", lambda _record: True) pointer = module._self_upgrade_store_compose_model( RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, _legacy_model(module), ) assert isinstance(pointer, dict) monkeypatch.setattr(module, "_self_upgrade_runtime", lambda **_kwargs: { "runtimeCasSha256": "6" * 64, "runtimeFingerprint": "5" * 64, "sourceFingerprint": "4" * 64, "containerId": CONTAINER_ID, "imageId": module.SELF_UPGRADE_IMAGE_ID, "revision": module.SELF_UPGRADE_OLD_REVISION, "runtimeEnvironment": _legacy_environment(), }) metadata = { "id": REVISION[:12], "repository": module.SELF_UPGRADE_REPOSITORY, "revision": REVISION, "tree": TREE, "treeDigest": hashlib.sha256(f"git-tree:{TREE}".encode()).hexdigest(), "builtAt": BUILT_AT, } snapshot = { "revision": module.SELF_UPGRADE_OLD_REVISION, "legacy_provenance": True, "receipt": {}, "compose_containers": ["Unraid-AppOps-Gateway"], "containers": [{ "name": "Unraid-AppOps-Gateway", "container_id": CONTAINER_ID, "image_id": module.SELF_UPGRADE_IMAGE_ID, "image_ref": module.SELF_UPGRADE_OLD_IMAGE_REF, "rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497", "was_running": True, }], "captured_at_epoch": 1, "appops_runtime_model": pointer, } now = 100 transaction_id = "22345678-1234-4234-8234-123456789abc" request = { "schema": module._SELF_UPGRADE_MODEL_TRANSACTION_SCHEMA, "transactionId": transaction_id, "recoveryId": RECOVERY_ID, "repository": module.SELF_UPGRADE_REPOSITORY, "kind": "recovery", "revision": REVISION, "metadata": metadata, "expectedStateSha256": "a" * 64, "registryFingerprint": "b" * 64, "preMutationRuntimeSha256": "6" * 64, "sourceFingerprint": "4" * 64, "priorModelPointer": pointer, "priorSnapshot": snapshot, "candidateModelPointer": None, "buildContexts": {}, "candidateImageId": None, "verifiedLiveReference": None, "verifiedReceipt": None, "phase": "prepared", "createdAt": now, "updatedAt": now, } assert module._self_upgrade_recovery_journal(RECOVERY_ID) == journal assert module._self_upgrade_recovery_request_payload(request_record) == request_payload assert module._self_upgrade_original_failed_delivery(semantic, REVISION, {"repository": module.SELF_UPGRADE_REPOSITORY}) == ( original_path, original_raw, original_record, ) assert module._self_upgrade_recovery_preimage_matches(recovery_delivery) is True read_legacy = module._self_upgrade_read_legacy_cache_json state_root = recovery_root / "state" state_root.mkdir(mode=0o700) state_path = state_root / "unraid-appops-gateway.json" restored_state_raw: bytes | None = None restored_state: dict[str, object] | None = None def read_test_inbox(path, **kwargs): if path == original_path: assert path.read_bytes() == original_raw return original_raw, original_record if path == state_path and restored_state_raw is not None and restored_state is not None: assert path.read_bytes() == restored_state_raw return restored_state_raw, restored_state return read_legacy(path, **kwargs) # The production reader pins the real legacy cache directory profile. For # this temp-root test, stub only that fixed-path adapter while preserving # the actual HMAC request, journal, authorization, and transaction checks. monkeypatch.setattr(module, "_self_upgrade_read_legacy_cache_json", read_test_inbox) actual_authorization = module._self_upgrade_recovery_authorization(RECOVERY_ID) assert actual_authorization is not None preimage = module._self_upgrade_transaction_live_preimage(RECOVERY_ID, "recovery") assert preimage is not None assert preimage["currentRevision"] == module.SELF_UPGRADE_OLD_REVISION assert preimage["runtimeEnvironment"] == _legacy_environment() assert module._self_upgrade_load_compose_model( RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, )["model"]["services"]["appops-gateway"]["environment"] == journal["runtimeEnvironment"] created = module._self_upgrade_begin_model_transaction(request) assert isinstance(created, dict) assert created["payload"]["priorSnapshot"]["appops_runtime_model"] == pointer assert stat.S_IMODE((recovery_root / "transactions" / f"{transaction_id}.json").stat().st_mode) == 0o600 assert module._self_upgrade_load_model_transaction(transaction_id) == created assert module._self_upgrade_update_model_transaction( transaction_id, "0" * 64, "recovery_required", ) is None candidate_pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) assert isinstance(candidate_pointer, dict) context = f"{module.EXACT_SOURCE_ROOT.as_posix()}/{cache}/unraid-appops-gateway/{REVISION}/tree" for bad_context in ( context + "/../outside", context.replace(REVISION, "f" * 40), context.replace(cache, "registered-builds-v3"), context + "-outside", ): assert module._self_upgrade_update_model_transaction( transaction_id, created["sha256"], "source_staged", {"candidateModelPointer": candidate_pointer, "buildContexts": {"appops-gateway": bad_context}}, ) is None staged = module._self_upgrade_update_model_transaction( transaction_id, created["sha256"], "source_staged", {"candidateModelPointer": candidate_pointer, "buildContexts": {"appops-gateway": context}}, ) assert isinstance(staged, dict) assert module._self_upgrade_load_model_transaction(transaction_id) == staged created = staged held = module._self_upgrade_update_model_transaction( transaction_id, created["sha256"], "recovery_required", ) assert isinstance(held, dict) assert held["payload"]["phase"] == "recovery_required" assert module._self_upgrade_pending_model_transaction() == held rollback_binding = { "containerId": CONTAINER_ID, "imageId": module.SELF_UPGRADE_IMAGE_ID, "runtimeFingerprint": "5" * 64, "sourceFingerprint": "4" * 64, "receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json({})).hexdigest(), } restored_reference = module._self_upgrade_store_live_model(pointer, rollback_binding) assert isinstance(restored_reference, dict) restored_state = { "repository": module.SELF_UPGRADE_REPOSITORY, "status": "failed", "live_revision": module.SELF_UPGRADE_OLD_REVISION, "live_status": "healthy", "live_legacy_provenance": True, "live_receipt": {}, "live_artifacts": { "appops_runtime_model": restored_reference, "containers": { "Unraid-AppOps-Gateway": { "container_id": CONTAINER_ID, "image_id": module.SELF_UPGRADE_IMAGE_ID, }, }, }, } restored_state_raw = module._self_upgrade_canonical_json(restored_state) + b"\n" state_path.write_bytes(restored_state_raw) state_path.chmod(0o600) monkeypatch.setattr(module, "STATE_ROOT", state_root) rollback_transaction_id = "32345678-1234-4234-8234-123456789abc" rollback_payload = dict(created["payload"]) rollback_payload.update({ "transactionId": rollback_transaction_id, "phase": "rolling_back", "failureCode": "rollback_started", "updatedAt": now + 1, }) transaction_path = module._self_upgrade_transaction_path(rollback_transaction_id) module._self_upgrade_write_private_record( transaction_path, module._self_upgrade_canonical_json( module._self_upgrade_model_transaction_envelope(rollback_payload, secret), ) + b"\n", ) rollback_before = module._self_upgrade_load_model_transaction(rollback_transaction_id) assert isinstance(rollback_before, dict) intent = { "schema": "appops-fixed-555-rollback-resume.v1", "stateSha256": "a" * 64, "candidateRuntimeSha256": "b" * 64, "candidateStableRuntimeSha256": "d" * 64, "snapshotSha256": hashlib.sha256(module._self_upgrade_canonical_json(rollback_payload["priorSnapshot"])).hexdigest(), } resumed = module._self_upgrade_update_model_transaction( rollback_transaction_id, rollback_before["sha256"], "rolling_back", {"rollbackResumeIntent": intent}, ) assert isinstance(resumed, dict) assert module._self_upgrade_load_model_transaction(rollback_transaction_id) == resumed assert module._self_upgrade_update_model_transaction( rollback_transaction_id, resumed["sha256"], "rolling_back", {"rollbackResumeIntent": {**intent, "stateSha256": "c" * 64}}, ) is None rollback_before = resumed rolled_back = module._self_upgrade_update_model_transaction( rollback_transaction_id, rollback_before["sha256"], "rolled_back", {"restoredLiveReference": restored_reference}, ) assert isinstance(rolled_back, dict) assert rolled_back["payload"]["phase"] == "rolled_back" assert rolled_back["payload"]["restoredLiveReference"] == restored_reference assert rolled_back["payload"]["restoredStateSha256"] == hashlib.sha256(restored_state_raw).hexdigest() assert module._self_upgrade_pending_model_transaction() == held