From 631b846397b6382cf190ee546b21faca02f62822 Mon Sep 17 00:00:00 2001 From: Public Source Publisher Date: Wed, 7 Oct 2026 20:07:49 +0000 Subject: [PATCH] Publish curated Unraid-AppOps-Gateway source from 2e6cf771e11a9cdc8b2e2453586a16d518adf40b --- PUBLIC_SOURCE_EXPORT.md | 2 +- PUBLIC_SOURCE_MANIFEST.json | 40 +- .../app/services/workmesh_secret_import.py | 327 +++++ ...ps_effective_transaction_reconciliation.py | 963 ++++++++++++++ .../test_appops_recovery_deploy_binding.py | 1137 +++++++++++++++++ .../test_appops_self_upgrade_controller.py | 1 + ...appops_self_upgrade_recovery_boundaries.py | 558 ++++++++ .../test_appops_self_upgrade_runtime_model.py | 815 ++++++++++++ ...rtifact_bootstrap_failed_child_recovery.py | 241 +++- backend/tests/test_workmesh_secret_import.py | 633 +++++++++ 10 files changed, 4709 insertions(+), 8 deletions(-) create mode 100644 backend/app/services/workmesh_secret_import.py create mode 100644 backend/tests/test_appops_effective_transaction_reconciliation.py create mode 100644 backend/tests/test_appops_recovery_deploy_binding.py create mode 100644 backend/tests/test_appops_self_upgrade_recovery_boundaries.py create mode 100644 backend/tests/test_appops_self_upgrade_runtime_model.py create mode 100644 backend/tests/test_workmesh_secret_import.py diff --git a/PUBLIC_SOURCE_EXPORT.md b/PUBLIC_SOURCE_EXPORT.md index 35ba438..f2d5206 100644 --- a/PUBLIC_SOURCE_EXPORT.md +++ b/PUBLIC_SOURCE_EXPORT.md @@ -1,3 +1,3 @@ # Public source export -This source snapshot was generated from the private canonical repository at revision `84f2a9958b2bdf1c1c13553925026d75f32703c3`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. +This source snapshot was generated from the private canonical repository at revision `2e6cf771e11a9cdc8b2e2453586a16d518adf40b`. It contains no private Git history or operational evidence. Changes are published from the canonical repository. diff --git a/PUBLIC_SOURCE_MANIFEST.json b/PUBLIC_SOURCE_MANIFEST.json index aff986d..f6e17bc 100644 --- a/PUBLIC_SOURCE_MANIFEST.json +++ b/PUBLIC_SOURCE_MANIFEST.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "sourceRevision": "84f2a9958b2bdf1c1c13553925026d75f32703c3", + "sourceRevision": "2e6cf771e11a9cdc8b2e2453586a16d518adf40b", "files": [ { "path": ".dockerignore", @@ -847,6 +847,11 @@ "sha256": "e1e1389c06dd38a8913c5d2d01cd851686e51e5ab5f3fafe4c671cd58be32e9e", "bytes": 29848 }, + { + "path": "backend/app/services/workmesh_secret_import.py", + "sha256": "392c0dd10e224246a9d89dd9dcad9201e3decadbc5857a5b59a3a9e5e659ee6c", + "bytes": 11662 + }, { "path": "backend/requirements-dev.txt", "sha256": "e7fe19d85b9e9453262d02c5ddd43c84152754a82af87fbaaa647e1bd2e82aec", @@ -872,6 +877,11 @@ "sha256": "80c9444a354c1bee6114ae004b0c7d9c1e72834afd2b2dc04ed7c6b738119150", "bytes": 47199 }, + { + "path": "backend/tests/test_appops_effective_transaction_reconciliation.py", + "sha256": "486e3cf3b6c2a39763402cd00c4a28b903de128498f2d489d76a0f6d08daaa00", + "bytes": 38457 + }, { "path": "backend/tests/test_appops_live_state_recovery.py", "sha256": "e65bf9968fba65318295ea3d2395da8f7a567657d8050029bd8db323cad5a0dc", @@ -882,10 +892,25 @@ "sha256": "11af3dfaf72d35affa1b26a52d7850c8e4d44d36183728f326e5e615bcbde102", "bytes": 11481 }, + { + "path": "backend/tests/test_appops_recovery_deploy_binding.py", + "sha256": "4902f2e16ed35b8bd52d8610ab61b9752aa10a4147e5aafc27404bbbfef61e0a", + "bytes": 50460 + }, { "path": "backend/tests/test_appops_self_upgrade_controller.py", - "sha256": "b41b936e6351a9e2a5a947b7d69e13eb05d2eb6b02bd5bee1b4d3fdfa2c63eea", - "bytes": 23121 + "sha256": "fa79f693efbb5178b7b40f395d526b9355f7ced8978486ee1637a8f75416f487", + "bytes": 23184 + }, + { + "path": "backend/tests/test_appops_self_upgrade_recovery_boundaries.py", + "sha256": "07c3a38c360f5b70b6f3efbc8ba62356d4937170667d07d31d82a6810ed58a0e", + "bytes": 21578 + }, + { + "path": "backend/tests/test_appops_self_upgrade_runtime_model.py", + "sha256": "cbc9febd7855b987395c7f352589a0e5cbddc8ea0c2f44cc25a24655c5fd3fad", + "bytes": 34600 }, { "path": "backend/tests/test_authentik_connector.py", @@ -909,8 +934,8 @@ }, { "path": "backend/tests/test_controller_artifact_bootstrap_failed_child_recovery.py", - "sha256": "394bbd68f8ac513a9b214317fefcc7cb5a547ca3baced8453a15ef9f9df1e7ce", - "bytes": 10598 + "sha256": "5da80eed7ef420bee44cec4e20fe45bd4918245c6236a4481aeffe71c939d82f", + "bytes": 18314 }, { "path": "backend/tests/test_controller_artifact_promoter.py", @@ -1662,6 +1687,11 @@ "sha256": "a1d2b759b8fb5538da36f8020e77292cea1442d63a69e300cd0aa73d433aa977", "bytes": 1365 }, + { + "path": "backend/tests/test_workmesh_secret_import.py", + "sha256": "efac50216b5050828196a55cdef15ad1ad1a224413c1e863ac55670614087a05", + "bytes": 21032 + }, { "path": "components.json", "sha256": "9820835141c54e5b5dfbb68b3c3790db73c9ebfb967884ae3fccd058d698a1a6", diff --git a/backend/app/services/workmesh_secret_import.py b/backend/app/services/workmesh_secret_import.py new file mode 100644 index 0000000..111d9bc --- /dev/null +++ b/backend/app/services/workmesh_secret_import.py @@ -0,0 +1,327 @@ +"""Fixed, write-only import of the existing WorkMesh controller credential.""" + +from __future__ import annotations + +import hmac +import json +import re +import sys +from typing import Any, TextIO +from urllib.parse import urlparse + +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlmodel import Session + +from ..core.time import utc_now +from ..db.models import Secret +from ..db.session import engine +from . import secret_store +from .audit_logger import log_event +from .deployment_broker import DeploymentBrokerClient +from .secret_store import get_secret, secret_storage_status +from .workmesh_promotion import _safe_preflight + +SECRET_REF = "WORKMESH_DEPLOYMENT_BROKER" +TARGET = "jens/workmesh" +_ACTION = "deployment.secret.import_workmesh_controller" +_PURPOSE = "fixed_workmesh_controller_promotion" +_USED_BY = ["deployment:workmesh_promotion"] +_API_VERSION = "itworx.workmesh-controller-secret-import/v1" +_REVISION = re.compile(r"^[0-9a-f]{40}$") +_AUDIT_ID = re.compile(r"^audit-[0-9a-f]{12}$") +_MIN_CREDENTIAL_BYTES = 32 +_MAX_CREDENTIAL_BYTES = 256 +_FAILURE_CODES = frozenset({ + "revision_invalid", "credential_invalid", "secret_reference_unreadable", + "secret_reference_conflict", "database_unavailable", "secret_storage_unavailable", + "session_not_clean", "controller_https_unverified", "controller_preflight_unavailable", + "controller_preflight_rejected", "audit_commit_failed", "secret_readback_failed", + "import_failed", +}) + + +def _safe_error_code(code: object) -> str: + return code if isinstance(code, str) and code in _FAILURE_CODES else "import_failed" + + +class WorkMeshSecretImportError(ValueError): + """Sanitized failure with a fixed operator-visible reason code.""" + + def __init__(self, code: str): + self.code = _safe_error_code(code) + super().__init__(self.code) + + +def _reject(code: str) -> None: + raise WorkMeshSecretImportError(code) + + +def _rollback_quietly(session: Session) -> bool: + try: + session.rollback() + except Exception: # noqa: BLE001 - rollback errors must not reach output. + return False + return True + + +def _validate_input(credential: object, revision: object) -> tuple[str, str]: + if ( + not isinstance(revision, str) + or len(revision) != 40 + or not _REVISION.fullmatch(revision) + ): + _reject("revision_invalid") + if not isinstance(credential, str): + _reject("credential_invalid") + if not _MIN_CREDENTIAL_BYTES <= len(credential) <= _MAX_CREDENTIAL_BYTES: + _reject("credential_invalid") + try: + raw = credential.encode("ascii") + except UnicodeEncodeError: + _reject("credential_invalid") + if ( + not _MIN_CREDENTIAL_BYTES <= len(raw) <= _MAX_CREDENTIAL_BYTES + or not credential.isprintable() + or any(character.isspace() for character in credential) + ): + _reject("credential_invalid") + return credential, revision + + +def _controller_is_verified_https(broker: DeploymentBrokerClient) -> bool: + try: + return ( + isinstance(broker.base_url, str) + and urlparse(broker.base_url).scheme == "https" + and broker.verify_tls is True + and broker.configuration_error() is None + ) + except Exception: # noqa: BLE001 - config errors may include sensitive values. + return False + + +def _read_existing(session: Session, credential: str) -> bool: + """Check the fixed row without touching its timestamps or metadata.""" + try: + row = session.get(Secret, SECRET_REF) + if row is None: + session.commit() + return False + existing = get_secret(session, SECRET_REF, touch=False) + if existing is None: + _reject("secret_reference_unreadable") + if not hmac.compare_digest(existing, credential): + _reject("secret_reference_conflict") + session.commit() + return True + except WorkMeshSecretImportError: + _rollback_quietly(session) + raise + except Exception: # noqa: BLE001 - database errors are not operator output. + _rollback_quietly(session) + _reject("database_unavailable") + + +def _store_if_absent( + session: Session, + credential: str, + *, + revision: str, + preflight_sha256: str, +) -> bool: + """Atomically add the fixed row; a concurrent value is never replaced.""" + try: + encrypted = secret_store._fernet().encrypt(credential.encode("ascii")).decode("ascii") + now = utc_now() + metadata = { + "purpose": _PURPOSE, + "repository": TARGET, + "revision": revision, + "preflightSha256": preflight_sha256, + } + insert = ( + sqlite_insert(Secret) + .values( + id=SECRET_REF, + name=SECRET_REF, + encrypted_value=encrypted, + created_at=now, + updated_at=now, + last_used_at=None, + used_by_json=list(_USED_BY), + metadata_json=metadata, + ) + .on_conflict_do_nothing(index_elements=[Secret.id]) + ) + inserted = session.exec(insert).rowcount == 1 + row = session.get(Secret, SECRET_REF) + if row is None: + _reject("database_unavailable") + stored = get_secret(session, SECRET_REF, touch=False) + if stored is None: + _reject("secret_reference_unreadable") + if not hmac.compare_digest(stored, credential): + _reject("secret_reference_conflict") + return inserted + except WorkMeshSecretImportError: + _rollback_quietly(session) + raise + except Exception: # noqa: BLE001 - crypto/database details stay private. + _rollback_quietly(session) + _reject("database_unavailable") + + +def import_workmesh_deployment_secret( + session: Session, + credential: str, + *, + revision: str, + broker: DeploymentBrokerClient | None = None, +) -> dict[str, Any]: + """Bind an existing target credential after exact fixed-target preflight.""" + value, requested_revision = _validate_input(credential, revision) + try: + if secret_storage_status().get("productionReady") is not True: + _reject("secret_storage_unavailable") + except WorkMeshSecretImportError: + raise + except Exception: # noqa: BLE001 - storage configuration may contain secrets. + _reject("secret_storage_unavailable") + + if session.new or session.dirty or session.deleted: + _reject("session_not_clean") + _read_existing(session, value) + + client = DeploymentBrokerClient() if broker is None else broker + if not _controller_is_verified_https(client): + _reject("controller_https_unverified") + try: + preflight = client.workmesh_promotion_preflight( + requested_revision, + secret=value, + ) + except Exception: # noqa: BLE001 - transport errors may contain the key. + _reject("controller_preflight_unavailable") + if not isinstance(preflight, dict): + _reject("controller_preflight_rejected") + try: + safe = _safe_preflight(preflight, requested_revision) + except Exception: # noqa: BLE001 - reject malformed or hostile response objects. + safe = None + if safe is None: + _reject("controller_preflight_rejected") + preflight_sha256 = safe["preflightSha256"] + + inserted = _store_if_absent( + session, + value, + revision=requested_revision, + preflight_sha256=preflight_sha256, + ) + outcome = "imported" if inserted else "already_present" + try: + audit = log_event( + session, + source="deployment", + identity="operator", + action_id=_ACTION, + target=TARGET, + risk="high", + policy_decision="operator_import", + reason="Imported the fixed WorkMesh controller credential after verified preflight.", + result="success", + execution_plan={ + "storeReference": SECRET_REF, + "repository": TARGET, + "revision": requested_revision, + "preflightSha256": preflight_sha256, + }, + after_state={ + "storeReference": SECRET_REF, + "status": "configured", + "repository": TARGET, + "revision": requested_revision, + "preflightSha256": preflight_sha256, + "outcome": outcome, + }, + redacted_metadata={ + "purpose": _PURPOSE, + "storeReference": SECRET_REF, + "repository": TARGET, + "revision": requested_revision, + "preflightSha256": preflight_sha256, + "outcome": outcome, + }, + ) + except Exception: # noqa: BLE001 - commit/audit failures must not echo details. + _rollback_quietly(session) + _reject("audit_commit_failed") + audit_id = audit.id if isinstance(audit.id, str) else "" + if not _AUDIT_ID.fullmatch(audit_id): + _reject("audit_commit_failed") + + try: + readback = get_secret(session, SECRET_REF, touch=False) + except Exception: # noqa: BLE001 - do not expose ciphertext/decryption details. + _reject("secret_readback_failed") + if readback is None or not hmac.compare_digest(readback, value): + _reject("secret_readback_failed") + return { + "apiVersion": _API_VERSION, + "status": outcome, + "secretRef": SECRET_REF, + "repository": TARGET, + "revision": requested_revision, + "preflightSha256": preflight_sha256, + "auditId": audit_id, + "readback": "verified", + } + + +def _write_result(stream: TextIO, value: dict[str, str]) -> None: + stream.write(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n") + stream.flush() + + +def main( + argv: list[str] | None = None, + *, + stdin=None, + stdout: TextIO | None = None, +) -> int: + """Import the single fixed reference from bounded stdin.""" + arguments = sys.argv[1:] if argv is None else argv + output = sys.stdout if stdout is None else stdout + if len(arguments) != 1 or not isinstance(arguments[0], str) or not _REVISION.fullmatch(arguments[0]): + _write_result(output, {"status": "blocked", "reason": "invalid_arguments"}) + return 2 + source = getattr(sys.stdin, "buffer", sys.stdin) if stdin is None else stdin + try: + raw = source.read(_MAX_CREDENTIAL_BYTES + 1) + if not isinstance(raw, bytes) or len(raw) > _MAX_CREDENTIAL_BYTES: + _write_result(output, {"status": "blocked", "reason": "invalid_input"}) + return 2 + try: + credential = raw.decode("ascii") + except UnicodeDecodeError: + _write_result(output, {"status": "blocked", "reason": "invalid_input"}) + return 2 + with Session(engine) as session: + result = import_workmesh_deployment_secret( + session, + credential, + revision=arguments[0], + broker=DeploymentBrokerClient(), + ) + _write_result(output, {key: str(value) for key, value in result.items()}) + return 0 + except WorkMeshSecretImportError as error: + _write_result(output, {"status": "blocked", "reason": _safe_error_code(error.code)}) + return 2 + except Exception: # noqa: BLE001 - never print traceback or arbitrary values. + _write_result(output, {"status": "blocked", "reason": "import_failed"}) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/test_appops_effective_transaction_reconciliation.py b/backend/tests/test_appops_effective_transaction_reconciliation.py new file mode 100644 index 0000000..274efd0 --- /dev/null +++ b/backend/tests/test_appops_effective_transaction_reconciliation.py @@ -0,0 +1,963 @@ +from __future__ import annotations + +import contextlib +import copy +import hashlib +import importlib.util +import json +import sys +import types +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +BROKER = ROOT / "ops" / "gitea-deploy-broker" +REPOSITORY = "jens/unraid-appops-gateway" +REVISION = "c" * 40 +PRIOR_REVISION = "d" * 40 +TREE = "e" * 40 +RECOVERY_ID = "12345678-1234-4234-8234-123456789abc" +TRANSACTION_ID = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" +SECRET = b"transaction-reconciliation-test-key-32-bytes" +REGISTRY_BYTES = b'{"repository":"jens/unraid-appops-gateway"}\n' +IMAGE_ID = "sha256:" + "7" * 64 + + +@pytest.fixture +def deploy(monkeypatch): + monkeypatch.syspath_prepend(str(BROKER)) + if "fcntl" not in sys.modules: + fcntl = types.ModuleType("fcntl") + fcntl.LOCK_EX = 2 + fcntl.LOCK_UN = 8 + fcntl.LOCK_NB = 4 + fcntl.flock = lambda *_args: None + monkeypatch.setitem(sys.modules, "fcntl", fcntl) + spec = importlib.util.spec_from_file_location( + "appops_effective_transaction_reconciliation_test", + BROKER / "deploy.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _digest(raw: bytes) -> str: + return hashlib.sha256(raw).hexdigest() + + +def _mac(secret: bytes, domain: str, raw: bytes) -> str: + return hashlib.sha256(secret + b"\0" + domain.encode() + b"\0" + raw).hexdigest() + + +def _metadata( + revision: str = REVISION, + tree: str = TREE, + built_at: str = "2026-10-07T12:00:00.000Z", +) -> dict[str, str]: + return { + "id": revision[:12], + "repository": REPOSITORY, + "revision": revision, + "tree": tree, + "treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(), + "builtAt": built_at, + } + + +def _pointer(revision: str) -> dict[str, str]: + return { + "recoveryId": RECOVERY_ID, + "revision": revision, + "modelSha256": "1" * 64, + "modelRecordSha256": "2" * 64, + } + + +def _state_bytes(state: dict[str, object]) -> bytes: + return json.dumps(state, sort_keys=True, separators=(",", ":")).encode() + b"\n" + + +def _case(deploy, monkeypatch, tmp_path, *, phase: str = "runtime_mutated"): + prior_pointer = _pointer(PRIOR_REVISION) + candidate_pointer = _pointer(REVISION) + backup = {"sha256": "3" * 64, "revision": REVISION} + prior_snapshot = { + "revision": PRIOR_REVISION, + "appops_runtime_model": prior_pointer, + "appops_data_backup": backup, + "containers": [{"name": "Unraid-AppOps-Gateway", "image_id": "sha256:" + "4" * 64}], + } + receipt = { + "schema": "gitea-deploy-receipt.v1", + "repository": REPOSITORY, + "revision": REVISION, + "tree": TREE, + "containers": { + "Unraid-AppOps-Gateway": { + "container_id": "5" * 64, + "image_id": IMAGE_ID, + }, + }, + } + live_reference = { + "recoveryId": RECOVERY_ID, + "revision": REVISION, + "liveRecordSha256": "6" * 64, + } + old_receipt = { + "schema": "gitea-deploy-receipt.v1", + "repository": REPOSITORY, + "revision": PRIOR_REVISION, + "tree": "9" * 40, + } + state: dict[str, object] = { + "repository": REPOSITORY, + "live_revision": PRIOR_REVISION, + "live_status": "healthy", + "live_legacy_provenance": False, + "live_receipt": old_receipt, + "rollback_snapshot": prior_snapshot, + "live_artifacts": { + "appops_runtime_model": {"recoveryId": RECOVERY_ID, "revision": PRIOR_REVISION}, + "containers": {"Unraid-AppOps-Gateway": {"container_id": "8" * 64}}, + }, + } + original_raw = _state_bytes(state) + original_state_sha = _digest(original_raw) + candidate_model = { + "services": { + "appops-gateway": { + "image": "itworx/appops-gateway:protected", + "build": {"context": "/controller/registered-builds/unraid-appops-gateway/" + REVISION + "/tree"}, + "environment": {"APPOPS_SOURCE_REVISION": REVISION}, + }, + }, + } + payload: dict[str, object] = { + "schema": "itworx.appops-self-upgrade-model-transaction/v1", + "repository": REPOSITORY, + "transactionId": TRANSACTION_ID, + "recoveryId": RECOVERY_ID, + "kind": "followup", + "revision": REVISION, + "metadata": _metadata(), + "originSemanticSha256": "a" * 64, + "originRawSha256": "b" * 64, + "originJournalSha256": "c" * 64, + "expectedStateSha256": original_state_sha, + "registryFingerprint": _mac(SECRET, "registry", REGISTRY_BYTES), + "preMutationRuntimeSha256": "d" * 64, + "preMutationRuntimeFingerprint": "e" * 64, + "sourceFingerprint": "f" * 64, + "priorModelPointer": prior_pointer, + "priorSnapshot": prior_snapshot, + "candidateSourceFingerprint": "0" * 64, + "candidateModelPointer": candidate_pointer, + "candidateImageId": IMAGE_ID, + "buildContexts": { + "appops-gateway": "/controller/registered-builds/unraid-appops-gateway/" + REVISION + "/tree", + }, + "phase": phase, + "createdAt": 1_791_378_000, + "updatedAt": 1_791_378_001, + } + if phase in {"verified", "committed"}: + payload["verifiedLiveReference"] = live_reference + payload["verifiedReceipt"] = receipt + if phase == "committed": + payload["committedStateSha256"] = "0" * 64 + envelope = {"payload": payload, "sha256": "9" * 64} + registry = {"bytes": REGISTRY_BYTES} + calls: dict[str, object] = { + "active": [], + "active_result": None, + "docker": [], + "git": [], + "backup": [], + "verify": [], + "current": [], + "updates": [], + "writes": [], + "state_sha_override": None, + "metadata_override": None, + "receipt_override": None, + "fail_commit_once": False, + } + + class StatusReader: + PROJECTS_PATH = tmp_path / "projects.json" + + @staticmethod + def _self_upgrade_pending_model_transaction(): + return copy.deepcopy(envelope) + + @staticmethod + def _self_upgrade_read_fixed_file(_path, **_kwargs): + return registry["bytes"] + + @staticmethod + def _self_upgrade_secret(): + return SECRET + + @staticmethod + def _self_upgrade_mac(secret, domain, raw): + return _mac(secret, domain, raw) + + @staticmethod + def _self_upgrade_load_compose_model(recovery_id, revision): + if recovery_id == RECOVERY_ID and revision == REVISION and calls.get("model_valid", True): + return {"model": candidate_model, "pointer": candidate_pointer} + return None + + @staticmethod + def _self_upgrade_candidate_model_binding(pointer, observed_receipt): + if (pointer == candidate_pointer and observed_receipt == receipt + and calls.get("model_valid", True)): + return {"modelPointer": pointer, "receipt": observed_receipt} + return None + + @staticmethod + def _self_upgrade_store_live_model(pointer, binding): + if pointer == candidate_pointer and binding.get("receipt") == receipt: + return live_reference + return None + + @staticmethod + def _self_upgrade_update_model_transaction(transaction_id, expected_sha, next_phase, bindings): + calls["updates"].append((next_phase, copy.deepcopy(bindings))) + if transaction_id != TRANSACTION_ID or expected_sha != envelope["sha256"]: + return None + if next_phase == "committed" and calls["fail_commit_once"]: + calls["fail_commit_once"] = False + return None + updated = dict(envelope["payload"]) + updated.update(bindings) + updated["phase"] = next_phase + updated["updatedAt"] = int(updated["updatedAt"]) + 1 + if next_phase == "committed": + updated["committedStateSha256"] = _digest(_state_bytes(state)) + envelope["payload"] = updated + envelope["sha256"] = _digest(json.dumps(updated, sort_keys=True).encode()) + return copy.deepcopy(envelope) + + @staticmethod + def _self_upgrade_active_compose_model(observed_state, observed_container): + calls["active"].append((copy.deepcopy(observed_state), copy.deepcopy(observed_container))) + active = calls.get("active_result") + return copy.deepcopy(active) if isinstance(active, dict) else None + + def read_preimage(_project): + raw = _state_bytes(state) + override = calls["state_sha_override"] + return copy.deepcopy(state), raw, override or _digest(raw) + + def write_state(repository, project, revision, status, _message, **kwargs): + calls["writes"].append((repository, revision, status, copy.deepcopy(kwargs))) + reference = project.get("_appops_runtime_model_binding") + state.update({ + "repository": repository, + "live_revision": revision, + "live_status": "healthy" if status == "success" else status, + "live_legacy_provenance": False, + "live_receipt": copy.deepcopy(kwargs["live_receipt"]), + "rollback_snapshot": copy.deepcopy(kwargs["rollback_snapshot"]), + "live_artifacts": { + "appops_runtime_model": copy.deepcopy(reference), + "containers": {"Unraid-AppOps-Gateway": {"container_id": receipt["containers"]["Unraid-AppOps-Gateway"]["container_id"]}}, + }, + }) + + def validate_commit(_repo, branch, candidate_revision): + calls["git"].append((branch, candidate_revision)) + if branch != "main" or candidate_revision != REVISION: + raise deploy.DeployError("pending source is not the requested main commit") + return candidate_revision + + def deployment_metadata(_repo, candidate_revision, repository, *, built_at=None): + if repository != REPOSITORY or candidate_revision != REVISION: + raise deploy.DeployError("pending target identity mismatch") + observed = _metadata(built_at=built_at or "2026-10-07T12:00:00.000Z") + override = calls["metadata_override"] + return copy.deepcopy(override or observed) + + def verify_backup(actual_backup, candidate_revision, previous_revision): + calls["backup"].append((actual_backup, candidate_revision, previous_revision)) + expected_previous = calls.get("backup_previous_revision", PRIOR_REVISION) + if actual_backup != backup or candidate_revision != REVISION or previous_revision != expected_previous: + raise deploy.DeployError("pending data backup proof changed") + + def verify_deployment(_project, observed_metadata, *, expected_image_ids): + calls["verify"].append((copy.deepcopy(observed_metadata), dict(expected_image_ids))) + if expected_image_ids != {"Unraid-AppOps-Gateway": IMAGE_ID}: + raise deploy.DeployError("sealed candidate image differs from the live image") + if observed_metadata != _metadata(): + raise deploy.DeployError("candidate receipt metadata changed") + return copy.deepcopy(calls["receipt_override"] or receipt) + + def verified_current(repository, _project, candidate_revision, tree): + calls["current"].append((repository, candidate_revision, tree)) + if (repository, candidate_revision, tree) == (REPOSITORY, REVISION, TREE): + return copy.deepcopy(receipt) + return None + + project = { + "_repository_name": REPOSITORY, + "slug": "unraid-appops-gateway", + "branch": "main", + "containers": ["Unraid-AppOps-Gateway"], + } + monkeypatch.setattr(deploy, "_appops_status_reader", lambda: StatusReader()) + monkeypatch.setattr(deploy, "read_appops_state_preimage", read_preimage) + monkeypatch.setattr(deploy, "write_state", write_state) + monkeypatch.setattr(deploy, "validate_commit", validate_commit) + monkeypatch.setattr(deploy, "deployment_metadata", deployment_metadata) + monkeypatch.setattr(deploy, "verify_appops_data_backup", verify_backup) + monkeypatch.setattr(deploy, "verify_deployment", verify_deployment) + monkeypatch.setattr(deploy, "verified_current_receipt", verified_current) + monkeypatch.setattr(deploy, "run", lambda *_args, **_kwargs: pytest.fail("reconciliation must not mutate Docker")) + monkeypatch.setattr(deploy, "inspect_container", lambda *_args, **_kwargs: pytest.fail("unexpected inspect path")) + monkeypatch.setattr(deploy, "restore_compose_snapshot", lambda *_args, **_kwargs: pytest.fail("unexpected rollback")) + return SimpleNamespace( + deploy=deploy, + project=project, + repo=tmp_path / "repository.git", + payload=payload, + envelope=envelope, + state=state, + state_sha=original_state_sha, + registry=registry, + metadata=_metadata(), + pointer=candidate_pointer, + reference=live_reference, + receipt=receipt, + backup=backup, + calls=calls, + ) + + +def _reconcile(case): + return case.deploy._appops_reconcile_pending_model_transaction( + case.project, REPOSITORY, case.repo, + ) + + +def test_runtime_mutated_transaction_finishes_state_publication_without_docker_mutation( + deploy, monkeypatch, tmp_path, +): + case = _case(deploy, monkeypatch, tmp_path) + + result = _reconcile(case) + + assert result == case.receipt + assert case.envelope["payload"]["phase"] == "committed" + assert [phase for phase, _bindings in case.calls["updates"]] == ["verified", "committed"] + assert len(case.calls["writes"]) == 1 + repo, revision, status, kwargs = case.calls["writes"][0] + assert (repo, revision, status) == (REPOSITORY, REVISION, "success") + assert kwargs["expected_state_sha256"] == case.state_sha + assert kwargs["live_receipt"] == case.receipt + assert case.state["live_legacy_provenance"] is False + assert case.state["live_artifacts"]["appops_runtime_model"] == case.reference + assert case.calls["git"] == [("main", REVISION)] + assert len(case.calls["backup"]) == 1 + assert case.calls["verify"] == [(case.metadata, {"Unraid-AppOps-Gateway": IMAGE_ID})] + assert case.calls["docker"] == [] + + +def test_verified_state_replay_only_reattests_and_marks_transaction_committed( + deploy, monkeypatch, tmp_path, +): + case = _case(deploy, monkeypatch, tmp_path, phase="verified") + case.state.update({ + "live_revision": REVISION, + "live_status": "healthy", + "live_legacy_provenance": False, + "live_receipt": copy.deepcopy(case.receipt), + "live_artifacts": {"appops_runtime_model": copy.deepcopy(case.reference)}, + }) + + result = _reconcile(case) + + assert result == case.receipt + assert case.envelope["payload"]["phase"] == "committed" + assert [phase for phase, _bindings in case.calls["updates"]] == ["committed"] + assert case.calls["current"] == [(REPOSITORY, REVISION, TREE)] + assert case.calls["writes"] == [] + assert case.calls["git"] == [] + assert case.calls["verify"] == [] + assert case.calls["docker"] == [] + + +def test_verified_state_replay_rejects_changed_registry_before_reattest( + deploy, monkeypatch, tmp_path, +): + case = _case(deploy, monkeypatch, tmp_path, phase="verified") + case.state.update({ + "live_revision": REVISION, + "live_status": "healthy", + "live_legacy_provenance": False, + "live_receipt": copy.deepcopy(case.receipt), + "live_artifacts": {"appops_runtime_model": copy.deepcopy(case.reference)}, + }) + case.registry["bytes"] = b'{"repository":"changed"}\n' + + with pytest.raises(deploy.DeployError, match="state or registry changed"): + _reconcile(case) + + assert case.calls["current"] == [] + assert case.calls["updates"] == [] + assert case.calls["writes"] == [] + assert case.calls["docker"] == [] + + +def test_state_commit_failure_restarts_from_verified_proof_without_second_state_write( + deploy, monkeypatch, tmp_path, +): + case = _case(deploy, monkeypatch, tmp_path) + case.calls["fail_commit_once"] = True + + with pytest.raises(deploy.DeployError, match="compare-and-swap failed"): + _reconcile(case) + + assert case.envelope["payload"]["phase"] == "verified" + assert len(case.calls["writes"]) == 1 + assert case.calls["docker"] == [] + + result = _reconcile(case) + + assert result == case.receipt + assert case.envelope["payload"]["phase"] == "committed" + assert [phase for phase, _bindings in case.calls["updates"]] == [ + "verified", "committed", "committed", + ] + assert len(case.calls["writes"]) == 1 + assert len(case.calls["current"]) == 1 + assert case.calls["docker"] == [] + + +@pytest.mark.parametrize( + "mismatch", + ["state", "registry", "model", "target_metadata", "backup", "image", "receipt", "unknown_phase"], +) +def test_pending_transaction_mismatches_fail_without_state_or_docker_mutation( + deploy, monkeypatch, tmp_path, mismatch, +): + case = _case(deploy, monkeypatch, tmp_path) + if mismatch == "state": + case.calls["state_sha_override"] = "0" * 64 + elif mismatch == "registry": + case.registry["bytes"] = b'{"repository":"different"}\n' + elif mismatch == "model": + case.calls["model_valid"] = False + elif mismatch == "target_metadata": + case.calls["metadata_override"] = _metadata(tree="9" * 40) + elif mismatch == "backup": + case.payload["priorSnapshot"]["appops_data_backup"] = {"sha256": "0" * 64} + elif mismatch == "image": + case.payload["candidateImageId"] = "sha256:" + "8" * 64 + elif mismatch == "receipt": + case.calls["receipt_override"] = {**case.receipt, "revision": "b" * 40} + elif mismatch == "unknown_phase": + case.payload["phase"] = "prepared" + + with pytest.raises(deploy.DeployError): + _reconcile(case) + + assert case.calls["writes"] == [] + assert case.calls["updates"] == [] + assert case.calls["docker"] == [] + + +def _prepare_rolling_back_case(deploy, monkeypatch, tmp_path): + case = _case(deploy, monkeypatch, tmp_path, phase="rolling_back") + prior_revision = deploy.APPOPS_LEGACY_OLD_REVISION + prior_pointer = { + **_pointer(prior_revision), + "schema": "itworx.appops-effective-compose-model.v1", + } + restored_reference = {**prior_pointer, "liveRecordSha256": "a" * 64} + prior_snapshot = { + "revision": prior_revision, + "legacy_provenance": True, + "receipt": {}, + "appops_runtime_model": prior_pointer, + "appops_data_backup": case.backup, + "containers": [{"name": "Unraid-AppOps-Gateway", "image_id": IMAGE_ID}], + } + case.payload.update({ + "phase": "rolling_back", + "priorModelPointer": prior_pointer, + "priorSnapshot": prior_snapshot, + }) + case.state.update({ + "repository": REPOSITORY, + "live_revision": prior_revision, + "live_status": "healthy", + "live_legacy_provenance": True, + "live_receipt": {}, + "live_artifacts": { + "appops_runtime_model": restored_reference, + "containers": {"Unraid-AppOps-Gateway": {"container_id": "b" * 64}}, + }, + }) + case.calls["active_result"] = {"pointer": copy.deepcopy(restored_reference)} + case.calls["backup_previous_revision"] = prior_revision + container = {"Id": "b" * 64, "Image": IMAGE_ID, "State": {"Running": True}} + monkeypatch.setattr(deploy, "inspect_container", lambda _name: copy.deepcopy(container)) + return case, prior_pointer, restored_reference, prior_snapshot, container + + +def test_rolling_back_transaction_seals_exact_restored_legacy_binding_without_mutation( + deploy, monkeypatch, tmp_path, +): + case, prior_pointer, restored_reference, prior_snapshot, container = _prepare_rolling_back_case( + deploy, monkeypatch, tmp_path, + ) + + result = _reconcile(case) + + assert result == {} + assert case.envelope["payload"]["phase"] == "rolled_back" + assert case.envelope["payload"]["restoredLiveReference"] == restored_reference + assert case.calls["updates"] == [("rolled_back", {"restoredLiveReference": restored_reference})] + assert case.calls["active"] == [(case.state, container)] + assert case.calls["backup"] == [(case.backup, REVISION, deploy.APPOPS_LEGACY_OLD_REVISION)] + assert case.calls["writes"] == [] + assert case.calls["git"] == [] + assert case.calls["docker"] == [] + assert case.payload["priorModelPointer"] == prior_pointer + assert case.payload["priorSnapshot"] == prior_snapshot + + +@pytest.mark.parametrize("mismatch", ["state", "reference", "runtime", "registry", "backup"]) +def test_rolling_back_transaction_rejects_unbound_prior_state_without_mutation( + deploy, monkeypatch, tmp_path, mismatch, +): + case, _prior_pointer, restored_reference, prior_snapshot, _container = _prepare_rolling_back_case( + deploy, monkeypatch, tmp_path, + ) + if mismatch == "state": + case.state["live_revision"] = "9" * 40 + elif mismatch == "reference": + case.state["live_artifacts"]["appops_runtime_model"]["liveRecordSha256"] = "c" * 64 + elif mismatch == "runtime": + case.calls["active_result"] = {"pointer": {**restored_reference, "liveRecordSha256": "d" * 64}} + elif mismatch == "registry": + case.registry["bytes"] = b'{"repository":"different"}\n' + elif mismatch == "backup": + prior_snapshot["appops_data_backup"] = {"sha256": "0" * 64} + + with pytest.raises(deploy.DeployError): + _reconcile(case) + + assert case.calls["updates"] == [] + assert case.calls["writes"] == [] + assert case.calls["git"] == [] + assert case.calls["docker"] == [] + + +def test_explicit_rollback_cli_routes_through_pending_model_reconciliation( + deploy, monkeypatch, tmp_path, capsys, +): + revision = REVISION + project = { + "slug": "unraid-appops-gateway", + "branch": "main", + "source_dir": str(tmp_path / "source"), + "compose_project": "unraid-appops-gateway", + "compose_files": ["docker-compose.yml"], + "containers": ["Unraid-AppOps-Gateway"], + "verify_oci_revision": True, + "environment": {}, + } + projects = {REPOSITORY: project} + events = [] + real_exists = deploy.Path.exists + + def path_exists(path): + if str(path) == "/controller-hub-releases/appops-self-upgrade-recoveries/transactions": + return True + return real_exists(path) + + def reconcile(actual_project, repository, repo): + assert actual_project["_repository_name"] == REPOSITORY + assert repository == REPOSITORY + assert repo == tmp_path / "repository.git" + events.append("reconcile") + return {"revision": revision, "proof": "signed-prior-runtime"} + + monkeypatch.setattr(deploy.Path, "exists", path_exists) + monkeypatch.setattr(deploy, "load_projects", lambda: projects) + monkeypatch.setattr(deploy, "effective_mcp_hub_project", lambda _repository, item: dict(item)) + monkeypatch.setattr(deploy, "repository_path", lambda *_args: tmp_path / "repository.git") + monkeypatch.setattr(deploy, "validate_commit", lambda *_args: revision) + monkeypatch.setattr(deploy, "deployment_metadata", lambda *_args: _metadata()) + monkeypatch.setattr(deploy, "validate_project_config", lambda *_args: None) + monkeypatch.setattr(deploy, "bind_appops_self_upgrade_recovery", lambda *_args: None) + monkeypatch.setattr(deploy, "bind_registered_legacy_request", lambda *_args: None) + monkeypatch.setattr(deploy, "exclusive_lock", lambda *_args, **_kwargs: contextlib.nullcontext()) + + @contextlib.contextmanager + def global_lock(): + events.append("global-lock") + yield + + monkeypatch.setattr(deploy, "exclusive_global_deploy_lock", global_lock) + monkeypatch.setattr(deploy, "_appops_reconcile_pending_model_transaction", reconcile) + monkeypatch.setattr(sys, "argv", ["deploy.py", "rollback", REPOSITORY, revision]) + + assert deploy.main() == 0 + assert events == ["global-lock", "reconcile"] + assert '"status": "already-current"' in capsys.readouterr().out + + +def _recreated_receipt_case(deploy, monkeypatch): + pointer = { + "schema": "itworx.appops-effective-compose-model.v1", + "recoveryId": RECOVERY_ID, + "revision": PRIOR_REVISION, + "modelSha256": "1" * 64, + "modelRecordSha256": "2" * 64, + "liveRecordSha256": "3" * 64, + } + recorded = { + "schema": "gitea-deploy-receipt.v1", + "repository": REPOSITORY, + "revision": PRIOR_REVISION, + "tree": "4" * 40, + "treeDigest": "5" * 64, + "build": { + "id": PRIOR_REVISION[:12], + "repository": REPOSITORY, + "revision": PRIOR_REVISION, + "tree": "4" * 40, + "treeDigest": "5" * 64, + "builtAt": "2026-10-07T11:00:00.000Z", + }, + "containers": { + "Unraid-AppOps-Gateway": { + "container_id": "a" * 64, + "image_id": IMAGE_ID, + "image_revision": PRIOR_REVISION, + "image_tree": "4" * 40, + "health": "healthy", + }, + }, + "live": {"status": "healthy"}, + } + observed = copy.deepcopy(recorded) + observed["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "b" * 64 + snapshot = { + "revision": PRIOR_REVISION, + "legacy_provenance": False, + "receipt": recorded, + "appops_runtime_model": pointer, + } + project = { + "_repository_name": REPOSITORY, + "_appops_effective_model_pointer": pointer, + } + calls = {"attestation": [], "attestation_valid": True} + + def verify_rollback_model(actual_pointer, actual_snapshot): + calls["attestation"].append((copy.deepcopy(actual_pointer), copy.deepcopy(actual_snapshot))) + return ( + calls["attestation_valid"] + and actual_pointer == pointer + and actual_snapshot == snapshot + ) + + monkeypatch.setattr( + deploy, + "_appops_status_reader", + lambda: SimpleNamespace(_self_upgrade_verify_rollback_model=verify_rollback_model), + ) + project["_appops_effective_rollback_snapshot"] = snapshot + return SimpleNamespace( + project=project, + pointer=pointer, + recorded=recorded, + observed=observed, + snapshot=snapshot, + calls=calls, + ) + + +def test_recreated_receipt_matcher_allows_only_new_valid_container_id( + deploy, monkeypatch, +): + case = _recreated_receipt_case(deploy, monkeypatch) + recorded_id = case.recorded["containers"]["Unraid-AppOps-Gateway"]["container_id"] + observed_id = case.observed["containers"]["Unraid-AppOps-Gateway"]["container_id"] + + assert len(recorded_id) == len(observed_id) == 64 + assert recorded_id != observed_id + assert deploy._appops_recreated_receipt_matches(case.project, case.observed, case.recorded) + assert case.calls["attestation"] == [(case.pointer, case.snapshot)] + + +@pytest.mark.parametrize( + "mismatch", + [ + "repository", "legacy_provenance", "snapshot_revision", "snapshot_receipt", + "project_model", "rollback_attestation", "observed_revision", "observed_tree", + "observed_image", "observed_container_id", "recorded_container_id", + "missing_container", "extra_container", + ], +) +def test_recreated_receipt_matcher_rejects_every_other_changed_fact( + deploy, monkeypatch, mismatch, +): + case = _recreated_receipt_case(deploy, monkeypatch) + if mismatch == "repository": + case.project["_repository_name"] = "jens/other" + elif mismatch == "legacy_provenance": + case.snapshot["legacy_provenance"] = True + elif mismatch == "snapshot_revision": + case.snapshot["revision"] = "9" * 40 + elif mismatch == "snapshot_receipt": + case.snapshot["receipt"] = {**case.recorded, "tree": "8" * 40} + elif mismatch == "project_model": + case.project["_appops_effective_model_pointer"] = {**case.pointer, "modelSha256": "9" * 64} + elif mismatch == "rollback_attestation": + case.calls["attestation_valid"] = False + elif mismatch == "observed_revision": + case.observed["revision"] = "9" * 40 + elif mismatch == "observed_tree": + case.observed["build"]["tree"] = "8" * 40 + elif mismatch == "observed_image": + case.observed["containers"]["Unraid-AppOps-Gateway"]["image_id"] = "sha256:" + "8" * 64 + elif mismatch == "observed_container_id": + case.observed["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "not-a-container-id" + elif mismatch == "recorded_container_id": + case.recorded["containers"]["Unraid-AppOps-Gateway"]["container_id"] = "not-a-container-id" + elif mismatch == "missing_container": + case.observed["containers"] = {} + elif mismatch == "extra_container": + case.observed["containers"]["unexpected"] = {"container_id": "c" * 64} + + assert not deploy._appops_recreated_receipt_matches(case.project, case.observed, case.recorded) + + +def test_known_model_restore_runs_real_receipt_comparison_and_rebinds_recreated_container( + deploy, monkeypatch, tmp_path, +): + case = _recreated_receipt_case(deploy, monkeypatch) + container_name = "Unraid-AppOps-Gateway" + old_container_id = case.recorded["containers"][container_name]["container_id"] + new_container_id = case.observed["containers"][container_name]["container_id"] + case.snapshot["containers"] = [{ + "name": container_name, + "container_id": old_container_id, + "image_id": IMAGE_ID, + "image_ref": "unraid-appops-gateway:production", + "was_running": True, + }] + case.project.update({ + "slug": "unraid-appops-gateway", + "source_dir": str(tmp_path / "source"), + "compose_project": "unraid-appops-gateway", + "compose_files": ["docker-compose.yml"], + "containers": [container_name], + "environment": {}, + }) + case.project["_appops_effective_rollback_snapshot"] = case.snapshot + model = {"services": {"appops-gateway": {"environment": {"SYNTHETIC": "safe"}}}} + live_binding = {**case.pointer, "liveRecordSha256": "f" * 64} + calls = {"events": [], "recorded_receipt": None} + + monkeypatch.setattr( + deploy, + "_appops_load_effective_model", + lambda _project, pointer: {"pointer": pointer, "model": model}, + ) + monkeypatch.setattr(deploy, "workmesh_transition_enabled", lambda _project: False) + monkeypatch.setattr( + deploy, + "run", + lambda command, **_kwargs: calls["events"].append(("docker-tag", list(command))), + ) + monkeypatch.setattr( + deploy, + "inspect_image", + lambda image_ref: {"Id": IMAGE_ID} if image_ref == "unraid-appops-gateway:production" else None, + ) + monkeypatch.setattr( + deploy, + "inspect_container", + lambda _name: {"Id": new_container_id, "Image": IMAGE_ID}, + ) + monkeypatch.setattr(deploy, "repository_path_from_project", lambda _project: tmp_path / "repository.git") + monkeypatch.setattr( + deploy, + "sync_source", + lambda *_args, **_kwargs: calls["events"].append(("sync-source", None)), + ) + monkeypatch.setattr( + deploy, + "copy_environment", + lambda *_args, **_kwargs: calls["events"].append(("copy-environment", None)), + ) + monkeypatch.setattr( + deploy, + "remove_attempt_only_compose_containers", + lambda *_args: calls["events"].append(("remove-attempt-only", None)), + ) + monkeypatch.setattr( + deploy, + "deployment_environment", + lambda *_args: {"SYNTHETIC": "safe"}, + ) + monkeypatch.setattr( + deploy, + "restore_snapshot_runtime", + lambda *_args: calls["events"].append(("restore-runtime", None)), + ) + monkeypatch.setattr( + deploy, + "validate_recorded_receipt", + lambda _project, receipt, repository, revision: ( + {"validated": receipt == case.recorded, "repository": repository, "revision": revision} + ), + ) + + def verify_deployment(_project, build, *, expected_image_ids): + calls["events"].append(("verify-deployment", copy.deepcopy(expected_image_ids))) + assert build == {"validated": True, "repository": REPOSITORY, "revision": PRIOR_REVISION} + assert expected_image_ids == {container_name: IMAGE_ID} + return copy.deepcopy(case.observed) + + monkeypatch.setattr(deploy, "verify_deployment", verify_deployment) + monkeypatch.setattr(deploy, "cleanup_rollback_image_tags", lambda *_args: None) + + def record_live_model(_project, receipt): + calls["recorded_receipt"] = copy.deepcopy(receipt) + return live_binding + + monkeypatch.setattr(deploy, "_appops_record_live_effective_model", record_live_model) + + deploy.restore_compose_snapshot(case.project, case.snapshot, REPOSITORY, REVISION) + + assert calls["recorded_receipt"] == case.observed + assert case.project["_rollback_receipt"] == case.observed + assert case.project["_appops_restored_model_binding"] == live_binding + assert case.project["_rollback_restored_revision"] == PRIOR_REVISION + assert "_appops_effective_rollback_snapshot" not in case.project + assert old_container_id != new_container_id + assert [event[0] for event in calls["events"]] == [ + "docker-tag", "sync-source", "copy-environment", "remove-attempt-only", + "restore-runtime", "verify-deployment", + ] + assert len(case.calls["attestation"]) == 2 + + +@pytest.mark.parametrize("legacy_provenance", [True, False]) +def test_model_rollback_rebinds_the_restored_live_proof( + deploy, monkeypatch, tmp_path, legacy_provenance, +): + old_revision = deploy.APPOPS_LEGACY_OLD_REVISION if legacy_provenance else PRIOR_REVISION + pointer = { + "schema": "itworx.appops-effective-compose-model.v1", + "recoveryId": RECOVERY_ID, + "revision": old_revision, + "modelSha256": "a" * 64, + "modelRecordSha256": "b" * 64, + } + if not legacy_provenance: + pointer["liveRecordSha256"] = "c" * 64 + model = { + "services": { + "appops-gateway": { + "image": "unraid-appops-gateway:production", + "environment": {"KNOWLEDGE_OPTIMIZER_MODEL": "synthetic-model"}, + }, + }, + } + snapshot_receipt = {} if legacy_provenance else { + "schema": "gitea-deploy-receipt.v1", + "repository": REPOSITORY, + "revision": old_revision, + "containers": {"Unraid-AppOps-Gateway": {"image_id": IMAGE_ID}}, + } + snapshot = { + "revision": old_revision, + "legacy_provenance": legacy_provenance, + "receipt": snapshot_receipt, + "containers": [{ + "name": "Unraid-AppOps-Gateway", + "container_id": "5" * 64, + "image_id": IMAGE_ID, + "image_ref": "unraid-appops-gateway:production", + }], + "appops_runtime_model": pointer, + } + reference = {**pointer, "liveRecordSha256": "d" * 64} + calls = {"plain_snapshot": None, "binding": None, "recorded_receipt": None} + + class StatusReader: + @staticmethod + def _self_upgrade_verify_rollback_model(observed_pointer, observed_snapshot): + return observed_pointer == pointer and observed_snapshot == snapshot + + @staticmethod + def _self_upgrade_legacy_model_binding(observed_pointer): + calls["binding"] = copy.deepcopy(observed_pointer) + return {"modelPointer": observed_pointer, "runtimeFingerprint": "e" * 64} + + @staticmethod + def _self_upgrade_store_live_model(observed_pointer, binding): + if binding.get("modelPointer") == observed_pointer: + return reference + return None + + project = {"_repository_name": REPOSITORY, "containers": ["Unraid-AppOps-Gateway"]} + monkeypatch.setattr(deploy, "_appops_status_reader", lambda: StatusReader()) + monkeypatch.setattr( + deploy, + "_appops_load_effective_model", + lambda _project, observed_pointer: {"pointer": observed_pointer, "model": model}, + ) + monkeypatch.setattr(deploy, "_appops_select_effective_model", lambda *_args: None) + monkeypatch.setattr(deploy, "verify_appops_legacy_runtime", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + deploy, "_appops_captured_environment", + lambda _container: {"KNOWLEDGE_OPTIMIZER_MODEL": "synthetic-model"}, + ) + monkeypatch.setattr(deploy, "inspect_container", lambda _name: {"Id": "6" * 64}) + monkeypatch.setattr(deploy, "inspect_image", lambda _image: {"Id": IMAGE_ID}) + monkeypatch.setattr( + deploy, + "_appops_record_live_effective_model", + lambda _project, receipt: calls.__setitem__("recorded_receipt", receipt) or reference, + ) + + original_restore = deploy.restore_compose_snapshot + + def restore_with_plain_snapshot(project_arg, observed_snapshot, repository, failed_revision): + if "appops_runtime_model" not in observed_snapshot: + calls["plain_snapshot"] = copy.deepcopy(observed_snapshot) + if not legacy_provenance: + project_arg["_rollback_receipt"] = snapshot_receipt + return None + return original_restore(project_arg, observed_snapshot, repository, failed_revision) + + monkeypatch.setattr(deploy, "restore_compose_snapshot", restore_with_plain_snapshot) + restore_with_plain_snapshot(project, snapshot, REPOSITORY, REVISION) + + plain_snapshot = calls["plain_snapshot"] + assert isinstance(plain_snapshot, dict) + assert "appops_runtime_model" not in plain_snapshot + if legacy_provenance: + assert plain_snapshot["containers"][0]["restore_image_ref"] == deploy.APPOPS_SOURCE_COMPOSE_IMAGE_REF + assert project["_appops_legacy_restored"] is True + assert calls["binding"] == pointer + else: + assert calls["recorded_receipt"] == snapshot_receipt + assert "_appops_legacy_restored" not in project + assert project["_appops_restored_model_binding"] == reference diff --git a/backend/tests/test_appops_recovery_deploy_binding.py b/backend/tests/test_appops_recovery_deploy_binding.py new file mode 100644 index 0000000..b51d8cf --- /dev/null +++ b/backend/tests/test_appops_recovery_deploy_binding.py @@ -0,0 +1,1137 @@ +from __future__ import annotations + +import copy +import contextlib +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import types +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +BROKER = ROOT / "ops" / "gitea-deploy-broker" +APP_OPS_REPOSITORY = "jens/unraid-appops-gateway" +RECOVERY_ID = "12345678-1234-4234-8234-123456789abc" +LEGACY_555_ENV_KEYS = ( + "APPOPS_AUTHENTIK_CLIENT_ID", "APPOPS_AUTHENTIK_CLIENT_SECRET_FILE", + "APPOPS_AUTHENTIK_ENTRA_SOURCE", "APPOPS_AUTHENTIK_ISSUER", + "APPOPS_AUTH_ADMIN_SUBJECTS", "APPOPS_AUTH_PUBLIC_URL", "APPOPS_BIND_HOST", + "APPOPS_DATA_DIR", "APPOPS_DB_PATH", "APPOPS_DEPLOYMENT_MODE", + "APPOPS_DISCOVERY_HISTORY_RUNS", "APPOPS_ENABLE_MCP", "APPOPS_ENABLE_N8N", + "APPOPS_ENV", "APPOPS_FRONTEND_ORIGIN", "APPOPS_GPU_METRICS_URL", + "APPOPS_LOG_LEVEL", "APPOPS_MCP_ALLOWED_HOSTS", "APPOPS_MCP_TOKEN_FILE", + "APPOPS_N8N_TOKEN_FILE", "APPOPS_OPERATOR_PROXY_AUTH", + "APPOPS_OPERATOR_PROXY_GROUPS", "APPOPS_OPERATOR_PROXY_TRUSTED_NETWORKS", + "APPOPS_OPERATOR_PROXY_USERS", "APPOPS_OPERATOR_TOKEN_FILE", "APPOPS_PORT", + "APPOPS_PRIVATEMUSE_COMFYUI_MODEL_ROOT", "APPOPS_REQUIRE_OPERATOR_AUTH", + "APPOPS_SECRET_KEY_FILE", "APPOPS_STATIC_DIR", "APP_OPS_VOICE_TOKEN", + "APP_OPS_VOICE_TOKEN_FILE", "DEPLOYMENT_BROKER_CA_FILE", "DEPLOYMENT_BROKER_URL", + "DEPLOYMENT_BROKER_VERIFY_TLS", "GITEA_ADMIN_TOKEN_FILE", + "GITEA_RUNNER_TARGETS_JSON", "GITEA_TIMEOUT_SECONDS", "GITEA_URL", + "GITEA_VERIFY_TLS", "GPG_KEY", "PATH", "PYTHONDONTWRITEBYTECODE", + "PYTHONUNBUFFERED", "PYTHON_SHA256", "PYTHON_VERSION", +) + + +@pytest.fixture +def deploy(monkeypatch): + monkeypatch.syspath_prepend(str(BROKER)) + if "fcntl" not in sys.modules: + fcntl = types.ModuleType("fcntl") + fcntl.LOCK_EX = 2 + fcntl.LOCK_UN = 8 + fcntl.LOCK_NB = 4 + fcntl.flock = lambda *_args: None + monkeypatch.setitem(sys.modules, "fcntl", fcntl) + spec = importlib.util.spec_from_file_location( + "appops_recovery_deploy_binding_test", + BROKER / "deploy.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _appops_project() -> dict[str, object]: + return { + "_repository_name": APP_OPS_REPOSITORY, + "slug": "unraid-appops-gateway", + "gitea_repository_id": 9, + "branch": "main", + } + + +@pytest.mark.parametrize( + ("mode", "repository", "recovery_id", "legacy_id"), + [ + ("deploy", "jens/other", RECOVERY_ID, None), + ("rollback", APP_OPS_REPOSITORY, RECOVERY_ID, None), + ("deploy", APP_OPS_REPOSITORY, "not-a-uuid", None), + ("deploy", APP_OPS_REPOSITORY, RECOVERY_ID, "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"), + ], +) +def test_cli_recovery_binding_rejects_before_mutating_git_environment( + deploy, monkeypatch, tmp_path, mode, repository, recovery_id, legacy_id, +): + project = _appops_project() + project["_repository_name"] = repository + monkeypatch.setattr(deploy, "load_projects", lambda: {repository: dict(project)}) + monkeypatch.setattr(deploy, "repository_path", lambda *_args: tmp_path / "repo.git") + monkeypatch.setattr( + deploy, + "validate_commit", + lambda *_args: pytest.fail("invalid recovery binding reached Git validation"), + ) + monkeypatch.setattr( + deploy, + "_appops_status_reader", + lambda: pytest.fail("invalid recovery binding read the authorization store"), + ) + monkeypatch.setattr(sys, "argv", ["deploy.py", mode, repository, "c" * 40]) + monkeypatch.setenv("APPOPS_SELF_UPGRADE_RECOVERY_ID", recovery_id) + if legacy_id is None: + monkeypatch.delenv("APPOPS_REGISTERED_LEGACY_REQUEST_ID", raising=False) + else: + monkeypatch.setenv("APPOPS_REGISTERED_LEGACY_REQUEST_ID", legacy_id) + monkeypatch.setenv("GIT_NO_REPLACE_OBJECTS", "0") + + with pytest.raises(deploy.DeployError, match="AppOps self-upgrade recovery binding is invalid"): + deploy.main() + + assert os.environ["GIT_NO_REPLACE_OBJECTS"] == "0" + + +def test_valid_cli_recovery_binding_carries_only_authorized_identity(deploy, monkeypatch): + project = _appops_project() + authorization = {"requestId": RECOVERY_ID, "revision": "c" * 40} + reader = types.SimpleNamespace( + _self_upgrade_recovery_authorization=lambda request_id, **_kwargs: ( + authorization if request_id == RECOVERY_ID else None + ) + ) + monkeypatch.setattr(deploy, "_appops_status_reader", lambda: reader) + monkeypatch.setenv("APPOPS_SELF_UPGRADE_RECOVERY_ID", RECOVERY_ID) + monkeypatch.delenv("APPOPS_REGISTERED_LEGACY_REQUEST_ID", raising=False) + + assert deploy.bind_appops_self_upgrade_recovery(project, "deploy") == RECOVERY_ID + assert project["_appops_self_upgrade_recovery_id"] == RECOVERY_ID + assert project["_appops_self_upgrade_recovery_authorization"] == authorization + + +def test_recovery_same_sha_fastpath_waits_for_global_preimage_cas( + deploy, monkeypatch, tmp_path, capsys, +): + project = { + **_appops_project(), + "source_dir": str(tmp_path / "prepared-source"), + "compose_project": "unraid-appops-gateway", + "containers": ["Unraid-AppOps-Gateway"], + "compose_files": ["docker-compose.yml"], + "verify_oci_revision": True, + "environment": {}, + } + revision = "c" * 40 + metadata = { + "id": revision[:12], + "repository": APP_OPS_REPOSITORY, + "revision": revision, + "tree": "d" * 40, + "treeDigest": "e" * 64, + "builtAt": "2026-10-07T12:00:00.000Z", + } + receipt = {"repository": APP_OPS_REPOSITORY, "revision": revision, "proof": "verified"} + state = { + "repository": APP_OPS_REPOSITORY, + "live_revision": revision, + "live_status": "healthy", + "live_artifacts": {"appops_runtime_model": {"pointer": "signed"}}, + } + projects = {APP_OPS_REPOSITORY: project} + events = [] + global_lock_held = False + + def acquire_project_lock(*_args, **_kwargs): + return contextlib.nullcontext() + + @contextlib.contextmanager + def acquire_global_lock(): + nonlocal global_lock_held + assert not global_lock_held + global_lock_held = True + events.append("global-lock-acquired") + try: + yield + finally: + global_lock_held = False + events.append("global-lock-released") + + def prepare_recovery(actual_project, repository, _repo, actual_revision, _metadata, _state): + assert global_lock_held + assert actual_project["_appops_self_upgrade_recovery_id"] == RECOVERY_ID + assert repository == APP_OPS_REPOSITORY + assert actual_revision == revision + events.append("recovery-preimage-cas") + + def check_current_receipt(repository, _project, actual_revision, tree): + assert global_lock_held + assert repository == APP_OPS_REPOSITORY + assert actual_revision == revision + assert tree == metadata["tree"] + events.append("same-sha-receipt-check") + return receipt + + monkeypatch.setattr(deploy, "load_projects", lambda: projects) + monkeypatch.setattr(deploy, "effective_mcp_hub_project", lambda _repo, item: dict(item)) + monkeypatch.setattr(deploy, "repository_path", lambda *_args: tmp_path / "repo.git") + monkeypatch.setattr(deploy, "validate_commit", lambda *_args: revision) + monkeypatch.setattr(deploy, "deployment_metadata", lambda *_args: metadata) + monkeypatch.setattr(deploy, "validate_project_config", lambda *_args: None) + monkeypatch.setattr(deploy, "exclusive_lock", acquire_project_lock) + monkeypatch.setattr(deploy, "exclusive_global_deploy_lock", acquire_global_lock) + monkeypatch.setattr(deploy, "read_state", lambda _project: state) + monkeypatch.setattr( + deploy, "state_live_artifacts", lambda value: dict(value.get("live_artifacts") or {}), + ) + monkeypatch.setattr(deploy, "state_live_revision", lambda _state: revision) + monkeypatch.setattr(deploy, "mcp_hub_prepare_current_receipt_model", lambda *_args: None) + monkeypatch.setattr(deploy, "validate_forward_revision", lambda *_args: None) + monkeypatch.setattr(deploy, "_appops_prepare_recovery", prepare_recovery) + monkeypatch.setattr(deploy, "_appops_legacy_reconcile_pending_rollback", lambda *_a, **_k: False) + monkeypatch.setattr(deploy, "reconcile_appops_deployed_intent", lambda *_args: None) + monkeypatch.setattr(deploy, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None) + monkeypatch.setattr(deploy, "verified_current_receipt", check_current_receipt) + monkeypatch.setattr( + deploy, + "sync_source", + lambda *_args, **_kwargs: pytest.fail("same-SHA recovery fastpath synced source"), + ) + monkeypatch.setattr( + deploy, + "copy_environment", + lambda *_args, **_kwargs: pytest.fail("same-SHA recovery fastpath copied environment"), + ) + reader = types.SimpleNamespace( + _self_upgrade_recovery_authorization=lambda request_id, **_kwargs: ( + { + "requestId": request_id, + "revision": revision, + "journal": {"revision": revision}, + } + if request_id == RECOVERY_ID else None + ) + ) + monkeypatch.setattr(deploy, "_appops_status_reader", lambda: reader) + monkeypatch.setenv("APPOPS_SELF_UPGRADE_RECOVERY_ID", RECOVERY_ID) + monkeypatch.delenv("APPOPS_REGISTERED_LEGACY_REQUEST_ID", raising=False) + monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", APP_OPS_REPOSITORY, revision]) + + assert deploy.main() == 0 + output = capsys.readouterr().out + assert '"status": "already-current"' in output + assert events == [ + "global-lock-acquired", + "recovery-preimage-cas", + "same-sha-receipt-check", + "global-lock-released", + ] + + +def test_effective_runtime_model_preserves_captured_environment_and_changes_only_provenance( + deploy, monkeypatch, +): + class Adoption: + @staticmethod + def _inspected_environment(container): + parsed = {} + for entry in container["Config"]["Env"]: + key, value = entry.split("=", 1) + if key in parsed: + raise ValueError("duplicate environment key") + parsed[key] = value + return parsed + + monkeypatch.setattr(deploy, "_load_registered_legacy_adoption", lambda: Adoption()) + provenance = { + "APPOPS_SOURCE_REVISION": "1" * 40, + "APPOPS_SOURCE_REPOSITORY": APP_OPS_REPOSITORY, + "APPOPS_SOURCE_TREE": "2" * 40, + "APPOPS_BUILD_ID": "333333333333", + "APPOPS_BUILD_DATE": "2026-10-07T11:30:00.000Z", + } + captured = { + **provenance, + "OPENRGB_HOST": "198.19.1.2", + "WATCH_HOST_DIR": "/srv/example/appdata/watch", + "MODEL_OVERRIDE": "operator-value-with-$-and-${literal}", + "EXTRA_RUNTIME_FLAG": "preserved", + } + source_model = { + "name": "unraid-appops-gateway", + "networks": {"default": {}, "appops_proxy": {"external": True}}, + "services": { + "appops-gateway": { + "image": "gitea.invalid/source-build:appops", + "environment": {**captured, "SOURCE_ONLY_DEFAULT": "must-not-drop-runtime"}, + "env_file": [".env.example", ".env"], + "ports": ["127.0.0.1:1216:1216"], + "volumes": ["/data:/data"], + "healthcheck": {"test": ["CMD", "python", "-m", "health"]}, + }, + }, + } + original_model = copy.deepcopy(source_model) + metadata = { + "id": "abcdef012345", + "repository": APP_OPS_REPOSITORY, + "revision": "c" * 40, + "tree": "d" * 40, + "builtAt": "2026-10-07T12:00:00.000Z", + } + + effective = deploy.appops_effective_runtime_model(source_model, captured, metadata) + + effective_environment = effective["services"]["appops-gateway"]["environment"] + assert set(effective_environment) == set(captured) + for key, value in captured.items(): + if key not in provenance: + assert effective_environment[key] == value + assert effective_environment["MODEL_OVERRIDE"] == "operator-value-with-$-and-${literal}" + assert {key: effective_environment[key] for key in provenance} == { + "APPOPS_SOURCE_REVISION": metadata["revision"], + "APPOPS_SOURCE_REPOSITORY": metadata["repository"], + "APPOPS_SOURCE_TREE": metadata["tree"], + "APPOPS_BUILD_ID": metadata["id"], + "APPOPS_BUILD_DATE": metadata["builtAt"], + } + assert source_model == original_model + assert "env_file" not in effective["services"]["appops-gateway"] + + +def test_legacy_555_duplicate_environment_is_normalized_but_raw_runtime_stays_cas_bound( + deploy, monkeypatch, +): + class Adoption: + @staticmethod + def _inspected_environment(container): + parsed = {} + for entry in container["Config"]["Env"]: + key, value = entry.split("=", 1) + if key in parsed: + raise ValueError("duplicate environment key") + parsed[key] = value + return parsed + + monkeypatch.setattr(deploy, "_load_registered_legacy_adoption", lambda: Adoption()) + values = {key: f"synthetic-{index:02d}" for index, key in enumerate(LEGACY_555_ENV_KEYS)} + original_origin = values["APPOPS_FRONTEND_ORIGIN"] + raw_environment = [f"{key}={value}" for key, value in values.items()] + raw_environment.append(f"APPOPS_FRONTEND_ORIGIN={original_origin}") + assert len(raw_environment) == 47 + assert len(values) == 46 + provenance_keys = { + "APPOPS_SOURCE_REVISION", "APPOPS_SOURCE_REPOSITORY", "APPOPS_SOURCE_TREE", + "APPOPS_BUILD_ID", "APPOPS_BUILD_DATE", + } + assert not provenance_keys.intersection(values) + + legacy_container = { + "Image": deploy.APPOPS_LEGACY_OLD_IMAGE_ID, + "Config": {"Env": raw_environment}, + "HostConfig": {}, + "NetworkSettings": {}, + "Mounts": [], + "State": {"Running": True}, + } + captured = deploy._appops_captured_environment(legacy_container) + assert captured == values + assert len(captured) == 46 + + collapsed_runtime = copy.deepcopy(legacy_container) + collapsed_runtime["Config"]["Env"] = [f"{key}={value}" for key, value in captured.items()] + assert deploy._appops_runtime_cas_digest(legacy_container) != deploy._appops_runtime_cas_digest( + collapsed_runtime, + ) + + source_model = {"services": {"appops-gateway": {"environment": {}}}} + metadata = { + "id": "abcdef012345", "repository": "jens/unraid-appops-gateway", + "revision": "c" * 40, "tree": "d" * 40, + "builtAt": "2026-10-07T12:00:00.000Z", + } + effective = deploy.appops_effective_runtime_model(source_model, captured, metadata) + candidate_environment = effective["services"]["appops-gateway"]["environment"] + assert len(candidate_environment) == 51 + assert {key: candidate_environment[key] for key in values} == values + assert set(candidate_environment) - set(values) == provenance_keys + + for image, entries in ( + (deploy.APPOPS_LEGACY_OLD_IMAGE_ID, [*raw_environment[:-1], "APPOPS_FRONTEND_ORIGIN=synthetic-conflict"]), + ("sha256:" + "8" * 64, raw_environment), + ): + container = {**legacy_container, "Image": image, "Config": {"Env": entries}} + with pytest.raises(deploy.DeployError, match="duplicate keys"): + deploy._appops_captured_environment(container) + + +def _appops_prepare_recovery_case(deploy, monkeypatch, tmp_path, capture_change=None): + values = {key: f"synthetic-value-{index:02d}" for index, key in enumerate(LEGACY_555_ENV_KEYS)} + raw_environment = [f"{key}={value}" for key, value in values.items()] + raw_environment.append(f"APPOPS_FRONTEND_ORIGIN={values['APPOPS_FRONTEND_ORIGIN']}") + old_image_id = deploy.APPOPS_LEGACY_OLD_IMAGE_ID + old_revision = deploy.APPOPS_LEGACY_OLD_REVISION + target_revision = "c" * 40 + container_id = "a" * 64 + state = { + "repository": APP_OPS_REPOSITORY, + "live_revision": old_revision, + "live_status": "healthy", + "live_legacy_provenance": True, + "live_receipt": {}, + "live_artifacts": {"containers": {"Unraid-AppOps-Gateway": { + "container_id": container_id, + "image_id": old_image_id, + }}}, + } + state_raw = json.dumps(state, sort_keys=True, separators=(",", ":")).encode() + b"\n" + state_sha = hashlib.sha256(state_raw).hexdigest() + journal = { + "recoveryId": RECOVERY_ID, + "revision": target_revision, + "stateSha256": state_sha, + "sourceFingerprint": "b" * 64, + "currentContainerId": container_id, + "currentImageId": old_image_id, + "runtimeEnvironment": values, + } + container = { + "Id": container_id, + "Image": old_image_id, + "Config": { + "Env": raw_environment, + "Labels": {"com.docker.compose.project": "unraid-appops-gateway"}, + "Hostname": "unraid-appops-gateway", + }, + "HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}}, + "NetworkSettings": {"Ports": {}, "Networks": {"default": {}, "appops_proxy": {}}}, + "Mounts": [{ + "Type": "bind", + "Source": "/srv/example/appdata/unraid-appops-gateway", + "Destination": "/data", + "RW": True, + }], + "State": {"Running": True, "Health": {"Status": "healthy", "FailingStreak": 0}}, + } + image = {"Id": old_image_id, "Config": {"Labels": {"synthetic": "fixed-555"}}} + pointer = { + "schema": "itworx.appops-effective-compose-model.v1", + "recoveryId": RECOVERY_ID, + "revision": old_revision, + "modelSha256": "1" * 64, + "modelRecordSha256": "2" * 64, + } + source_model = { + "name": "unraid-appops-gateway", + "services": {"appops-gateway": { + "image": deploy.APPOPS_SOURCE_COMPOSE_IMAGE_REF, + "environment": {}, + }}, + } + target_metadata = { + "id": target_revision[:12], + "repository": APP_OPS_REPOSITORY, + "revision": target_revision, + "tree": "d" * 40, + "treeDigest": "e" * 64, + "builtAt": "2026-10-07T12:00:00.000Z", + } + backup = {"schema": "synthetic-backup-proof", "sha256": "f" * 64} + captured_snapshot = { + "revision": old_revision, + "legacy_provenance": True, + "receipt": {}, + "containers": [{ + "name": "Unraid-AppOps-Gateway", + "container_id": container_id, + "image_id": old_image_id, + "image_ref": deploy.APPOPS_LEGACY_OLD_IMAGE_REF, + "rollback_ref": f"gitea-deploy-rollback/unraid-appops-gateway-0:{old_revision[:12]}", + "was_running": True, + }], + } + if capture_change is not None: + capture_change(captured_snapshot, container_id, old_image_id) + calls = {"authorizations": [], "source_checks": [], "capture": [], "models": [], "backup": [], "begin": [], "tags": []} + + class Adoption: + @staticmethod + def _inspected_environment(observed): + result = {} + for entry in observed["Config"]["Env"]: + key, value = entry.split("=", 1) + if key in result: + raise ValueError("test expects normalized unique environment") + result[key] = value + return result + + class StatusReader: + @staticmethod + def _self_upgrade_recovery_authorization(request_id, **_kwargs): + calls["authorizations"].append(request_id) + if request_id != RECOVERY_ID: + return None + return {"requestId": RECOVERY_ID, "revision": target_revision, "journal": journal} + + @staticmethod + def _self_upgrade_store_compose_model(recovery_id, model_revision, model): + calls["models"].append((recovery_id, model_revision, copy.deepcopy(model))) + return pointer + + @staticmethod + def _self_upgrade_load_compose_model(recovery_id, model_revision): + if recovery_id != RECOVERY_ID or model_revision != old_revision or not calls["models"]: + return None + return {"pointer": pointer, "model": calls["models"][-1][2]} + + project = { + "_repository_name": APP_OPS_REPOSITORY, + "_appops_self_upgrade_recovery_id": RECOVERY_ID, + "slug": "unraid-appops-gateway", + "source_dir": str(tmp_path / "source"), + "compose_project": "unraid-appops-gateway", + "containers": ["Unraid-AppOps-Gateway"], + } + monkeypatch.setattr(deploy, "_load_registered_legacy_adoption", lambda: Adoption()) + monkeypatch.setattr(deploy, "_appops_status_reader", lambda: StatusReader()) + monkeypatch.setattr( + deploy, + "verify_appops_legacy_source", + lambda actual_repo, actual_project, revision: calls["source_checks"].append( + (actual_repo, actual_project, revision) + ), + ) + monkeypatch.setattr( + deploy, + "read_appops_state_preimage", + lambda _project: (copy.deepcopy(state), state_raw, state_sha), + ) + monkeypatch.setattr(deploy, "inspect_container", lambda _name: copy.deepcopy(container)) + monkeypatch.setattr(deploy, "inspect_image", lambda _image_id: copy.deepcopy(image)) + monkeypatch.setattr(deploy, "verify_appops_legacy_runtime", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy, "appops_legacy_compose_model", lambda *_args: (source_model, "0" * 64, "1" * 64)) + monkeypatch.setattr( + deploy, "sync_source", + lambda *_args, **_kwargs: pytest.fail("recovery capture must not sync source"), + ) + monkeypatch.setattr( + deploy, "copy_environment", + lambda *_args, **_kwargs: pytest.fail("recovery capture must not copy source environment"), + ) + + def capture_rollback(actual_project, *, previous_state_override, tag_snapshot_images): + calls["capture"].append((actual_project, copy.deepcopy(previous_state_override), tag_snapshot_images)) + return copy.deepcopy(captured_snapshot) + + monkeypatch.setattr(deploy, "capture_compose_rollback", capture_rollback) + + def capture_backup(candidate_revision, previous_revision): + calls["backup"].append((candidate_revision, previous_revision)) + return backup + + monkeypatch.setattr(deploy, "capture_appops_data_backup", capture_backup) + + def begin_transaction(actual_project, revision, metadata, source_fingerprint, kind): + calls["begin"].append(( + actual_project["_repository_name"], revision, copy.deepcopy(metadata), source_fingerprint, kind, + )) + assert actual_project["_compose_rollback"]["containers"] == [{ + **captured_snapshot["containers"][0], + "restore_image_ref": deploy.APPOPS_SOURCE_COMPOSE_IMAGE_REF, + }] + assert actual_project["_compose_rollback"]["appops_runtime_model"] == pointer + assert actual_project["_appops_pre_mutation_runtime_sha256"] == deploy._appops_runtime_cas_digest(container) + assert actual_project["_appops_self_upgrade_recovery_id"] == RECOVERY_ID + assert actual_project["_appops_self_upgrade_recovery_authorization"]["journal"] is journal + stored_environment = calls["models"][-1][2]["services"]["appops-gateway"]["environment"] + assert stored_environment == values + assert len(stored_environment) == 46 + assert not { + "APPOPS_SOURCE_REVISION", "APPOPS_SOURCE_REPOSITORY", "APPOPS_SOURCE_TREE", + "APPOPS_BUILD_ID", "APPOPS_BUILD_DATE", + }.intersection(stored_environment) + + monkeypatch.setattr(deploy, "_appops_begin_model_transaction", begin_transaction) + monkeypatch.setattr(deploy, "run", lambda *args, **kwargs: calls["tags"].append((args, kwargs))) + return types.SimpleNamespace( + project=project, + repo=tmp_path / "repository.git", + revision=target_revision, + metadata=target_metadata, + state=state, + state_sha=state_sha, + container=container, + image=image, + raw_environment=raw_environment, + values=values, + captured_snapshot=captured_snapshot, + calls=calls, + ) + + +def test_prepare_recovery_persists_exact_555_environment_and_adds_restore_ref_before_transaction( + deploy, monkeypatch, tmp_path, +): + case = _appops_prepare_recovery_case(deploy, monkeypatch, tmp_path) + assert len(case.raw_environment) == 47 + assert len(case.values) == 46 + + deploy._appops_prepare_recovery( + case.project, APP_OPS_REPOSITORY, case.repo, case.revision, case.metadata, case.state, + ) + + snapshot_item = case.project["_compose_rollback"]["containers"][0] + assert "restore_image_ref" not in case.captured_snapshot["containers"][0] + assert snapshot_item["restore_image_ref"] == deploy.APPOPS_SOURCE_COMPOSE_IMAGE_REF + assert case.project["_compose_rollback"]["appops_runtime_model"]["recoveryId"] == RECOVERY_ID + assert case.project["_compose_rollback"]["appops_runtime_model"]["revision"] == deploy.APPOPS_LEGACY_OLD_REVISION + assert case.project["_appops_pre_mutation_runtime_sha256"] == deploy._appops_runtime_cas_digest(case.container) + collapsed_container = copy.deepcopy(case.container) + collapsed_container["Config"]["Env"] = [ + f"{key}={value}" for key, value in case.values.items() + ] + assert deploy._appops_runtime_cas_digest(case.container) != deploy._appops_runtime_cas_digest( + collapsed_container, + ) + assert case.calls["capture"] == [(case.project, case.state, False)] + assert case.calls["begin"] == [( + APP_OPS_REPOSITORY, case.revision, case.metadata, "b" * 64, "recovery", + )] + assert case.calls["authorizations"] == [RECOVERY_ID, RECOVERY_ID] + assert case.calls["backup"] == [(case.revision, deploy.APPOPS_LEGACY_OLD_REVISION)] + assert case.calls["tags"] == [] + + +@pytest.mark.parametrize("mismatch", ["image", "container_id", "name"]) +def test_prepare_recovery_rejects_changed_capture_identity_before_transaction_or_tagging( + deploy, monkeypatch, tmp_path, mismatch, +): + def alter_snapshot(snapshot, container_id, image_id): + item = snapshot["containers"][0] + if mismatch == "image": + item["image_id"] = "sha256:" + "8" * 64 + elif mismatch == "container_id": + item["container_id"] = "c" * 64 + elif mismatch == "name": + item["name"] = "unexpected-container" + + case = _appops_prepare_recovery_case(deploy, monkeypatch, tmp_path, alter_snapshot) + + with pytest.raises(deploy.DeployError, match="rollback snapshot changed during capture"): + deploy._appops_prepare_recovery( + case.project, APP_OPS_REPOSITORY, case.repo, case.revision, case.metadata, case.state, + ) + + assert case.calls["begin"] == [] + assert case.calls["tags"] == [] + assert case.calls["backup"] == [] + assert case.project.get("_compose_rollback") is None + + +def test_capture_compose_rollback_wrapper_forwards_tagging_choice(deploy, monkeypatch): + project = {"compose_project": "appops-test", "containers": ["appops-gateway"]} + state = {"live_revision": "c" * 40, "live_status": "healthy"} + calls = [] + + def capture_inner(actual_project, **kwargs): + calls.append((actual_project, kwargs)) + return {"snapshot": "captured"} + + monkeypatch.setattr(deploy, "_capture_compose_rollback_inner", capture_inner) + + result = deploy.capture_compose_rollback( + project, + previous_state_override=state, + tag_snapshot_images=False, + ) + + assert result == {"snapshot": "captured"} + assert calls == [( + project, + {"previous_state_override": state, "tag_snapshot_images": False}, + )] + + +def _git(path: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(path), *args], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ).stdout.strip() + + +def _effective_candidate_case(deploy, tmp_path, monkeypatch, drift): + checkout = tmp_path / "source" + checkout.mkdir() + _git(checkout, "init", "-q") + _git(checkout, "config", "user.email", "test@example.invalid") + _git(checkout, "config", "user.name", "AppOps test") + (checkout / "server").mkdir() + (checkout / "server/Dockerfile").write_text("FROM scratch\nCOPY . /app\n") + (checkout / "server/main.py").write_text("print('tracked candidate')\n") + _git(checkout, "add", ".") + _git(checkout, "commit", "-qm", "protected candidate") + revision = _git(checkout, "rev-parse", "HEAD") + bare = tmp_path / "repository.git" + subprocess.run( + ["git", "clone", "--bare", "--quiet", str(checkout), str(bare)], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + (checkout / "server/stale.py").write_text("untracked source must not enter image\n") + (checkout / ".env").write_text("PRIVATE_VALUE=excluded\n") + + source_model = { + "services": { + "appops-gateway": { + "build": {"context": str(checkout), "dockerfile": "server/Dockerfile"}, + }, + }, + } + project = { + **_appops_project(), + "source_dir": str(checkout), + "compose_dir": "", + "compose_project": "unraid-appops-gateway", + "containers": ["Unraid-AppOps-Gateway"], + "_repository_path": str(bare), + "_registry_project": "internal-test-identity", + } + exact_root = tmp_path / "controller-builds" + monkeypatch.setattr(deploy, "EXACT_SOURCE_ROOT", exact_root) + build_context = deploy._load_registered_build_context().prepare_build_contexts( + bare, revision, project, source_model, exact_root, + )["appops-gateway"] + effective_model = copy.deepcopy(source_model) + effective_model["services"]["appops-gateway"]["build"]["context"] = build_context + pointer = { + "recoveryId": RECOVERY_ID, + "revision": revision, + "modelSha256": "a" * 64, + "modelRecordSha256": "b" * 64, + } + + def load_effective_model(recovery_id, value): + if drift == "private_model": + return None + if recovery_id == RECOVERY_ID and value == revision: + return {"model": effective_model, "pointer": pointer} + return None + + def fetch_green(_repository, value, _policy): + calls["green"] += 1 + calls["green_revision"] = value + return "pass", {"revision": value, "run": "canonical-full"} + + reader = types.SimpleNamespace( + _validation_policy=lambda _repository, _project: {"policy": "current-full"}, + _self_upgrade_protected_head=lambda value: value == revision, + _is_current_branch_head=lambda _repository, _project, value: value == revision, + _fetch_gitea_validation=fetch_green, + _self_upgrade_load_compose_model=load_effective_model, + ) + registered_project = {"slug": "unraid-appops-gateway", "branch": "main"} + project["_registry_project"] = registered_project + project["_appops_registry_project"] = copy.deepcopy(registered_project) + project["_appops_effective_model_pointer"] = dict(pointer) + project["_appops_candidate_source_model"] = copy.deepcopy(source_model) + raw_state = b'{"repository":"jens/unraid-appops-gateway","status":"success"}\n' + original_state_sha = hashlib.sha256(raw_state).hexdigest() + project["_appops_runtime_model_state_sha256"] = original_state_sha + + runtime = { + "Id": "container-before-build", + "Image": "sha256:" + "c" * 64, + "Config": {"Env": ["MODEL=known", "TOKEN=private"], "Labels": {"x": "y"}}, + "HostConfig": {"Memory": 0, "Binds": ["/data:/data"]}, + "NetworkSettings": {"Networks": {"default": {"NetworkID": "network-before"}}}, + "Mounts": [{"Type": "volume", "Name": "appops-data", "Destination": "/data"}], + "State": {"Running": True, "Health": {"Status": "healthy"}}, + } + project["_appops_pre_mutation_runtime_sha256"] = deploy._appops_runtime_cas_digest(runtime) + + state_observation = {"raw": raw_state, "sha": original_state_sha} + projects = {APP_OPS_REPOSITORY: registered_project} + calls = {"commands": [], "green": 0, "green_revision": None} + + def read_preimage(_project): + if drift == "state" and "build" in calls["commands"]: + changed = b'{"repository":"jens/unraid-appops-gateway","status":"other"}\n' + return {"repository": APP_OPS_REPOSITORY}, changed, hashlib.sha256(changed).hexdigest() + return {"repository": APP_OPS_REPOSITORY}, state_observation["raw"], state_observation["sha"] + + def load_projects(): + if drift == "registry" and "build" in calls["commands"]: + return {APP_OPS_REPOSITORY: {**registered_project, "branch": "release"}} + return projects + + def mutate_after_build(): + if drift == "environment": + runtime["Config"]["Env"].append("MODEL=external-change") + elif drift == "config": + runtime["Config"]["Labels"]["x"] = "externally-changed" + elif drift == "hostconfig": + runtime["HostConfig"]["Memory"] = 128 + elif drift == "container_id": + runtime["Id"] = "container-replaced-during-build" + elif drift == "image": + runtime["Image"] = "sha256:" + "d" * 64 + elif drift == "network": + runtime["NetworkSettings"]["Networks"]["default"]["NetworkID"] = "network-drifted" + elif drift == "mount": + runtime["Mounts"][0]["Source"] = "/unexpected/volume" + elif drift == "running": + runtime["State"]["Running"] = False + elif drift == "archive": + Path(build_context, "server/main.py").write_text("tampered after build\n") + + def run(command, **_kwargs): + calls["commands"].append(command[-1]) + if command[-1] == "build": + assert not (Path(build_context) / "server/stale.py").exists() + assert not (Path(build_context) / ".env").exists() + assert (Path(build_context) / "server/main.py").read_text() == "print('tracked candidate')\n" + mutate_after_build() + if "up" in command: + calls["commands"].append("up-mutated") + return None + + monkeypatch.setattr(deploy, "_appops_status_reader", lambda: reader) + monkeypatch.setattr(deploy, "read_appops_state_preimage", read_preimage) + monkeypatch.setattr(deploy, "load_projects", load_projects) + monkeypatch.setattr(deploy, "inspect_container", lambda _name: runtime) + monkeypatch.setattr(deploy, "deployment_environment", lambda *_args, **_kwargs: {}) + monkeypatch.setattr(deploy, "compose_command", lambda _project: (checkout, ["docker", "compose"])) + monkeypatch.setattr(deploy, "run", run) + monkeypatch.setattr(deploy, "prepare_registered_legacy_build", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy, "write_appops_progress", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy, "prepare_compose_migration", lambda *_args, **_kwargs: []) + monkeypatch.setattr(deploy, "finish_compose_migration", lambda *_args, **_kwargs: None) + monkeypatch.setattr(deploy, "workmesh_transition_enabled", lambda _project: False) + return project, revision, runtime, calls, original_state_sha, reader + + +@pytest.mark.parametrize( + "drift", [ + "environment", "config", "hostconfig", "container_id", "image", "network", + "mount", "running", "state", "registry", "archive", "private_model", + ], +) +def test_deploy_project_rejects_after_build_drift_before_up_without_rollback( + deploy, monkeypatch, tmp_path, drift, +): + project, revision, runtime, calls, _state_sha, _reader = _effective_candidate_case( + deploy, tmp_path, monkeypatch, drift, + ) + + with pytest.raises(deploy.AppOpsEffectiveNoMutation): + deploy.deploy_project(project, revision, APP_OPS_REPOSITORY) + + assert calls["commands"] == ["--quiet", "build"] + assert calls["green"] == 1 + assert calls["green_revision"] == revision + if drift in {"environment", "hostconfig", "container_id", "network"}: + assert runtime["Id"] in {"container-before-build", "container-replaced-during-build"} + + +@pytest.mark.parametrize("proof_failure", ["protected_head", "branch_head", "full_validation"]) +def test_deploy_project_requires_canonical_green_validation_before_up( + deploy, monkeypatch, tmp_path, proof_failure, +): + project, revision, _runtime, calls, _state_sha, reader = _effective_candidate_case( + deploy, tmp_path, monkeypatch, "none", + ) + if proof_failure == "protected_head": + reader._self_upgrade_protected_head = lambda _value: False + elif proof_failure == "branch_head": + reader._is_current_branch_head = lambda *_args: False + else: + def fetch_failed(*_args): + calls["green"] += 1 + calls["green_revision"] = revision + return "fail", {"reason": "full validation failed"} + + reader._fetch_gitea_validation = fetch_failed + + with pytest.raises(deploy.AppOpsEffectiveNoMutation): + deploy.deploy_project(project, revision, APP_OPS_REPOSITORY) + + assert calls["commands"] == ["--quiet", "build"] + assert calls["green"] == (1 if proof_failure == "full_validation" else 0) + assert calls["green_revision"] == (revision if proof_failure == "full_validation" else None) + + +def test_actual_registered_archive_is_clean_and_rejects_cached_byte_tampering( + deploy, tmp_path, monkeypatch, +): + project, revision, _runtime, _calls, _state_sha, _reader = _effective_candidate_case( + deploy, tmp_path, monkeypatch, "none", + ) + model = project["_appops_candidate_source_model"] + context = deploy._load_registered_build_context().prepare_build_contexts( + Path(project["_repository_path"]), revision, project, model, + Path(deploy.EXACT_SOURCE_ROOT), + )["appops-gateway"] + tree = Path(context) + assert (tree / "server/main.py").read_text() == "print('tracked candidate')\n" + assert not (tree / "server/stale.py").exists() + assert not (tree / ".env").exists() + + (tree / "server/main.py").write_text("changed archive bytes\n") + with pytest.raises(deploy._load_registered_build_context().RegisteredBuildContextError): + deploy._load_registered_build_context().prepare_build_contexts( + Path(project["_repository_path"]), revision, project, model, + Path(deploy.EXACT_SOURCE_ROOT), + ) + + +def test_main_does_not_clean_up_foreign_drift_after_effective_pre_mutation_rejection( + deploy, monkeypatch, tmp_path, capsys, +): + revision = "c" * 40 + prior_revision = "d" * 40 + raw_state = b'{"repository":"jens/unraid-appops-gateway","status":"success"}\n' + state = { + "repository": APP_OPS_REPOSITORY, + "live_revision": prior_revision, + "live_status": "healthy", + "live_artifacts": {"appops_runtime_model": {"pointer": "signed-model"}}, + } + registered = { + **_appops_project(), + "source_dir": str(tmp_path / "source"), + "compose_project": "unraid-appops-gateway", + "containers": ["Unraid-AppOps-Gateway"], + "compose_files": ["docker-compose.yml"], + "verify_oci_revision": True, + "environment": {}, + } + projects = {APP_OPS_REPOSITORY: registered} + cleanup_calls = [] + progress = [] + + @contextlib.contextmanager + def held_lock(*_args, **_kwargs): + yield + + def no_cleanup(name): + def fail_if_called(*_args, **_kwargs): + cleanup_calls.append(name) + pytest.fail(f"foreign drift rejection entered {name}") + return fail_if_called + + def prepare_followup(actual_project, *_args, **_kwargs): + actual_project["_compose_rollback"] = {"containers": [{"name": "Unraid-AppOps-Gateway"}]} + actual_project["_appops_runtime_model_state_sha256"] = hashlib.sha256(raw_state).hexdigest() + actual_project["_appops_data_backup"] = {"sha256": "e" * 64} + return None + + monkeypatch.setattr(deploy, "load_projects", lambda: projects) + monkeypatch.setattr(deploy, "effective_mcp_hub_project", lambda _repo, item: dict(item)) + monkeypatch.setattr(deploy, "repository_path", lambda *_args: tmp_path / "repo.git") + monkeypatch.setattr(deploy, "validate_commit", lambda *_args: revision) + monkeypatch.setattr(deploy, "deployment_metadata", lambda *_args: { + "id": revision[:12], "repository": APP_OPS_REPOSITORY, "revision": revision, + "tree": "f" * 40, "treeDigest": "a" * 64, "builtAt": "2026-10-07T12:00:00.000Z", + }) + monkeypatch.setattr(deploy, "validate_project_config", lambda *_args: None) + monkeypatch.setattr(deploy, "exclusive_lock", lambda *_args, **_kwargs: held_lock()) + monkeypatch.setattr(deploy, "exclusive_global_deploy_lock", lambda: held_lock()) + monkeypatch.setattr(deploy, "read_state", lambda _project: state) + monkeypatch.setattr(deploy, "state_live_artifacts", lambda value: dict(value.get("live_artifacts") or {})) + monkeypatch.setattr(deploy, "state_live_revision", lambda _value: prior_revision) + monkeypatch.setattr(deploy, "state_live_status", lambda _value: "healthy") + monkeypatch.setattr(deploy, "state_live_legacy_provenance", lambda _value: False) + monkeypatch.setattr(deploy, "mcp_hub_prepare_current_receipt_model", lambda *_args: None) + monkeypatch.setattr(deploy, "verified_current_receipt", lambda *_args: None) + monkeypatch.setattr(deploy, "validate_forward_revision", lambda *_args: None) + monkeypatch.setattr(deploy, "_appops_prepare_model_followup", prepare_followup) + monkeypatch.setattr(deploy, "_appops_prepare_effective_candidate", lambda *_args: None) + monkeypatch.setattr(deploy, "_appops_legacy_reconcile_pending_rollback", lambda *_args, **_kwargs: False) + monkeypatch.setattr(deploy, "_mcp_hub_assert_no_unresolved_migration_intent", lambda *_args: None) + monkeypatch.setattr(deploy, "reconcile_appops_deployed_intent", lambda *_args: None) + monkeypatch.setattr(deploy, "revalidate_deployment_candidate", lambda *_args: None) + monkeypatch.setattr(deploy, "validate_managed_v1_runtime", lambda *_args: None) + monkeypatch.setattr(deploy, "capture_appops_data_backup", lambda *_args: pytest.fail("unexpected backup")) + monkeypatch.setattr(deploy, "ensure_docker_capacity", lambda: None) + monkeypatch.setattr(deploy, "write_appops_progress", lambda _revision, phase, **_kwargs: progress.append(phase)) + monkeypatch.setattr( + deploy, "deploy_project", + lambda *_args: (_ for _ in ()).throw(deploy.AppOpsEffectiveNoMutation("runtime changed after build")), + ) + monkeypatch.setattr(deploy, "write_state", no_cleanup("write_state")) + monkeypatch.setattr(deploy, "restore_compose_snapshot", no_cleanup("restore_compose_snapshot")) + monkeypatch.setattr(deploy, "restore_live_source", no_cleanup("restore_live_source")) + monkeypatch.setattr(deploy, "restore_compose_migration", no_cleanup("restore_compose_migration")) + monkeypatch.setattr(deploy, "cleanup_failed_first_deploy", no_cleanup("cleanup_failed_first_deploy")) + monkeypatch.setattr(deploy, "_appops_legacy_checkpoint_forward_rollback", no_cleanup("legacy_rollback_checkpoint")) + monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", APP_OPS_REPOSITORY, revision]) + monkeypatch.delenv("APPOPS_SELF_UPGRADE_RECOVERY_ID", raising=False) + monkeypatch.delenv("APPOPS_REGISTERED_LEGACY_REQUEST_ID", raising=False) + + with pytest.raises(deploy.AppOpsEffectiveNoMutation, match="runtime changed after build"): + deploy.main() + + assert cleanup_calls == [] + assert state["live_revision"] == prior_revision + assert progress == ["preflighting"] + assert capsys.readouterr().out == "" + + +def test_followup_transaction_journal_failure_precedes_all_source_runtime_and_state_writes( + deploy, monkeypatch, tmp_path, +): + revision = "c" * 40 + prior_revision = "d" * 40 + source = tmp_path / "source" + source.mkdir() + marker = source / "deployed-marker.txt" + marker.write_text("prior source remains\n") + raw_state = b'{"repository":"jens/unraid-appops-gateway","status":"success"}\n' + state = { + "repository": APP_OPS_REPOSITORY, + "live_revision": prior_revision, + "live_status": "healthy", + "live_artifacts": {"appops_runtime_model": {"pointer": "signed-model"}}, + } + registered = { + **_appops_project(), + "source_dir": str(source), + "compose_project": "unraid-appops-gateway", + "containers": ["Unraid-AppOps-Gateway"], + "compose_files": ["docker-compose.yml"], + "verify_oci_revision": True, + "environment": {}, + } + projects = {APP_OPS_REPOSITORY: registered} + pointer = { + "recoveryId": RECOVERY_ID, + "revision": prior_revision, + "modelSha256": "a" * 64, + "modelRecordSha256": "b" * 64, + } + effective_model = {"services": {"appops-gateway": {"environment": {"MODEL": "kept"}}}} + events = [] + runtime = { + "Id": "container-stable", + "Image": "sha256:" + "c" * 64, + "Config": {"Env": ["MODEL=kept"], "Labels": {}}, + "HostConfig": {"Binds": ["/data:/data"]}, + "NetworkSettings": {"Networks": {"default": {"NetworkID": "network-stable"}}}, + "Mounts": [{"Type": "volume", "Name": "data", "Destination": "/data"}], + "State": {"Running": True, "Health": {"Status": "healthy"}}, + } + + class StatusReader: + @staticmethod + def _validation_policy(_repository, _project): + return {"policy": "fixed-main-full"} + + @staticmethod + def _self_upgrade_protected_head(value): + return value == revision + + @staticmethod + def _is_current_branch_head(_repository, _project, value): + return value == revision + + @staticmethod + def _fetch_gitea_validation(_repository, value, _policy): + events.append(("green", value)) + return "pass", {"revision": value} + + @staticmethod + def _self_upgrade_active_compose_model(_state, _runtime): + return { + "pointer": pointer, + "live": {"sourceFingerprint": "e" * 64}, + } + + @staticmethod + def _self_upgrade_load_compose_model(recovery_id, value): + if recovery_id == RECOVERY_ID and value == prior_revision: + return {"model": effective_model, "pointer": pointer} + return None + + @contextlib.contextmanager + def held_lock(*_args, **_kwargs): + yield + + def forbidden(name): + def fail(*_args, **_kwargs): + events.append(("forbidden", name)) + pytest.fail(f"journal failure reached {name}") + return fail + + def begin_transaction(*_args, **_kwargs): + events.append(("journal",)) + raise deploy.DeployError("test journal store unavailable") + + monkeypatch.setattr(deploy, "load_projects", lambda: projects) + monkeypatch.setattr(deploy, "effective_mcp_hub_project", lambda _repo, item: dict(item)) + monkeypatch.setattr(deploy, "repository_path", lambda *_args: tmp_path / "repo.git") + monkeypatch.setattr(deploy, "validate_commit", lambda *_args: revision) + monkeypatch.setattr(deploy, "deployment_metadata", lambda *_args: { + "id": revision[:12], "repository": APP_OPS_REPOSITORY, "revision": revision, + "tree": "f" * 40, "treeDigest": "a" * 64, "builtAt": "2026-10-07T12:00:00.000Z", + }) + monkeypatch.setattr(deploy, "validate_project_config", lambda *_args: None) + monkeypatch.setattr(deploy, "exclusive_lock", lambda *_args, **_kwargs: held_lock()) + monkeypatch.setattr(deploy, "exclusive_global_deploy_lock", lambda: held_lock()) + monkeypatch.setattr(deploy, "read_state", lambda _project: state) + monkeypatch.setattr(deploy, "state_live_artifacts", lambda value: dict(value.get("live_artifacts") or {})) + monkeypatch.setattr(deploy, "state_live_revision", lambda _value: prior_revision) + monkeypatch.setattr(deploy, "mcp_hub_prepare_current_receipt_model", lambda *_args: None) + monkeypatch.setattr(deploy, "verified_current_receipt", lambda *_args: None) + monkeypatch.setattr(deploy, "validate_forward_revision", lambda *_args: None) + monkeypatch.setattr(deploy, "_appops_status_reader", StatusReader) + monkeypatch.setattr( + deploy, "read_appops_state_preimage", + lambda _project: (state, raw_state, hashlib.sha256(raw_state).hexdigest()), + ) + monkeypatch.setattr(deploy, "inspect_container", lambda _name: runtime) + monkeypatch.setattr( + deploy, "capture_compose_rollback", + lambda _project, **_kwargs: (events.append(("snapshot",)) or {"containers": [{"name": "Unraid-AppOps-Gateway"}]}), + ) + monkeypatch.setattr( + deploy, "capture_appops_data_backup", + lambda *_args: (events.append(("backup",)) or {"sha256": "1" * 64}), + ) + monkeypatch.setattr(deploy, "_appops_begin_model_transaction", begin_transaction) + monkeypatch.setattr(deploy, "sync_source", forbidden("sync_source")) + monkeypatch.setattr(deploy, "copy_environment", forbidden("copy_environment")) + monkeypatch.setattr(deploy, "run", forbidden("compose/docker")) + monkeypatch.setattr(deploy, "write_state", forbidden("write_state")) + monkeypatch.setattr(deploy, "deploy_project", forbidden("deploy_project")) + monkeypatch.setattr(deploy, "restore_compose_snapshot", forbidden("restore_compose_snapshot")) + monkeypatch.setattr(deploy, "restore_live_source", forbidden("restore_live_source")) + monkeypatch.setattr(sys, "argv", ["deploy.py", "deploy", APP_OPS_REPOSITORY, revision]) + monkeypatch.delenv("APPOPS_SELF_UPGRADE_RECOVERY_ID", raising=False) + monkeypatch.delenv("APPOPS_REGISTERED_LEGACY_REQUEST_ID", raising=False) + + with pytest.raises(deploy.DeployError, match="test journal store unavailable"): + deploy.main() + + assert [event[0] for event in events] == ["green", "snapshot", "backup", "journal"] + assert marker.read_text() == "prior source remains\n" + assert state["live_revision"] == prior_revision diff --git a/backend/tests/test_appops_self_upgrade_controller.py b/backend/tests/test_appops_self_upgrade_controller.py index 3df871b..28c1b5e 100644 --- a/backend/tests/test_appops_self_upgrade_controller.py +++ b/backend/tests/test_appops_self_upgrade_controller.py @@ -168,6 +168,7 @@ def test_runtime_requires_exact_container_mount_port_and_health(controller, monk "Config": { "Image": "unraid-appops-gateway:platform-5553c04ce497", "Hostname": "unraid-appops-gateway", + "Env": ["APPOPS_RUNTIME_FIXTURE=synthetic-value"], "Labels": { "com.docker.compose.project": "unraid-appops-gateway", "com.docker.compose.service": "appops-gateway", diff --git a/backend/tests/test_appops_self_upgrade_recovery_boundaries.py b/backend/tests/test_appops_self_upgrade_recovery_boundaries.py new file mode 100644 index 0000000..5c4380a --- /dev/null +++ b/backend/tests/test_appops_self_upgrade_recovery_boundaries.py @@ -0,0 +1,558 @@ +from __future__ import annotations + +import http.server +import importlib.util +import json +import os +import stat +import subprocess +import sys +import threading +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +CONTROLLER = ROOT / "ops" / "gitea-deploy-broker" + + +@pytest.fixture +def controller(monkeypatch, tmp_path): + monkeypatch.syspath_prepend(str(CONTROLLER)) + spec = importlib.util.spec_from_file_location( + "appops_self_upgrade_recovery_boundaries_test", + CONTROLLER / "status_server.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", tmp_path / "inbox") + module.WEBHOOK_INBOX_ROOT.mkdir(mode=0o700) + monkeypatch.setattr(module, "APPOPS_PROGRESS_ROOT", tmp_path / "progress") + monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_ROOT", tmp_path / "recovery") + monkeypatch.setattr( + module, + "SELF_UPGRADE_RECOVERY_REQUEST_ROOT", + tmp_path / "recovery" / "requests", + ) + return module + + +def _private_recovery_payload(module, recovery_id: str) -> dict[str, object]: + revision = "c" * 40 + return { + "schema": module._SELF_UPGRADE_RECOVERY_SCHEMA, + "recoveryId": recovery_id, + "semanticSha256": module._webhook_semantic_digest( + 9, "refs/heads/main", revision, + ), + "repository": module.SELF_UPGRADE_REPOSITORY, + "revision": revision, + "originalRawSha256": "1" * 64, + "idempotencyKeySha256": "2" * 64, + "preflightSha256": "3" * 64, + "expectedCurrentRevision": "4" * 40, + "currentContainerId": "5" * 64, + "currentImageId": "sha256:" + "6" * 64, + "currentSourceTree": "7" * 40, + "runtimeFingerprint": "8" * 64, + "registryFingerprint": "9" * 64, + "sourceFingerprint": "a" * 64, + "stateSha256": "b" * 64, + "runtimeEnvironment": {"API_TOKEN": "recovery-secret-sentinel"}, + "createdAt": 1, + } + + +def _recovery_delivery(module, *, secret: str) -> dict[str, object]: + return { + "schema": module._WEBHOOK_SELF_UPGRADE_RECOVERY_SCHEMA, + "semantic_sha256": "a" * 64, + "recovery_of_sha256": "b" * 64, + "recovery_id": str(uuid.UUID("12345678-1234-4234-8234-123456789abc")), + "idempotency_key_sha256": "c" * 64, + "preflight_sha256": "d" * 64, + "body_sha256": "e" * 64, + "delivery": "22345678-1234-4234-8234-123456789abc", + "gitea_repository_id": 9, + "ref": "refs/heads/main", + "repository": module.SELF_UPGRADE_REPOSITORY, + "revision": "f" * 40, + "source": "typed_self_upgrade_recovery", + "status": "awaiting_validation", + "validation": { + "last_result": "wait", + "required_contexts": ["Managed validation / full (push)"], + "observed": [], + "checked_at": 1, + "deadline_at": 2, + }, + "audit": [ + {"at": 1, "stage": "self_upgrade_recovery_requested"}, + {"at": 2, "stage": "awaiting_validation"}, + ], + "received_at": 1, + "updated_at": 2, + # These fields must remain private even if a caller or record contains them. + "runtimeEnvironment": {"API_TOKEN": secret}, + "privateCredential": secret, + } + + +def test_ordinary_deploy_strips_ambient_recovery_binding_and_rejects_bad_bindings( + controller, monkeypatch, +): + ambient_secret = "ambient-recovery-secret-sentinel" + monkeypatch.setenv("APPOPS_SELF_UPGRADE_RECOVERY_ID", ambient_secret) + monkeypatch.setenv("APPOPS_REGISTERED_LEGACY_REQUEST_ID", ambient_secret) + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs)) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(controller.subprocess, "run", fake_run) + ordinary = controller._run_registered_deploy("jens/quiz-monitor", "c" * 40) + assert ordinary.returncode == 0 + command, kwargs = calls.pop() + assert command[-3:] == ["deploy", "jens/quiz-monitor", "c" * 40] + assert "APPOPS_SELF_UPGRADE_RECOVERY_ID" not in kwargs["env"] + assert "APPOPS_REGISTERED_LEGACY_REQUEST_ID" not in kwargs["env"] + assert ambient_secret not in json.dumps(kwargs["env"]) + + recovery_id = str(uuid.UUID("12345678-1234-4234-8234-123456789abc")) + authorized = controller._run_registered_deploy( + controller.SELF_UPGRADE_REPOSITORY, + "c" * 40, + self_upgrade_recovery_id=recovery_id, + ) + assert authorized.returncode == 0 + _, kwargs = calls.pop() + assert kwargs["env"]["APPOPS_SELF_UPGRADE_RECOVERY_ID"] == recovery_id + assert "APPOPS_REGISTERED_LEGACY_REQUEST_ID" not in kwargs["env"] + + invalid_bindings = [ + (controller.SELF_UPGRADE_REPOSITORY, "not-a-uuid", None), + ("jens/quiz-monitor", recovery_id, None), + (controller.SELF_UPGRADE_REPOSITORY, recovery_id, recovery_id), + ] + for repository, bad_recovery_id, registered_id in invalid_bindings: + with pytest.raises(ValueError): + controller._run_registered_deploy( + repository, + "c" * 40, + self_upgrade_recovery_id=bad_recovery_id, + registered_legacy_request_id=registered_id, + ) + assert calls == [] + + +def test_preflight_projects_runtime_identity_without_environment_values( + controller, monkeypatch, +): + project = { + "branch": "main", + "gitea_repository_id": 9, + "containers": ["Unraid-AppOps-Gateway"], + "compose_project": "unraid-appops-gateway", + "compose_files": ["docker-compose.yml"], + "environment": { + "APPOPS_DATA_PATH": controller.SELF_UPGRADE_DATA_PATH, + "APPOPS_PUBLISH_HOST": controller.SELF_UPGRADE_PUBLISH_HOST, + }, + "verify_oci_revision": True, + "validation": { + "validation_gate_version": 1, + "required": True, + "contexts": ["Managed validation / full (push)"], + "timeout_seconds": 1800, + "poll_interval_seconds": 5, + "max_poll_interval_seconds": 30, + }, + } + private_environment = {"API_TOKEN": "preflight-secret-sentinel"} + monkeypatch.setattr( + controller, + "_load_projects", + lambda: {controller.SELF_UPGRADE_REPOSITORY: project}, + ) + monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda _revision: True) + monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: True) + monkeypatch.setattr( + controller, "_fetch_gitea_validation", lambda *_args: ("pass", []), + ) + monkeypatch.setattr( + controller, + "_self_upgrade_runtime", + lambda: { + "revision": "4" * 40, + "imageId": "sha256:" + "5" * 64, + "containerId": "6" * 64, + "imageRef": controller.SELF_UPGRADE_PRODUCTION_IMAGE_REF, + "sourceTree": "7" * 40, + "runtimeFingerprint": "8" * 64, + "runtimeEnvironment": private_environment, + }, + ) + monkeypatch.setattr( + controller.shutil, + "disk_usage", + lambda _path: type("Usage", (), {"free": 50 * 1024**3})(), + ) + monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda _revision: True) + monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: True) + monkeypatch.setattr( + controller, "_fetch_gitea_validation", lambda *_args: ("pass", []), + ) + monkeypatch.setattr( + controller, "_validation_policy", + lambda _repository, _project: { + "contexts": ["Managed validation / full (push)"], + }, + ) + monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *_a, **_k: True) + monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None) + + result = controller._self_upgrade_preflight("c" * 40) + serialized = json.dumps(result) + assert result["status"] == "preflight_ok" + assert "runtimeEnvironment" not in result + assert "API_TOKEN" not in serialized + assert "preflight-secret-sentinel" not in serialized + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + body = json.dumps({ + "repository": controller.SELF_UPGRADE_REPOSITORY, + "revision": "c" * 40, + "reason": "preflight boundary test", + "expectedCurrentRevision": None, + }).encode("utf-8") + request = urllib.request.Request( + f"http://127.0.0.1:{server.server_port}{controller.SELF_UPGRADE_PREFLIGHT_PATH}", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=5) as response: + response_body = response.read() + assert response.status == 200 + response_payload = json.loads(response_body) + serialized_response = response_body.decode("utf-8") + assert response_payload["status"] == "preflight_ok" + assert "runtimeEnvironment" not in response_payload + assert "API_TOKEN" not in serialized_response + assert "preflight-secret-sentinel" not in serialized_response + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_public_recovery_status_projects_away_private_environment(controller, monkeypatch): + secret = "public-status-secret-sentinel" + record = _recovery_delivery(controller, secret=secret) + monkeypatch.setattr(controller, "_self_upgrade_delivery_record", lambda _request_id: record) + monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *_a, **_k: True) + monkeypatch.setattr(controller, "_read_inbox_record", lambda _path: None) + monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None) + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + request = urllib.request.Request( + f"http://127.0.0.1:{server.server_port}" + f"{controller.SELF_UPGRADE_STATUS_PREFIX}{'a' * 64}", + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=5) as response: + status = response.status + body = response.read() + except urllib.error.HTTPError as error: + status = error.code + body = error.read() + assert status == 200, body.decode("utf-8", errors="replace") + view = json.loads(body) + assert view["repository"] == controller.SELF_UPGRADE_REPOSITORY + assert view["revision"] == "f" * 40 + assert view["attemptType"] == "self-upgrade-recovery" + assert view["recoveryOf"] == "b" * 64 + assert not { + "recovery_id", "recovery_of_sha256", "idempotency_key_sha256", + "preflight_sha256", "body_sha256", "recovery_hmac", + } & view.keys() + assert "runtimeEnvironment" not in view + assert "privateCredential" not in view + assert "recovery_hmac" not in view + assert "idempotency_key_sha256" not in view + assert secret.encode() not in body + assert "API_TOKEN" not in body.decode("utf-8") + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_private_recovery_journal_is_0600_and_hmac_tampering_fails_closed( + controller, monkeypatch, tmp_path, +): + if os.name == "nt" or os.geteuid() != 0: + pytest.skip("The controller recovery store requires root-owned POSIX files") + recovery_root = tmp_path / "recovery-store" + recovery_root.mkdir(mode=0o700) + recovery_root.chmod(0o700) + monkeypatch.setattr(controller, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root) + secret = b"fixed-self-upgrade-recovery-test-key-0123456789" + monkeypatch.setattr(controller, "_self_upgrade_secret", lambda: secret) + recovery_id = str(uuid.UUID("12345678-1234-4234-8234-123456789abc")) + payload = _private_recovery_payload(controller, recovery_id) + + controller._self_upgrade_write_private_journal(recovery_id, payload, secret) + marker = recovery_root / f"{recovery_id}.json" + assert stat.S_IMODE(marker.stat().st_mode) == 0o600 + assert controller._self_upgrade_recovery_journal(recovery_id) == payload + + envelope = json.loads(marker.read_text(encoding="utf-8")) + envelope["payload"]["runtimeEnvironment"]["API_TOKEN"] = "changed-secret" + marker.write_text(json.dumps(envelope, sort_keys=True, separators=(",", ":")) + "\n") + marker.chmod(0o600) + assert controller._self_upgrade_recovery_journal(recovery_id) is None + + envelope = controller._self_upgrade_recovery_envelope(payload, secret) + marker.write_text(json.dumps(envelope, sort_keys=True, separators=(",", ":")) + "\n") + marker.chmod(0o644) + assert controller._self_upgrade_recovery_journal(recovery_id) is None + + +def test_recovery_request_is_immutable_and_idempotent_for_one_key( + controller, monkeypatch, tmp_path, +): + if os.name == "nt" or os.geteuid() != 0: + pytest.skip("The controller recovery store requires root-owned POSIX files") + + recovery_root = tmp_path / "recovery-store" + recovery_root.mkdir(mode=0o700) + recovery_root.chmod(0o700) + request_root = recovery_root / "requests" + monkeypatch.setattr(controller, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root) + monkeypatch.setattr(controller, "SELF_UPGRADE_RECOVERY_REQUEST_ROOT", request_root) + + secret = b"fixed-self-upgrade-recovery-test-key-0123456789" + monkeypatch.setattr(controller, "_self_upgrade_secret", lambda: secret) + revision = "c" * 40 + semantic = controller._webhook_semantic_digest(9, "refs/heads/main", revision) + original_path = tmp_path / "original-failed-delivery.json" + original_raw = b'{"immutable":"original failed delivery"}\n' + original_path.write_bytes(original_raw) + original = {"revision": revision} + original_info = (original_path, original_raw, original) + monkeypatch.setattr( + controller, + "_self_upgrade_original_failed_delivery", + lambda *_args: original_info, + ) + monkeypatch.setattr( + controller, "_self_upgrade_delivery_revision", lambda _semantic: revision, + ) + + expected_preflight = "3" * 64 + expected_current_revision = controller.SELF_UPGRADE_OLD_REVISION + expected_container_id = "5" * 64 + expected_image_id = "sha256:" + "6" * 64 + expected_source_tree = "7" * 40 + expected_runtime_fingerprint = "8" * 64 + private_environment = {"API_TOKEN": "journal-secret-sentinel"} + fresh = { + "status": "preflight_ok", + "preflightSha256": expected_preflight, + "currentRevision": expected_current_revision, + "currentContainerId": expected_container_id, + "currentImageId": expected_image_id, + "currentSourceTree": expected_source_tree, + "currentRuntimeFingerprint": expected_runtime_fingerprint, + } + bindings = { + "currentRevision": expected_current_revision, + "currentContainerId": expected_container_id, + "currentImageId": expected_image_id, + "currentSourceTree": expected_source_tree, + "runtimeFingerprint": expected_runtime_fingerprint, + "registryFingerprint": "9" * 64, + "sourceFingerprint": "a" * 64, + "stateSha256": "b" * 64, + "runtimeEnvironment": private_environment, + } + monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _revision: fresh) + monkeypatch.setattr( + controller, "_self_upgrade_recovery_current_bindings", lambda: bindings, + ) + + policy = { + "repository": controller.SELF_UPGRADE_REPOSITORY, + "validation_gate_version": 1, + "contexts": ["Managed validation / full (push)"], + "timeout_seconds": 1800, + "poll_interval_seconds": 5, + "max_poll_interval_seconds": 30, + } + project = {"slug": "unraid-appops-gateway"} + monkeypatch.setattr( + controller, "_load_projects", + lambda: {controller.SELF_UPGRADE_REPOSITORY: project}, + ) + monkeypatch.setattr( + controller, + "_validation_policy", + lambda _repository, _project: policy, + ) + + original_sha = controller.hashlib.sha256(original_raw).hexdigest() + idempotency_key_sha = "2" * 64 + reason_sha = "4" * 64 + first = controller._self_upgrade_recovery_request_status( + semantic, + original_sha, + idempotency_key_sha, + expected_preflight, + expected_current_revision, + reason_sha, + policy, + ) + assert first == ("awaiting_validation", semantic) + assert original_path.read_bytes() == original_raw + + def store_snapshot() -> dict[str, bytes]: + return { + path.relative_to(recovery_root).as_posix(): path.read_bytes() + for path in sorted(recovery_root.rglob("*.json")) + } + + accepted_bytes = store_snapshot() + assert accepted_bytes + assert stat.S_IMODE(recovery_root.stat().st_mode) == 0o700 + assert stat.S_IMODE(request_root.stat().st_mode) == 0o700 + assert all( + stat.S_IMODE(path.stat().st_mode) == 0o600 + for path in recovery_root.rglob("*.json") + ) + # The secret is intentionally present only in the root-owned HMAC journal. + recovery_id = str(uuid.UUID(bytes=bytes.fromhex(idempotency_key_sha)[:16], version=4)) + journal_bytes = accepted_bytes[f"{recovery_id}.json"] + assert b"journal-secret-sentinel" in journal_bytes + assert b"journal-secret-sentinel" not in accepted_bytes[ + f"{semantic}.delivery.json" + ] + assert b"journal-secret-sentinel" not in accepted_bytes[ + f"requests/{idempotency_key_sha}.json" + ] + + same_key = controller._self_upgrade_recovery_request_status( + semantic, + original_sha, + idempotency_key_sha, + expected_preflight, + expected_current_revision, + reason_sha, + policy, + ) + assert same_key == ("awaiting_validation", semantic) + assert store_snapshot() == accepted_bytes + assert original_path.read_bytes() == original_raw + + different_key = controller._self_upgrade_recovery_request_status( + semantic, + original_sha, + "d" * 64, + expected_preflight, + expected_current_revision, + reason_sha, + policy, + ) + assert different_key == ("conflict", semantic) + assert store_snapshot() == accepted_bytes + assert original_path.read_bytes() == original_raw + + +@pytest.mark.parametrize( + "malformed", + [b"{", b"[]", b'{"duplicate":1,"duplicate":2}'], +) +def test_legacy_json_parser_rejects_malformed_or_duplicate_records( + controller, monkeypatch, malformed, +): + monkeypatch.setattr( + controller, + "_self_upgrade_read_legacy_cache_file", + lambda *_args, **_kwargs: malformed, + ) + with pytest.raises((OSError, ValueError)): + controller._self_upgrade_read_legacy_cache_json( + Path("/mnt/cache/appdata/gitea-deploy-runners/state/webhook-inbox") + / ("a" * 64 + ".json"), + maximum_bytes=65536, + allowed_modes=frozenset({0o600}), + ) + + +def test_original_failed_delivery_rejects_modified_legacy_record(controller, monkeypatch): + revision = "c" * 40 + semantic = controller._webhook_semantic_digest(9, "refs/heads/main", revision) + policy = { + "repository": controller.SELF_UPGRADE_REPOSITORY, + "validation_gate_version": 1, + "contexts": ["Managed validation / full (push)"], + "timeout_seconds": 1800, + "poll_interval_seconds": 5, + "max_poll_interval_seconds": 30, + } + validation = controller._validation_snapshot(policy, 1) + validation["last_result"] = "pass" + validation["observed"] = [{ + "context": "Managed validation / full (push)", + "status": "success", + }] + record = { + "schema": controller._WEBHOOK_SCHEMA, + "semantic_sha256": semantic, + "gitea_repository_id": 9, + "ref": "refs/heads/main", + "repository": controller.SELF_UPGRADE_REPOSITORY, + "revision": revision, + "status": "failed", + "reason": "deployment_failed", + "source": "gitea_webhook", + "body_sha256": "1" * 64, + "validation": validation, + "audit": [ + {"stage": "validation_passed"}, + {"stage": "deployment_started"}, + {"stage": "deployment_failed", "reason": "deployment_failed"}, + ], + } + raw = json.dumps(record, sort_keys=True, separators=(",", ":")).encode() + monkeypatch.setattr( + controller, + "_self_upgrade_read_legacy_cache_json", + lambda *_args, **_kwargs: (raw, record), + ) + assert controller._self_upgrade_original_failed_delivery( + semantic, revision, policy, + ) == (controller.WEBHOOK_INBOX_ROOT / f"{semantic}.json", raw, record) + + tampered = {**record, "revision": "d" * 40} + monkeypatch.setattr( + controller, + "_self_upgrade_read_legacy_cache_json", + lambda *_args, **_kwargs: (raw, tampered), + ) + assert controller._self_upgrade_original_failed_delivery( + semantic, revision, policy, + ) is None diff --git a/backend/tests/test_appops_self_upgrade_runtime_model.py b/backend/tests/test_appops_self_upgrade_runtime_model.py new file mode 100644 index 0000000..660133a --- /dev/null +++ b/backend/tests/test_appops_self_upgrade_runtime_model.py @@ -0,0 +1,815 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import os +import stat +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +CONTROLLER = ROOT / "ops" / "gitea-deploy-broker" +RECOVERY_ID = "12345678-1234-4234-8234-123456789abc" +REVISION = "c" * 40 +TREE = "d" * 40 +IMAGE_ID = "sha256:" + "e" * 64 +CONTAINER_ID = "f" * 64 +BUILT_AT = "2026-10-07T00:00:00.000Z" +SECRET_SENTINEL = "model-private-secret$literal$$value" + + +@pytest.fixture +def controller(monkeypatch, tmp_path): + monkeypatch.syspath_prepend(str(CONTROLLER)) + spec = importlib.util.spec_from_file_location( + "appops_self_upgrade_runtime_model_test", + CONTROLLER / "status_server.py", + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + + recovery_root = tmp_path / "recovery" + monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root) + monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_REQUEST_ROOT", recovery_root / "requests") + monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_MODEL_ROOT", recovery_root / "models") + monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_TRANSACTION_ROOT", recovery_root / "transactions") + secret = b"runtime-model-test-secret-key-0123456789" + monkeypatch.setattr(module, "_self_upgrade_secret", lambda: secret) + return module, recovery_root, secret + + +def _environment(revision: str = REVISION, tree: str = TREE) -> dict[str, str]: + return { + "APPOPS_SOURCE_REPOSITORY": "jens/unraid-appops-gateway", + "APPOPS_SOURCE_REVISION": revision, + "APPOPS_SOURCE_TREE": tree, + "APPOPS_BUILD_ID": revision[:12], + "APPOPS_BUILD_DATE": BUILT_AT, + "KNOWLEDGE_OPTIMIZER_MODEL": "gpt-6-luna", + "OPERATOR_TOKEN": SECRET_SENTINEL, + } + + +def _model(revision: str = REVISION, tree: str = TREE) -> dict[str, object]: + return { + "name": "unraid-appops-gateway", + "services": { + "appops-gateway": { + "image": "unraid-appops-gateway:production", + "container_name": "Unraid-AppOps-Gateway", + "environment": _environment(revision, tree), + "ports": [{"target": 1216, "published": "1216", "host_ip": "198.18.10.150"}], + }, + }, + } + + +def _legacy_environment() -> dict[str, str]: + return { + **{f"LEGACY_SETTING_{index:02d}": f"fixture-value-{index:02d}" for index in range(43)}, + "KNOWLEDGE_OPTIMIZER_MODEL": "gpt-6-luna", + "APPOPS_FRONTEND_ORIGIN": "https://apps.example.test", + "OPERATOR_TOKEN": SECRET_SENTINEL, + } + + +def _legacy_model(module, image_ref: str | None = None) -> dict[str, object]: + return { + "name": "unraid-appops-gateway", + "services": { + "appops-gateway": { + "image": image_ref or module.SELF_UPGRADE_OLD_IMAGE_REF, + "container_name": "Unraid-AppOps-Gateway", + "environment": _legacy_environment(), + "ports": [{ + "target": 1216, "published": "1216", "host_ip": "198.18.10.150", + }], + }, + }, + } + + +def _mock_legacy_docker(monkeypatch, module): + environment = _legacy_environment() + raw_env = [f"{key}={value}" for key, value in environment.items()] + raw_env.append(f"APPOPS_FRONTEND_ORIGIN={environment['APPOPS_FRONTEND_ORIGIN']}") + image_labels = { + "org.opencontainers.image.source": module.SELF_UPGRADE_REPOSITORY, + "org.opencontainers.image.revision": module.SELF_UPGRADE_OLD_REVISION, + "org.opencontainers.image.source-tree": module.SELF_UPGRADE_OLD_TREE, + "org.opencontainers.image.version": module.SELF_UPGRADE_OLD_REVISION[:12], + "org.opencontainers.image.created": "2026-09-01T00:00:00.000Z", + } + container = { + "Id": CONTAINER_ID, + "Image": module.SELF_UPGRADE_IMAGE_ID, + "State": {"Running": True, "Health": {"Status": "healthy"}}, + "Mounts": [ + {"Type": "bind", "Source": module.SELF_UPGRADE_DATA_PATH, + "Destination": "/data", "RW": True}, + {"Type": "bind", "Source": module.SELF_UPGRADE_MODELS_PATH, + "Destination": module.SELF_UPGRADE_MODELS_TARGET, "RW": True}, + ], + "Config": { + "Image": module.SELF_UPGRADE_OLD_IMAGE_REF, + "Hostname": "unraid-appops-gateway", + "Env": raw_env, + "Labels": { + **image_labels, + "com.docker.compose.project": "unraid-appops-gateway", + "com.docker.compose.service": "appops-gateway", + "com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source", + "com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml", + }, + }, + "HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}}, + "NetworkSettings": { + "Ports": {"1216/tcp": [{"HostIp": module.SELF_UPGRADE_PUBLISH_HOST, "HostPort": "1216"}]}, + "Networks": { + "itworx-mcp-hub-npm": {}, + "unraid-appops-gateway_default": {}, + }, + }, + } + image = {"Id": module.SELF_UPGRADE_IMAGE_ID, "Config": {"Labels": image_labels}} + + def fake_run(args, **_kwargs): + if args[:3] == ["docker", "ps", "-a"]: + return SimpleNamespace(stdout="Unraid-AppOps-Gateway\n") + if args[:2] == ["docker", "inspect"]: + return SimpleNamespace(stdout=json.dumps([container])) + if args[:3] == ["docker", "image", "inspect"]: + return SimpleNamespace(stdout=json.dumps([image])) + raise AssertionError("unexpected Docker command category") + + monkeypatch.setattr(module.subprocess, "run", fake_run) + monkeypatch.setattr(module, "_load_projects", lambda: { + module.SELF_UPGRADE_REPOSITORY: { + "containers": ["Unraid-AppOps-Gateway"], "verify_oci_revision": True, + }, + }) + monkeypatch.setattr( + module, "_self_upgrade_read_legacy_cache_json", + lambda *_args, **_kwargs: (b"{}", {"live_artifacts": {"containers": {}}}), + ) + monkeypatch.setattr(module, "_self_upgrade_source_is_valid", lambda _source: True) + monkeypatch.setattr(module, "_self_upgrade_source_fingerprint", lambda *_args: "2" * 64) + return container + + +def _receipt(module, revision: str = REVISION, tree: str = TREE) -> dict[str, object]: + digest = hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest() + return { + "schema": "gitea-deploy-receipt.v1", + "repository": module.SELF_UPGRADE_REPOSITORY, + "revision": revision, + "tree": tree, + "treeDigest": digest, + "build": { + "id": revision[:12], + "repository": module.SELF_UPGRADE_REPOSITORY, + "revision": revision, + "tree": tree, + "treeDigest": digest, + "builtAt": BUILT_AT, + }, + "containers": { + "Unraid-AppOps-Gateway": { + "image_id": IMAGE_ID, + "image_revision": revision, + "image_source": module.SELF_UPGRADE_REPOSITORY, + "image_tree": tree, + "image_version": revision[:12], + "image_created": BUILT_AT, + }, + }, + "live": None, + } + + +def _binding(module, receipt, *, container_id: str = CONTAINER_ID) -> dict[str, object]: + return { + "containerId": container_id, + "imageId": IMAGE_ID, + "runtimeFingerprint": "1" * 64, + "sourceFingerprint": "2" * 64, + "receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json(receipt)).hexdigest(), + } + + +def test_plain_compose_file_preserves_literal_dollars_under_private_attestation( + controller, +): + module, recovery_root, _secret = controller + if os.name == "nt" or os.geteuid() != 0: + pytest.skip("The controller model store requires root-owned POSIX files") + + pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) + assert isinstance(pointer, dict) + loaded = module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION) + assert loaded is not None + assert loaded["model"] == _model() + assert loaded["pointer"] == pointer + assert Path(loaded["path"]).name == f"{RECOVERY_ID}-{REVISION}.compose.json" + + compose_path = Path(loaded["path"]) + attestation_path = compose_path.with_name( + f"{RECOVERY_ID}-{REVISION}.model.json" + ) + compose_raw = compose_path.read_bytes() + attestation_raw = attestation_path.read_bytes() + assert b"model-private-secret$$literal$$$$value" in compose_raw + assert SECRET_SENTINEL.encode() not in compose_raw + assert SECRET_SENTINEL.encode() not in attestation_raw + assert b"OPERATOR_TOKEN" not in attestation_raw + assert stat.S_IMODE(compose_path.stat().st_mode) == 0o600 + assert stat.S_IMODE(attestation_path.stat().st_mode) == 0o600 + assert stat.S_IMODE(recovery_root.stat().st_mode) == 0o700 + assert stat.S_IMODE((recovery_root / "models").stat().st_mode) == 0o700 + assert module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) == pointer + assert module._self_upgrade_store_compose_model( + RECOVERY_ID, REVISION, _model(tree="a" * 40), + ) is None + + compose_path.write_text(compose_path.read_text(encoding="utf-8").replace("gpt-6-luna", "gpt-5.5")) + compose_path.chmod(0o600) + assert module._self_upgrade_load_compose_model(RECOVERY_ID, REVISION) is None + + +def test_live_model_records_are_content_addressed_and_activate_only_from_state( + controller, monkeypatch, +): + module, _recovery_root, _secret = controller + if os.name == "nt" or os.geteuid() != 0: + pytest.skip("The controller model store requires root-owned POSIX files") + pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) + assert isinstance(pointer, dict) + receipt = _receipt(module) + live_pointer = module._self_upgrade_store_live_model(pointer, _binding(module, receipt)) + assert isinstance(live_pointer, dict) + first_live = module._self_upgrade_read_live_model_reference(live_pointer) + assert first_live is not None + assert first_live["model"] == _model() + + fresh_container_id = "a" * 64 + restored_pointer = module._self_upgrade_store_live_model( + pointer, _binding(module, receipt, container_id=fresh_container_id), + ) + assert isinstance(restored_pointer, dict) + assert restored_pointer["liveRecordSha256"] != live_pointer["liveRecordSha256"] + assert module._self_upgrade_read_live_model_reference(live_pointer) == first_live + restored = module._self_upgrade_read_live_model_reference(restored_pointer) + assert restored is not None + assert restored["live"]["containerId"] == fresh_container_id + + state = { + "schema_version": 3, + "repository": module.SELF_UPGRADE_REPOSITORY, + "status": "failed", + "live_revision": REVISION, + "live_status": "healthy", + "live_legacy_provenance": False, + "live_receipt": receipt, + "live_artifacts": { + "containers": { + "Unraid-AppOps-Gateway": { + "container_id": fresh_container_id, + "image_id": IMAGE_ID, + }, + }, + "appops_runtime_model": restored_pointer, + }, + } + project = { + "containers": ["Unraid-AppOps-Gateway"], + "verify_oci_revision": True, + } + active = module._self_upgrade_active_compose_model(state, project=project) + assert active is not None + assert active["model"] == _model() + runtime = { + "revision": REVISION, + "containerId": fresh_container_id, + "imageId": IMAGE_ID, + "runtimeFingerprint": "1" * 64, + "sourceFingerprint": "2" * 64, + "runtimeEnvironment": _environment(), + } + assert module._self_upgrade_active_compose_model(state, runtime, project=project) is not None + runtime["runtimeEnvironment"] = {**_environment(), "KNOWLEDGE_OPTIMIZER_MODEL": "gpt-5.5"} + assert module._self_upgrade_active_compose_model(state, runtime, project=project) is None + state["live_status"] = "unhealthy" + assert module._self_upgrade_active_compose_model(state, project=project) is None + + +def test_candidate_binding_rechecks_exact_model_receipt_and_runtime( + controller, monkeypatch, +): + module, _recovery_root, _secret = controller + if os.name == "nt" or os.geteuid() != 0: + pytest.skip("The controller model store requires root-owned POSIX files") + pointer = module._self_upgrade_store_compose_model(RECOVERY_ID, REVISION, _model()) + assert isinstance(pointer, dict) + receipt = _receipt(module) + project = { + "containers": ["Unraid-AppOps-Gateway"], + "verify_oci_revision": True, + } + monkeypatch.setattr(module, "_load_projects", lambda: {module.SELF_UPGRADE_REPOSITORY: project}) + monkeypatch.setattr(module, "_self_upgrade_runtime", lambda **_kwargs: { + "revision": REVISION, + "sourceRepository": module.SELF_UPGRADE_REPOSITORY, + "sourceTree": TREE, + "imageId": IMAGE_ID, + "containerId": CONTAINER_ID, + "runtimeFingerprint": "1" * 64, + "sourceFingerprint": "2" * 64, + "runtimeEnvironment": _environment(), + }) + binding = module._self_upgrade_candidate_model_binding(pointer, receipt) + assert binding == _binding(module, receipt) + wrong_receipt = _receipt(module, tree="b" * 40) + assert module._self_upgrade_candidate_model_binding(pointer, wrong_receipt) is None + bad_pointer = {**pointer, "modelSha256": "0" * 64} + assert module._self_upgrade_candidate_model_binding(bad_pointer, receipt) is None + + +def test_fixed_555_model_preserves_exact_environment_and_duplicate_origin( + controller, monkeypatch, +): + module, _recovery_root, _secret = controller + if os.name == "nt" or os.geteuid() != 0: + pytest.skip("The controller model store requires root-owned POSIX files") + journal = { + "expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION, + "currentImageId": module.SELF_UPGRADE_IMAGE_ID, + "runtimeEnvironment": _legacy_environment(), + } + monkeypatch.setattr(module, "_self_upgrade_recovery_journal", lambda _rid: journal) + pointer = module._self_upgrade_store_compose_model( + RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, _legacy_model(module), + ) + assert isinstance(pointer, dict) + loaded = module._self_upgrade_load_compose_model( + RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, + ) + assert loaded is not None + assert loaded["model"] == _legacy_model(module) + changed = _legacy_model(module) + changed["services"]["appops-gateway"]["environment"]["APPOPS_BUILD_ID"] = "untrusted" + assert not module._self_upgrade_compose_model_valid( + changed, module.SELF_UPGRADE_OLD_REVISION, + ) + + container = _mock_legacy_docker(monkeypatch, module) + initial_runtime = module._self_upgrade_runtime() + assert initial_runtime is not None + assert initial_runtime["runtimeEnvironment"] == _legacy_environment() + assert len(container["Config"]["Env"]) == 47 + assert len(initial_runtime["runtimeEnvironment"]) == 46 + runtime = module._self_upgrade_runtime( + authorized_model_pointer=pointer, + allow_legacy_canonical_config=True, + ) + assert runtime is not None + assert runtime["runtimeEnvironment"] == _legacy_environment() + assert len(runtime["runtimeEnvironment"]) == 46 + assert len(container["Config"]["Env"]) == 47 + assert runtime["imageId"] == module.SELF_UPGRADE_IMAGE_ID + assert runtime["containerId"] == container["Id"] + binding = module._self_upgrade_legacy_model_binding(pointer) + assert binding == { + "containerId": CONTAINER_ID, + "imageId": module.SELF_UPGRADE_IMAGE_ID, + "runtimeFingerprint": runtime["runtimeFingerprint"], + "sourceFingerprint": "2" * 64, + "receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json({})).hexdigest(), + } + + live_pointer = module._self_upgrade_store_live_model(pointer, binding) + assert isinstance(live_pointer, dict) + state = { + "schema_version": 3, + "repository": module.SELF_UPGRADE_REPOSITORY, + "status": "rolled-back", + "live_revision": module.SELF_UPGRADE_OLD_REVISION, + "live_status": "healthy", + "live_legacy_provenance": True, + "live_receipt": {}, + "live_artifacts": { + "containers": { + "Unraid-AppOps-Gateway": { + "container_id": CONTAINER_ID, + "image_id": module.SELF_UPGRADE_IMAGE_ID, + }, + }, + "appops_runtime_model": live_pointer, + }, + } + project = { + "containers": ["Unraid-AppOps-Gateway"], + "verify_oci_revision": True, + } + active = module._self_upgrade_active_compose_model(state, project=project) + assert active is not None + assert active["legacy"] is True + assert active["model"] == _legacy_model(module) + assert module._self_upgrade_active_compose_model(state, runtime, project=project) is not None + + container["State"]["Health"]["Status"] = "unhealthy" + assert module._self_upgrade_runtime() is None + assert module._self_upgrade_model_runtime_identity(pointer, module.SELF_UPGRADE_IMAGE_ID) is None + recovery_identity = module._self_upgrade_model_runtime_identity( + pointer, module.SELF_UPGRADE_IMAGE_ID, require_healthy=False, + ) + assert isinstance(recovery_identity, dict) + assert set(recovery_identity) == { + "containerId", "imageId", "revision", "sourceRepository", "sourceTree", + "runtimeFingerprint", "sourceFingerprint", "runtimeCasSha256", + } + assert module._self_upgrade_model_runtime_identity( + pointer, module.SELF_UPGRADE_IMAGE_ID, require_healthy="false", + ) is None + container["State"]["Health"]["Status"] = "healthy" + + # The identical duplicate is accepted only for the exact immutable old + # image; conflicting duplicates remain fail-closed. + raw_env = container["Config"]["Env"] + raw_env[-1] = "APPOPS_FRONTEND_ORIGIN=https://attacker.invalid" + assert module._self_upgrade_runtime( + authorized_model_pointer=pointer, + allow_legacy_canonical_config=True, + ) is None + + +def test_fixed_555_production_reference_is_valid_only_under_signed_legacy_model( + controller, monkeypatch, +): + module, _recovery_root, _secret = controller + if os.name == "nt" or os.geteuid() != 0: + pytest.skip("The controller model store requires root-owned POSIX files") + journal = { + "expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION, + "currentImageId": module.SELF_UPGRADE_IMAGE_ID, + "runtimeEnvironment": _legacy_environment(), + } + monkeypatch.setattr(module, "_self_upgrade_recovery_journal", lambda _rid: journal) + model = _legacy_model(module, module.SELF_UPGRADE_PRODUCTION_IMAGE_REF) + pointer = module._self_upgrade_store_compose_model( + "42345678-1234-4234-8234-123456789abc", + module.SELF_UPGRADE_OLD_REVISION, + model, + ) + assert isinstance(pointer, dict) + container = _mock_legacy_docker(monkeypatch, module) + # The pre-capture container can retain the old platform tag while the + # signed effective rollback model uses the closed production tag. + container["Config"]["Image"] = module.SELF_UPGRADE_OLD_IMAGE_REF + initial_runtime = module._self_upgrade_runtime() + assert initial_runtime is not None + assert initial_runtime["imageRef"] == module.SELF_UPGRADE_OLD_IMAGE_REF + + runtime = module._self_upgrade_runtime( + authorized_model_pointer=pointer, + allow_legacy_canonical_config=True, + ) + assert isinstance(runtime, dict) + assert runtime["imageId"] == module.SELF_UPGRADE_IMAGE_ID + assert runtime["imageRef"] == module.SELF_UPGRADE_OLD_IMAGE_REF + binding = module._self_upgrade_legacy_model_binding(pointer) + assert isinstance(binding, dict) + reference = module._self_upgrade_store_live_model(pointer, binding) + assert isinstance(reference, dict) + + project = { + "containers": ["Unraid-AppOps-Gateway"], + "verify_oci_revision": True, + } + monkeypatch.setattr(module, "_load_projects", lambda: { + module.SELF_UPGRADE_REPOSITORY: project, + }) + snapshot = { + "revision": module.SELF_UPGRADE_OLD_REVISION, + "legacy_provenance": True, + "receipt": {}, + "appops_runtime_model": reference, + "containers": [{ + "name": "Unraid-AppOps-Gateway", + "container_id": CONTAINER_ID, + "image_id": module.SELF_UPGRADE_IMAGE_ID, + "image_ref": module.SELF_UPGRADE_OLD_IMAGE_REF, + "rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497", + "restore_image_ref": module.SELF_UPGRADE_PRODUCTION_IMAGE_REF, + "was_running": True, + }], + } + active_state = { + "schema_version": 3, + "repository": module.SELF_UPGRADE_REPOSITORY, + "status": "rolled-back", + "live_revision": module.SELF_UPGRADE_OLD_REVISION, + "live_status": "healthy", + "live_legacy_provenance": True, + "live_receipt": {}, + "live_artifacts": { + "appops_runtime_model": reference, + "containers": { + "Unraid-AppOps-Gateway": { + "container_id": CONTAINER_ID, + "image_id": module.SELF_UPGRADE_IMAGE_ID, + }, + }, + }, + } + monkeypatch.setattr( + module, + "_self_upgrade_read_legacy_cache_json", + lambda *_args, **_kwargs: (b"{}", active_state), + ) + assert module._self_upgrade_runtime() is not None + verified = module._self_upgrade_verify_rollback_model(reference, snapshot) + assert verified is not None + assert verified["model"] == model + assert verified["legacy"] is True + + container["Config"]["Image"] = module.SELF_UPGRADE_PRODUCTION_IMAGE_REF + production_model = module._self_upgrade_load_compose_model( + pointer["recoveryId"], pointer["revision"], + ) + assert production_model is not None + container["Config"]["Labels"]["com.docker.compose.project.config_files"] = str( + production_model["path"], + ) + restored_runtime = module._self_upgrade_runtime( + authorized_model_pointer=pointer, + allow_legacy_canonical_config=True, + ) + assert isinstance(restored_runtime, dict) + assert restored_runtime["imageRef"] == module.SELF_UPGRADE_PRODUCTION_IMAGE_REF + + +def test_recovery_transaction_binds_authorized_555_preimage_and_holds_pending_state( + controller, monkeypatch, +): + module, recovery_root, secret = controller + if os.name == "nt" or os.geteuid() != 0: + pytest.skip("The controller transaction store requires root-owned POSIX files") + recovery_root.mkdir(mode=0o700, parents=True, exist_ok=True) + semantic = module._webhook_semantic_digest(9, "refs/heads/main", REVISION) + original_record = { + "schema": module._WEBHOOK_SCHEMA, + "semantic_sha256": semantic, + "revision": REVISION, + "status": "failed", + "reason": "deployment_failed", + } + original_raw = module._self_upgrade_canonical_json(original_record) + b"\n" + original_sha = hashlib.sha256(original_raw).hexdigest() + journal = { + "schema": module._SELF_UPGRADE_RECOVERY_SCHEMA, + "recoveryId": RECOVERY_ID, + "semanticSha256": semantic, + "repository": module.SELF_UPGRADE_REPOSITORY, + "revision": REVISION, + "originalRawSha256": original_sha, + "idempotencyKeySha256": "3" * 64, + "preflightSha256": "7" * 64, + "expectedCurrentRevision": module.SELF_UPGRADE_OLD_REVISION, + "runtimeEnvironment": _legacy_environment(), + "sourceFingerprint": "4" * 64, + "runtimeFingerprint": "5" * 64, + "registryFingerprint": "b" * 64, + "currentContainerId": CONTAINER_ID, + "currentImageId": module.SELF_UPGRADE_IMAGE_ID, + "stateSha256": "a" * 64, + "currentSourceTree": module.SELF_UPGRADE_OLD_TREE, + "createdAt": 1, + } + monkeypatch.setattr(module, "_self_upgrade_model_transaction_origin_valid", lambda _payload: True) + module._self_upgrade_write_private_journal(RECOVERY_ID, journal, secret) + request_root = recovery_root / "requests" + request_root.mkdir(mode=0o700) + request_payload = { + "schema": module._SELF_UPGRADE_RECOVERY_REQUEST_SCHEMA, + "revision": REVISION, + "preflightSha256": "7" * 64, + "operationId": semantic, + "reasonSha256": "8" * 64, + "idempotencyKeySha256": "3" * 64, + "recoveryId": RECOVERY_ID, + "recoveryOfSha256": original_sha, + "createdAt": 1, + } + request_record = { + **request_payload, + "hmac": module._self_upgrade_mac( + secret, "recovery-request", module._self_upgrade_canonical_json(request_payload), + ), + } + module._self_upgrade_write_private_record( + request_root / f"{'3' * 64}.json", + module._self_upgrade_canonical_json(request_record) + b"\n", + ) + inbox_root = recovery_root / "inbox" + inbox_root.mkdir(mode=0o700) + monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", inbox_root) + original_path = inbox_root / f"{semantic}.json" + original_path.write_bytes(original_raw) + original_path.chmod(0o600) + recovery_delivery = { + "status": "deploying", + "recovery_id": RECOVERY_ID, + "validation": {"last_result": "pass"}, + "audit": [ + {"stage": "validation_passed"}, + {"stage": "deployment_started"}, + ], + } + module._self_upgrade_write_private_record( + recovery_root / f"{semantic}.delivery.json", + module._self_upgrade_canonical_json(recovery_delivery) + b"\n", + ) + project = {"branch": "main", "gitea_repository_id": 9} + monkeypatch.setattr(module, "_load_projects", lambda: {module.SELF_UPGRADE_REPOSITORY: project}) + monkeypatch.setattr(module, "_validation_policy", lambda *_args: {"repository": module.SELF_UPGRADE_REPOSITORY}) + monkeypatch.setattr(module, "_self_upgrade_original_failed_delivery", lambda *_args: ( + original_path, original_raw, original_record, + )) + monkeypatch.setattr(module, "_self_upgrade_recovery_record_valid", lambda *_args, **_kwargs: True) + monkeypatch.setattr(module, "_self_upgrade_recovery_preimage_matches", lambda _record: True) + pointer = module._self_upgrade_store_compose_model( + RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, _legacy_model(module), + ) + assert isinstance(pointer, dict) + monkeypatch.setattr(module, "_self_upgrade_runtime", lambda **_kwargs: { + "runtimeCasSha256": "6" * 64, + "runtimeFingerprint": "5" * 64, + "sourceFingerprint": "4" * 64, + "containerId": CONTAINER_ID, + "imageId": module.SELF_UPGRADE_IMAGE_ID, + "revision": module.SELF_UPGRADE_OLD_REVISION, + "runtimeEnvironment": _legacy_environment(), + }) + + metadata = { + "id": REVISION[:12], "repository": module.SELF_UPGRADE_REPOSITORY, + "revision": REVISION, "tree": TREE, + "treeDigest": hashlib.sha256(f"git-tree:{TREE}".encode()).hexdigest(), + "builtAt": BUILT_AT, + } + snapshot = { + "revision": module.SELF_UPGRADE_OLD_REVISION, + "legacy_provenance": True, + "receipt": {}, + "compose_containers": ["Unraid-AppOps-Gateway"], + "containers": [{ + "name": "Unraid-AppOps-Gateway", "container_id": CONTAINER_ID, + "image_id": module.SELF_UPGRADE_IMAGE_ID, + "image_ref": module.SELF_UPGRADE_OLD_IMAGE_REF, + "rollback_ref": "gitea-deploy-rollback/unraid-appops-gateway-0:5553c04ce497", + "was_running": True, + }], + "captured_at_epoch": 1, + "appops_runtime_model": pointer, + } + now = 100 + transaction_id = "22345678-1234-4234-8234-123456789abc" + request = { + "schema": module._SELF_UPGRADE_MODEL_TRANSACTION_SCHEMA, + "transactionId": transaction_id, + "recoveryId": RECOVERY_ID, + "repository": module.SELF_UPGRADE_REPOSITORY, + "kind": "recovery", + "revision": REVISION, + "metadata": metadata, + "expectedStateSha256": "a" * 64, + "registryFingerprint": "b" * 64, + "preMutationRuntimeSha256": "6" * 64, + "sourceFingerprint": "4" * 64, + "priorModelPointer": pointer, + "priorSnapshot": snapshot, + "candidateModelPointer": None, + "buildContexts": {}, + "candidateImageId": None, + "verifiedLiveReference": None, + "verifiedReceipt": None, + "phase": "prepared", + "createdAt": now, + "updatedAt": now, + } + assert module._self_upgrade_recovery_journal(RECOVERY_ID) == journal + assert module._self_upgrade_recovery_request_payload(request_record) == request_payload + assert module._self_upgrade_original_failed_delivery(semantic, REVISION, {"repository": module.SELF_UPGRADE_REPOSITORY}) == ( + original_path, original_raw, original_record, + ) + assert module._self_upgrade_recovery_preimage_matches(recovery_delivery) is True + read_legacy = module._self_upgrade_read_legacy_cache_json + state_root = recovery_root / "state" + state_root.mkdir(mode=0o700) + state_path = state_root / "unraid-appops-gateway.json" + restored_state_raw: bytes | None = None + restored_state: dict[str, object] | None = None + + def read_test_inbox(path, **kwargs): + if path == original_path: + assert path.read_bytes() == original_raw + return original_raw, original_record + if path == state_path and restored_state_raw is not None and restored_state is not None: + assert path.read_bytes() == restored_state_raw + return restored_state_raw, restored_state + return read_legacy(path, **kwargs) + + # The production reader pins the real legacy cache directory profile. For + # this temp-root test, stub only that fixed-path adapter while preserving + # the actual HMAC request, journal, authorization, and transaction checks. + monkeypatch.setattr(module, "_self_upgrade_read_legacy_cache_json", read_test_inbox) + actual_authorization = module._self_upgrade_recovery_authorization(RECOVERY_ID) + assert actual_authorization is not None + preimage = module._self_upgrade_transaction_live_preimage(RECOVERY_ID, "recovery") + assert preimage is not None + assert preimage["currentRevision"] == module.SELF_UPGRADE_OLD_REVISION + assert preimage["runtimeEnvironment"] == _legacy_environment() + assert module._self_upgrade_load_compose_model( + RECOVERY_ID, module.SELF_UPGRADE_OLD_REVISION, + )["model"]["services"]["appops-gateway"]["environment"] == journal["runtimeEnvironment"] + created = module._self_upgrade_begin_model_transaction(request) + assert isinstance(created, dict) + assert created["payload"]["priorSnapshot"]["appops_runtime_model"] == pointer + assert stat.S_IMODE((recovery_root / "transactions" / f"{transaction_id}.json").stat().st_mode) == 0o600 + assert module._self_upgrade_load_model_transaction(transaction_id) == created + + assert module._self_upgrade_update_model_transaction( + transaction_id, "0" * 64, "recovery_required", + ) is None + held = module._self_upgrade_update_model_transaction( + transaction_id, created["sha256"], "recovery_required", + ) + assert isinstance(held, dict) + assert held["payload"]["phase"] == "recovery_required" + assert module._self_upgrade_pending_model_transaction() == held + + rollback_binding = { + "containerId": CONTAINER_ID, + "imageId": module.SELF_UPGRADE_IMAGE_ID, + "runtimeFingerprint": "5" * 64, + "sourceFingerprint": "4" * 64, + "receiptSha256": hashlib.sha256(module._self_upgrade_canonical_json({})).hexdigest(), + } + restored_reference = module._self_upgrade_store_live_model(pointer, rollback_binding) + assert isinstance(restored_reference, dict) + restored_state = { + "repository": module.SELF_UPGRADE_REPOSITORY, + "status": "failed", + "live_revision": module.SELF_UPGRADE_OLD_REVISION, + "live_status": "healthy", + "live_legacy_provenance": True, + "live_receipt": {}, + "live_artifacts": { + "appops_runtime_model": restored_reference, + "containers": { + "Unraid-AppOps-Gateway": { + "container_id": CONTAINER_ID, + "image_id": module.SELF_UPGRADE_IMAGE_ID, + }, + }, + }, + } + restored_state_raw = module._self_upgrade_canonical_json(restored_state) + b"\n" + state_path.write_bytes(restored_state_raw) + state_path.chmod(0o600) + monkeypatch.setattr(module, "STATE_ROOT", state_root) + + rollback_transaction_id = "32345678-1234-4234-8234-123456789abc" + rollback_payload = dict(created["payload"]) + rollback_payload.update({ + "transactionId": rollback_transaction_id, + "phase": "rolling_back", + "failureCode": "rollback_started", + "updatedAt": now + 1, + }) + transaction_path = module._self_upgrade_transaction_path(rollback_transaction_id) + module._self_upgrade_write_private_record( + transaction_path, + module._self_upgrade_canonical_json( + module._self_upgrade_model_transaction_envelope(rollback_payload, secret), + ) + b"\n", + ) + rollback_before = module._self_upgrade_load_model_transaction(rollback_transaction_id) + assert isinstance(rollback_before, dict) + rolled_back = module._self_upgrade_update_model_transaction( + rollback_transaction_id, + rollback_before["sha256"], + "rolled_back", + {"restoredLiveReference": restored_reference}, + ) + assert isinstance(rolled_back, dict) + assert rolled_back["payload"]["phase"] == "rolled_back" + assert rolled_back["payload"]["restoredLiveReference"] == restored_reference + assert rolled_back["payload"]["restoredStateSha256"] == hashlib.sha256(restored_state_raw).hexdigest() + assert module._self_upgrade_pending_model_transaction() == held diff --git a/backend/tests/test_controller_artifact_bootstrap_failed_child_recovery.py b/backend/tests/test_controller_artifact_bootstrap_failed_child_recovery.py index ff1d2d9..5c2255e 100644 --- a/backend/tests/test_controller_artifact_bootstrap_failed_child_recovery.py +++ b/backend/tests/test_controller_artifact_bootstrap_failed_child_recovery.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import os import subprocess import pytest @@ -110,6 +111,235 @@ def run(docker, git): ) +_MOUNT_ORDERS = { + ("name", 1): (3, 0, 4, 1, 2), + ("name", 2): (4, 2, 0, 3, 1), + ("name", 3): (1, 3, 0, 4, 2), + ("id", 1): (2, 4, 1, 0, 3), + ("id", 2): (4, 1, 3, 2, 0), +} + + +def configure_reinspection( + docker, + *, + drift=None, + drift_reference="id", + drift_pair=1, + identifier=FAILED_ID, +): + """Model Docker returning the same inspect with independent mount ordering.""" + original_inspect = docker.inspect_container + counts = {"name": 0, "id": 0} + reads = [] + + def inspect(reference): + if reference == bootstrap.MODERN_HELPER_NAME: + scope = "name" + elif reference == identifier: + scope = "id" + else: + return original_inspect(reference) + + counts[scope] += 1 + reads.append((scope, counts[scope])) + if docker.existing_child is None: + return None + + snapshot = copy.deepcopy(docker.existing_child) + order = _MOUNT_ORDERS.get((scope, counts[scope])) + if order is not None: + snapshot["Mounts"] = [snapshot["Mounts"][index] for index in order] + + pair_number = ( + 1 + if (scope == "name" and counts[scope] == 2) + or (scope == "id" and counts[scope] == 1) + else 2 + if (scope == "name" and counts[scope] == 3) + or (scope == "id" and counts[scope] == 2) + else None + ) + is_drift_read = pair_number == drift_pair and ( + drift_reference == "both" or scope == drift_reference + ) + if drift is not None and is_drift_read: + mount = snapshot["Mounts"][0] + if drift == "mount_type": + mount["Type"] = "volume" + elif drift == "mount_source": + mount["Source"] = "/tmp/changed-source" + elif drift == "mount_destination": + mount["Destination"] = "/tmp/changed-destination" + elif drift == "mount_rw": + mount["RW"] = not mount["RW"] + elif drift == "mount_propagation": + mount["Propagation"] = "shared" + elif drift == "mount_name": + mount["Name"] = "unexpected-volume" + elif drift == "missing_mount": + snapshot["Mounts"].pop() + elif drift == "extra_mount": + extra = copy.deepcopy(mount) + extra["Destination"] = "/unexpected" + snapshot["Mounts"].append(extra) + elif drift == "duplicate_destination": + snapshot["Mounts"].append(copy.deepcopy(mount)) + elif drift == "metadata_field": + cache_mount = next( + item + for item in snapshot["Mounts"] + if item["Destination"] == "/mnt/cache" + ) + cache_mount["Mode"] = "ro" + elif drift == "environment_order": + snapshot["Config"]["Env"] = list(reversed(snapshot["Config"]["Env"])) + elif drift == "binds_order": + snapshot["HostConfig"]["Binds"] = list( + reversed(snapshot["HostConfig"]["Binds"]) + ) + elif drift == "state_drift": + snapshot["State"]["ExitCode"] = 2 + else: + raise AssertionError(f"unhandled test drift: {drift}") + return snapshot + + docker.inspect_container = inspect + return counts, reads + + +def historical_inputs(): + helper_source = ( + original.ROOT / "backend/app/services/controller_artifact_helper_spec.py" + ).read_text(encoding="utf-8") + child = original._retired_failed_child(helper_source) + docker = original.FakeDocker( + original._legacy_helper_spec(), existing_child=child + ) + docker.containers["gitea-deploy-control"] = original._modern_controller_source_inspect( + bootstrap.MODERN_CONTROLLER_REVISION + ) + git = original.FakeProtectedGit(helper_source) + return docker, git + + +def run_historical(docker, git): + return original._run_bootstrap( + docker, + git, + health_sequence=[ + original._modern_baseline_health(), + original._modern_baseline_health(), + original._modern_baseline_health(), + original._modern_baseline_health(), + original._health(), + original._health(), + ], + ) + + +def test_failed_child_inspect_ignores_mount_order_across_name_and_id_reads(): + docker, git, _child = inputs() + counts, reads = configure_reinspection(docker) + + result = run(docker, git) + + assert result["status"] == "promoted" + assert counts == {"name": 4, "id": 2} + assert reads[:5] == [ + ("name", 1), + ("name", 2), + ("id", 1), + ("name", 3), + ("id", 2), + ] + assert docker.removed == [FAILED_ID] + assert len(docker.created) == len(docker.started) == 1 + + +def test_historical_failed_child_mount_order_is_ignored_across_name_and_id_reads(): + docker, git = historical_inputs() + counts, reads = configure_reinspection( + docker, identifier=original.RETIRED_HELPER_ID + ) + + result = run_historical(docker, git) + + assert result["status"] == "promoted" + assert counts == {"name": 4, "id": 2} + assert reads[:5] == [ + ("name", 1), + ("name", 2), + ("id", 1), + ("name", 3), + ("id", 2), + ] + assert docker.removed == [original.RETIRED_HELPER_ID] + assert len(docker.created) == len(docker.started) == 1 + + +@pytest.mark.parametrize("drift_pair", [1, 2]) +def test_historical_failed_child_rejects_mount_metadata_change_from_initial_snapshot( + drift_pair, +): + docker, git = historical_inputs() + configure_reinspection( + docker, + drift="metadata_field", + drift_reference="both", + drift_pair=drift_pair, + identifier=original.RETIRED_HELPER_ID, + ) + + with pytest.raises(bootstrap.BootstrapError): + run_historical(docker, git) + + assert docker.removed == docker.created == docker.started == [] + + +@pytest.mark.parametrize( + "drift", + [ + "mount_type", + "mount_source", + "mount_destination", + "mount_rw", + "mount_propagation", + "mount_name", + "missing_mount", + "extra_mount", + "duplicate_destination", + "metadata_field", + "environment_order", + "binds_order", + "state_drift", + ], +) +@pytest.mark.parametrize("drift_reference", ["name", "id"]) +def test_failed_child_reinspection_rejects_any_non_mount_order_drift( + drift, drift_reference +): + docker, git, _child = inputs() + # Make the environment order check meaningful while keeping the initial + # image and child Config.Env identical. + docker.existing_child["Config"]["Env"] = ["PATH=/usr/bin", "LANG=C"] + docker.images[bootstrap.LEGACY_CONTROLLER_IMAGE_ID]["Config"]["Env"] = [ + "PATH=/usr/bin", + "LANG=C", + ] + _counts, _reads = configure_reinspection( + docker, drift=drift, drift_reference=drift_reference + ) + + if drift == "binds_order": + assert len(docker.existing_child["HostConfig"]["Binds"]) > 1 + + with pytest.raises(bootstrap.BootstrapError): + run(docker, git) + + assert docker.removed == docker.created == docker.started == [] + + def test_noncurrent_protected_ancestor_failure_releases_only_its_verified_name(): docker, git, child = inputs() result = run(docker, git) @@ -274,13 +504,20 @@ def test_delete_adapter_rejects_noncanonical_full_container_identity( def test_git_ancestor_proof_reads_original_protected_objects( tmp_path, monkeypatch, scenario ): - def git(*args): + def git(*args, allow_replace_objects=False): + environment = None + if allow_replace_objects: + # Only the attack demonstration opts out of the inherited guard. + # The production proof still reads original protected Git objects. + environment = dict(os.environ) + environment.pop("GIT_NO_REPLACE_OBJECTS", None) return ( subprocess.run( ["git", "-C", str(tmp_path), *args], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=environment, ) .stdout.decode() .strip() @@ -302,7 +539,7 @@ def test_git_ancestor_proof_reads_original_protected_objects( proof = bootstrap.ProtectedGitRepository() if scenario == "replace_ref": git("replace", old, current) - assert git("rev-parse", f"{old}^{{tree}}") != tree + assert git("rev-parse", f"{old}^{{tree}}", allow_replace_objects=True) != tree if scenario == "unrelated": git("checkout", "-qb", "side", old) path.write_text("unrelated child") diff --git a/backend/tests/test_workmesh_secret_import.py b/backend/tests/test_workmesh_secret_import.py new file mode 100644 index 0000000..3811eb2 --- /dev/null +++ b/backend/tests/test_workmesh_secret_import.py @@ -0,0 +1,633 @@ +from __future__ import annotations + +import io +import json +from datetime import datetime, timezone + +import pytest +from sqlmodel import Session, SQLModel, create_engine, select + +from backend.app.db.models import AuditEvent, Secret +from backend.app.services import workmesh_secret_import as importer +from backend.app.services.secret_store import get_secret, set_secret + +REVISION = "a" * 40 +CREDENTIAL = "W" * 48 +PREFLIGHT_SHA = "b" * 64 +BACKUP_SHA = "c" * 64 +SECRET_REF = "WORKMESH_DEPLOYMENT_BROKER" +TARGET = "jens/workmesh" +CHECKS = { + "exactMain": "passed", + "pushValidation": "passed", + "controllerPreflight": "passed", + "dataBackup": "passed", +} + + +def _valid_preflight(revision: str = REVISION) -> dict[str, object]: + return { + "apiVersion": "itworx.workmesh-promotion/v2", + "status": "preflight_ok", + "repository": TARGET, + "revision": revision, + "preflightSha256": PREFLIGHT_SHA, + "backupInputsSha256": BACKUP_SHA, + "checks": dict(CHECKS), + "diagnostic": CREDENTIAL, + } + + +class FakeBroker: + def __init__( + self, + *, + response: object | None = None, + error: Exception | None = None, + base_url: str = "https://controller.example", + verify_tls: bool = True, + config_error: str | None = None, + before_response=None, + ): + self.response = _valid_preflight() if response is None else response + self.error = error + self.base_url = base_url + self.verify_tls = verify_tls + self._config_error = config_error + self.before_response = before_response + self.preflight_calls: list[tuple[str, str]] = [] + self.apply_calls = 0 + + def configuration_error(self): + return self._config_error + + def workmesh_promotion_preflight(self, revision: str, *, secret: str): + self.preflight_calls.append((revision, secret)) + if self.before_response is not None: + self.before_response() + if self.error is not None: + raise self.error + return self.response + + def workmesh_promotion_apply(self, *args, **kwargs): + self.apply_calls += 1 + raise AssertionError("the credential importer must not invoke promotion apply") + + +@pytest.fixture +def database(tmp_path, monkeypatch): + engine = create_engine(f"sqlite:///{tmp_path / 'workmesh-secret-import.db'}") + SQLModel.metadata.create_all(engine) + monkeypatch.setattr( + importer, "secret_storage_status", lambda: {"productionReady": True} + ) + with Session(engine) as session: + yield engine, session + engine.dispose() + + +def _rows(session: Session) -> tuple[Secret | None, list[AuditEvent]]: + return ( + session.get(Secret, SECRET_REF), + list(session.exec(select(AuditEvent)).all()), + ) + + +def _database_dump(session: Session) -> str: + secret, audits = _rows(session) + return repr( + { + "secret": secret.model_dump() if secret is not None else None, + "audits": [event.model_dump() for event in audits], + } + ) + + +def _import( + session: Session, + *, + credential: str = CREDENTIAL, + revision: str = REVISION, + broker: FakeBroker | None = None, +): + return importer.import_workmesh_deployment_secret( + session, + credential, + revision=revision, + broker=broker or FakeBroker(), + ) + + +def test_import_creates_encrypted_secret_after_verified_fixed_preflight_and_audits( + database, +): + _, session = database + broker = FakeBroker() + + result = _import(session, broker=broker) + + row = session.get(Secret, SECRET_REF) + assert result["status"] == "imported" + assert result["secretRef"] == SECRET_REF + assert result["repository"] == TARGET + assert result["revision"] == REVISION + assert result["preflightSha256"] == PREFLIGHT_SHA + assert result["readback"] == "verified" + assert row is not None + assert row.encrypted_value != CREDENTIAL + assert get_secret(session, SECRET_REF, touch=False) == CREDENTIAL + assert broker.preflight_calls == [(REVISION, CREDENTIAL)] + assert broker.apply_calls == 0 + audits = session.exec(select(AuditEvent)).all() + assert len(audits) == 1 + audit = audits[0] + assert audit.target == TARGET + assert audit.result == "success" + assert audit.redacted_metadata["storeReference"] == SECRET_REF + assert audit.redacted_metadata["revision"] == REVISION + assert audit.redacted_metadata["preflightSha256"] == PREFLIGHT_SHA + assert CREDENTIAL not in _database_dump(session) + + +def test_identical_import_verifies_and_audits_without_mutating_secret(database): + _, session = database + assert _import(session)["status"] == "imported" + row = session.get(Secret, SECRET_REF) + assert row is not None + before = { + "encrypted_value": row.encrypted_value, + "created_at": row.created_at, + "updated_at": row.updated_at, + "last_used_at": row.last_used_at, + "used_by_json": list(row.used_by_json), + "metadata_json": dict(row.metadata_json), + } + + result = _import(session) + + session.refresh(row) + assert result["status"] == "already_present" + assert { + "encrypted_value": row.encrypted_value, + "created_at": row.created_at, + "updated_at": row.updated_at, + "last_used_at": row.last_used_at, + "used_by_json": list(row.used_by_json), + "metadata_json": dict(row.metadata_json), + } == before + assert len(session.exec(select(AuditEvent)).all()) == 2 + + +@pytest.mark.parametrize("credential", ["W" * 32, "W" * 256]) +def test_credential_length_contract_accepts_exact_boundaries(database, credential): + _, session = database + + result = _import(session, credential=credential) + + assert result["status"] == "imported" + assert get_secret(session, SECRET_REF, touch=False) == credential + + +def test_different_existing_value_is_rejected_before_controller_preflight(database): + _, session = database + set_secret(session, SECRET_REF, "D" * 48, metadata={"owner": "keep"}) + original = session.get(Secret, SECRET_REF) + assert original is not None + before = original.model_dump() + broker = FakeBroker() + + with pytest.raises(importer.WorkMeshSecretImportError): + _import(session, broker=broker) + + session.rollback() + current = session.get(Secret, SECRET_REF) + assert current is not None and current.model_dump() == before + assert broker.preflight_calls == [] + assert session.exec(select(AuditEvent)).all() == [] + + +def test_undecryptable_existing_value_is_rejected_without_replacement(database): + _, session = database + set_secret(session, SECRET_REF, "D" * 48) + row = session.get(Secret, SECRET_REF) + assert row is not None + row.encrypted_value = "unreadable-ciphertext" + session.add(row) + session.commit() + broker = FakeBroker() + + with pytest.raises(importer.WorkMeshSecretImportError): + _import(session, broker=broker) + + assert session.get(Secret, SECRET_REF).encrypted_value == "unreadable-ciphertext" + assert broker.preflight_calls == [] + + +def test_concurrent_identical_insert_winner_is_retained_and_audited(database): + engine, session = database + winner_timestamp = datetime(2026, 10, 4, 12, 0, 0, tzinfo=timezone.utc) + + def concurrent_winner(): + with Session(engine) as winner_session: + set_secret( + winner_session, + SECRET_REF, + CREDENTIAL, + used_by=["concurrent-owner"], + metadata={"owner": "concurrent-winner"}, + ) + winner = winner_session.get(Secret, SECRET_REF) + assert winner is not None + winner.created_at = winner_timestamp + winner.updated_at = winner_timestamp + winner_session.add(winner) + winner_session.commit() + + broker = FakeBroker(before_response=concurrent_winner) + result = _import(session, broker=broker) + + winner = session.get(Secret, SECRET_REF) + assert result["status"] == "already_present" + assert winner is not None + assert winner.created_at.replace(tzinfo=timezone.utc) == winner_timestamp + assert winner.updated_at.replace(tzinfo=timezone.utc) == winner_timestamp + assert winner.used_by_json == ["concurrent-owner"] + assert winner.metadata_json == {"owner": "concurrent-winner"} + assert get_secret(session, SECRET_REF, touch=False) == CREDENTIAL + assert len(session.exec(select(AuditEvent)).all()) == 1 + + +def test_concurrent_different_insert_winner_is_never_overwritten(database): + engine, session = database + + def concurrent_winner(): + with Session(engine) as winner_session: + set_secret( + winner_session, + SECRET_REF, + "D" * 48, + metadata={"owner": "concurrent-winner"}, + ) + + broker = FakeBroker(before_response=concurrent_winner) + with pytest.raises(importer.WorkMeshSecretImportError): + _import(session, broker=broker) + + session.rollback() + assert get_secret(session, SECRET_REF, touch=False) == "D" * 48 + assert session.get(Secret, SECRET_REF).metadata_json == { + "owner": "concurrent-winner" + } + assert session.exec(select(AuditEvent)).all() == [] + + +def test_production_secret_storage_must_be_ready(database, monkeypatch): + _, session = database + monkeypatch.setattr( + importer, "secret_storage_status", lambda: {"productionReady": False} + ) + broker = FakeBroker() + + with pytest.raises(importer.WorkMeshSecretImportError): + _import(session, broker=broker) + + assert broker.preflight_calls == [] + assert session.get(Secret, SECRET_REF) is None + + +@pytest.mark.parametrize( + "credential", + [ + "W" * 31, + "W" * 257, + "W" * 31 + " ", + "W" * 31 + "\n", + "W" * 31 + "\u00e9", + "W" * 31 + "\x00", + ], +) +def test_credential_must_be_bounded_printable_ascii_without_whitespace( + database, credential +): + _, session = database + broker = FakeBroker() + + with pytest.raises(importer.WorkMeshSecretImportError): + _import(session, credential=credential, broker=broker) + + assert broker.preflight_calls == [] + assert session.get(Secret, SECRET_REF) is None + + +@pytest.mark.parametrize("revision", ["main", "A" * 40, "a" * 39, "a" * 41]) +def test_revision_must_be_an_exact_full_lowercase_commit(database, revision): + _, session = database + broker = FakeBroker() + + with pytest.raises(importer.WorkMeshSecretImportError): + _import(session, revision=revision, broker=broker) + + assert broker.preflight_calls == [] + assert session.get(Secret, SECRET_REF) is None + + +@pytest.mark.parametrize( + ("base_url", "verify_tls", "config_error"), + [ + ("http://controller.example", True, None), + ("https://controller.example", False, None), + ("https://controller.example", True, "deployment_broker_url_invalid"), + ], +) +def test_preflight_requires_verified_https_configuration( + database, base_url, verify_tls, config_error +): + _, session = database + broker = FakeBroker( + base_url=base_url, + verify_tls=verify_tls, + config_error=config_error, + ) + + with pytest.raises(importer.WorkMeshSecretImportError): + _import(session, broker=broker) + + assert broker.preflight_calls == [] + assert session.get(Secret, SECRET_REF) is None + + +@pytest.mark.parametrize( + "mutate", + [ + lambda value: value.update(status="preflight_failed"), + lambda value: value.update(apiVersion="wrong/v1"), + lambda value: value.update(repository="jens/other"), + lambda value: value.update(revision="f" * 40), + lambda value: value.update(preflightSha256="invalid"), + lambda value: value.update(backupInputsSha256="invalid"), + lambda value: value.update(checks={"exactMain": "passed"}), + lambda value: value["checks"].update(pushValidation="blocked"), + ], +) +def test_invalid_fixed_preflight_never_writes_secret_or_audit(database, mutate): + _, session = database + preflight = _valid_preflight() + mutate(preflight) + broker = FakeBroker(response=preflight) + + with pytest.raises(importer.WorkMeshSecretImportError): + _import(session, broker=broker) + + session.rollback() + assert session.get(Secret, SECRET_REF) is None + assert session.exec(select(AuditEvent)).all() == [] + assert broker.apply_calls == 0 + + +def test_non_mapping_preflight_is_rejected_without_storage_write(database): + _, session = database + broker = FakeBroker(response=["not", "a", "mapping"]) + + with pytest.raises(importer.WorkMeshSecretImportError): + _import(session, broker=broker) + + session.rollback() + assert session.get(Secret, SECRET_REF) is None + + +def test_broker_exception_is_sanitized_and_never_persisted(database): + _, session = database + broker = FakeBroker(error=RuntimeError(f"upstream leaked {CREDENTIAL}")) + + with pytest.raises(importer.WorkMeshSecretImportError) as error: + _import(session, broker=broker) + + assert CREDENTIAL not in str(error.value) + assert broker.preflight_calls == [(REVISION, CREDENTIAL)] + session.rollback() + assert session.get(Secret, SECRET_REF) is None + assert session.exec(select(AuditEvent)).all() == [] + + +def test_audit_insert_failure_rolls_back_secret_and_audit_together(database): + engine, session = database + + def fail_audit_insert(conn, cursor, statement, parameters, context, executemany): + if statement.lstrip().lower().startswith("insert into auditevent"): + raise RuntimeError(f"database detail contained {CREDENTIAL}") + + from sqlalchemy import event + + event.listen(engine, "before_cursor_execute", fail_audit_insert) + try: + with pytest.raises(importer.WorkMeshSecretImportError) as error: + _import(session) + finally: + event.remove(engine, "before_cursor_execute", fail_audit_insert) + + assert CREDENTIAL not in str(error.value) + session.rollback() + with Session(engine) as verify: + assert verify.get(Secret, SECRET_REF) is None + assert verify.exec(select(AuditEvent)).all() == [] + + +def test_uncertain_readback_returns_blocked_without_replacing_committed_secret( + database, monkeypatch +): + engine, session = database + actual_get_secret = importer.get_secret + calls = 0 + + def fail_readback_once(db_session, name, *, touch=False): + nonlocal calls + calls += 1 + if calls == 2: + return None + return actual_get_secret(db_session, name, touch=touch) + + monkeypatch.setattr(importer, "get_secret", fail_readback_once) + broker = FakeBroker() + + with pytest.raises(importer.WorkMeshSecretImportError) as error: + _import(session, broker=broker) + + assert CREDENTIAL not in str(error.value) + monkeypatch.setattr(importer, "get_secret", actual_get_secret) + session.rollback() + with Session(engine) as verify: + row = verify.get(Secret, SECRET_REF) + assert row is not None + before = row.model_dump() + assert get_secret(verify, SECRET_REF, touch=False) == CREDENTIAL + assert len(verify.exec(select(AuditEvent)).all()) == 1 + + outcome = _import(session, broker=FakeBroker()) + assert outcome["status"] == "already_present" + session.expire_all() + after = session.get(Secret, SECRET_REF) + assert after is not None and after.model_dump() == before + + +def test_uncertain_commit_returns_blocked_and_same_value_replay_does_not_replace( + database, monkeypatch +): + engine, session = database + actual_commit = session.commit + + def commit_then_lose_acknowledgement(): + if any(isinstance(item, AuditEvent) for item in session.new): + actual_commit() + raise RuntimeError(f"commit acknowledgement lost for {CREDENTIAL}") + actual_commit() + + monkeypatch.setattr(session, "commit", commit_then_lose_acknowledgement) + with pytest.raises(importer.WorkMeshSecretImportError) as error: + _import(session) + assert CREDENTIAL not in str(error.value) + + monkeypatch.setattr(session, "commit", actual_commit) + with Session(engine) as verify: + row = verify.get(Secret, SECRET_REF) + assert row is not None + before = row.model_dump() + assert get_secret(verify, SECRET_REF, touch=False) == CREDENTIAL + assert len(verify.exec(select(AuditEvent)).all()) == 1 + + result = _import(session, broker=FakeBroker()) + assert result["status"] == "already_present" + session.expire_all() + after = session.get(Secret, SECRET_REF) + assert after is not None and after.model_dump() == before + assert len(session.exec(select(AuditEvent)).all()) == 2 + + +def test_cli_imports_stdin_only_and_emits_sanitized_verification(database, monkeypatch): + engine, _ = database + broker = FakeBroker() + monkeypatch.setattr(importer, "engine", engine) + monkeypatch.setattr(importer, "DeploymentBrokerClient", lambda: broker) + output = io.StringIO() + + exit_code = importer.main( + [REVISION], + stdin=io.BytesIO(CREDENTIAL.encode("ascii")), + stdout=output, + ) + + result = json.loads(output.getvalue()) + assert exit_code == 0 + assert result["status"] == "imported" + assert result["secretRef"] == SECRET_REF + assert result["revision"] == REVISION + assert CREDENTIAL not in output.getvalue() + assert broker.preflight_calls == [(REVISION, CREDENTIAL)] + + +def test_cli_rejects_extra_arguments_before_reading_secret(database, monkeypatch): + engine, _ = database + broker = FakeBroker() + monkeypatch.setattr(importer, "engine", engine) + monkeypatch.setattr(importer, "DeploymentBrokerClient", lambda: broker) + output = io.StringIO() + + class UnreadableInput: + def read(self, _size): + pytest.fail("invalid command line must be rejected before reading stdin") + + exit_code = importer.main( + [REVISION, CREDENTIAL], + stdin=UnreadableInput(), + stdout=output, + ) + + assert exit_code != 0 + assert json.loads(output.getvalue()) == { + "reason": "invalid_arguments", + "status": "blocked", + } + assert broker.preflight_calls == [] + + +def test_cli_rejects_non_ascii_stdin_without_calling_controller(database, monkeypatch): + engine, _ = database + broker = FakeBroker() + monkeypatch.setattr(importer, "engine", engine) + monkeypatch.setattr(importer, "DeploymentBrokerClient", lambda: broker) + output = io.StringIO() + + exit_code = importer.main( + [REVISION], + stdin=io.BytesIO(b"W" * 47 + b"\xff"), + stdout=output, + ) + + assert exit_code != 0 + assert json.loads(output.getvalue()) == { + "reason": "invalid_input", + "status": "blocked", + } + assert broker.preflight_calls == [] + + +def test_cli_rejects_oversized_stdin_without_calling_controller(database, monkeypatch): + engine, _ = database + broker = FakeBroker() + monkeypatch.setattr(importer, "engine", engine) + monkeypatch.setattr(importer, "DeploymentBrokerClient", lambda: broker) + output = io.StringIO() + + exit_code = importer.main( + [REVISION], + stdin=io.BytesIO(b"W" * 4097), + stdout=output, + ) + + assert exit_code != 0 + assert json.loads(output.getvalue())["status"] == "blocked" + assert "W" * 48 not in output.getvalue() + assert broker.preflight_calls == [] + + +def test_cli_unexpected_error_never_echoes_secret(database, monkeypatch): + engine, _ = database + broker = FakeBroker(error=RuntimeError(f"unexpected {CREDENTIAL}")) + monkeypatch.setattr(importer, "engine", engine) + monkeypatch.setattr(importer, "DeploymentBrokerClient", lambda: broker) + output = io.StringIO() + + exit_code = importer.main( + [REVISION], + stdin=io.BytesIO(CREDENTIAL.encode("ascii")), + stdout=output, + ) + + result = json.loads(output.getvalue()) + assert exit_code != 0 + assert result["status"] == "blocked" + assert CREDENTIAL not in output.getvalue() + assert "unexpected" not in output.getvalue() + assert "traceback" not in output.getvalue().lower() + + +@pytest.mark.parametrize("tampered", [False, True]) +def test_cli_unknown_typed_error_code_is_normalized(database, monkeypatch, tampered): + engine, _ = database + monkeypatch.setattr(importer, "engine", engine) + monkeypatch.setattr(importer, "DeploymentBrokerClient", FakeBroker) + + def fail_import(*args, **kwargs): + error = importer.WorkMeshSecretImportError(CREDENTIAL) + if tampered: + error.code = CREDENTIAL + raise error + + monkeypatch.setattr(importer, "import_workmesh_deployment_secret", fail_import) + output = io.StringIO() + assert importer.main( + [REVISION], stdin=io.BytesIO(CREDENTIAL.encode("ascii")), stdout=output + ) == 2 + assert json.loads(output.getvalue()) == { + "status": "blocked", "reason": "import_failed" + } + assert CREDENTIAL not in output.getvalue()