from __future__ import annotations import http.server import importlib.util import json import os import stat import subprocess import sys import threading import urllib.error import urllib.request import uuid from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[2] CONTROLLER = ROOT / "ops" / "gitea-deploy-broker" @pytest.fixture def controller(monkeypatch, tmp_path): monkeypatch.syspath_prepend(str(CONTROLLER)) spec = importlib.util.spec_from_file_location( "appops_self_upgrade_recovery_boundaries_test", CONTROLLER / "status_server.py", ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", tmp_path / "inbox") module.WEBHOOK_INBOX_ROOT.mkdir(mode=0o700) monkeypatch.setattr(module, "APPOPS_PROGRESS_ROOT", tmp_path / "progress") monkeypatch.setattr(module, "SELF_UPGRADE_RECOVERY_ROOT", tmp_path / "recovery") monkeypatch.setattr( module, "SELF_UPGRADE_RECOVERY_REQUEST_ROOT", tmp_path / "recovery" / "requests", ) return module def _private_recovery_payload(module, recovery_id: str) -> dict[str, object]: revision = "c" * 40 return { "schema": module._SELF_UPGRADE_RECOVERY_SCHEMA, "recoveryId": recovery_id, "semanticSha256": module._webhook_semantic_digest( 9, "refs/heads/main", revision, ), "repository": module.SELF_UPGRADE_REPOSITORY, "revision": revision, "originalRawSha256": "1" * 64, "idempotencyKeySha256": "2" * 64, "preflightSha256": "3" * 64, "expectedCurrentRevision": "4" * 40, "currentContainerId": "5" * 64, "currentImageId": "sha256:" + "6" * 64, "currentSourceTree": "7" * 40, "runtimeFingerprint": "8" * 64, "registryFingerprint": "9" * 64, "sourceFingerprint": "a" * 64, "stateSha256": "b" * 64, "runtimeEnvironment": {"API_TOKEN": "recovery-secret-sentinel"}, "createdAt": 1, } def _recovery_delivery(module, *, secret: str) -> dict[str, object]: return { "schema": module._WEBHOOK_SELF_UPGRADE_RECOVERY_SCHEMA, "semantic_sha256": "a" * 64, "recovery_of_sha256": "b" * 64, "recovery_id": str(uuid.UUID("12345678-1234-4234-8234-123456789abc")), "idempotency_key_sha256": "c" * 64, "preflight_sha256": "d" * 64, "body_sha256": "e" * 64, "delivery": "22345678-1234-4234-8234-123456789abc", "gitea_repository_id": 9, "ref": "refs/heads/main", "repository": module.SELF_UPGRADE_REPOSITORY, "revision": "f" * 40, "source": "typed_self_upgrade_recovery", "status": "awaiting_validation", "validation": { "last_result": "wait", "required_contexts": ["Managed validation / full (push)"], "observed": [], "checked_at": 1, "deadline_at": 2, }, "audit": [ {"at": 1, "stage": "self_upgrade_recovery_requested"}, {"at": 2, "stage": "awaiting_validation"}, ], "received_at": 1, "updated_at": 2, # These fields must remain private even if a caller or record contains them. "runtimeEnvironment": {"API_TOKEN": secret}, "privateCredential": secret, } def test_ordinary_deploy_strips_ambient_recovery_binding_and_rejects_bad_bindings( controller, monkeypatch, ): ambient_secret = "ambient-recovery-secret-sentinel" monkeypatch.setenv("APPOPS_SELF_UPGRADE_RECOVERY_ID", ambient_secret) monkeypatch.setenv("APPOPS_REGISTERED_LEGACY_REQUEST_ID", ambient_secret) calls = [] def fake_run(command, **kwargs): calls.append((command, kwargs)) return subprocess.CompletedProcess(command, 0) monkeypatch.setattr(controller.subprocess, "run", fake_run) ordinary = controller._run_registered_deploy("jens/quiz-monitor", "c" * 40) assert ordinary.returncode == 0 command, kwargs = calls.pop() assert command[-3:] == ["deploy", "jens/quiz-monitor", "c" * 40] assert "APPOPS_SELF_UPGRADE_RECOVERY_ID" not in kwargs["env"] assert "APPOPS_REGISTERED_LEGACY_REQUEST_ID" not in kwargs["env"] assert ambient_secret not in json.dumps(kwargs["env"]) recovery_id = str(uuid.UUID("12345678-1234-4234-8234-123456789abc")) authorized = controller._run_registered_deploy( controller.SELF_UPGRADE_REPOSITORY, "c" * 40, self_upgrade_recovery_id=recovery_id, ) assert authorized.returncode == 0 _, kwargs = calls.pop() assert kwargs["env"]["APPOPS_SELF_UPGRADE_RECOVERY_ID"] == recovery_id assert "APPOPS_REGISTERED_LEGACY_REQUEST_ID" not in kwargs["env"] invalid_bindings = [ (controller.SELF_UPGRADE_REPOSITORY, "not-a-uuid", None), ("jens/quiz-monitor", recovery_id, None), (controller.SELF_UPGRADE_REPOSITORY, recovery_id, recovery_id), ] for repository, bad_recovery_id, registered_id in invalid_bindings: with pytest.raises(ValueError): controller._run_registered_deploy( repository, "c" * 40, self_upgrade_recovery_id=bad_recovery_id, registered_legacy_request_id=registered_id, ) assert calls == [] def test_preflight_projects_runtime_identity_without_environment_values( controller, monkeypatch, ): project = { "branch": "main", "gitea_repository_id": 9, "containers": ["Unraid-AppOps-Gateway"], "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], "environment": { "APPOPS_DATA_PATH": controller.SELF_UPGRADE_DATA_PATH, "APPOPS_PUBLISH_HOST": controller.SELF_UPGRADE_PUBLISH_HOST, }, "verify_oci_revision": True, "validation": { "validation_gate_version": 1, "required": True, "contexts": ["Managed validation / full (push)"], "timeout_seconds": 1800, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30, }, } private_environment = {"API_TOKEN": "preflight-secret-sentinel"} monkeypatch.setattr( controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: project}, ) monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda _revision: True) monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: True) monkeypatch.setattr( controller, "_fetch_gitea_validation", lambda *_args: ("pass", []), ) monkeypatch.setattr( controller, "_self_upgrade_runtime", lambda: { "revision": "4" * 40, "imageId": "sha256:" + "5" * 64, "containerId": "6" * 64, "imageRef": controller.SELF_UPGRADE_PRODUCTION_IMAGE_REF, "sourceTree": "7" * 40, "runtimeFingerprint": "8" * 64, "runtimeEnvironment": private_environment, }, ) monkeypatch.setattr( controller.shutil, "disk_usage", lambda _path: type("Usage", (), {"free": 50 * 1024**3})(), ) monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda _revision: True) monkeypatch.setattr(controller, "_is_current_branch_head", lambda *_args: True) monkeypatch.setattr( controller, "_fetch_gitea_validation", lambda *_args: ("pass", []), ) monkeypatch.setattr( controller, "_validation_policy", lambda _repository, _project: { "contexts": ["Managed validation / full (push)"], }, ) monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *_a, **_k: True) monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None) result = controller._self_upgrade_preflight("c" * 40) serialized = json.dumps(result) assert result["status"] == "preflight_ok" assert "runtimeEnvironment" not in result assert "API_TOKEN" not in serialized assert "preflight-secret-sentinel" not in serialized server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() try: body = json.dumps({ "repository": controller.SELF_UPGRADE_REPOSITORY, "revision": "c" * 40, "reason": "preflight boundary test", "expectedCurrentRevision": None, }).encode("utf-8") request = urllib.request.Request( f"http://127.0.0.1:{server.server_port}{controller.SELF_UPGRADE_PREFLIGHT_PATH}", data=body, headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=5) as response: response_body = response.read() assert response.status == 200 response_payload = json.loads(response_body) serialized_response = response_body.decode("utf-8") assert response_payload["status"] == "preflight_ok" assert "runtimeEnvironment" not in response_payload assert "API_TOKEN" not in serialized_response assert "preflight-secret-sentinel" not in serialized_response finally: server.shutdown() server.server_close() thread.join(timeout=5) def test_public_recovery_status_projects_away_private_environment(controller, monkeypatch): secret = "public-status-secret-sentinel" record = _recovery_delivery(controller, secret=secret) monkeypatch.setattr(controller, "_self_upgrade_delivery_record", lambda _request_id: record) monkeypatch.setattr(controller, "_self_upgrade_signature_is_valid", lambda *_a, **_k: True) monkeypatch.setattr(controller, "_read_inbox_record", lambda _path: None) monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None) server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() try: request = urllib.request.Request( f"http://127.0.0.1:{server.server_port}" f"{controller.SELF_UPGRADE_STATUS_PREFIX}{'a' * 64}", method="GET", ) try: with urllib.request.urlopen(request, timeout=5) as response: status = response.status body = response.read() except urllib.error.HTTPError as error: status = error.code body = error.read() assert status == 200, body.decode("utf-8", errors="replace") view = json.loads(body) assert view["repository"] == controller.SELF_UPGRADE_REPOSITORY assert view["revision"] == "f" * 40 assert view["attemptType"] == "self-upgrade-recovery" assert view["recoveryOf"] == "b" * 64 assert not { "recovery_id", "recovery_of_sha256", "idempotency_key_sha256", "preflight_sha256", "body_sha256", "recovery_hmac", } & view.keys() assert "runtimeEnvironment" not in view assert "privateCredential" not in view assert "recovery_hmac" not in view assert "idempotency_key_sha256" not in view assert secret.encode() not in body assert "API_TOKEN" not in body.decode("utf-8") finally: server.shutdown() server.server_close() thread.join(timeout=5) def test_private_recovery_journal_is_0600_and_hmac_tampering_fails_closed( controller, monkeypatch, tmp_path, ): if os.name == "nt" or os.geteuid() != 0: pytest.skip("The controller recovery store requires root-owned POSIX files") recovery_root = tmp_path / "recovery-store" recovery_root.mkdir(mode=0o700) recovery_root.chmod(0o700) monkeypatch.setattr(controller, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root) secret = b"fixed-self-upgrade-recovery-test-key-0123456789" monkeypatch.setattr(controller, "_self_upgrade_secret", lambda: secret) recovery_id = str(uuid.UUID("12345678-1234-4234-8234-123456789abc")) payload = _private_recovery_payload(controller, recovery_id) controller._self_upgrade_write_private_journal(recovery_id, payload, secret) marker = recovery_root / f"{recovery_id}.json" assert stat.S_IMODE(marker.stat().st_mode) == 0o600 assert controller._self_upgrade_recovery_journal(recovery_id) == payload envelope = json.loads(marker.read_text(encoding="utf-8")) envelope["payload"]["runtimeEnvironment"]["API_TOKEN"] = "changed-secret" marker.write_text(json.dumps(envelope, sort_keys=True, separators=(",", ":")) + "\n") marker.chmod(0o600) assert controller._self_upgrade_recovery_journal(recovery_id) is None envelope = controller._self_upgrade_recovery_envelope(payload, secret) marker.write_text(json.dumps(envelope, sort_keys=True, separators=(",", ":")) + "\n") marker.chmod(0o644) assert controller._self_upgrade_recovery_journal(recovery_id) is None @pytest.mark.parametrize("failed_revalidation", [False, True]) def test_recovery_request_is_immutable_and_idempotent_for_one_key( controller, monkeypatch, tmp_path, failed_revalidation, ): if os.name == "nt" or os.geteuid() != 0: pytest.skip("The controller recovery store requires root-owned POSIX files") recovery_root = tmp_path / "recovery-store" recovery_root.mkdir(mode=0o700) recovery_root.chmod(0o700) request_root = recovery_root / "requests" monkeypatch.setattr(controller, "SELF_UPGRADE_RECOVERY_ROOT", recovery_root) monkeypatch.setattr(controller, "SELF_UPGRADE_RECOVERY_REQUEST_ROOT", request_root) secret = b"fixed-self-upgrade-recovery-test-key-0123456789" monkeypatch.setattr(controller, "_self_upgrade_secret", lambda: secret) revision = "c" * 40 semantic = controller._webhook_semantic_digest(9, "refs/heads/main", revision) original_path = tmp_path / "original-failed-delivery.json" original_raw = b'{"immutable":"original failed delivery"}\n' original_path.write_bytes(original_raw) original = {"revision": revision} if failed_revalidation: original["status"] = "validation_timeout" monkeypatch.setattr(controller, "_self_upgrade_failed_revalidation_sha256", lambda *_args: "e" * 64) monkeypatch.setattr(controller, "_self_upgrade_read_legacy_cache_json", lambda *_a, **_k: (original_raw, original)) original_info = (original_path, original_raw, original) monkeypatch.setattr( controller, "_self_upgrade_original_failed_delivery", lambda *_args: original_info, ) monkeypatch.setattr( controller, "_self_upgrade_delivery_revision", lambda _semantic: revision, ) expected_preflight = "3" * 64 expected_current_revision = controller.SELF_UPGRADE_OLD_REVISION expected_container_id = "5" * 64 expected_image_id = "sha256:" + "6" * 64 expected_source_tree = "7" * 40 expected_runtime_fingerprint = "8" * 64 private_environment = {"API_TOKEN": "journal-secret-sentinel"} fresh = { "status": "preflight_ok", "preflightSha256": expected_preflight, "currentRevision": expected_current_revision, "currentContainerId": expected_container_id, "currentImageId": expected_image_id, "currentSourceTree": expected_source_tree, "currentRuntimeFingerprint": expected_runtime_fingerprint, } bindings = { "currentRevision": expected_current_revision, "currentContainerId": expected_container_id, "currentImageId": expected_image_id, "currentSourceTree": expected_source_tree, "runtimeFingerprint": expected_runtime_fingerprint, "registryFingerprint": "9" * 64, "sourceFingerprint": "a" * 64, "stateSha256": "b" * 64, "runtimeEnvironment": private_environment, } monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _revision: fresh) monkeypatch.setattr( controller, "_self_upgrade_recovery_current_bindings", lambda: bindings, ) policy = { "repository": controller.SELF_UPGRADE_REPOSITORY, "validation_gate_version": 1, "contexts": ["Managed validation / full (push)"], "timeout_seconds": 1800, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30, } project = {"slug": "unraid-appops-gateway"} monkeypatch.setattr( controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: project}, ) monkeypatch.setattr( controller, "_validation_policy", lambda _repository, _project: policy, ) original_sha = controller.hashlib.sha256(original_raw).hexdigest() idempotency_key_sha = "2" * 64 reason_sha = "4" * 64 first = controller._self_upgrade_recovery_request_status( semantic, original_sha, idempotency_key_sha, expected_preflight, expected_current_revision, reason_sha, policy, ) assert first == ("awaiting_validation", semantic) assert original_path.read_bytes() == original_raw def store_snapshot() -> dict[str, bytes]: return { path.relative_to(recovery_root).as_posix(): path.read_bytes() for path in sorted(recovery_root.rglob("*.json")) } accepted_bytes = store_snapshot() assert accepted_bytes assert stat.S_IMODE(recovery_root.stat().st_mode) == 0o700 assert stat.S_IMODE(request_root.stat().st_mode) == 0o700 assert all( stat.S_IMODE(path.stat().st_mode) == 0o600 for path in recovery_root.rglob("*.json") ) # The secret is intentionally present only in the root-owned HMAC journal. recovery_id = str(uuid.UUID(bytes=bytes.fromhex(idempotency_key_sha)[:16], version=4)) journal_bytes = accepted_bytes[f"{recovery_id}.json"] assert b"journal-secret-sentinel" in journal_bytes assert b"journal-secret-sentinel" not in accepted_bytes[ f"{semantic}.delivery.json" ] assert b"journal-secret-sentinel" not in accepted_bytes[ f"requests/{idempotency_key_sha}.json" ] journal = json.loads(journal_bytes)["payload"] request = json.loads(accepted_bytes[f"requests/{idempotency_key_sha}.json"]) if failed_revalidation: assert journal["failedRevalidationRawSha256"] == request["failedRevalidationRawSha256"] == "e" * 64 else: assert "failedRevalidationRawSha256" not in journal assert "failedRevalidationRawSha256" not in request same_key = controller._self_upgrade_recovery_request_status( semantic, original_sha, idempotency_key_sha, expected_preflight, expected_current_revision, reason_sha, policy, ) assert same_key == ("awaiting_validation", semantic) assert store_snapshot() == accepted_bytes assert original_path.read_bytes() == original_raw different_key = controller._self_upgrade_recovery_request_status( semantic, original_sha, "d" * 64, expected_preflight, expected_current_revision, reason_sha, policy, ) assert different_key == ("conflict", semantic) assert store_snapshot() == accepted_bytes assert original_path.read_bytes() == original_raw if failed_revalidation: monkeypatch.setattr(controller, "_self_upgrade_failed_revalidation_sha256", lambda *_args: "f" * 64) changed = controller._self_upgrade_recovery_request_status( semantic, original_sha, idempotency_key_sha, expected_preflight, expected_current_revision, reason_sha, policy, ) assert changed == ("conflict", semantic) assert store_snapshot() == accepted_bytes @pytest.mark.parametrize( "malformed", [b"{", b"[]", b'{"duplicate":1,"duplicate":2}'], ) def test_legacy_json_parser_rejects_malformed_or_duplicate_records( controller, monkeypatch, malformed, ): monkeypatch.setattr( controller, "_self_upgrade_read_legacy_cache_file", lambda *_args, **_kwargs: malformed, ) with pytest.raises((OSError, ValueError)): controller._self_upgrade_read_legacy_cache_json( Path("/mnt/cache/appdata/gitea-deploy-runners/state/webhook-inbox") / ("a" * 64 + ".json"), maximum_bytes=65536, allowed_modes=frozenset({0o600}), ) def test_original_failed_delivery_rejects_modified_legacy_record(controller, monkeypatch): revision = "c" * 40 semantic = controller._webhook_semantic_digest(9, "refs/heads/main", revision) policy = { "repository": controller.SELF_UPGRADE_REPOSITORY, "validation_gate_version": 1, "contexts": ["Managed validation / full (push)"], "timeout_seconds": 1800, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30, } validation = controller._validation_snapshot(policy, 1) validation["last_result"] = "pass" validation["observed"] = [{ "context": "Managed validation / full (push)", "status": "success", }] record = { "schema": controller._WEBHOOK_SCHEMA, "semantic_sha256": semantic, "gitea_repository_id": 9, "ref": "refs/heads/main", "repository": controller.SELF_UPGRADE_REPOSITORY, "revision": revision, "status": "failed", "reason": "deployment_failed", "source": "gitea_webhook", "body_sha256": "1" * 64, "validation": validation, "audit": [ {"stage": "validation_passed"}, {"stage": "deployment_started"}, {"stage": "deployment_failed", "reason": "deployment_failed"}, ], } raw = json.dumps(record, sort_keys=True, separators=(",", ":")).encode() monkeypatch.setattr( controller, "_self_upgrade_read_legacy_cache_json", lambda *_args, **_kwargs: (raw, record), ) assert controller._self_upgrade_original_failed_delivery( semantic, revision, policy, ) == (controller.WEBHOOK_INBOX_ROOT / f"{semantic}.json", raw, record) tampered = {**record, "revision": "d" * 40} monkeypatch.setattr( controller, "_self_upgrade_read_legacy_cache_json", lambda *_args, **_kwargs: (raw, tampered), ) assert controller._self_upgrade_original_failed_delivery( semantic, revision, policy, ) is None @pytest.mark.parametrize(("status", "extra", "valid"), [ ("deploying", {}, True), ("failed", {"reason": "deployment_failed"}, True), ("deployed", {"receipt_sha256": "f" * 64}, True), ("deployed", {"receipt_sha256": "invalid"}, False), ("deploying", {"reason": "deployment_failed"}, False), ("failed", {"reason": "private-secret-value"}, False), ("failed", {"receipt_sha256": "f" * 64}, False), ("failed", {"unknown": "value"}, False), ("failed", {"reason": ["deployment_failed"]}, False), ("awaiting_validation", {"reason": "gitea_validation_unavailable"}, True), ("failed", {"reason": "recovery_preimage_changed"}, True), ]) def test_signed_recovery_reader_accepts_only_typed_worker_outcome_fields( controller, monkeypatch, tmp_path, status, extra, valid, ): record = _recovery_delivery(controller, secret="sentinel") record.pop("runtimeEnvironment") record.pop("privateCredential") record.update(status=status, **extra) secret = b"fixture-recovery-signing-key" journal = { "recoveryId": record["recovery_id"], "revision": record["revision"], "idempotencyKeySha256": record["idempotency_key_sha256"], "preflightSha256": record["preflight_sha256"], "semanticSha256": record["semantic_sha256"], "originalRawSha256": record["recovery_of_sha256"], } payload = { "operationId": record["semantic_sha256"], "revision": record["revision"], "preflightSha256": record["preflight_sha256"], "idempotencyKeySha256": record["idempotency_key_sha256"], "recoveryId": record["recovery_id"], "recoveryOfSha256": record["recovery_of_sha256"], } record["body_sha256"] = controller.hashlib.sha256(controller._self_upgrade_canonical_json({ "semanticSha256": record["semantic_sha256"], "recoveryId": record["recovery_id"], "idempotencyKeySha256": record["idempotency_key_sha256"], })).hexdigest() record["recovery_hmac"] = controller._self_upgrade_public_recovery_signature(record, secret) monkeypatch.setattr(controller, "_self_upgrade_secret", lambda: secret) monkeypatch.setattr(controller, "_self_upgrade_recovery_journal", lambda _id: journal) monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: {}}) monkeypatch.setattr(controller, "_validation_policy", lambda *_a: {}) monkeypatch.setattr(controller, "_validation_snapshot_matches", lambda *_a: True) monkeypatch.setattr(controller, "_self_upgrade_request_root", lambda **_k: tmp_path) monkeypatch.setattr(controller, "_self_upgrade_read_private_json", lambda _path: (b"", {})) monkeypatch.setattr(controller, "_self_upgrade_recovery_request_payload", lambda _record: payload) def verify(): return controller._self_upgrade_recovery_record_valid( record, semantic_digest=record["semantic_sha256"], original_raw_sha256=record["recovery_of_sha256"], ) assert verify() is valid record["updated_at"] += 1 assert verify() is False # Even permitted outcome fields require the original complete HMAC. @pytest.mark.parametrize(("uid", "gid", "mode"), [(99, 100, 0o777), (0, 0, 0o755)]) @pytest.mark.parametrize("drift", ["none", "owner", "mode", "symlink", "foreign_cache", "inode"]) def test_legacy_cache_reader_and_state_cas_bind_both_exact_appdata_profiles( controller, monkeypatch, tmp_path, uid, gid, mode, drift, ): if os.name == "nt" or os.geteuid() != 0: pytest.skip("Pinned dirfd ownership contract requires root POSIX") root = tmp_path / "virtual-host" root.mkdir(mode=0o755) dirs = [ ("mnt", 0, 0, 0o755), ("mnt/cache", 99, 100, 0o777), ("mnt/cache/appdata", uid, gid, mode), ("mnt/cache/appdata/gitea-deploy-runners", 0, 0, 0o755), ("mnt/cache/appdata/gitea-deploy-runners/state", 0, 0, 0o755), ] for name, owner, group, permissions in dirs: path = root / name path.mkdir() os.chown(path, owner, group) path.chmod(permissions) appdata = root / "mnt/cache/appdata" state_root = appdata / "gitea-deploy-runners/state" state_path = state_root / "unraid-appops-gateway.json" before = controller._self_upgrade_canonical_json({ "schema_version": 3, "repository": controller.SELF_UPGRADE_REPOSITORY, "marker": "old", }) + b"\n" state_path.write_bytes(before) state_path.chmod(0o600) after = before.replace(b'"old"', b'"new"') if drift == "owner": os.chown(appdata, 1000, 1000) elif drift == "mode": appdata.chmod(0o775) elif drift == "foreign_cache": os.chown(root / "mnt/cache", 0, 0) (root / "mnt/cache").chmod(0o755) elif drift == "inode": replacement = appdata.with_name("different-inode") replacement.mkdir() os.chown(replacement, uid, gid) replacement.chmod(mode) elif drift == "symlink": renamed = appdata.with_name("appdata-elsewhere") appdata.rename(renamed) appdata.symlink_to(renamed, target_is_directory=True) monkeypatch.setattr(controller, "STATE_ROOT", Path("/mnt/cache/appdata/gitea-deploy-runners/state")) real_open = os.open def virtual_root_open(path, flags, *args, **kwargs): if drift == "inode" and str(path) == "appdata": return real_open(replacement, flags) return real_open(root if str(path) == "/" else path, flags, *args, **kwargs) monkeypatch.setattr(controller.os, "open", virtual_root_open) requested = controller.STATE_ROOT / state_path.name if drift == "none": assert controller._self_upgrade_read_legacy_cache_file( requested, maximum_bytes=65536, allowed_modes=frozenset({0o600}), ) == before assert not controller._self_upgrade_write_legacy_state_cas("0" * 64, after) assert state_path.read_bytes() == before assert controller._self_upgrade_write_legacy_state_cas( controller.hashlib.sha256(before).hexdigest(), after, ) assert state_path.read_bytes() == after else: with pytest.raises(OSError): controller._self_upgrade_read_legacy_cache_file( requested, maximum_bytes=65536, allowed_modes=frozenset({0o600}), ) assert not controller._self_upgrade_write_legacy_state_cas( controller.hashlib.sha256(before).hexdigest(), after, ) @pytest.mark.parametrize(("profile", "valid"), [ ((99, 100, 0o777), True), ((0, 0, 0o755), True), ((0, 0, 0o775), False), ((99, 100, 0o755), False), ]) def test_source_root_allows_only_legacy_or_root_owned_appdata_anchor( controller, monkeypatch, profile, valid, ): from types import SimpleNamespace root = Path("/srv/example/appdata/unraid-appops-gateway/source") profiles = { Path("/mnt"): (0, 0, 0o755), Path("/mnt/user"): (99, 100, 0o777), Path("/srv/example/appdata"): profile, root.parent: (0, 0, 0o755), root: (100, 101, 0o755), } def lstat(path): uid, gid, mode = profiles[path] return SimpleNamespace(st_uid=uid, st_gid=gid, st_mode=stat.S_IFDIR | mode) monkeypatch.setattr(Path, "lstat", lstat) monkeypatch.setattr(Path, "is_symlink", lambda _path: False) assert controller._self_upgrade_source_root_safe(root) is valid monkeypatch.setattr(Path, "is_symlink", lambda path: path == root.parent) assert controller._self_upgrade_source_root_safe(root) is False def _failed_revalidation_pair(controller, monkeypatch): revision = "c" * 40 semantic = controller._webhook_semantic_digest(9, "refs/heads/main", revision) policy = { "repository": controller.SELF_UPGRADE_REPOSITORY, "validation_gate_version": 1, "contexts": ["Managed validation / full (push)"], "timeout_seconds": 1800, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30, } original = { "schema": controller._WEBHOOK_SCHEMA, "semantic_sha256": semantic, "gitea_repository_id": 9, "ref": "refs/heads/main", "repository": controller.SELF_UPGRADE_REPOSITORY, "revision": revision, "status": "validation_timeout", "reason": "validation_timeout", "source": "gitea_webhook", "body_sha256": "1" * 64, "delivery": "12345678-1234-4234-8234-123456789abc", "validation": controller._validation_snapshot(policy, 1), "audit": [{"stage": "validation_timeout", "reason": "validation_timeout"}], } original["validation"]["last_result"] = "timeout" validation = controller._validation_snapshot(policy, 2) validation.update(last_result="pass", observed=[{ "context": "Managed validation / full (push)", "status": "success", }]) sibling = { **original, "schema": controller._WEBHOOK_REVALIDATION_SCHEMA, "revalidation_of": semantic, "status": "failed", "reason": "deployment_failed", "validation": validation, "audit": [{"stage": stage} for stage in ( "revalidation_requested", "validation_passed", "deployment_started", )] + [{"stage": "deployment_failed", "reason": "deployment_failed"}], } sibling.pop("source") def read(path, **_kwargs): if path == controller.WEBHOOK_INBOX_ROOT / f"{semantic}.json": record = original elif path == controller.WEBHOOK_INBOX_ROOT / f"{semantic}.revalidation.json": record = sibling else: raise AssertionError("Only the exact immutable original and fixed sibling may be read") return controller._self_upgrade_canonical_json(record), record monkeypatch.setattr(controller, "_self_upgrade_read_legacy_cache_json", read) return semantic, revision, policy, original, sibling def test_failed_revalidation_origin_binds_both_records_and_raw_sibling_bytes(controller, monkeypatch): semantic, revision, policy, original, sibling = _failed_revalidation_pair(controller, monkeypatch) selected = controller._self_upgrade_original_failed_delivery(semantic, revision, policy) assert selected is not None and selected[2] == original assert selected[1] == controller._self_upgrade_canonical_json(original) sibling_hash = controller.hashlib.sha256(controller._self_upgrade_canonical_json(sibling)).hexdigest() binding = {"failedRevalidationRawSha256": sibling_hash} assert controller._self_upgrade_origin_binding_matches(semantic, original, policy, binding) assert not controller._self_upgrade_origin_binding_matches(semantic, original, policy, {}) sibling["updated_at"] = 42 # Valid semantic data with different raw bytes must still fail the signed CAS. assert controller._self_upgrade_original_failed_delivery(semantic, revision, policy) is not None assert not controller._self_upgrade_origin_binding_matches(semantic, original, policy, binding) original["revision"] = "d" * 40 assert controller._self_upgrade_original_failed_delivery(semantic, revision, policy) is None @pytest.mark.parametrize("mutation", [ "schema", "semantic_sha256", "revalidation_of", "gitea_repository_id", "ref", "repository", "revision", "body_sha256", "delivery", "active", "interrupted", "missing_gate", "wrong_gate", "wrong_policy", "missing_audit", "unordered_audit", "null_audit", "source", "registered_legacy_request_id", "projectbrain_delivery_revalidation", "hub_delivery_revalidation", "original_not_timeout", "original_wrong_reason", "original_wrong_source", "original_bad_delivery", "original_missing_audit", "original_passed", "original_started", "original_null_status", ]) def test_failed_revalidation_origin_rejects_foreign_or_unproven_pair(controller, monkeypatch, mutation): semantic, revision, policy, original, sibling = _failed_revalidation_pair(controller, monkeypatch) if mutation == "active": sibling["status"] = "deploying" elif mutation == "interrupted": sibling["reason"] = "deployment_interrupted" elif mutation == "missing_gate": sibling["validation"]["observed"] = [] elif mutation == "wrong_gate": sibling["validation"]["observed"][0]["status"] = "pending" elif mutation == "wrong_policy": sibling["validation"]["required_contexts"] = ["Other gate"] elif mutation == "null_audit": sibling["audit"] = None elif mutation == "original_missing_audit": original["audit"] = [] elif mutation == "original_passed": original["validation"]["last_result"] = "pass" elif mutation == "original_started": original["audit"].append({"stage": "deployment_started"}) elif mutation == "original_null_status": original["status"] = [] elif mutation == "missing_audit": sibling["audit"].pop(0) elif mutation == "unordered_audit": sibling["audit"].reverse() elif mutation == "original_not_timeout": original["status"] = "validation_failed" elif mutation == "original_wrong_reason": original["reason"] = "other" elif mutation == "original_wrong_source": original["source"] = "foreign" elif mutation == "original_bad_delivery": original["delivery"] = sibling["delivery"] = "invalid" else: sibling[mutation] = "foreign" assert controller._self_upgrade_original_failed_delivery(semantic, revision, policy) is None @pytest.mark.parametrize("boundary", ["preimage", "authorization"]) def test_failed_revalidation_rechecks_pair_at_execution_boundaries(controller, monkeypatch, boundary): semantic, revision, policy, original, sibling = _failed_revalidation_pair(controller, monkeypatch) recovery_id = "12345678-1234-4234-8234-123456789abc" origin_raw = controller._self_upgrade_canonical_json(original) journal = _private_recovery_payload(controller, recovery_id) journal.update( expectedCurrentRevision=controller.SELF_UPGRADE_OLD_REVISION, originalRawSha256=controller.hashlib.sha256(origin_raw).hexdigest(), failedRevalidationRawSha256=controller.hashlib.sha256(controller._self_upgrade_canonical_json(sibling)).hexdigest(), ) monkeypatch.setattr(controller, "_self_upgrade_recovery_journal", lambda _id: journal) monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: {}}) monkeypatch.setattr(controller, "_validation_policy", lambda *_args: policy) fresh = {"status": "preflight_ok", **{key: journal[key] for key in ( "preflightSha256", "currentContainerId", "currentImageId", "currentSourceTree", )}, "currentRevision": journal["expectedCurrentRevision"], "currentRuntimeFingerprint": journal["runtimeFingerprint"]} monkeypatch.setattr(controller, "_self_upgrade_preflight", lambda _revision: fresh) bindings = {key: journal[key] for key in ( "currentContainerId", "currentImageId", "currentSourceTree", "runtimeFingerprint", "registryFingerprint", "sourceFingerprint", "stateSha256", "runtimeEnvironment", )} bindings["currentRevision"] = journal["expectedCurrentRevision"] monkeypatch.setattr(controller, "_self_upgrade_recovery_current_bindings", lambda: bindings) delivery = {"semantic_sha256": semantic, "revision": revision, "recovery_id": recovery_id, "status": "deploying", "validation": {"last_result": "pass"}, "audit": [{"stage": "validation_passed"}, {"stage": "deployment_started"}]} request = { "schema": controller._SELF_UPGRADE_RECOVERY_REQUEST_SCHEMA, "revision": revision, "preflightSha256": journal["preflightSha256"], "operationId": semantic, "reasonSha256": "f" * 64, "idempotencyKeySha256": journal["idempotencyKeySha256"], "recoveryId": recovery_id, "recoveryOfSha256": journal["originalRawSha256"], "failedRevalidationRawSha256": journal["failedRevalidationRawSha256"], "createdAt": 1, } secret = b"fixed-recovery-boundary-key-0123456789" monkeypatch.setattr(controller, "_self_upgrade_secret", lambda: secret) signed_request = {**request, "hmac": controller._self_upgrade_mac( secret, "recovery-request", controller._self_upgrade_canonical_json(request))} monkeypatch.setattr(controller, "_self_upgrade_request_root", lambda **_kwargs: controller.SELF_UPGRADE_RECOVERY_REQUEST_ROOT) def read_private(path, **_kwargs): record = delivery if path.name.endswith(".delivery.json") else signed_request return controller._self_upgrade_canonical_json(record), record monkeypatch.setattr(controller, "_self_upgrade_read_private_json", read_private) # The separate HMAC admission is covered by journal/request tests; this isolates the immutable pair CAS. monkeypatch.setattr(controller, "_self_upgrade_recovery_record_valid", lambda *_args, **_kwargs: True) def check(): if boundary == "preimage": return controller._self_upgrade_recovery_preimage_matches(delivery) return controller._self_upgrade_recovery_authorization(recovery_id) is not None assert check() sibling["updated_at"] = 99 assert not check() sibling.pop("updated_at") assert check() original["updated_at"] = 99 assert not check()