from __future__ import annotations import hashlib import importlib.util import json import subprocess import sys import tarfile import threading import time import urllib.error import urllib.request from http.server import ThreadingHTTPServer from pathlib import Path from types import SimpleNamespace import pytest ROOT = Path(__file__).resolve().parents[2] CONTROLLER = ROOT / "ops" / "gitea-deploy-broker" @pytest.fixture def controller(monkeypatch, tmp_path): monkeypatch.syspath_prepend(str(CONTROLLER)) spec = importlib.util.spec_from_file_location("self_upgrade_controller_test", CONTROLLER / "status_server.py") assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", tmp_path / "inbox") module.WEBHOOK_INBOX_ROOT.mkdir(mode=0o700) return module @pytest.fixture def deploy_module(monkeypatch): if sys.platform == "win32": pytest.skip("controller deploy.py uses Linux fcntl") monkeypatch.syspath_prepend(str(CONTROLLER)) spec = importlib.util.spec_from_file_location("self_upgrade_deploy_test", CONTROLLER / "deploy.py") assert spec and spec.loader module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) return module def _project(): return { "branch": "main", "gitea_repository_id": 9, "gitea_webhook_secret_file": "jens__unraid-appops-gateway", "containers": ["Unraid-AppOps-Gateway"], "compose_project": "unraid-appops-gateway", "compose_files": ["docker-compose.yml"], "environment": {"APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216"}, "verify_oci_revision": True, "validation": { "validation_gate_version": 1, "required": True, "contexts": ["Managed validation / full (push)"], "timeout_seconds": 1800, "poll_interval_seconds": 5, "max_poll_interval_seconds": 30, }, } def _ready(controller, monkeypatch): monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: _project()}) monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: True) monkeypatch.setattr(controller, "_is_current_branch_head", lambda repository, project, revision: True) monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda repository, revision, policy: ("pass", [])) monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: { "revision": "a" * 40, "imageId": "sha256:" + "b" * 64, "dataPath": "/srv/example/appdata/unraid-appops-gateway-1216", }) monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3)) def test_fixed_self_upgrade_preflight_accepts_exact_validated_head(controller, monkeypatch): _ready(controller, monkeypatch) result = controller._self_upgrade_preflight("c" * 40) assert result["status"] == "preflight_ok" assert result["currentRevision"] == "a" * 40 assert len(result["preflightSha256"]) == 64 assert result["executed"] is False assert all(check["passed"] for check in result["checks"]) def test_timed_out_delivery_recovers_only_after_exact_protected_push_pass(controller, monkeypatch): _ready(controller, monkeypatch) revision = "c" * 40 digest = controller._webhook_semantic_digest(9, "refs/heads/main", revision) policy = controller._validation_policy(controller.SELF_UPGRADE_REPOSITORY, _project()) original = { "schema": controller._WEBHOOK_SCHEMA, "semantic_sha256": digest, "body_sha256": "b" * 64, "gitea_repository_id": 9, "ref": "refs/heads/main", "repository": controller.SELF_UPGRADE_REPOSITORY, "revision": revision, "status": "validation_timeout", "validation": controller._validation_snapshot(policy, int(time.time())), } marker = controller.WEBHOOK_INBOX_ROOT / f"{digest}.json" marker.write_text(json.dumps(original), encoding="utf-8") monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("wait", [])) controller._auto_revalidate_timed_out_appops(marker) revalidation = controller.WEBHOOK_INBOX_ROOT / f"{digest}.revalidation.json" assert not revalidation.exists() controller._appops_revalidation_next_check.clear() monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("pass", [])) monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: False) controller._auto_revalidate_timed_out_appops(marker) assert not revalidation.exists() controller._appops_revalidation_next_check.clear() monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: True) controller._auto_revalidate_timed_out_appops(marker) assert json.loads(revalidation.read_text())["status"] == "awaiting_validation" assert json.loads(marker.read_text())["status"] == "validation_timeout" updated = json.loads(revalidation.read_text()) updated["status"] = "deployed" revalidation.write_text(json.dumps(updated), encoding="utf-8") assert controller._self_upgrade_delivery_record(digest)["status"] == "deployed" @pytest.mark.parametrize("gate", ["head", "validation", "runtime", "disk"]) def test_self_upgrade_preflight_fails_closed_on_gate(controller, monkeypatch, gate): _ready(controller, monkeypatch) if gate == "head": monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: False) elif gate == "validation": monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("wait", [])) elif gate == "runtime": monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: None) else: monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=1)) result = controller._self_upgrade_preflight("c" * 40) assert result["status"] == "blocked" assert "preflightSha256" not in result def test_self_upgrade_preflight_rejects_non_sha_and_conflicting_upgrade(controller, monkeypatch): _ready(controller, monkeypatch) assert controller._self_upgrade_preflight("main")["status"] == "blocked" marker = controller.WEBHOOK_INBOX_ROOT / ("e" * 64 + ".json") marker.write_text(json.dumps({ "repository": controller.SELF_UPGRADE_REPOSITORY, "revision": "d" * 40, "status": "deploying", }), encoding="utf-8") result = controller._self_upgrade_preflight("c" * 40) assert result["status"] == "blocked" assert result["checks"][-1] == {"name": "no_conflicting_upgrade", "passed": False} def test_runtime_requires_exact_container_mount_port_and_health(controller, monkeypatch): payload = [{ "State": {"Running": True, "Health": {"Status": "healthy"}}, "Mounts": [{"Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True}], "Config": {"Labels": {"org.opencontainers.image.revision": "a" * 40, "org.opencontainers.image.source": "jens/unraid-appops-gateway"}}, "NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}}, "Image": "sha256:" + "b" * 64, }] names = ["Unraid-AppOps-Gateway"] def fake_run(args, **kwargs): return subprocess.CompletedProcess(args, 0, json.dumps(payload) if args[1] == "inspect" else "\n".join(names), "") monkeypatch.setattr(controller.subprocess, "run", fake_run) assert controller._self_upgrade_runtime()["revision"] == "a" * 40 payload[0]["Mounts"][0]["Source"] = "/wrong" assert controller._self_upgrade_runtime() is None payload[0]["Mounts"][0]["Source"] = "/srv/example/appdata/unraid-appops-gateway-1216" names.append("Unraid-AppOps-Gateway-copy") assert controller._self_upgrade_runtime() is None def test_mcp_self_upgrade_rejects_arbitrary_repository(monkeypatch): from backend.app import mcp_server with pytest.raises(ValueError, match="repository must be"): mcp_server.appops_self_upgrade_preflight("other/repo", "a" * 40, "test reason") with pytest.raises(ValueError, match="repository must be"): mcp_server.appops_self_upgrade_apply("other/repo", "a" * 40, "test reason", "b" * 64, "key-12345") def test_self_upgrade_signature_binds_endpoint_and_rejects_replay(controller, monkeypatch, tmp_path): secret = b"fixed-test-self-upgrade-capability-0123456789" token_file = tmp_path / "mcp-token" token_file.write_bytes(secret + b"\n") token_file.chmod(0o600) monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file) monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces") stamp = str(int(time.time())) nonce = "a" * 32 body = b'{"revision":"' + b"b" * 40 + b'"}' path = controller.SELF_UPGRADE_PREFLIGHT_PATH signature = controller._request_signature( secret, method="POST", path=path, timestamp=stamp, nonce=nonce, raw_body=body, ) assert not controller._self_upgrade_signature_is_valid( body, stamp, nonce, signature, method="POST", path=controller.SELF_UPGRADE_APPLY_PATH, ) assert controller._self_upgrade_signature_is_valid( body, stamp, nonce, signature, method="POST", path=path, ) assert not controller._self_upgrade_signature_is_valid( body, stamp, nonce, signature, method="POST", path=path, ) def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller, monkeypatch, tmp_path): _ready(controller, monkeypatch) monkeypatch.setattr(controller, "SELF_UPGRADE_REQUEST_ROOT", tmp_path / "requests") monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces") monkeypatch.setattr(controller, "APPOPS_PROGRESS_ROOT", tmp_path / "progress") secret = b"fixed-test-self-upgrade-capability-0123456789" token_file = tmp_path / "mcp-token" token_file.write_bytes(secret + b"\n") token_file.chmod(0o600) monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file) monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None) server = ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() base = f"http://127.0.0.1:{server.server_port}" counter = 0 def call(path, payload, *, method="POST"): nonlocal counter counter += 1 body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() if payload is not None else b"" nonce = f"{counter:032x}" stamp = str(int(time.time())) signature = controller._request_signature( secret, method=method, path=path, timestamp=stamp, nonce=nonce, raw_body=body, ) request = urllib.request.Request( base + path, data=body if method == "POST" else None, method=method, headers={ "Content-Type": "application/json", "X-ITWorx-Deployment-Timestamp": stamp, "X-ITWorx-Deployment-Nonce": nonce, "X-ITWorx-Deployment-Signature": signature, }, ) try: with urllib.request.urlopen(request, timeout=5) as response: return response.status, json.load(response) except urllib.error.HTTPError as error: return error.code, json.load(error) try: intent = { "repository": "jens/unraid-appops-gateway", "revision": "c" * 40, "reason": "Validated production upgrade", "expectedCurrentRevision": "a" * 40, } invalid = {**intent, "image": "attacker/image:latest"} assert call(controller.SELF_UPGRADE_PREFLIGHT_PATH, invalid)[0] == 400 code, preflight = call(controller.SELF_UPGRADE_PREFLIGHT_PATH, intent) assert code == 200 and preflight["status"] == "preflight_ok" apply = {**intent, "preflightSha256": preflight["preflightSha256"], "idempotencyKey": "fixed-request-12345"} code, accepted = call(controller.SELF_UPGRADE_APPLY_PATH, apply) assert code == 202 and accepted["status"] == "accepted", accepted operation_id = accepted["operationId"] assert len(operation_id) == 64 assert (controller.SELF_UPGRADE_REQUEST_ROOT / ( hashlib.sha256(apply["idempotencyKey"].encode()).hexdigest() + ".json" )).exists() code, replay = call(controller.SELF_UPGRADE_APPLY_PATH, apply) assert code == 200 and replay["status"] == "already_queued" assert replay["operationId"] == operation_id assert call(controller.SELF_UPGRADE_APPLY_PATH, {**apply, "revision": "d" * 40})[0] == 409 code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET") assert code == 200 and status["state"] == "awaiting_validation" assert status["operationState"] == "queued" assert status["revision"] == "c" * 40 controller.APPOPS_PROGRESS_ROOT.mkdir(mode=0o700) (controller.APPOPS_PROGRESS_ROOT / ("c" * 40 + ".json")).write_text(json.dumps({ "schema": "itworx.appops-upgrade-progress/v1", "revision": "c" * 40, "phase": "building", }), encoding="utf-8") delivery_path = controller.WEBHOOK_INBOX_ROOT / f"{operation_id}.json" delivery_record = json.loads(delivery_path.read_text(encoding="utf-8")) delivery_record["status"] = "deploying" delivery_path.write_text(json.dumps(delivery_record), encoding="utf-8") code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET") assert code == 200 and status["operationState"] == "building" delivery_record["status"] = "validation_timeout" delivery_path.write_text(json.dumps(delivery_record), encoding="utf-8") recovered = {**delivery_record, "schema": controller._WEBHOOK_REVALIDATION_SCHEMA, "revalidation_of": operation_id, "status": "deployed"} (controller.WEBHOOK_INBOX_ROOT / f"{operation_id}.revalidation.json").write_text( json.dumps(recovered), encoding="utf-8" ) monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: { "revision": "c" * 40, "imageId": "sha256:" + "b" * 64, "dataPath": "/srv/example/appdata/unraid-appops-gateway-1216", }) monkeypatch.setattr(controller, "_deployment_state", lambda slug: {}) monkeypatch.setattr(controller, "_verified_receipt", lambda *args: {"revision": "c" * 40}) code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET") assert code == 200 and status["operationState"] == "succeeded" assert status["state"] == "deployed" code, preflight = call(controller.SELF_UPGRADE_PREFLIGHT_PATH, { **intent, "expectedCurrentRevision": "c" * 40, }) assert code == 200 and preflight["status"] == "already_deployed" already = {**intent, "expectedCurrentRevision": "c" * 40, "preflightSha256": preflight["preflightSha256"], "idempotencyKey": "already-deployed-request-12345"} code, result = call(controller.SELF_UPGRADE_APPLY_PATH, already) assert code == 200 and result["status"] == "already_deployed" assert result["operationId"] == operation_id code, result = call(controller.SELF_UPGRADE_APPLY_PATH, already) assert code == 200 and result["status"] == "already_deployed" finally: server.shutdown() thread.join(timeout=5) server.server_close() def test_appops_deploy_rechecks_exact_main_head_before_mutation(deploy_module, monkeypatch): revision = "c" * 40 metadata = {"builtAt": "2026-09-28T00:00:00.000Z"} monkeypatch.setattr(deploy_module, "validate_commit", lambda repo, branch, value: value) monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *args, **kwargs: metadata) monkeypatch.setattr(deploy_module, "output", lambda args: "d" * 40) with pytest.raises(deploy_module.DeployError, match="exact protected main head"): deploy_module.revalidate_deployment_candidate( Path("/repo.git"), {"branch": "main"}, revision, "jens/unraid-appops-gateway", metadata, ) monkeypatch.setattr(deploy_module, "output", lambda args: revision) deploy_module.revalidate_deployment_candidate( Path("/repo.git"), {"branch": "main"}, revision, "jens/unraid-appops-gateway", metadata, ) def test_appops_backup_is_fixed_and_digest_verified(deploy_module, monkeypatch, tmp_path): data = tmp_path / "appops-data" data.mkdir() (data / "state.txt").write_text("preserved", encoding="utf-8") monkeypatch.setattr(deploy_module, "APPOPS_DATA_ROOT", data) monkeypatch.setattr(deploy_module, "APPOPS_BACKUP_ROOT", tmp_path / "backups") monkeypatch.setattr( deploy_module.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3) ) receipt = deploy_module.capture_appops_data_backup("c" * 40, "a" * 40) archive = Path(receipt["archive"]) assert hashlib.sha256(archive.read_bytes()).hexdigest() == receipt["sha256"] assert receipt["previousRevision"] == "a" * 40 assert json.loads((archive.parent / "receipt.json").read_text()) == receipt with tarfile.open(archive, "r:gz") as backup: assert backup.extractfile("./state.txt").read() == b"preserved" def test_appops_backup_rejects_symlinked_data(deploy_module, monkeypatch, tmp_path): data = tmp_path / "real-data" data.mkdir() link = tmp_path / "link-data" link.symlink_to(data, target_is_directory=True) monkeypatch.setattr(deploy_module, "APPOPS_DATA_ROOT", link) with pytest.raises(deploy_module.DeployError, match="unsafe"): deploy_module.capture_appops_data_backup("c" * 40, "a" * 40) def test_appops_progress_is_durable_and_bounded(deploy_module, monkeypatch, tmp_path): monkeypatch.setattr(deploy_module, "APPOPS_PROGRESS_ROOT", tmp_path / "progress") deploy_module.write_appops_progress("c" * 40, "building") path = deploy_module.APPOPS_PROGRESS_ROOT / ("c" * 40 + ".json") assert json.loads(path.read_text())["phase"] == "building" deploy_module.write_appops_progress("c" * 40, "verifying") assert json.loads(path.read_text())["phase"] == "verifying" with pytest.raises(deploy_module.DeployError, match="invalid"): deploy_module.write_appops_progress("c" * 40, "arbitrary-command")