Files
Unraid-AppOps-Gateway-Public/backend/tests/test_appops_self_upgrade_controller.py
T

461 lines
23 KiB
Python

from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import subprocess
import sys
import tarfile
import threading
import time
import urllib.error
import urllib.request
from http.server import ThreadingHTTPServer
from pathlib import Path
from types import SimpleNamespace
import pytest
ROOT = Path(__file__).resolve().parents[2]
CONTROLLER = ROOT / "ops" / "gitea-deploy-broker"
@pytest.fixture
def controller(monkeypatch, tmp_path):
monkeypatch.syspath_prepend(str(CONTROLLER))
spec = importlib.util.spec_from_file_location("self_upgrade_controller_test", CONTROLLER / "status_server.py")
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", tmp_path / "inbox")
module.WEBHOOK_INBOX_ROOT.mkdir(mode=0o700)
return module
@pytest.fixture
def deploy_module(monkeypatch):
if sys.platform == "win32":
pytest.skip("controller deploy.py uses Linux fcntl")
monkeypatch.syspath_prepend(str(CONTROLLER))
spec = importlib.util.spec_from_file_location("self_upgrade_deploy_test", CONTROLLER / "deploy.py")
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _project():
return {
"branch": "main", "gitea_repository_id": 9,
"gitea_webhook_secret_file": "jens__unraid-appops-gateway",
"containers": ["Unraid-AppOps-Gateway"],
"compose_project": "unraid-appops-gateway",
"compose_files": ["docker-compose.yml"],
"environment": {
"APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216",
"APPOPS_PUBLISH_HOST": "198.18.10.150",
},
"verify_oci_revision": True,
"validation": {
"validation_gate_version": 1, "required": True,
"contexts": ["Managed validation / full (push)"],
"timeout_seconds": 1800, "poll_interval_seconds": 5,
"max_poll_interval_seconds": 30,
},
}
def _ready(controller, monkeypatch):
monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: _project()})
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: True)
monkeypatch.setattr(controller, "_is_current_branch_head", lambda repository, project, revision: True)
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda repository, revision, policy: ("pass", []))
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {
"revision": "a" * 40, "imageId": "sha256:" + "b" * 64,
"containerId": "c" * 64,
"imageRef": "unraid-appops-gateway:production",
"sourceTree": "d" * 40,
"dataPath": "/srv/example/appdata/unraid-appops-gateway-1216",
})
monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3))
def test_fixed_self_upgrade_preflight_accepts_exact_validated_head(controller, monkeypatch):
_ready(controller, monkeypatch)
result = controller._self_upgrade_preflight("c" * 40)
assert result["status"] == "preflight_ok"
assert result["currentRevision"] == "a" * 40
assert len(result["preflightSha256"]) == 64
assert result["executed"] is False
assert all(check["passed"] for check in result["checks"])
def test_timed_out_delivery_recovers_only_after_exact_protected_push_pass(controller, monkeypatch):
_ready(controller, monkeypatch)
revision = "c" * 40
digest = controller._webhook_semantic_digest(9, "refs/heads/main", revision)
policy = controller._validation_policy(controller.SELF_UPGRADE_REPOSITORY, _project())
original = {
"schema": controller._WEBHOOK_SCHEMA, "semantic_sha256": digest,
"body_sha256": "b" * 64, "gitea_repository_id": 9,
"ref": "refs/heads/main", "repository": controller.SELF_UPGRADE_REPOSITORY,
"revision": revision, "status": "validation_timeout",
"validation": controller._validation_snapshot(policy, int(time.time())),
}
marker = controller.WEBHOOK_INBOX_ROOT / f"{digest}.json"
marker.write_text(json.dumps(original), encoding="utf-8")
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("wait", []))
controller._auto_revalidate_timed_out_appops(marker)
revalidation = controller.WEBHOOK_INBOX_ROOT / f"{digest}.revalidation.json"
assert not revalidation.exists()
controller._appops_revalidation_next_check.clear()
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("pass", []))
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: False)
controller._auto_revalidate_timed_out_appops(marker)
assert not revalidation.exists()
controller._appops_revalidation_next_check.clear()
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: True)
controller._auto_revalidate_timed_out_appops(marker)
assert json.loads(revalidation.read_text())["status"] == "awaiting_validation"
assert json.loads(marker.read_text())["status"] == "validation_timeout"
updated = json.loads(revalidation.read_text())
updated["status"] = "deployed"
revalidation.write_text(json.dumps(updated), encoding="utf-8")
assert controller._self_upgrade_delivery_record(digest)["status"] == "deployed"
@pytest.mark.parametrize("gate", ["head", "validation", "runtime", "disk"])
def test_self_upgrade_preflight_fails_closed_on_gate(controller, monkeypatch, gate):
_ready(controller, monkeypatch)
if gate == "head":
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: False)
elif gate == "validation":
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("wait", []))
elif gate == "runtime":
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: None)
else:
monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=1))
result = controller._self_upgrade_preflight("c" * 40)
assert result["status"] == "blocked"
assert "preflightSha256" not in result
def test_self_upgrade_preflight_rejects_non_sha_and_conflicting_upgrade(controller, monkeypatch):
_ready(controller, monkeypatch)
assert controller._self_upgrade_preflight("main")["status"] == "blocked"
marker = controller.WEBHOOK_INBOX_ROOT / ("e" * 64 + ".json")
marker.write_text(json.dumps({
"repository": controller.SELF_UPGRADE_REPOSITORY,
"revision": "d" * 40, "status": "deploying",
}), encoding="utf-8")
result = controller._self_upgrade_preflight("c" * 40)
assert result["status"] == "blocked"
assert result["checks"][-1] == {"name": "no_conflicting_upgrade", "passed": False}
def test_runtime_requires_exact_container_mount_port_and_health(controller, monkeypatch):
monkeypatch.setattr(controller, "_gitea_validation_settings", lambda: ("https://gitea.example.com/api/v1", b"t" * 32))
payload = [{
"State": {"Running": True, "Health": {"Status": "healthy"}},
"Id": "1" * 64,
"Mounts": [
{"Type": "bind", "Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True},
{"Type": "bind", "Destination": "/mnt/privatemuse-comfyui-models", "Source": "/srv/example/appdata/comfyui-nvidia/basedir/models", "RW": True},
],
"Config": {
"Image": "unraid-appops-gateway:platform-5553c04ce497",
"Hostname": "unraid-appops-gateway",
"Labels": {
"com.docker.compose.project": "unraid-appops-gateway",
"com.docker.compose.service": "appops-gateway",
"com.docker.compose.project.working_dir": "/srv/example/appdata/unraid-appops-gateway/source",
"com.docker.compose.project.config_files": "/srv/example/appdata/unraid-appops-gateway/source/docker-compose.yml",
"org.opencontainers.image.revision": "5553c04ce497c08a35ed301052dc5e0b02e5d9b4",
"org.opencontainers.image.source": "https://gitea.example.com/Jens/Unraid-AppOps-Gateway",
"org.opencontainers.image.source-tree": "1624e1748053f998d8427c8c8a4203277d22f13c",
"org.opencontainers.image.version": "1.8.0-platform-5553c04ce497",
"org.opencontainers.image.created": "2026-10-01T16:32:01Z",
},
},
"HostConfig": {"RestartPolicy": {"Name": "unless-stopped"}},
"NetworkSettings": {
"Ports": {"1216/tcp": [{"HostIp": "198.18.10.150", "HostPort": "1216"}]},
"Networks": {"itworx-mcp-hub-npm": {}, "unraid-appops-gateway_default": {}},
},
"Image": "sha256:" + "b" * 64,
}]
image = [{"Id": payload[0]["Image"], "Config": {"Labels": {
key: value for key, value in payload[0]["Config"]["Labels"].items()
if key.startswith("org.opencontainers.image.")
}}}]
names = ["Unraid-AppOps-Gateway"]
def fake_run(args, **kwargs):
if args[1] == "inspect":
result = payload
elif args[1] == "image":
result = image
else:
result = "\n".join(names)
return subprocess.CompletedProcess(args, 0, json.dumps(result) if isinstance(result, list) else result, "")
monkeypatch.setattr(controller.subprocess, "run", fake_run)
runtime = controller._self_upgrade_runtime()
assert runtime["revision"] == "5553c04ce497c08a35ed301052dc5e0b02e5d9b4"
assert runtime["imageRef"] == "unraid-appops-gateway:platform-5553c04ce497"
assert runtime["sourceTree"] == "1624e1748053f998d8427c8c8a4203277d22f13c"
payload[0]["Mounts"][0]["Source"] = "/wrong"
assert controller._self_upgrade_runtime() is None
payload[0]["Mounts"][0]["Source"] = "/srv/example/appdata/unraid-appops-gateway-1216"
payload[0]["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostIp"] = "127.0.0.1"
assert controller._self_upgrade_runtime() is None
payload[0]["NetworkSettings"]["Ports"]["1216/tcp"][0]["HostIp"] = "198.18.10.150"
image[0]["Id"] = "sha256:" + "0" * 64
assert controller._self_upgrade_runtime() is None
image[0]["Id"] = payload[0]["Image"]
names.append("Unraid-AppOps-Gateway-copy")
assert controller._self_upgrade_runtime() is None
@pytest.mark.parametrize(
("source", "accepted"),
[
("jens/unraid-appops-gateway", True),
("https://gitea.example.com/Jens/Unraid-AppOps-Gateway", True),
("other/repository", False),
("https://evil.example/Jens/Unraid-AppOps-Gateway", False),
("https://gitea.example.com/Jens/Other-App", False),
("https://gitea.example.com/Jens/Unraid-AppOps-Gateway.evil", False),
("http://gitea.example.com/Jens/Unraid-AppOps-Gateway", False),
],
)
def test_source_validation_accepts_only_fixed_appops_oci_sources(controller, monkeypatch, source, accepted):
monkeypatch.setattr(
controller,
"_gitea_validation_settings",
lambda: ("https://gitea.example.com/api/v1", b"t" * 32),
)
assert controller._self_upgrade_source_is_valid(source) is accepted
def test_mcp_self_upgrade_rejects_arbitrary_repository(monkeypatch):
from backend.app import mcp_server
with pytest.raises(ValueError, match="repository must be"):
mcp_server.appops_self_upgrade_preflight("other/repo", "a" * 40, "test reason")
with pytest.raises(ValueError, match="repository must be"):
mcp_server.appops_self_upgrade_apply("other/repo", "a" * 40, "test reason", "b" * 64, "key-12345")
@pytest.mark.skipif(sys.platform == 'win32', reason='Linux controller pinned directory descriptors')
def test_self_upgrade_signature_binds_endpoint_and_rejects_replay(controller, monkeypatch, tmp_path):
secret = b"fixed-test-self-upgrade-capability-0123456789"
token_file = tmp_path / "mcp-token"
token_file.write_bytes(secret + b"\n")
token_file.chmod(0o600)
import runner_statistics_binding
monkeypatch.setattr(runner_statistics_binding, 'APPOPS_DATA', tmp_path)
monkeypatch.setattr(runner_statistics_binding, 'ADMIN_TOKEN_FILE', token_file)
monkeypatch.setattr(runner_statistics_binding, 'ROOT_UID', os.getuid())
monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
stamp = str(int(time.time()))
nonce = "a" * 32
body = b'{"revision":"' + b"b" * 40 + b'"}'
path = controller.SELF_UPGRADE_PREFLIGHT_PATH
signature = controller._request_signature(
secret, method="POST", path=path, timestamp=stamp,
nonce=nonce, raw_body=body,
)
assert not controller._self_upgrade_signature_is_valid(
body, stamp, nonce, signature, method="POST", path=controller.SELF_UPGRADE_APPLY_PATH,
)
assert controller._self_upgrade_signature_is_valid(
body, stamp, nonce, signature, method="POST", path=path,
)
assert not controller._self_upgrade_signature_is_valid(
body, stamp, nonce, signature, method="POST", path=path,
)
@pytest.mark.skipif(sys.platform == 'win32', reason='Linux controller pinned directory descriptors')
def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller, monkeypatch, tmp_path):
_ready(controller, monkeypatch)
monkeypatch.setattr(controller, "SELF_UPGRADE_REQUEST_ROOT", tmp_path / "requests")
monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
monkeypatch.setattr(controller, "APPOPS_PROGRESS_ROOT", tmp_path / "progress")
secret = b"fixed-test-self-upgrade-capability-0123456789"
token_file = tmp_path / "mcp-token"
token_file.write_bytes(secret + b"\n")
token_file.chmod(0o600)
import runner_statistics_binding
monkeypatch.setattr(runner_statistics_binding, 'APPOPS_DATA', tmp_path)
monkeypatch.setattr(runner_statistics_binding, 'ADMIN_TOKEN_FILE', token_file)
monkeypatch.setattr(runner_statistics_binding, 'ROOT_UID', os.getuid())
monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None)
server = ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
base = f"http://127.0.0.1:{server.server_port}"
counter = 0
def call(path, payload, *, method="POST"):
nonlocal counter
counter += 1
body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() if payload is not None else b""
nonce = f"{counter:032x}"
stamp = str(int(time.time()))
signature = controller._request_signature(
secret, method=method, path=path, timestamp=stamp,
nonce=nonce, raw_body=body,
)
request = urllib.request.Request(
base + path, data=body if method == "POST" else None,
method=method,
headers={
"Content-Type": "application/json", "X-ITWorx-Deployment-Timestamp": stamp,
"X-ITWorx-Deployment-Nonce": nonce,
"X-ITWorx-Deployment-Signature": signature,
},
)
try:
with urllib.request.urlopen(request, timeout=5) as response:
return response.status, json.load(response)
except urllib.error.HTTPError as error:
return error.code, json.load(error)
try:
intent = {
"repository": "jens/unraid-appops-gateway", "revision": "c" * 40,
"reason": "Validated production upgrade", "expectedCurrentRevision": "a" * 40,
}
invalid = {**intent, "image": "attacker/image:latest"}
assert call(controller.SELF_UPGRADE_PREFLIGHT_PATH, invalid)[0] == 400
code, preflight = call(controller.SELF_UPGRADE_PREFLIGHT_PATH, intent)
assert code == 200 and preflight["status"] == "preflight_ok"
apply = {**intent, "preflightSha256": preflight["preflightSha256"],
"idempotencyKey": "fixed-request-12345"}
code, accepted = call(controller.SELF_UPGRADE_APPLY_PATH, apply)
assert code == 202 and accepted["status"] == "accepted", accepted
operation_id = accepted["operationId"]
assert len(operation_id) == 64
assert (controller.SELF_UPGRADE_REQUEST_ROOT / (
hashlib.sha256(apply["idempotencyKey"].encode()).hexdigest() + ".json"
)).exists()
code, replay = call(controller.SELF_UPGRADE_APPLY_PATH, apply)
assert code == 200 and replay["status"] == "already_queued"
assert replay["operationId"] == operation_id
assert call(controller.SELF_UPGRADE_APPLY_PATH, {**apply, "revision": "d" * 40})[0] == 409
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
assert code == 200 and status["state"] == "awaiting_validation"
assert status["operationState"] == "queued"
assert status["revision"] == "c" * 40
controller.APPOPS_PROGRESS_ROOT.mkdir(mode=0o700)
(controller.APPOPS_PROGRESS_ROOT / ("c" * 40 + ".json")).write_text(json.dumps({
"schema": "itworx.appops-upgrade-progress/v1", "revision": "c" * 40,
"phase": "building",
}), encoding="utf-8")
delivery_path = controller.WEBHOOK_INBOX_ROOT / f"{operation_id}.json"
delivery_record = json.loads(delivery_path.read_text(encoding="utf-8"))
delivery_record["status"] = "deploying"
delivery_path.write_text(json.dumps(delivery_record), encoding="utf-8")
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
assert code == 200 and status["operationState"] == "building"
delivery_record["status"] = "validation_timeout"
delivery_path.write_text(json.dumps(delivery_record), encoding="utf-8")
recovered = {**delivery_record, "schema": controller._WEBHOOK_REVALIDATION_SCHEMA,
"revalidation_of": operation_id, "status": "deployed"}
(controller.WEBHOOK_INBOX_ROOT / f"{operation_id}.revalidation.json").write_text(
json.dumps(recovered), encoding="utf-8"
)
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {
"revision": "c" * 40, "imageId": "sha256:" + "b" * 64,
"containerId": "d" * 64,
"imageRef": "unraid-appops-gateway:production",
"sourceTree": "e" * 40,
"dataPath": "/srv/example/appdata/unraid-appops-gateway-1216",
})
monkeypatch.setattr(controller, "_deployment_state", lambda slug: {})
monkeypatch.setattr(controller, "_verified_receipt", lambda *args: {"revision": "c" * 40})
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
assert code == 200 and status["operationState"] == "succeeded"
assert status["state"] == "deployed"
code, preflight = call(controller.SELF_UPGRADE_PREFLIGHT_PATH, {
**intent, "expectedCurrentRevision": "c" * 40,
})
assert code == 200 and preflight["status"] == "already_deployed"
already = {**intent, "expectedCurrentRevision": "c" * 40,
"preflightSha256": preflight["preflightSha256"],
"idempotencyKey": "already-deployed-request-12345"}
code, result = call(controller.SELF_UPGRADE_APPLY_PATH, already)
assert code == 200 and result["status"] == "already_deployed"
assert result["operationId"] == operation_id
code, result = call(controller.SELF_UPGRADE_APPLY_PATH, already)
assert code == 200 and result["status"] == "already_deployed"
finally:
server.shutdown()
thread.join(timeout=5)
server.server_close()
def test_appops_deploy_rechecks_exact_main_head_before_mutation(deploy_module, monkeypatch):
revision = "c" * 40
metadata = {"builtAt": "2026-09-28T00:00:00.000Z"}
monkeypatch.setattr(deploy_module, "validate_commit", lambda repo, branch, value: value)
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *args, **kwargs: metadata)
monkeypatch.setattr(deploy_module, "output", lambda args: "d" * 40)
with pytest.raises(deploy_module.DeployError, match="exact protected main head"):
deploy_module.revalidate_deployment_candidate(
Path("/repo.git"), {"branch": "main"}, revision,
"jens/unraid-appops-gateway", metadata,
)
monkeypatch.setattr(deploy_module, "output", lambda args: revision)
deploy_module.revalidate_deployment_candidate(
Path("/repo.git"), {"branch": "main"}, revision,
"jens/unraid-appops-gateway", metadata,
)
def test_appops_backup_is_fixed_and_digest_verified(deploy_module, monkeypatch, tmp_path):
data = tmp_path / "appops-data"
data.mkdir()
(data / "state.txt").write_text("preserved", encoding="utf-8")
monkeypatch.setattr(deploy_module, "APPOPS_DATA_ROOT", data)
monkeypatch.setattr(deploy_module, "APPOPS_BACKUP_ROOT", tmp_path / "backups")
monkeypatch.setattr(
deploy_module.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3)
)
receipt = deploy_module.capture_appops_data_backup("c" * 40, "a" * 40)
archive = Path(receipt["archive"])
assert hashlib.sha256(archive.read_bytes()).hexdigest() == receipt["sha256"]
assert receipt["previousRevision"] == "a" * 40
stored_receipt = json.loads((archive.parent / "receipt.json").read_text())
assert {**stored_receipt, "receiptSha256": hashlib.sha256(
json.dumps(stored_receipt, sort_keys=True, separators=(",", ":")).encode()
).hexdigest()} == receipt
with tarfile.open(archive, "r:gz") as backup:
assert backup.extractfile("./state.txt").read() == b"preserved"
def test_appops_backup_rejects_symlinked_data(deploy_module, monkeypatch, tmp_path):
data = tmp_path / "real-data"
data.mkdir()
link = tmp_path / "link-data"
link.symlink_to(data, target_is_directory=True)
monkeypatch.setattr(deploy_module, "APPOPS_DATA_ROOT", link)
with pytest.raises(deploy_module.DeployError, match="unsafe"):
deploy_module.capture_appops_data_backup("c" * 40, "a" * 40)
def test_appops_progress_is_durable_and_bounded(deploy_module, monkeypatch, tmp_path):
monkeypatch.setattr(deploy_module, "APPOPS_PROGRESS_ROOT", tmp_path / "progress")
deploy_module.write_appops_progress("c" * 40, "building")
path = deploy_module.APPOPS_PROGRESS_ROOT / ("c" * 40 + ".json")
assert json.loads(path.read_text())["phase"] == "building"
deploy_module.write_appops_progress("c" * 40, "verifying")
assert json.loads(path.read_text())["phase"] == "verifying"
with pytest.raises(deploy_module.DeployError, match="invalid"):
deploy_module.write_appops_progress("c" * 40, "arbitrary-command")