416 lines
20 KiB
Python
416 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from http.server import ThreadingHTTPServer
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
CONTROLLER = ROOT / "ops" / "gitea-deploy-broker"
|
|
|
|
|
|
@pytest.fixture
|
|
def controller(monkeypatch, tmp_path):
|
|
monkeypatch.syspath_prepend(str(CONTROLLER))
|
|
spec = importlib.util.spec_from_file_location("self_upgrade_controller_test", CONTROLLER / "status_server.py")
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", tmp_path / "inbox")
|
|
module.WEBHOOK_INBOX_ROOT.mkdir(mode=0o700)
|
|
return module
|
|
|
|
|
|
@pytest.fixture
|
|
def deploy_module(monkeypatch):
|
|
if sys.platform == "win32":
|
|
pytest.skip("controller deploy.py uses Linux fcntl")
|
|
monkeypatch.syspath_prepend(str(CONTROLLER))
|
|
spec = importlib.util.spec_from_file_location("self_upgrade_deploy_test", CONTROLLER / "deploy.py")
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _project():
|
|
return {
|
|
"branch": "main", "gitea_repository_id": 9,
|
|
"gitea_webhook_secret_file": "jens__unraid-appops-gateway",
|
|
"containers": ["Unraid-AppOps-Gateway"],
|
|
"compose_project": "unraid-appops-gateway",
|
|
"compose_files": ["docker-compose.yml"],
|
|
"environment": {"APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216"},
|
|
"verify_oci_revision": True,
|
|
"validation": {
|
|
"validation_gate_version": 1, "required": True,
|
|
"contexts": ["Managed validation / full (push)"],
|
|
"timeout_seconds": 1800, "poll_interval_seconds": 5,
|
|
"max_poll_interval_seconds": 30,
|
|
},
|
|
}
|
|
|
|
|
|
def _ready(controller, monkeypatch):
|
|
monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: _project()})
|
|
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: True)
|
|
monkeypatch.setattr(controller, "_is_current_branch_head", lambda repository, project, revision: True)
|
|
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda repository, revision, policy: ("pass", []))
|
|
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {
|
|
"revision": "a" * 40, "imageId": "sha256:" + "b" * 64,
|
|
"dataPath": "/srv/example/appdata/unraid-appops-gateway-1216",
|
|
})
|
|
monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3))
|
|
|
|
|
|
def test_fixed_self_upgrade_preflight_accepts_exact_validated_head(controller, monkeypatch):
|
|
_ready(controller, monkeypatch)
|
|
result = controller._self_upgrade_preflight("c" * 40)
|
|
assert result["status"] == "preflight_ok"
|
|
assert result["currentRevision"] == "a" * 40
|
|
assert len(result["preflightSha256"]) == 64
|
|
assert result["executed"] is False
|
|
assert all(check["passed"] for check in result["checks"])
|
|
|
|
|
|
def test_timed_out_delivery_recovers_only_after_exact_protected_push_pass(controller, monkeypatch):
|
|
_ready(controller, monkeypatch)
|
|
revision = "c" * 40
|
|
digest = controller._webhook_semantic_digest(9, "refs/heads/main", revision)
|
|
policy = controller._validation_policy(controller.SELF_UPGRADE_REPOSITORY, _project())
|
|
original = {
|
|
"schema": controller._WEBHOOK_SCHEMA, "semantic_sha256": digest,
|
|
"body_sha256": "b" * 64, "gitea_repository_id": 9,
|
|
"ref": "refs/heads/main", "repository": controller.SELF_UPGRADE_REPOSITORY,
|
|
"revision": revision, "status": "validation_timeout",
|
|
"validation": controller._validation_snapshot(policy, int(time.time())),
|
|
}
|
|
marker = controller.WEBHOOK_INBOX_ROOT / f"{digest}.json"
|
|
marker.write_text(json.dumps(original), encoding="utf-8")
|
|
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("wait", []))
|
|
controller._auto_revalidate_timed_out_appops(marker)
|
|
revalidation = controller.WEBHOOK_INBOX_ROOT / f"{digest}.revalidation.json"
|
|
assert not revalidation.exists()
|
|
controller._appops_revalidation_next_check.clear()
|
|
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("pass", []))
|
|
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: False)
|
|
controller._auto_revalidate_timed_out_appops(marker)
|
|
assert not revalidation.exists()
|
|
controller._appops_revalidation_next_check.clear()
|
|
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: True)
|
|
controller._auto_revalidate_timed_out_appops(marker)
|
|
assert json.loads(revalidation.read_text())["status"] == "awaiting_validation"
|
|
assert json.loads(marker.read_text())["status"] == "validation_timeout"
|
|
updated = json.loads(revalidation.read_text())
|
|
updated["status"] = "deployed"
|
|
revalidation.write_text(json.dumps(updated), encoding="utf-8")
|
|
assert controller._self_upgrade_delivery_record(digest)["status"] == "deployed"
|
|
|
|
|
|
@pytest.mark.parametrize("gate", ["head", "validation", "runtime", "disk"])
|
|
def test_self_upgrade_preflight_fails_closed_on_gate(controller, monkeypatch, gate):
|
|
_ready(controller, monkeypatch)
|
|
if gate == "head":
|
|
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: False)
|
|
elif gate == "validation":
|
|
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("wait", []))
|
|
elif gate == "runtime":
|
|
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: None)
|
|
else:
|
|
monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=1))
|
|
result = controller._self_upgrade_preflight("c" * 40)
|
|
assert result["status"] == "blocked"
|
|
assert "preflightSha256" not in result
|
|
|
|
|
|
def test_self_upgrade_preflight_rejects_non_sha_and_conflicting_upgrade(controller, monkeypatch):
|
|
_ready(controller, monkeypatch)
|
|
assert controller._self_upgrade_preflight("main")["status"] == "blocked"
|
|
marker = controller.WEBHOOK_INBOX_ROOT / ("e" * 64 + ".json")
|
|
marker.write_text(json.dumps({
|
|
"repository": controller.SELF_UPGRADE_REPOSITORY,
|
|
"revision": "d" * 40, "status": "deploying",
|
|
}), encoding="utf-8")
|
|
result = controller._self_upgrade_preflight("c" * 40)
|
|
assert result["status"] == "blocked"
|
|
assert result["checks"][-1] == {"name": "no_conflicting_upgrade", "passed": False}
|
|
|
|
|
|
def test_runtime_requires_exact_container_mount_port_and_health(controller, monkeypatch):
|
|
payload = [{
|
|
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
|
"Mounts": [{"Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True}],
|
|
"Config": {"Labels": {"org.opencontainers.image.revision": "a" * 40,
|
|
"org.opencontainers.image.source": "jens/unraid-appops-gateway"}},
|
|
"NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}},
|
|
"Image": "sha256:" + "b" * 64,
|
|
}]
|
|
names = ["Unraid-AppOps-Gateway"]
|
|
|
|
def fake_run(args, **kwargs):
|
|
return subprocess.CompletedProcess(args, 0, json.dumps(payload) if args[1] == "inspect" else "\n".join(names), "")
|
|
|
|
monkeypatch.setattr(controller.subprocess, "run", fake_run)
|
|
assert controller._self_upgrade_runtime()["revision"] == "a" * 40
|
|
payload[0]["Mounts"][0]["Source"] = "/wrong"
|
|
assert controller._self_upgrade_runtime() is None
|
|
payload[0]["Mounts"][0]["Source"] = "/srv/example/appdata/unraid-appops-gateway-1216"
|
|
names.append("Unraid-AppOps-Gateway-copy")
|
|
assert controller._self_upgrade_runtime() is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("source", "accepted"),
|
|
[
|
|
("jens/unraid-appops-gateway", True),
|
|
("https://gitea.example.com/Jens/Unraid-AppOps-Gateway", True),
|
|
("other/repository", False),
|
|
("https://evil.example/Jens/Unraid-AppOps-Gateway", False),
|
|
("https://gitea.example.com/Jens/Other-App", False),
|
|
("https://gitea.example.com/Jens/Unraid-AppOps-Gateway.evil", False),
|
|
("http://gitea.example.com/Jens/Unraid-AppOps-Gateway", False),
|
|
],
|
|
)
|
|
def test_runtime_accepts_only_fixed_appops_oci_sources(controller, monkeypatch, source, accepted):
|
|
monkeypatch.setattr(
|
|
controller,
|
|
"_gitea_validation_settings",
|
|
lambda: ("https://gitea.example.com/api/v1", b"t" * 32),
|
|
)
|
|
payload = [{
|
|
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
|
"Mounts": [{"Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True}],
|
|
"Config": {"Labels": {
|
|
"org.opencontainers.image.revision": "a" * 40,
|
|
"org.opencontainers.image.source": source,
|
|
}},
|
|
"NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}},
|
|
"Image": "sha256:" + "b" * 64,
|
|
}]
|
|
|
|
def fake_run(args, **kwargs):
|
|
stdout = json.dumps(payload) if args[1] == "inspect" else "Unraid-AppOps-Gateway"
|
|
return subprocess.CompletedProcess(args, 0, stdout, "")
|
|
|
|
monkeypatch.setattr(controller.subprocess, "run", fake_run)
|
|
|
|
assert (controller._self_upgrade_runtime() is not None) is accepted
|
|
|
|
|
|
def test_mcp_self_upgrade_rejects_arbitrary_repository(monkeypatch):
|
|
from backend.app import mcp_server
|
|
|
|
with pytest.raises(ValueError, match="repository must be"):
|
|
mcp_server.appops_self_upgrade_preflight("other/repo", "a" * 40, "test reason")
|
|
with pytest.raises(ValueError, match="repository must be"):
|
|
mcp_server.appops_self_upgrade_apply("other/repo", "a" * 40, "test reason", "b" * 64, "key-12345")
|
|
|
|
|
|
def test_self_upgrade_signature_binds_endpoint_and_rejects_replay(controller, monkeypatch, tmp_path):
|
|
secret = b"fixed-test-self-upgrade-capability-0123456789"
|
|
token_file = tmp_path / "mcp-token"
|
|
token_file.write_bytes(secret + b"\n")
|
|
token_file.chmod(0o600)
|
|
monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file)
|
|
monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
|
|
stamp = str(int(time.time()))
|
|
nonce = "a" * 32
|
|
body = b'{"revision":"' + b"b" * 40 + b'"}'
|
|
path = controller.SELF_UPGRADE_PREFLIGHT_PATH
|
|
signature = controller._request_signature(
|
|
secret, method="POST", path=path, timestamp=stamp,
|
|
nonce=nonce, raw_body=body,
|
|
)
|
|
assert not controller._self_upgrade_signature_is_valid(
|
|
body, stamp, nonce, signature, method="POST", path=controller.SELF_UPGRADE_APPLY_PATH,
|
|
)
|
|
assert controller._self_upgrade_signature_is_valid(
|
|
body, stamp, nonce, signature, method="POST", path=path,
|
|
)
|
|
assert not controller._self_upgrade_signature_is_valid(
|
|
body, stamp, nonce, signature, method="POST", path=path,
|
|
)
|
|
|
|
|
|
def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller, monkeypatch, tmp_path):
|
|
_ready(controller, monkeypatch)
|
|
monkeypatch.setattr(controller, "SELF_UPGRADE_REQUEST_ROOT", tmp_path / "requests")
|
|
monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
|
|
monkeypatch.setattr(controller, "APPOPS_PROGRESS_ROOT", tmp_path / "progress")
|
|
secret = b"fixed-test-self-upgrade-capability-0123456789"
|
|
token_file = tmp_path / "mcp-token"
|
|
token_file.write_bytes(secret + b"\n")
|
|
token_file.chmod(0o600)
|
|
monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file)
|
|
monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None)
|
|
server = ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
base = f"http://127.0.0.1:{server.server_port}"
|
|
counter = 0
|
|
|
|
def call(path, payload, *, method="POST"):
|
|
nonlocal counter
|
|
counter += 1
|
|
body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() if payload is not None else b""
|
|
nonce = f"{counter:032x}"
|
|
stamp = str(int(time.time()))
|
|
signature = controller._request_signature(
|
|
secret, method=method, path=path, timestamp=stamp,
|
|
nonce=nonce, raw_body=body,
|
|
)
|
|
request = urllib.request.Request(
|
|
base + path, data=body if method == "POST" else None,
|
|
method=method,
|
|
headers={
|
|
"Content-Type": "application/json", "X-ITWorx-Deployment-Timestamp": stamp,
|
|
"X-ITWorx-Deployment-Nonce": nonce,
|
|
"X-ITWorx-Deployment-Signature": signature,
|
|
},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=5) as response:
|
|
return response.status, json.load(response)
|
|
except urllib.error.HTTPError as error:
|
|
return error.code, json.load(error)
|
|
|
|
try:
|
|
intent = {
|
|
"repository": "jens/unraid-appops-gateway", "revision": "c" * 40,
|
|
"reason": "Validated production upgrade", "expectedCurrentRevision": "a" * 40,
|
|
}
|
|
invalid = {**intent, "image": "attacker/image:latest"}
|
|
assert call(controller.SELF_UPGRADE_PREFLIGHT_PATH, invalid)[0] == 400
|
|
code, preflight = call(controller.SELF_UPGRADE_PREFLIGHT_PATH, intent)
|
|
assert code == 200 and preflight["status"] == "preflight_ok"
|
|
apply = {**intent, "preflightSha256": preflight["preflightSha256"],
|
|
"idempotencyKey": "fixed-request-12345"}
|
|
code, accepted = call(controller.SELF_UPGRADE_APPLY_PATH, apply)
|
|
assert code == 202 and accepted["status"] == "accepted", accepted
|
|
operation_id = accepted["operationId"]
|
|
assert len(operation_id) == 64
|
|
assert (controller.SELF_UPGRADE_REQUEST_ROOT / (
|
|
hashlib.sha256(apply["idempotencyKey"].encode()).hexdigest() + ".json"
|
|
)).exists()
|
|
code, replay = call(controller.SELF_UPGRADE_APPLY_PATH, apply)
|
|
assert code == 200 and replay["status"] == "already_queued"
|
|
assert replay["operationId"] == operation_id
|
|
assert call(controller.SELF_UPGRADE_APPLY_PATH, {**apply, "revision": "d" * 40})[0] == 409
|
|
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
|
|
assert code == 200 and status["state"] == "awaiting_validation"
|
|
assert status["operationState"] == "queued"
|
|
assert status["revision"] == "c" * 40
|
|
controller.APPOPS_PROGRESS_ROOT.mkdir(mode=0o700)
|
|
(controller.APPOPS_PROGRESS_ROOT / ("c" * 40 + ".json")).write_text(json.dumps({
|
|
"schema": "itworx.appops-upgrade-progress/v1", "revision": "c" * 40,
|
|
"phase": "building",
|
|
}), encoding="utf-8")
|
|
delivery_path = controller.WEBHOOK_INBOX_ROOT / f"{operation_id}.json"
|
|
delivery_record = json.loads(delivery_path.read_text(encoding="utf-8"))
|
|
delivery_record["status"] = "deploying"
|
|
delivery_path.write_text(json.dumps(delivery_record), encoding="utf-8")
|
|
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
|
|
assert code == 200 and status["operationState"] == "building"
|
|
delivery_record["status"] = "validation_timeout"
|
|
delivery_path.write_text(json.dumps(delivery_record), encoding="utf-8")
|
|
recovered = {**delivery_record, "schema": controller._WEBHOOK_REVALIDATION_SCHEMA,
|
|
"revalidation_of": operation_id, "status": "deployed"}
|
|
(controller.WEBHOOK_INBOX_ROOT / f"{operation_id}.revalidation.json").write_text(
|
|
json.dumps(recovered), encoding="utf-8"
|
|
)
|
|
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {
|
|
"revision": "c" * 40, "imageId": "sha256:" + "b" * 64,
|
|
"dataPath": "/srv/example/appdata/unraid-appops-gateway-1216",
|
|
})
|
|
monkeypatch.setattr(controller, "_deployment_state", lambda slug: {})
|
|
monkeypatch.setattr(controller, "_verified_receipt", lambda *args: {"revision": "c" * 40})
|
|
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
|
|
assert code == 200 and status["operationState"] == "succeeded"
|
|
assert status["state"] == "deployed"
|
|
code, preflight = call(controller.SELF_UPGRADE_PREFLIGHT_PATH, {
|
|
**intent, "expectedCurrentRevision": "c" * 40,
|
|
})
|
|
assert code == 200 and preflight["status"] == "already_deployed"
|
|
already = {**intent, "expectedCurrentRevision": "c" * 40,
|
|
"preflightSha256": preflight["preflightSha256"],
|
|
"idempotencyKey": "already-deployed-request-12345"}
|
|
code, result = call(controller.SELF_UPGRADE_APPLY_PATH, already)
|
|
assert code == 200 and result["status"] == "already_deployed"
|
|
assert result["operationId"] == operation_id
|
|
code, result = call(controller.SELF_UPGRADE_APPLY_PATH, already)
|
|
assert code == 200 and result["status"] == "already_deployed"
|
|
finally:
|
|
server.shutdown()
|
|
thread.join(timeout=5)
|
|
server.server_close()
|
|
|
|
|
|
def test_appops_deploy_rechecks_exact_main_head_before_mutation(deploy_module, monkeypatch):
|
|
revision = "c" * 40
|
|
metadata = {"builtAt": "2026-09-28T00:00:00.000Z"}
|
|
monkeypatch.setattr(deploy_module, "validate_commit", lambda repo, branch, value: value)
|
|
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *args, **kwargs: metadata)
|
|
monkeypatch.setattr(deploy_module, "output", lambda args: "d" * 40)
|
|
with pytest.raises(deploy_module.DeployError, match="exact protected main head"):
|
|
deploy_module.revalidate_deployment_candidate(
|
|
Path("/repo.git"), {"branch": "main"}, revision,
|
|
"jens/unraid-appops-gateway", metadata,
|
|
)
|
|
monkeypatch.setattr(deploy_module, "output", lambda args: revision)
|
|
deploy_module.revalidate_deployment_candidate(
|
|
Path("/repo.git"), {"branch": "main"}, revision,
|
|
"jens/unraid-appops-gateway", metadata,
|
|
)
|
|
|
|
|
|
def test_appops_backup_is_fixed_and_digest_verified(deploy_module, monkeypatch, tmp_path):
|
|
data = tmp_path / "appops-data"
|
|
data.mkdir()
|
|
(data / "state.txt").write_text("preserved", encoding="utf-8")
|
|
monkeypatch.setattr(deploy_module, "APPOPS_DATA_ROOT", data)
|
|
monkeypatch.setattr(deploy_module, "APPOPS_BACKUP_ROOT", tmp_path / "backups")
|
|
monkeypatch.setattr(
|
|
deploy_module.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3)
|
|
)
|
|
receipt = deploy_module.capture_appops_data_backup("c" * 40, "a" * 40)
|
|
archive = Path(receipt["archive"])
|
|
assert hashlib.sha256(archive.read_bytes()).hexdigest() == receipt["sha256"]
|
|
assert receipt["previousRevision"] == "a" * 40
|
|
assert json.loads((archive.parent / "receipt.json").read_text()) == receipt
|
|
with tarfile.open(archive, "r:gz") as backup:
|
|
assert backup.extractfile("./state.txt").read() == b"preserved"
|
|
|
|
|
|
def test_appops_backup_rejects_symlinked_data(deploy_module, monkeypatch, tmp_path):
|
|
data = tmp_path / "real-data"
|
|
data.mkdir()
|
|
link = tmp_path / "link-data"
|
|
link.symlink_to(data, target_is_directory=True)
|
|
monkeypatch.setattr(deploy_module, "APPOPS_DATA_ROOT", link)
|
|
with pytest.raises(deploy_module.DeployError, match="unsafe"):
|
|
deploy_module.capture_appops_data_backup("c" * 40, "a" * 40)
|
|
|
|
|
|
def test_appops_progress_is_durable_and_bounded(deploy_module, monkeypatch, tmp_path):
|
|
monkeypatch.setattr(deploy_module, "APPOPS_PROGRESS_ROOT", tmp_path / "progress")
|
|
deploy_module.write_appops_progress("c" * 40, "building")
|
|
path = deploy_module.APPOPS_PROGRESS_ROOT / ("c" * 40 + ".json")
|
|
assert json.loads(path.read_text())["phase"] == "building"
|
|
deploy_module.write_appops_progress("c" * 40, "verifying")
|
|
assert json.loads(path.read_text())["phase"] == "verifying"
|
|
with pytest.raises(deploy_module.DeployError, match="invalid"):
|
|
deploy_module.write_appops_progress("c" * 40, "arbitrary-command")
|