Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d
This commit is contained in:
Public Source Publisher committed
commit
1dfbca08a1
528 files changed
+131054
No files matched your search
@@ -0,0 +1,377 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from http.server import ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CONTROLLER = ROOT / "ops" / "gitea-deploy-broker"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def controller(monkeypatch, tmp_path):
|
||||
monkeypatch.syspath_prepend(str(CONTROLLER))
|
||||
spec = importlib.util.spec_from_file_location("self_upgrade_controller_test", CONTROLLER / "status_server.py")
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", tmp_path / "inbox")
|
||||
module.WEBHOOK_INBOX_ROOT.mkdir(mode=0o700)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deploy_module(monkeypatch):
|
||||
if sys.platform == "win32":
|
||||
pytest.skip("controller deploy.py uses Linux fcntl")
|
||||
monkeypatch.syspath_prepend(str(CONTROLLER))
|
||||
spec = importlib.util.spec_from_file_location("self_upgrade_deploy_test", CONTROLLER / "deploy.py")
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _project():
|
||||
return {
|
||||
"branch": "main", "gitea_repository_id": 9,
|
||||
"gitea_webhook_secret_file": "jens__unraid-appops-gateway",
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"compose_project": "unraid-appops-gateway",
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"environment": {"APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216"},
|
||||
"verify_oci_revision": True,
|
||||
"validation": {
|
||||
"validation_gate_version": 1, "required": True,
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"timeout_seconds": 1800, "poll_interval_seconds": 5,
|
||||
"max_poll_interval_seconds": 30,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _ready(controller, monkeypatch):
|
||||
monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: _project()})
|
||||
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: True)
|
||||
monkeypatch.setattr(controller, "_is_current_branch_head", lambda repository, project, revision: True)
|
||||
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda repository, revision, policy: ("pass", []))
|
||||
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {
|
||||
"revision": "a" * 40, "imageId": "sha256:" + "b" * 64,
|
||||
"dataPath": "/srv/example/appdata/unraid-appops-gateway-1216",
|
||||
})
|
||||
monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3))
|
||||
|
||||
|
||||
def test_fixed_self_upgrade_preflight_accepts_exact_validated_head(controller, monkeypatch):
|
||||
_ready(controller, monkeypatch)
|
||||
result = controller._self_upgrade_preflight("c" * 40)
|
||||
assert result["status"] == "preflight_ok"
|
||||
assert result["currentRevision"] == "a" * 40
|
||||
assert len(result["preflightSha256"]) == 64
|
||||
assert result["executed"] is False
|
||||
assert all(check["passed"] for check in result["checks"])
|
||||
|
||||
|
||||
def test_timed_out_delivery_recovers_only_after_exact_protected_push_pass(controller, monkeypatch):
|
||||
_ready(controller, monkeypatch)
|
||||
revision = "c" * 40
|
||||
digest = controller._webhook_semantic_digest(9, "refs/heads/main", revision)
|
||||
policy = controller._validation_policy(controller.SELF_UPGRADE_REPOSITORY, _project())
|
||||
original = {
|
||||
"schema": controller._WEBHOOK_SCHEMA, "semantic_sha256": digest,
|
||||
"body_sha256": "b" * 64, "gitea_repository_id": 9,
|
||||
"ref": "refs/heads/main", "repository": controller.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": revision, "status": "validation_timeout",
|
||||
"validation": controller._validation_snapshot(policy, int(time.time())),
|
||||
}
|
||||
marker = controller.WEBHOOK_INBOX_ROOT / f"{digest}.json"
|
||||
marker.write_text(json.dumps(original), encoding="utf-8")
|
||||
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("wait", []))
|
||||
controller._auto_revalidate_timed_out_appops(marker)
|
||||
revalidation = controller.WEBHOOK_INBOX_ROOT / f"{digest}.revalidation.json"
|
||||
assert not revalidation.exists()
|
||||
controller._appops_revalidation_next_check.clear()
|
||||
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("pass", []))
|
||||
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: False)
|
||||
controller._auto_revalidate_timed_out_appops(marker)
|
||||
assert not revalidation.exists()
|
||||
controller._appops_revalidation_next_check.clear()
|
||||
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: True)
|
||||
controller._auto_revalidate_timed_out_appops(marker)
|
||||
assert json.loads(revalidation.read_text())["status"] == "awaiting_validation"
|
||||
assert json.loads(marker.read_text())["status"] == "validation_timeout"
|
||||
updated = json.loads(revalidation.read_text())
|
||||
updated["status"] = "deployed"
|
||||
revalidation.write_text(json.dumps(updated), encoding="utf-8")
|
||||
assert controller._self_upgrade_delivery_record(digest)["status"] == "deployed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gate", ["head", "validation", "runtime", "disk"])
|
||||
def test_self_upgrade_preflight_fails_closed_on_gate(controller, monkeypatch, gate):
|
||||
_ready(controller, monkeypatch)
|
||||
if gate == "head":
|
||||
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: False)
|
||||
elif gate == "validation":
|
||||
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("wait", []))
|
||||
elif gate == "runtime":
|
||||
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: None)
|
||||
else:
|
||||
monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=1))
|
||||
result = controller._self_upgrade_preflight("c" * 40)
|
||||
assert result["status"] == "blocked"
|
||||
assert "preflightSha256" not in result
|
||||
|
||||
|
||||
def test_self_upgrade_preflight_rejects_non_sha_and_conflicting_upgrade(controller, monkeypatch):
|
||||
_ready(controller, monkeypatch)
|
||||
assert controller._self_upgrade_preflight("main")["status"] == "blocked"
|
||||
marker = controller.WEBHOOK_INBOX_ROOT / ("e" * 64 + ".json")
|
||||
marker.write_text(json.dumps({
|
||||
"repository": controller.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": "d" * 40, "status": "deploying",
|
||||
}), encoding="utf-8")
|
||||
result = controller._self_upgrade_preflight("c" * 40)
|
||||
assert result["status"] == "blocked"
|
||||
assert result["checks"][-1] == {"name": "no_conflicting_upgrade", "passed": False}
|
||||
|
||||
|
||||
def test_runtime_requires_exact_container_mount_port_and_health(controller, monkeypatch):
|
||||
payload = [{
|
||||
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
||||
"Mounts": [{"Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True}],
|
||||
"Config": {"Labels": {"org.opencontainers.image.revision": "a" * 40,
|
||||
"org.opencontainers.image.source": "jens/unraid-appops-gateway"}},
|
||||
"NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}},
|
||||
"Image": "sha256:" + "b" * 64,
|
||||
}]
|
||||
names = ["Unraid-AppOps-Gateway"]
|
||||
|
||||
def fake_run(args, **kwargs):
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps(payload) if args[1] == "inspect" else "\n".join(names), "")
|
||||
|
||||
monkeypatch.setattr(controller.subprocess, "run", fake_run)
|
||||
assert controller._self_upgrade_runtime()["revision"] == "a" * 40
|
||||
payload[0]["Mounts"][0]["Source"] = "/wrong"
|
||||
assert controller._self_upgrade_runtime() is None
|
||||
payload[0]["Mounts"][0]["Source"] = "/srv/example/appdata/unraid-appops-gateway-1216"
|
||||
names.append("Unraid-AppOps-Gateway-copy")
|
||||
assert controller._self_upgrade_runtime() is None
|
||||
|
||||
|
||||
def test_mcp_self_upgrade_rejects_arbitrary_repository(monkeypatch):
|
||||
from backend.app import mcp_server
|
||||
|
||||
with pytest.raises(ValueError, match="repository must be"):
|
||||
mcp_server.appops_self_upgrade_preflight("other/repo", "a" * 40, "test reason")
|
||||
with pytest.raises(ValueError, match="repository must be"):
|
||||
mcp_server.appops_self_upgrade_apply("other/repo", "a" * 40, "test reason", "b" * 64, "key-12345")
|
||||
|
||||
|
||||
def test_self_upgrade_signature_binds_endpoint_and_rejects_replay(controller, monkeypatch, tmp_path):
|
||||
secret = b"fixed-test-self-upgrade-capability-0123456789"
|
||||
token_file = tmp_path / "mcp-token"
|
||||
token_file.write_bytes(secret + b"\n")
|
||||
token_file.chmod(0o600)
|
||||
monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file)
|
||||
monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
|
||||
stamp = str(int(time.time()))
|
||||
nonce = "a" * 32
|
||||
body = b'{"revision":"' + b"b" * 40 + b'"}'
|
||||
path = controller.SELF_UPGRADE_PREFLIGHT_PATH
|
||||
signature = controller._request_signature(
|
||||
secret, method="POST", path=path, timestamp=stamp,
|
||||
nonce=nonce, raw_body=body,
|
||||
)
|
||||
assert not controller._self_upgrade_signature_is_valid(
|
||||
body, stamp, nonce, signature, method="POST", path=controller.SELF_UPGRADE_APPLY_PATH,
|
||||
)
|
||||
assert controller._self_upgrade_signature_is_valid(
|
||||
body, stamp, nonce, signature, method="POST", path=path,
|
||||
)
|
||||
assert not controller._self_upgrade_signature_is_valid(
|
||||
body, stamp, nonce, signature, method="POST", path=path,
|
||||
)
|
||||
|
||||
|
||||
def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller, monkeypatch, tmp_path):
|
||||
_ready(controller, monkeypatch)
|
||||
monkeypatch.setattr(controller, "SELF_UPGRADE_REQUEST_ROOT", tmp_path / "requests")
|
||||
monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
|
||||
monkeypatch.setattr(controller, "APPOPS_PROGRESS_ROOT", tmp_path / "progress")
|
||||
secret = b"fixed-test-self-upgrade-capability-0123456789"
|
||||
token_file = tmp_path / "mcp-token"
|
||||
token_file.write_bytes(secret + b"\n")
|
||||
token_file.chmod(0o600)
|
||||
monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file)
|
||||
monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None)
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
base = f"http://127.0.0.1:{server.server_port}"
|
||||
counter = 0
|
||||
|
||||
def call(path, payload, *, method="POST"):
|
||||
nonlocal counter
|
||||
counter += 1
|
||||
body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() if payload is not None else b""
|
||||
nonce = f"{counter:032x}"
|
||||
stamp = str(int(time.time()))
|
||||
signature = controller._request_signature(
|
||||
secret, method=method, path=path, timestamp=stamp,
|
||||
nonce=nonce, raw_body=body,
|
||||
)
|
||||
request = urllib.request.Request(
|
||||
base + path, data=body if method == "POST" else None,
|
||||
method=method,
|
||||
headers={
|
||||
"Content-Type": "application/json", "X-ITWorx-Deployment-Timestamp": stamp,
|
||||
"X-ITWorx-Deployment-Nonce": nonce,
|
||||
"X-ITWorx-Deployment-Signature": signature,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return response.status, json.load(response)
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, json.load(error)
|
||||
|
||||
try:
|
||||
intent = {
|
||||
"repository": "jens/unraid-appops-gateway", "revision": "c" * 40,
|
||||
"reason": "Validated production upgrade", "expectedCurrentRevision": "a" * 40,
|
||||
}
|
||||
invalid = {**intent, "image": "attacker/image:latest"}
|
||||
assert call(controller.SELF_UPGRADE_PREFLIGHT_PATH, invalid)[0] == 400
|
||||
code, preflight = call(controller.SELF_UPGRADE_PREFLIGHT_PATH, intent)
|
||||
assert code == 200 and preflight["status"] == "preflight_ok"
|
||||
apply = {**intent, "preflightSha256": preflight["preflightSha256"],
|
||||
"idempotencyKey": "fixed-request-12345"}
|
||||
code, accepted = call(controller.SELF_UPGRADE_APPLY_PATH, apply)
|
||||
assert code == 202 and accepted["status"] == "accepted", accepted
|
||||
operation_id = accepted["operationId"]
|
||||
assert len(operation_id) == 64
|
||||
assert (controller.SELF_UPGRADE_REQUEST_ROOT / (
|
||||
hashlib.sha256(apply["idempotencyKey"].encode()).hexdigest() + ".json"
|
||||
)).exists()
|
||||
code, replay = call(controller.SELF_UPGRADE_APPLY_PATH, apply)
|
||||
assert code == 200 and replay["status"] == "already_queued"
|
||||
assert replay["operationId"] == operation_id
|
||||
assert call(controller.SELF_UPGRADE_APPLY_PATH, {**apply, "revision": "d" * 40})[0] == 409
|
||||
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
|
||||
assert code == 200 and status["state"] == "awaiting_validation"
|
||||
assert status["operationState"] == "queued"
|
||||
assert status["revision"] == "c" * 40
|
||||
controller.APPOPS_PROGRESS_ROOT.mkdir(mode=0o700)
|
||||
(controller.APPOPS_PROGRESS_ROOT / ("c" * 40 + ".json")).write_text(json.dumps({
|
||||
"schema": "itworx.appops-upgrade-progress/v1", "revision": "c" * 40,
|
||||
"phase": "building",
|
||||
}), encoding="utf-8")
|
||||
delivery_path = controller.WEBHOOK_INBOX_ROOT / f"{operation_id}.json"
|
||||
delivery_record = json.loads(delivery_path.read_text(encoding="utf-8"))
|
||||
delivery_record["status"] = "deploying"
|
||||
delivery_path.write_text(json.dumps(delivery_record), encoding="utf-8")
|
||||
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
|
||||
assert code == 200 and status["operationState"] == "building"
|
||||
delivery_record["status"] = "validation_timeout"
|
||||
delivery_path.write_text(json.dumps(delivery_record), encoding="utf-8")
|
||||
recovered = {**delivery_record, "schema": controller._WEBHOOK_REVALIDATION_SCHEMA,
|
||||
"revalidation_of": operation_id, "status": "deployed"}
|
||||
(controller.WEBHOOK_INBOX_ROOT / f"{operation_id}.revalidation.json").write_text(
|
||||
json.dumps(recovered), encoding="utf-8"
|
||||
)
|
||||
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {
|
||||
"revision": "c" * 40, "imageId": "sha256:" + "b" * 64,
|
||||
"dataPath": "/srv/example/appdata/unraid-appops-gateway-1216",
|
||||
})
|
||||
monkeypatch.setattr(controller, "_deployment_state", lambda slug: {})
|
||||
monkeypatch.setattr(controller, "_verified_receipt", lambda *args: {"revision": "c" * 40})
|
||||
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
|
||||
assert code == 200 and status["operationState"] == "succeeded"
|
||||
assert status["state"] == "deployed"
|
||||
code, preflight = call(controller.SELF_UPGRADE_PREFLIGHT_PATH, {
|
||||
**intent, "expectedCurrentRevision": "c" * 40,
|
||||
})
|
||||
assert code == 200 and preflight["status"] == "already_deployed"
|
||||
already = {**intent, "expectedCurrentRevision": "c" * 40,
|
||||
"preflightSha256": preflight["preflightSha256"],
|
||||
"idempotencyKey": "already-deployed-request-12345"}
|
||||
code, result = call(controller.SELF_UPGRADE_APPLY_PATH, already)
|
||||
assert code == 200 and result["status"] == "already_deployed"
|
||||
assert result["operationId"] == operation_id
|
||||
code, result = call(controller.SELF_UPGRADE_APPLY_PATH, already)
|
||||
assert code == 200 and result["status"] == "already_deployed"
|
||||
finally:
|
||||
server.shutdown()
|
||||
thread.join(timeout=5)
|
||||
server.server_close()
|
||||
|
||||
|
||||
def test_appops_deploy_rechecks_exact_main_head_before_mutation(deploy_module, monkeypatch):
|
||||
revision = "c" * 40
|
||||
metadata = {"builtAt": "2026-09-28T00:00:00.000Z"}
|
||||
monkeypatch.setattr(deploy_module, "validate_commit", lambda repo, branch, value: value)
|
||||
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *args, **kwargs: metadata)
|
||||
monkeypatch.setattr(deploy_module, "output", lambda args: "d" * 40)
|
||||
with pytest.raises(deploy_module.DeployError, match="exact protected main head"):
|
||||
deploy_module.revalidate_deployment_candidate(
|
||||
Path("/repo.git"), {"branch": "main"}, revision,
|
||||
"jens/unraid-appops-gateway", metadata,
|
||||
)
|
||||
monkeypatch.setattr(deploy_module, "output", lambda args: revision)
|
||||
deploy_module.revalidate_deployment_candidate(
|
||||
Path("/repo.git"), {"branch": "main"}, revision,
|
||||
"jens/unraid-appops-gateway", metadata,
|
||||
)
|
||||
|
||||
|
||||
def test_appops_backup_is_fixed_and_digest_verified(deploy_module, monkeypatch, tmp_path):
|
||||
data = tmp_path / "appops-data"
|
||||
data.mkdir()
|
||||
(data / "state.txt").write_text("preserved", encoding="utf-8")
|
||||
monkeypatch.setattr(deploy_module, "APPOPS_DATA_ROOT", data)
|
||||
monkeypatch.setattr(deploy_module, "APPOPS_BACKUP_ROOT", tmp_path / "backups")
|
||||
monkeypatch.setattr(
|
||||
deploy_module.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3)
|
||||
)
|
||||
receipt = deploy_module.capture_appops_data_backup("c" * 40, "a" * 40)
|
||||
archive = Path(receipt["archive"])
|
||||
assert hashlib.sha256(archive.read_bytes()).hexdigest() == receipt["sha256"]
|
||||
assert receipt["previousRevision"] == "a" * 40
|
||||
assert json.loads((archive.parent / "receipt.json").read_text()) == receipt
|
||||
with tarfile.open(archive, "r:gz") as backup:
|
||||
assert backup.extractfile("./state.txt").read() == b"preserved"
|
||||
|
||||
|
||||
def test_appops_backup_rejects_symlinked_data(deploy_module, monkeypatch, tmp_path):
|
||||
data = tmp_path / "real-data"
|
||||
data.mkdir()
|
||||
link = tmp_path / "link-data"
|
||||
link.symlink_to(data, target_is_directory=True)
|
||||
monkeypatch.setattr(deploy_module, "APPOPS_DATA_ROOT", link)
|
||||
with pytest.raises(deploy_module.DeployError, match="unsafe"):
|
||||
deploy_module.capture_appops_data_backup("c" * 40, "a" * 40)
|
||||
|
||||
|
||||
def test_appops_progress_is_durable_and_bounded(deploy_module, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(deploy_module, "APPOPS_PROGRESS_ROOT", tmp_path / "progress")
|
||||
deploy_module.write_appops_progress("c" * 40, "building")
|
||||
path = deploy_module.APPOPS_PROGRESS_ROOT / ("c" * 40 + ".json")
|
||||
assert json.loads(path.read_text())["phase"] == "building"
|
||||
deploy_module.write_appops_progress("c" * 40, "verifying")
|
||||
assert json.loads(path.read_text())["phase"] == "verifying"
|
||||
with pytest.raises(deploy_module.DeployError, match="invalid"):
|
||||
deploy_module.write_appops_progress("c" * 40, "arbitrary-command")
|
||||
Reference in new issue
Block a user