Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d
This commit is contained in:
commit
1dfbca08a1
528 files changed
+131054
No files matched your search
@@ -0,0 +1,345 @@
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, Session, create_engine
|
||||
|
||||
|
||||
def _reset_db(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "ai_remediation_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
|
||||
previous = {
|
||||
"apis": dict(settings.apis),
|
||||
"environment": settings.environment,
|
||||
"secret_key": settings.secret_key,
|
||||
"secret_key_source": settings.secret_key_source,
|
||||
}
|
||||
for key in settings.apis:
|
||||
settings.apis[key] = None if key.endswith(("_URL", "_API_KEY", "_TOKEN", "_SECRET", "_PASSWORD")) else settings.apis[key]
|
||||
settings.environment = "development"
|
||||
settings.secret_key = "test-ai-remediation-key"
|
||||
settings.secret_key_source = "env"
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
return previous
|
||||
|
||||
|
||||
def _restore_config(previous):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous["apis"])
|
||||
settings.environment = previous["environment"]
|
||||
settings.secret_key = previous["secret_key"]
|
||||
settings.secret_key_source = previous["secret_key_source"]
|
||||
|
||||
|
||||
def _client(tmp_path, monkeypatch):
|
||||
previous = _reset_db(tmp_path, monkeypatch)
|
||||
from backend.app.main import app
|
||||
|
||||
return TestClient(app), previous
|
||||
|
||||
|
||||
def _npm_exposure():
|
||||
return {
|
||||
"success": True,
|
||||
"status": "ok",
|
||||
"totalProxyHosts": 2,
|
||||
"enabledProxyHosts": 2,
|
||||
"publicHosts": 2,
|
||||
"hostsWithoutCertificate": 1,
|
||||
"hostsWithExpiredCertificate": 0,
|
||||
"hostsWithExpiringCertificate": 1,
|
||||
"hostsWithoutAccessList": 1,
|
||||
"hostsWithoutSslForce": 1,
|
||||
"hostsWithWebsocket": 1,
|
||||
"warnings": [
|
||||
{"type": "host_without_certificate", "severity": "high", "title": "open.example.com has no certificate", "evidence": ["open.example.com"], "suggestedAction": "review_certificate"},
|
||||
{"type": "certificate_expiring_soon", "severity": "warning", "title": "plex.example.com certificate expires soon", "evidence": ["12"], "suggestedAction": "review_certificate_in_npm"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _auth_coverage():
|
||||
return {
|
||||
"success": True,
|
||||
"status": "ok",
|
||||
"summary": {"totalPublicHosts": 2, "protectedHosts": 1, "likelyProtectedHosts": 0, "unprotectedHosts": 1, "npmStatus": "ok"},
|
||||
"publicHosts": [
|
||||
{"hostId": 1, "domains": ["plex.example.com"], "target": "http://plex:32400", "coverageStatus": "protected", "authentikMatches": ["plex.example.com"]},
|
||||
{"hostId": 2, "domains": ["open.example.com"], "target": "http://unknown:8080", "coverageStatus": "unprotected", "authentikMatches": []},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _seed_signals(monkeypatch):
|
||||
monkeypatch.setattr("backend.app.connectors.nginx_proxy_manager.NginxProxyManagerConnector.exposure_summary", lambda self: _npm_exposure(), raising=True)
|
||||
monkeypatch.setattr("backend.app.connectors.nginx_proxy_manager.NginxProxyManagerConnector.proxy_hosts", lambda self: {"success": True, "items": [{"id": 2, "domainNames": ["open.example.com"], "enabled": True}]}, raising=True)
|
||||
monkeypatch.setattr("backend.app.connectors.authentik.AuthentikConnector.auth_coverage", lambda self: _auth_coverage(), raising=True)
|
||||
monkeypatch.setattr("backend.app.connectors.authentik.AuthentikConnector.security_warnings", lambda self: {"success": True, "items": []}, raising=True)
|
||||
monkeypatch.setattr("backend.app.connectors.authentik.AuthentikConnector.outpost_health", lambda self: {"success": True, "available": True, "embeddedExpected": True, "items": []}, raising=True)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.services.media_pipeline_service.issues",
|
||||
lambda session: {
|
||||
"success": True,
|
||||
"issues": [
|
||||
{"id": "deluge-stalled", "severity": "warning", "title": "Deluge stalled downloads", "description": "2 stalled downloads", "affectedConnectors": ["deluge"], "evidence": ["stalled=2"]},
|
||||
{"id": "widefrog-failed", "severity": "warning", "title": "Widefrog failed jobs", "description": "1 failed job", "affectedConnectors": ["widefrog"], "evidence": ["failed=1"]},
|
||||
],
|
||||
},
|
||||
raising=False,
|
||||
)
|
||||
|
||||
|
||||
def test_generate_plans_from_security_findings_and_media(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
_seed_signals(monkeypatch)
|
||||
|
||||
with client:
|
||||
payload = client.post("/api/ai/remediation/plans/generate").json()
|
||||
|
||||
plan_ids = {plan["id"] for plan in payload["plans"]}
|
||||
assert "rem-ai-sec-auth-public-host-without-authentik" in plan_ids
|
||||
assert "rem-ai-sec-npm-certificate-expiring-soon" in plan_ids
|
||||
assert "rem-media-deluge-stalled" in plan_ids
|
||||
assert "rem-media-widefrog-failed" in plan_ids
|
||||
assert all(plan["executable"] is False for plan in payload["plans"])
|
||||
assert all(step["canExecute"] is False for plan in payload["plans"] for step in plan["steps"])
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_generated_media_security_plan_uses_media_steps_not_inventory(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
_seed_signals(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.services.media_pipeline_service.summary",
|
||||
lambda session=None: {"success": True, "status": "partial", "message": "Media pipeline needs review."},
|
||||
raising=False,
|
||||
)
|
||||
|
||||
with client:
|
||||
payload = client.post("/api/ai/remediation/plans/generate").json()
|
||||
|
||||
plans = [
|
||||
item for item in payload["plans"]
|
||||
if item["sourceId"] == "ai-sec-media-pipeline-issue"
|
||||
]
|
||||
|
||||
assert plans
|
||||
for plan in plans:
|
||||
step_ids = {step["id"] for step in plan["steps"]}
|
||||
assert plan["title"] == "Review media pipeline issue"
|
||||
assert "run-diagnose" in step_ids
|
||||
assert "review-connectors" in step_ids
|
||||
assert "mark-protected" not in step_ids
|
||||
assert any(check["endpoint"] == "/api/media-pipeline/summary" for check in plan["verificationChecks"])
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_generate_plans_deduplicates_recommendations_covered_by_findings(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
_seed_signals(monkeypatch)
|
||||
|
||||
with client:
|
||||
payload = client.get("/api/ai/remediation/plans").json()
|
||||
|
||||
source_ids = [plan["sourceId"] for plan in payload["plans"]]
|
||||
assert "ai-sec-auth-public-host-without-authentik" in source_ids
|
||||
finding_source_ids = {source_id for source_id in source_ids if source_id.startswith("ai-sec-")}
|
||||
duplicate_recommendations = [
|
||||
source_id
|
||||
for source_id in source_ids
|
||||
if source_id.startswith("rec-") and source_id.removeprefix("rec-") in finding_source_ids
|
||||
]
|
||||
assert duplicate_recommendations == []
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_generate_missing_secret_plan_has_no_secret_values(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
secret = "AUTHENTIK_TOKEN_VALUE"
|
||||
|
||||
from backend.app.db.models import Connector
|
||||
from backend.app.db.session import engine
|
||||
|
||||
with Session(engine) as session:
|
||||
connector = session.get(Connector, "authentik")
|
||||
connector.secret_ref = "AUTHENTIK_TOKEN"
|
||||
session.add(connector)
|
||||
session.commit()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.authentik.AuthentikConnector.security_warnings",
|
||||
lambda self: {"success": True, "items": [{"type": "token_leak", "severity": "critical", "title": f"leaked {secret}", "evidence": [secret]}]},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
with client:
|
||||
finding_payload = client.get("/api/ai/security/findings?scope=secrets").json()
|
||||
finding_id = next(item["id"] for item in finding_payload if item["category"] == "connector_config")
|
||||
plan = client.post(f"/api/ai/remediation/plans/from-finding/{finding_id}").json()
|
||||
|
||||
assert "secret" in plan["title"].lower()
|
||||
assert secret not in str(plan)
|
||||
assert "[REDACTED]" not in plan["affectedResources"]
|
||||
assert any(step["type"] == "configure_secret" for step in plan["steps"])
|
||||
configure_step = next(step for step in plan["steps"] if step["id"] == "configure-secret")
|
||||
test_step = next(step for step in plan["steps"] if step["id"] == "test-connector")
|
||||
secret_owner = plan["affectedResources"][0].removesuffix("_API_KEY").removesuffix("_TOKEN").lower()
|
||||
assert configure_step["href"] == f"/setup?step=connectors&connector={secret_owner}"
|
||||
assert test_step["href"] == f"/setup?step=connectors&connector={secret_owner}"
|
||||
assert plan["executable"] is False
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_generate_plans_builds_security_review_once(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
|
||||
from backend.app.services import ai_remediation_planner
|
||||
|
||||
calls = {"count": 0}
|
||||
original_generate_review = ai_remediation_planner.ai_security_reviewer.generate_review
|
||||
|
||||
def counted_generate_review(session, scope="all"):
|
||||
calls["count"] += 1
|
||||
return original_generate_review(session, scope)
|
||||
|
||||
monkeypatch.setattr(ai_remediation_planner.ai_security_reviewer, "generate_review", counted_generate_review)
|
||||
monkeypatch.setattr("backend.app.services.media_pipeline_service.issues", lambda session: {"success": True, "issues": []}, raising=False)
|
||||
|
||||
with client:
|
||||
payload = client.post("/api/ai/remediation/plans/generate").json()
|
||||
|
||||
assert payload["status"] == "ok"
|
||||
assert calls["count"] == 1
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_dry_run_evaluates_policy_without_execution(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
_seed_signals(monkeypatch)
|
||||
|
||||
with client:
|
||||
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-auth-public-host-without-authentik").json()
|
||||
dry_run = client.post(f"/api/ai/remediation/plans/{plan['id']}/dry-run").json()
|
||||
|
||||
assert dry_run["executed"] is False
|
||||
assert dry_run["status"] in {"pass", "warning", "blocked"}
|
||||
assert any(step["policyDecision"] == "blocked" for step in dry_run["stepResults"])
|
||||
assert any(item["actionId"].startswith("authentik.") for item in dry_run["blockedActions"])
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_auth_coverage_plan_is_host_aware_and_non_executing(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
_seed_signals(monkeypatch)
|
||||
|
||||
with client:
|
||||
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-auth-public-host-without-authentik").json()
|
||||
|
||||
assert plan["title"] == "Add identity coverage for 1 public host"
|
||||
assert "open.example.com" in plan["description"]
|
||||
assert plan["affectedResources"] == ["open.example.com"]
|
||||
assert any("host-by-host" in step["description"] for step in plan["steps"])
|
||||
assert any(step["id"] == "manual-auth-change" and step["canExecute"] is False for step in plan["steps"])
|
||||
assert any(check["endpoint"] == "/api/apps/authentik/auth-coverage" for check in plan["verificationChecks"])
|
||||
assert any(item["actionId"] == "authentik.create_application" for item in plan["approvalRequirements"])
|
||||
assert plan["executable"] is False
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_media_pipeline_security_finding_uses_media_plan_not_inventory(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
_seed_signals(monkeypatch)
|
||||
|
||||
with client:
|
||||
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-media-pipeline-issue").json()
|
||||
|
||||
assert plan["title"] == "Review media pipeline issue"
|
||||
assert plan["affectedResources"] == ["media_pipeline"]
|
||||
assert any(step["href"] == "/media-pipeline" for step in plan["steps"])
|
||||
assert not any(step["href"] == "/inventory" for step in plan["steps"])
|
||||
assert any(check["endpoint"] == "/api/media-pipeline/summary" for check in plan["verificationChecks"])
|
||||
assert plan["executable"] is False
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_ssl_force_finding_has_specific_npm_review_plan(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
|
||||
with client:
|
||||
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-npm-host-without-ssl-force").json()
|
||||
|
||||
assert plan["title"] == "Review NPM SSL enforcement"
|
||||
assert "Force SSL" in plan["description"]
|
||||
assert any(step["actionId"] == "npm.proxy_hosts" for step in plan["steps"])
|
||||
assert any(step["id"] == "manual-ssl-change" and step["canExecute"] is False for step in plan["steps"])
|
||||
assert any(check["endpoint"] == "/api/apps/nginx-proxy-manager/proxy-hosts" for check in plan["verificationChecks"])
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_authentik_outpost_finding_has_specific_review_plan(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
|
||||
with client:
|
||||
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-auth-outpost-unavailable").json()
|
||||
|
||||
assert plan["title"] == "Review Authentik outpost health"
|
||||
assert any(step["actionId"] == "authentik.outpost_health" for step in plan["steps"])
|
||||
assert any(step["id"] == "manual-outpost-review" and step["canExecute"] is False for step in plan["steps"])
|
||||
assert any(check["endpoint"] == "/api/apps/authentik/outposts/health" for check in plan["verificationChecks"])
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_approval_proposal_is_draft_only_and_lifecycle_audited(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
_seed_signals(monkeypatch)
|
||||
|
||||
with client:
|
||||
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-policy-unraid-mover-voice-allowed").json()
|
||||
reviewed = client.post(f"/api/ai/remediation/plans/{plan['id']}/review").json()
|
||||
proposal = client.post(f"/api/ai/remediation/plans/{plan['id']}/create-approval-proposal").json()
|
||||
dismissed = client.post(f"/api/ai/remediation/plans/{plan['id']}/dismiss").json()
|
||||
approvals = client.get("/api/approvals").json()
|
||||
audit = client.get("/api/audit").json()
|
||||
|
||||
assert reviewed["status"] == "reviewed"
|
||||
assert proposal["created"] is True
|
||||
assert proposal["executed"] is False
|
||||
assert proposal["approvalId"] is None
|
||||
assert dismissed["status"] == "dismissed"
|
||||
assert approvals == []
|
||||
assert any(event["action_id"] == "ai.remediation.approval_proposal" for event in audit)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_routes_return_404_for_unknown_plan(tmp_path, monkeypatch):
|
||||
client, previous = _client(tmp_path, monkeypatch)
|
||||
|
||||
with client:
|
||||
response = client.get("/api/ai/remediation/plans/rem-missing")
|
||||
|
||||
assert response.status_code == 404
|
||||
|
||||
_restore_config(previous)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,377 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from http.server import ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CONTROLLER = ROOT / "ops" / "gitea-deploy-broker"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def controller(monkeypatch, tmp_path):
|
||||
monkeypatch.syspath_prepend(str(CONTROLLER))
|
||||
spec = importlib.util.spec_from_file_location("self_upgrade_controller_test", CONTROLLER / "status_server.py")
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
monkeypatch.setattr(module, "WEBHOOK_INBOX_ROOT", tmp_path / "inbox")
|
||||
module.WEBHOOK_INBOX_ROOT.mkdir(mode=0o700)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deploy_module(monkeypatch):
|
||||
if sys.platform == "win32":
|
||||
pytest.skip("controller deploy.py uses Linux fcntl")
|
||||
monkeypatch.syspath_prepend(str(CONTROLLER))
|
||||
spec = importlib.util.spec_from_file_location("self_upgrade_deploy_test", CONTROLLER / "deploy.py")
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _project():
|
||||
return {
|
||||
"branch": "main", "gitea_repository_id": 9,
|
||||
"gitea_webhook_secret_file": "jens__unraid-appops-gateway",
|
||||
"containers": ["Unraid-AppOps-Gateway"],
|
||||
"compose_project": "unraid-appops-gateway",
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"environment": {"APPOPS_DATA_PATH": "/srv/example/appdata/unraid-appops-gateway-1216"},
|
||||
"verify_oci_revision": True,
|
||||
"validation": {
|
||||
"validation_gate_version": 1, "required": True,
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"timeout_seconds": 1800, "poll_interval_seconds": 5,
|
||||
"max_poll_interval_seconds": 30,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _ready(controller, monkeypatch):
|
||||
monkeypatch.setattr(controller, "_load_projects", lambda: {controller.SELF_UPGRADE_REPOSITORY: _project()})
|
||||
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: True)
|
||||
monkeypatch.setattr(controller, "_is_current_branch_head", lambda repository, project, revision: True)
|
||||
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda repository, revision, policy: ("pass", []))
|
||||
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {
|
||||
"revision": "a" * 40, "imageId": "sha256:" + "b" * 64,
|
||||
"dataPath": "/srv/example/appdata/unraid-appops-gateway-1216",
|
||||
})
|
||||
monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3))
|
||||
|
||||
|
||||
def test_fixed_self_upgrade_preflight_accepts_exact_validated_head(controller, monkeypatch):
|
||||
_ready(controller, monkeypatch)
|
||||
result = controller._self_upgrade_preflight("c" * 40)
|
||||
assert result["status"] == "preflight_ok"
|
||||
assert result["currentRevision"] == "a" * 40
|
||||
assert len(result["preflightSha256"]) == 64
|
||||
assert result["executed"] is False
|
||||
assert all(check["passed"] for check in result["checks"])
|
||||
|
||||
|
||||
def test_timed_out_delivery_recovers_only_after_exact_protected_push_pass(controller, monkeypatch):
|
||||
_ready(controller, monkeypatch)
|
||||
revision = "c" * 40
|
||||
digest = controller._webhook_semantic_digest(9, "refs/heads/main", revision)
|
||||
policy = controller._validation_policy(controller.SELF_UPGRADE_REPOSITORY, _project())
|
||||
original = {
|
||||
"schema": controller._WEBHOOK_SCHEMA, "semantic_sha256": digest,
|
||||
"body_sha256": "b" * 64, "gitea_repository_id": 9,
|
||||
"ref": "refs/heads/main", "repository": controller.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": revision, "status": "validation_timeout",
|
||||
"validation": controller._validation_snapshot(policy, int(time.time())),
|
||||
}
|
||||
marker = controller.WEBHOOK_INBOX_ROOT / f"{digest}.json"
|
||||
marker.write_text(json.dumps(original), encoding="utf-8")
|
||||
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("wait", []))
|
||||
controller._auto_revalidate_timed_out_appops(marker)
|
||||
revalidation = controller.WEBHOOK_INBOX_ROOT / f"{digest}.revalidation.json"
|
||||
assert not revalidation.exists()
|
||||
controller._appops_revalidation_next_check.clear()
|
||||
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("pass", []))
|
||||
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: False)
|
||||
controller._auto_revalidate_timed_out_appops(marker)
|
||||
assert not revalidation.exists()
|
||||
controller._appops_revalidation_next_check.clear()
|
||||
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: True)
|
||||
controller._auto_revalidate_timed_out_appops(marker)
|
||||
assert json.loads(revalidation.read_text())["status"] == "awaiting_validation"
|
||||
assert json.loads(marker.read_text())["status"] == "validation_timeout"
|
||||
updated = json.loads(revalidation.read_text())
|
||||
updated["status"] = "deployed"
|
||||
revalidation.write_text(json.dumps(updated), encoding="utf-8")
|
||||
assert controller._self_upgrade_delivery_record(digest)["status"] == "deployed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gate", ["head", "validation", "runtime", "disk"])
|
||||
def test_self_upgrade_preflight_fails_closed_on_gate(controller, monkeypatch, gate):
|
||||
_ready(controller, monkeypatch)
|
||||
if gate == "head":
|
||||
monkeypatch.setattr(controller, "_self_upgrade_protected_head", lambda revision: False)
|
||||
elif gate == "validation":
|
||||
monkeypatch.setattr(controller, "_fetch_gitea_validation", lambda *args: ("wait", []))
|
||||
elif gate == "runtime":
|
||||
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: None)
|
||||
else:
|
||||
monkeypatch.setattr(controller.shutil, "disk_usage", lambda path: SimpleNamespace(free=1))
|
||||
result = controller._self_upgrade_preflight("c" * 40)
|
||||
assert result["status"] == "blocked"
|
||||
assert "preflightSha256" not in result
|
||||
|
||||
|
||||
def test_self_upgrade_preflight_rejects_non_sha_and_conflicting_upgrade(controller, monkeypatch):
|
||||
_ready(controller, monkeypatch)
|
||||
assert controller._self_upgrade_preflight("main")["status"] == "blocked"
|
||||
marker = controller.WEBHOOK_INBOX_ROOT / ("e" * 64 + ".json")
|
||||
marker.write_text(json.dumps({
|
||||
"repository": controller.SELF_UPGRADE_REPOSITORY,
|
||||
"revision": "d" * 40, "status": "deploying",
|
||||
}), encoding="utf-8")
|
||||
result = controller._self_upgrade_preflight("c" * 40)
|
||||
assert result["status"] == "blocked"
|
||||
assert result["checks"][-1] == {"name": "no_conflicting_upgrade", "passed": False}
|
||||
|
||||
|
||||
def test_runtime_requires_exact_container_mount_port_and_health(controller, monkeypatch):
|
||||
payload = [{
|
||||
"State": {"Running": True, "Health": {"Status": "healthy"}},
|
||||
"Mounts": [{"Destination": "/data", "Source": "/srv/example/appdata/unraid-appops-gateway-1216", "RW": True}],
|
||||
"Config": {"Labels": {"org.opencontainers.image.revision": "a" * 40,
|
||||
"org.opencontainers.image.source": "jens/unraid-appops-gateway"}},
|
||||
"NetworkSettings": {"Ports": {"1216/tcp": [{"HostPort": "1216"}]}},
|
||||
"Image": "sha256:" + "b" * 64,
|
||||
}]
|
||||
names = ["Unraid-AppOps-Gateway"]
|
||||
|
||||
def fake_run(args, **kwargs):
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps(payload) if args[1] == "inspect" else "\n".join(names), "")
|
||||
|
||||
monkeypatch.setattr(controller.subprocess, "run", fake_run)
|
||||
assert controller._self_upgrade_runtime()["revision"] == "a" * 40
|
||||
payload[0]["Mounts"][0]["Source"] = "/wrong"
|
||||
assert controller._self_upgrade_runtime() is None
|
||||
payload[0]["Mounts"][0]["Source"] = "/srv/example/appdata/unraid-appops-gateway-1216"
|
||||
names.append("Unraid-AppOps-Gateway-copy")
|
||||
assert controller._self_upgrade_runtime() is None
|
||||
|
||||
|
||||
def test_mcp_self_upgrade_rejects_arbitrary_repository(monkeypatch):
|
||||
from backend.app import mcp_server
|
||||
|
||||
with pytest.raises(ValueError, match="repository must be"):
|
||||
mcp_server.appops_self_upgrade_preflight("other/repo", "a" * 40, "test reason")
|
||||
with pytest.raises(ValueError, match="repository must be"):
|
||||
mcp_server.appops_self_upgrade_apply("other/repo", "a" * 40, "test reason", "b" * 64, "key-12345")
|
||||
|
||||
|
||||
def test_self_upgrade_signature_binds_endpoint_and_rejects_replay(controller, monkeypatch, tmp_path):
|
||||
secret = b"fixed-test-self-upgrade-capability-0123456789"
|
||||
token_file = tmp_path / "mcp-token"
|
||||
token_file.write_bytes(secret + b"\n")
|
||||
token_file.chmod(0o600)
|
||||
monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file)
|
||||
monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
|
||||
stamp = str(int(time.time()))
|
||||
nonce = "a" * 32
|
||||
body = b'{"revision":"' + b"b" * 40 + b'"}'
|
||||
path = controller.SELF_UPGRADE_PREFLIGHT_PATH
|
||||
signature = controller._request_signature(
|
||||
secret, method="POST", path=path, timestamp=stamp,
|
||||
nonce=nonce, raw_body=body,
|
||||
)
|
||||
assert not controller._self_upgrade_signature_is_valid(
|
||||
body, stamp, nonce, signature, method="POST", path=controller.SELF_UPGRADE_APPLY_PATH,
|
||||
)
|
||||
assert controller._self_upgrade_signature_is_valid(
|
||||
body, stamp, nonce, signature, method="POST", path=path,
|
||||
)
|
||||
assert not controller._self_upgrade_signature_is_valid(
|
||||
body, stamp, nonce, signature, method="POST", path=path,
|
||||
)
|
||||
|
||||
|
||||
def test_self_upgrade_http_contract_is_typed_idempotent_and_durable(controller, monkeypatch, tmp_path):
|
||||
_ready(controller, monkeypatch)
|
||||
monkeypatch.setattr(controller, "SELF_UPGRADE_REQUEST_ROOT", tmp_path / "requests")
|
||||
monkeypatch.setattr(controller, "APPOPS_NONCE_ROOT", tmp_path / "nonces")
|
||||
monkeypatch.setattr(controller, "APPOPS_PROGRESS_ROOT", tmp_path / "progress")
|
||||
secret = b"fixed-test-self-upgrade-capability-0123456789"
|
||||
token_file = tmp_path / "mcp-token"
|
||||
token_file.write_bytes(secret + b"\n")
|
||||
token_file.chmod(0o600)
|
||||
monkeypatch.setattr(controller, "SELF_UPGRADE_TOKEN_FILE", token_file)
|
||||
monkeypatch.setattr(controller.Handler, "log_message", lambda *args: None)
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), controller.Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
base = f"http://127.0.0.1:{server.server_port}"
|
||||
counter = 0
|
||||
|
||||
def call(path, payload, *, method="POST"):
|
||||
nonlocal counter
|
||||
counter += 1
|
||||
body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() if payload is not None else b""
|
||||
nonce = f"{counter:032x}"
|
||||
stamp = str(int(time.time()))
|
||||
signature = controller._request_signature(
|
||||
secret, method=method, path=path, timestamp=stamp,
|
||||
nonce=nonce, raw_body=body,
|
||||
)
|
||||
request = urllib.request.Request(
|
||||
base + path, data=body if method == "POST" else None,
|
||||
method=method,
|
||||
headers={
|
||||
"Content-Type": "application/json", "X-ITWorx-Deployment-Timestamp": stamp,
|
||||
"X-ITWorx-Deployment-Nonce": nonce,
|
||||
"X-ITWorx-Deployment-Signature": signature,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return response.status, json.load(response)
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, json.load(error)
|
||||
|
||||
try:
|
||||
intent = {
|
||||
"repository": "jens/unraid-appops-gateway", "revision": "c" * 40,
|
||||
"reason": "Validated production upgrade", "expectedCurrentRevision": "a" * 40,
|
||||
}
|
||||
invalid = {**intent, "image": "attacker/image:latest"}
|
||||
assert call(controller.SELF_UPGRADE_PREFLIGHT_PATH, invalid)[0] == 400
|
||||
code, preflight = call(controller.SELF_UPGRADE_PREFLIGHT_PATH, intent)
|
||||
assert code == 200 and preflight["status"] == "preflight_ok"
|
||||
apply = {**intent, "preflightSha256": preflight["preflightSha256"],
|
||||
"idempotencyKey": "fixed-request-12345"}
|
||||
code, accepted = call(controller.SELF_UPGRADE_APPLY_PATH, apply)
|
||||
assert code == 202 and accepted["status"] == "accepted", accepted
|
||||
operation_id = accepted["operationId"]
|
||||
assert len(operation_id) == 64
|
||||
assert (controller.SELF_UPGRADE_REQUEST_ROOT / (
|
||||
hashlib.sha256(apply["idempotencyKey"].encode()).hexdigest() + ".json"
|
||||
)).exists()
|
||||
code, replay = call(controller.SELF_UPGRADE_APPLY_PATH, apply)
|
||||
assert code == 200 and replay["status"] == "already_queued"
|
||||
assert replay["operationId"] == operation_id
|
||||
assert call(controller.SELF_UPGRADE_APPLY_PATH, {**apply, "revision": "d" * 40})[0] == 409
|
||||
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
|
||||
assert code == 200 and status["state"] == "awaiting_validation"
|
||||
assert status["operationState"] == "queued"
|
||||
assert status["revision"] == "c" * 40
|
||||
controller.APPOPS_PROGRESS_ROOT.mkdir(mode=0o700)
|
||||
(controller.APPOPS_PROGRESS_ROOT / ("c" * 40 + ".json")).write_text(json.dumps({
|
||||
"schema": "itworx.appops-upgrade-progress/v1", "revision": "c" * 40,
|
||||
"phase": "building",
|
||||
}), encoding="utf-8")
|
||||
delivery_path = controller.WEBHOOK_INBOX_ROOT / f"{operation_id}.json"
|
||||
delivery_record = json.loads(delivery_path.read_text(encoding="utf-8"))
|
||||
delivery_record["status"] = "deploying"
|
||||
delivery_path.write_text(json.dumps(delivery_record), encoding="utf-8")
|
||||
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
|
||||
assert code == 200 and status["operationState"] == "building"
|
||||
delivery_record["status"] = "validation_timeout"
|
||||
delivery_path.write_text(json.dumps(delivery_record), encoding="utf-8")
|
||||
recovered = {**delivery_record, "schema": controller._WEBHOOK_REVALIDATION_SCHEMA,
|
||||
"revalidation_of": operation_id, "status": "deployed"}
|
||||
(controller.WEBHOOK_INBOX_ROOT / f"{operation_id}.revalidation.json").write_text(
|
||||
json.dumps(recovered), encoding="utf-8"
|
||||
)
|
||||
monkeypatch.setattr(controller, "_self_upgrade_runtime", lambda: {
|
||||
"revision": "c" * 40, "imageId": "sha256:" + "b" * 64,
|
||||
"dataPath": "/srv/example/appdata/unraid-appops-gateway-1216",
|
||||
})
|
||||
monkeypatch.setattr(controller, "_deployment_state", lambda slug: {})
|
||||
monkeypatch.setattr(controller, "_verified_receipt", lambda *args: {"revision": "c" * 40})
|
||||
code, status = call(controller.SELF_UPGRADE_STATUS_PREFIX + operation_id, None, method="GET")
|
||||
assert code == 200 and status["operationState"] == "succeeded"
|
||||
assert status["state"] == "deployed"
|
||||
code, preflight = call(controller.SELF_UPGRADE_PREFLIGHT_PATH, {
|
||||
**intent, "expectedCurrentRevision": "c" * 40,
|
||||
})
|
||||
assert code == 200 and preflight["status"] == "already_deployed"
|
||||
already = {**intent, "expectedCurrentRevision": "c" * 40,
|
||||
"preflightSha256": preflight["preflightSha256"],
|
||||
"idempotencyKey": "already-deployed-request-12345"}
|
||||
code, result = call(controller.SELF_UPGRADE_APPLY_PATH, already)
|
||||
assert code == 200 and result["status"] == "already_deployed"
|
||||
assert result["operationId"] == operation_id
|
||||
code, result = call(controller.SELF_UPGRADE_APPLY_PATH, already)
|
||||
assert code == 200 and result["status"] == "already_deployed"
|
||||
finally:
|
||||
server.shutdown()
|
||||
thread.join(timeout=5)
|
||||
server.server_close()
|
||||
|
||||
|
||||
def test_appops_deploy_rechecks_exact_main_head_before_mutation(deploy_module, monkeypatch):
|
||||
revision = "c" * 40
|
||||
metadata = {"builtAt": "2026-09-28T00:00:00.000Z"}
|
||||
monkeypatch.setattr(deploy_module, "validate_commit", lambda repo, branch, value: value)
|
||||
monkeypatch.setattr(deploy_module, "deployment_metadata", lambda *args, **kwargs: metadata)
|
||||
monkeypatch.setattr(deploy_module, "output", lambda args: "d" * 40)
|
||||
with pytest.raises(deploy_module.DeployError, match="exact protected main head"):
|
||||
deploy_module.revalidate_deployment_candidate(
|
||||
Path("/repo.git"), {"branch": "main"}, revision,
|
||||
"jens/unraid-appops-gateway", metadata,
|
||||
)
|
||||
monkeypatch.setattr(deploy_module, "output", lambda args: revision)
|
||||
deploy_module.revalidate_deployment_candidate(
|
||||
Path("/repo.git"), {"branch": "main"}, revision,
|
||||
"jens/unraid-appops-gateway", metadata,
|
||||
)
|
||||
|
||||
|
||||
def test_appops_backup_is_fixed_and_digest_verified(deploy_module, monkeypatch, tmp_path):
|
||||
data = tmp_path / "appops-data"
|
||||
data.mkdir()
|
||||
(data / "state.txt").write_text("preserved", encoding="utf-8")
|
||||
monkeypatch.setattr(deploy_module, "APPOPS_DATA_ROOT", data)
|
||||
monkeypatch.setattr(deploy_module, "APPOPS_BACKUP_ROOT", tmp_path / "backups")
|
||||
monkeypatch.setattr(
|
||||
deploy_module.shutil, "disk_usage", lambda path: SimpleNamespace(free=50 * 1024**3)
|
||||
)
|
||||
receipt = deploy_module.capture_appops_data_backup("c" * 40, "a" * 40)
|
||||
archive = Path(receipt["archive"])
|
||||
assert hashlib.sha256(archive.read_bytes()).hexdigest() == receipt["sha256"]
|
||||
assert receipt["previousRevision"] == "a" * 40
|
||||
assert json.loads((archive.parent / "receipt.json").read_text()) == receipt
|
||||
with tarfile.open(archive, "r:gz") as backup:
|
||||
assert backup.extractfile("./state.txt").read() == b"preserved"
|
||||
|
||||
|
||||
def test_appops_backup_rejects_symlinked_data(deploy_module, monkeypatch, tmp_path):
|
||||
data = tmp_path / "real-data"
|
||||
data.mkdir()
|
||||
link = tmp_path / "link-data"
|
||||
link.symlink_to(data, target_is_directory=True)
|
||||
monkeypatch.setattr(deploy_module, "APPOPS_DATA_ROOT", link)
|
||||
with pytest.raises(deploy_module.DeployError, match="unsafe"):
|
||||
deploy_module.capture_appops_data_backup("c" * 40, "a" * 40)
|
||||
|
||||
|
||||
def test_appops_progress_is_durable_and_bounded(deploy_module, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(deploy_module, "APPOPS_PROGRESS_ROOT", tmp_path / "progress")
|
||||
deploy_module.write_appops_progress("c" * 40, "building")
|
||||
path = deploy_module.APPOPS_PROGRESS_ROOT / ("c" * 40 + ".json")
|
||||
assert json.loads(path.read_text())["phase"] == "building"
|
||||
deploy_module.write_appops_progress("c" * 40, "verifying")
|
||||
assert json.loads(path.read_text())["phase"] == "verifying"
|
||||
with pytest.raises(deploy_module.DeployError, match="invalid"):
|
||||
deploy_module.write_appops_progress("c" * 40, "arbitrary-command")
|
||||
@@ -0,0 +1,673 @@
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
def _reset_db(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "authentik_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
|
||||
def _set_authentik_config(
|
||||
url: str | None = "https://auth.example.test",
|
||||
token: str | None = "AUTHENTIK_TOKEN_VALUE",
|
||||
outpost_url: str | None = "https://outpost.example.test",
|
||||
):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = dict(settings.apis)
|
||||
settings.apis["AUTHENTIK_URL"] = url
|
||||
settings.apis["AUTHENTIK_TOKEN"] = token
|
||||
settings.apis["AUTHENTIK_VERIFY_TLS"] = False
|
||||
settings.apis["AUTHENTIK_TIMEOUT_SECONDS"] = "10"
|
||||
settings.apis["AUTHENTIK_OUTPOST_URL"] = outpost_url
|
||||
settings.apis["AUTHENTIK_EXPECT_EMBEDDED_OUTPOST"] = True
|
||||
return previous
|
||||
|
||||
|
||||
def _set_npm_config():
|
||||
from backend.app.core.config import settings
|
||||
|
||||
settings.apis["NPM_URL"] = "https://npm.example.test"
|
||||
settings.apis["NPM_IDENTITY"] = "admin@example.test"
|
||||
settings.apis["NPM_SECRET"] = "NPM_SECRET_VALUE"
|
||||
settings.apis["NPM_VERIFY_TLS"] = False
|
||||
settings.apis["NPM_TIMEOUT_SECONDS"] = "10"
|
||||
|
||||
|
||||
def _restore_config(previous):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous)
|
||||
|
||||
|
||||
def _applications():
|
||||
return [
|
||||
{"pk": "app-media", "name": "Media", "slug": "media", "launch_url": "https://media.example.com", "provider": 101},
|
||||
{"pk": "app-orphan", "name": "Orphan", "slug": "orphan", "launch_url": "https://orphan.example.com"},
|
||||
]
|
||||
|
||||
|
||||
def _providers():
|
||||
return [
|
||||
{"pk": 101, "name": "Media proxy", "type": "authentik_providers_proxy.proxyprovider", "assigned_application_slug": "media"},
|
||||
{"pk": 202, "name": "Dashboard oauth", "type": "authentik_providers_oauth2.oauth2provider", "assigned_application_slug": "dashboard"},
|
||||
]
|
||||
|
||||
|
||||
def _proxy_providers():
|
||||
return [
|
||||
{"pk": 101, "name": "Media proxy", "external_host": "https://media.example.com", "assigned_application_slug": "media"},
|
||||
{"pk": 303, "name": "Unused proxy", "external_host": "https://unused.example.com"},
|
||||
]
|
||||
|
||||
|
||||
def _oauth2_providers():
|
||||
return [
|
||||
{
|
||||
"pk": 202,
|
||||
"name": "Dashboard oauth",
|
||||
"assigned_application_slug": "dashboard",
|
||||
"redirect_uris": "https://a.example.com/cb\nhttps://b.example.com/cb\nhttps://c.example.com/cb\nhttps://d.example.com/cb",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _outposts():
|
||||
return [{"pk": "outpost-1", "name": "Embedded Outpost", "type": "proxy", "health": "healthy", "providers": [101]}]
|
||||
|
||||
|
||||
def _events():
|
||||
return [
|
||||
{"pk": "evt-1", "action": "login", "created": "2026-01-01T10:00:00Z", "client_ip": "198.18.10.25", "user": {"username": "jens"}},
|
||||
{"pk": "evt-2", "action": "authorize_application", "created": "2026-01-01T10:05:00Z", "client_ip": "198.19.0.2"},
|
||||
]
|
||||
|
||||
|
||||
def test_authentik_connector_missing_config_reports_required_env(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
previous = _set_authentik_config(url=None, token=None)
|
||||
|
||||
from backend.app.main import app
|
||||
|
||||
with TestClient(app) as client:
|
||||
payload = client.post("/api/connectors/authentik/test").json()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "missing_config"
|
||||
assert payload["missing_config"] is True
|
||||
assert payload["required_env_vars"] == ["AUTHENTIK_URL", "AUTHENTIK_TOKEN"]
|
||||
assert payload["secretRef"] == "AUTHENTIK_TOKEN"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_authentik_token_accepts_optional_bearer_prefix():
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
|
||||
previous = _set_authentik_config(token="Bearer AUTHENTIK_TOKEN_VALUE")
|
||||
|
||||
try:
|
||||
config = AuthentikConnector()._config()
|
||||
|
||||
assert config is not None
|
||||
assert config.token == "AUTHENTIK_TOKEN_VALUE"
|
||||
finally:
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_oidc_template_requires_the_exact_enrolled_role_claim():
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = _set_authentik_config()
|
||||
settings.apis['AUTHENTIK_OIDC_AUTHORIZATION_FLOW'] = 'flow-default-auth'
|
||||
settings.apis['AUTHENTIK_OIDC_INVALIDATION_FLOW'] = 'flow-default-invalidation'
|
||||
settings.apis['AUTHENTIK_OIDC_GROUP_CLAIM_MAPPING'] = 'mapping-groups'
|
||||
settings.apis['AUTHENTIK_OIDC_ROLE_CLAIM'] = 'groups'
|
||||
try:
|
||||
assert AuthentikConnector.deployment_oidc_template_status(role_claim='roles') == {
|
||||
'status': 'missing_config',
|
||||
'missing': ['AUTHENTIK_OIDC_ROLE_CLAIM'],
|
||||
}
|
||||
settings.apis['AUTHENTIK_OIDC_ROLE_CLAIM'] = 'roles'
|
||||
assert AuthentikConnector.deployment_oidc_template_status(role_claim='roles') == {
|
||||
'status': 'ready',
|
||||
'missing': [],
|
||||
}
|
||||
finally:
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_typed_oidc_provisioning_is_bounded_and_never_returns_client_secret(monkeypatch):
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = _set_authentik_config()
|
||||
settings.apis['AUTHENTIK_OIDC_AUTHORIZATION_FLOW'] = 'flow-default-auth'
|
||||
settings.apis['AUTHENTIK_OIDC_INVALIDATION_FLOW'] = 'flow-default-invalidation'
|
||||
settings.apis['AUTHENTIK_OIDC_GROUP_CLAIM_MAPPING'] = 'mapping-groups'
|
||||
writes = []
|
||||
|
||||
def paged(self, path, params=None):
|
||||
assert params is None
|
||||
return {
|
||||
'/core/groups/': [{'name': 'Administrator'}, {'name': 'SupportAgent'}, {'name': 'ReadOnly'}],
|
||||
'/core/applications/': [],
|
||||
}[path]
|
||||
|
||||
def write(self, method, path, payload=None):
|
||||
writes.append((method, path, payload))
|
||||
return {'pk': 'provider-1'} if path == '/providers/oauth2/' else {'pk': 'application-1'}
|
||||
|
||||
monkeypatch.setattr(AuthentikConnector, '_paged', paged, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, '_write_request', write, raising=True)
|
||||
secret = 'generated-client-secret-value'
|
||||
result = AuthentikConnector().ensure_deployment_oidc_application(
|
||||
application_slug='wsbc-portal',
|
||||
launch_url='https://support.itworx.test',
|
||||
redirect_uri='https://support.itworx.test/signin-oidc',
|
||||
client_secret=secret,
|
||||
administrator_group='Administrator',
|
||||
support_agent_group='SupportAgent',
|
||||
read_only_group='ReadOnly',
|
||||
)
|
||||
|
||||
assert result == {
|
||||
'success': True,
|
||||
'status': 'created',
|
||||
'message': 'oidc_application_created_from_typed_template',
|
||||
'applicationId': 'application-1',
|
||||
'providerId': 'provider-1',
|
||||
'clientSecretCreated': True,
|
||||
}
|
||||
assert writes[0][0:2] == ('POST', '/providers/oauth2/')
|
||||
assert writes[0][2]['client_secret'] == secret
|
||||
assert secret not in str(result)
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_typed_oidc_provisioning_removes_new_provider_when_application_create_fails(monkeypatch):
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = _set_authentik_config()
|
||||
settings.apis['AUTHENTIK_OIDC_AUTHORIZATION_FLOW'] = 'flow-default-auth'
|
||||
settings.apis['AUTHENTIK_OIDC_INVALIDATION_FLOW'] = 'flow-default-invalidation'
|
||||
settings.apis['AUTHENTIK_OIDC_GROUP_CLAIM_MAPPING'] = 'mapping-groups'
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
AuthentikConnector,
|
||||
'_paged',
|
||||
lambda self, path, params=None: (
|
||||
[{'name': 'Administrator'}, {'name': 'SupportAgent'}, {'name': 'ReadOnly'}]
|
||||
if path == '/core/groups/' else []
|
||||
),
|
||||
raising=True,
|
||||
)
|
||||
|
||||
def write(self, method, path, payload=None):
|
||||
calls.append((method, path, payload))
|
||||
if path == '/providers/oauth2/':
|
||||
return {'pk': 'provider-1'}
|
||||
if path == '/core/applications/':
|
||||
raise RuntimeError('request failed client_secret=never-return-this')
|
||||
assert (method, path, payload) == ('DELETE', '/providers/oauth2/provider-1/', None)
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(AuthentikConnector, '_write_request', write, raising=True)
|
||||
result = AuthentikConnector().ensure_deployment_oidc_application(
|
||||
application_slug='wsbc-portal',
|
||||
launch_url='https://support.itworx.test',
|
||||
redirect_uri='https://support.itworx.test/signin-oidc',
|
||||
client_secret='never-return-this',
|
||||
administrator_group='Administrator',
|
||||
support_agent_group='SupportAgent',
|
||||
read_only_group='ReadOnly',
|
||||
)
|
||||
|
||||
assert result == {
|
||||
'success': False,
|
||||
'status': 'failed',
|
||||
'message': 'oidc_application_creation_failed_provider_removed',
|
||||
}
|
||||
assert calls[-1] == ('DELETE', '/providers/oauth2/provider-1/', None)
|
||||
assert 'never-return-this' not in str(result)
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_typed_oidc_existing_application_requires_matching_provider_contract(monkeypatch):
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = _set_authentik_config()
|
||||
settings.apis['AUTHENTIK_OIDC_AUTHORIZATION_FLOW'] = 'flow-default-auth'
|
||||
settings.apis['AUTHENTIK_OIDC_INVALIDATION_FLOW'] = 'flow-default-invalidation'
|
||||
settings.apis['AUTHENTIK_OIDC_GROUP_CLAIM_MAPPING'] = 'mapping-groups'
|
||||
monkeypatch.setattr(
|
||||
AuthentikConnector,
|
||||
'_paged',
|
||||
lambda self, path, params=None: {
|
||||
'/core/groups/': [{'name': 'Administrator'}, {'name': 'SupportAgent'}, {'name': 'ReadOnly'}],
|
||||
'/core/applications/': [{'pk': 'application-1', 'slug': 'wsbc-portal', 'meta_launch_url': 'https://support.itworx.test', 'provider': 'provider-1'}],
|
||||
'/providers/oauth2/': [{'pk': 'provider-1', 'client_id': 'unexpected-client', 'client_type': 'confidential', 'redirect_uris': ['https://support.itworx.test/signin-oidc'], 'property_mappings': ['mapping-groups'], 'authorization_flow': 'flow-default-auth', 'invalidation_flow': 'flow-default-invalidation'}],
|
||||
}[path],
|
||||
raising=True,
|
||||
)
|
||||
try:
|
||||
result = AuthentikConnector().ensure_deployment_oidc_application(
|
||||
application_slug='wsbc-portal',
|
||||
launch_url='https://support.itworx.test',
|
||||
redirect_uri='https://support.itworx.test/signin-oidc',
|
||||
client_secret='never-return-this',
|
||||
administrator_group='Administrator',
|
||||
support_agent_group='SupportAgent',
|
||||
read_only_group='ReadOnly',
|
||||
)
|
||||
finally:
|
||||
_restore_config(previous)
|
||||
|
||||
assert result == {
|
||||
'success': False,
|
||||
'status': 'conflict',
|
||||
'message': 'existing_oidc_provider_does_not_match_managed_contract',
|
||||
}
|
||||
|
||||
|
||||
def test_authentik_health_success_and_auth_failure_redact_token(monkeypatch):
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
|
||||
previous = _set_authentik_config()
|
||||
requested_paths = []
|
||||
|
||||
def ok_request(self, path, params=None):
|
||||
requested_paths.append(path)
|
||||
return {"version": "2026.1.0"} if path == "/root/config/" else {}
|
||||
|
||||
monkeypatch.setattr(AuthentikConnector, "_request", ok_request, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "applications", lambda self: {"success": True, "items": _applications(), "total": 2}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "outpost_health", lambda self: {"success": True, "status": "ok", "available": True, "items": []}, raising=True)
|
||||
|
||||
healthy = AuthentikConnector().health()
|
||||
|
||||
assert healthy["success"] is True
|
||||
assert healthy["status"] == "connected"
|
||||
assert healthy["version"] == "2026.1.0"
|
||||
assert healthy["applicationCount"] == 2
|
||||
assert healthy["url"] == "https://auth.example.test"
|
||||
assert "AUTHENTIK_TOKEN_VALUE" not in str(healthy)
|
||||
assert "/root/config/" in requested_paths
|
||||
assert "/core/root/" not in requested_paths
|
||||
|
||||
def fail_request(self, path, params=None):
|
||||
raise RuntimeError("auth_failed token=AUTHENTIK_TOKEN_VALUE")
|
||||
|
||||
monkeypatch.setattr(AuthentikConnector, "_request", fail_request, raising=True)
|
||||
failed = AuthentikConnector().health()
|
||||
|
||||
assert failed["success"] is False
|
||||
assert failed["status"] == "auth_failed"
|
||||
assert "AUTHENTIK_TOKEN_VALUE" not in str(failed)
|
||||
assert "[REDACTED]" in str(failed)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_authentik_resources_normalize_and_events_mask_ips(monkeypatch):
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
|
||||
previous = _set_authentik_config()
|
||||
monkeypatch.setattr(
|
||||
AuthentikConnector,
|
||||
"_paged",
|
||||
lambda self, path, params=None: {
|
||||
"/core/applications/": _applications(),
|
||||
"/core/providers/": _providers(),
|
||||
"/providers/proxy/": _proxy_providers(),
|
||||
"/providers/oauth2/": _oauth2_providers(),
|
||||
"/outposts/instances/": _outposts(),
|
||||
"/events/events/": _events(),
|
||||
"/flows/instances/": [{"pk": "flow-1", "name": "Login", "designation": "authentication"}],
|
||||
"/policies/all/": [{"pk": "policy-1", "name": "Require MFA", "enabled": True}],
|
||||
}[path],
|
||||
raising=True,
|
||||
)
|
||||
|
||||
connector = AuthentikConnector()
|
||||
apps = connector.applications()
|
||||
providers = connector.providers()
|
||||
proxy = connector.proxy_providers()
|
||||
oauth = connector.oauth2_providers()
|
||||
outposts = connector.outposts()
|
||||
events = connector.events()
|
||||
flows = connector.flows_summary()
|
||||
policies = connector.policies_summary()
|
||||
|
||||
assert apps["items"][0]["slug"] == "media"
|
||||
assert providers["items"][0]["providerType"] == "proxy"
|
||||
assert proxy["items"][0]["externalHost"] == "https://media.example.com"
|
||||
assert oauth["items"][0]["redirectUriCount"] == 4
|
||||
assert outposts["items"][0]["providerCount"] == 1
|
||||
assert events["items"][0]["clientIp"] == "192.168.10.x"
|
||||
assert flows["total"] == 1
|
||||
assert policies["enabled"] == 1
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_authentik_auth_coverage_and_security_warnings_with_npm(monkeypatch):
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
|
||||
previous = _set_authentik_config()
|
||||
_set_npm_config()
|
||||
monkeypatch.setattr(AuthentikConnector, "applications", lambda self: {"success": True, "items": AuthentikConnector._map_applications(_applications())}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "providers", lambda self: {"success": True, "items": AuthentikConnector._map_providers(_providers())}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "proxy_providers", lambda self: {"success": True, "items": AuthentikConnector._map_proxy_providers(_proxy_providers())}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "oauth2_providers", lambda self: {"success": True, "items": AuthentikConnector._map_oauth2_providers(_oauth2_providers())}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "outposts", lambda self: {"success": True, "items": []}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "outpost_health", lambda self: {"success": True, "available": False, "embeddedExpected": True, "status": "warning", "items": []}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "events", lambda self, limit=25: {"success": True, "items": []}, raising=True)
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
"proxy_hosts",
|
||||
lambda self: {
|
||||
"success": True,
|
||||
"items": [
|
||||
{"id": 1, "domainNames": ["media.example.com"], "enabled": True, "accessListId": 2, "forwardHost": "plex", "forwardPort": 32400, "forwardScheme": "http"},
|
||||
{"id": 2, "domainNames": ["open.example.com"], "enabled": True, "accessListId": None, "forwardHost": "app", "forwardPort": 8080, "forwardScheme": "http"},
|
||||
],
|
||||
},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
coverage = AuthentikConnector().auth_coverage()
|
||||
warnings = AuthentikConnector().security_warnings()
|
||||
|
||||
assert coverage["summary"]["totalPublicHosts"] == 2
|
||||
assert coverage["summary"]["protectedHosts"] == 1
|
||||
assert coverage["summary"]["unprotectedHosts"] == 1
|
||||
assert any(item["coverageStatus"] == "protected" for item in coverage["publicHosts"])
|
||||
assert any(item["coverageStatus"] == "unprotected" for item in coverage["publicHosts"])
|
||||
assert any(item["type"] == "no_outposts" for item in warnings["items"])
|
||||
assert any(item["type"] == "public_host_without_authentik" for item in warnings["items"])
|
||||
assert any(item["type"] == "oauth_many_redirect_uris" for item in warnings["items"])
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_authentik_reuses_read_results_within_connector_instance(monkeypatch):
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
|
||||
previous = _set_authentik_config()
|
||||
_set_npm_config()
|
||||
calls = []
|
||||
|
||||
def paged(self, path, params=None):
|
||||
calls.append(path)
|
||||
return {
|
||||
"/core/applications/": _applications(),
|
||||
"/core/providers/": _providers(),
|
||||
"/providers/proxy/": _proxy_providers(),
|
||||
"/providers/oauth2/": _oauth2_providers(),
|
||||
"/outposts/instances/": _outposts(),
|
||||
"/events/events/": _events(),
|
||||
}[path]
|
||||
|
||||
monkeypatch.setattr(AuthentikConnector, "_paged", paged, raising=True)
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
"proxy_hosts",
|
||||
lambda self: {
|
||||
"success": True,
|
||||
"status": "ok",
|
||||
"items": [
|
||||
{"id": 1, "domainNames": ["media.example.com"], "enabled": True, "accessListId": 2, "forwardHost": "plex", "forwardPort": 32400, "forwardScheme": "http"},
|
||||
{"id": 2, "domainNames": ["open.example.com"], "enabled": True, "accessListId": None, "forwardHost": "app", "forwardPort": 8080, "forwardScheme": "http"},
|
||||
],
|
||||
},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
connector = AuthentikConnector()
|
||||
coverage = connector.auth_coverage()
|
||||
warnings = connector.security_warnings()
|
||||
outpost = connector.outpost_health()
|
||||
|
||||
assert coverage["summary"]["totalPublicHosts"] == 2
|
||||
assert warnings["total"] >= 1
|
||||
assert outpost["available"] is True
|
||||
assert calls.count("/core/applications/") == 1
|
||||
assert calls.count("/core/providers/") == 1
|
||||
assert calls.count("/providers/proxy/") == 1
|
||||
assert calls.count("/providers/oauth2/") == 1
|
||||
assert calls.count("/outposts/instances/") == 1
|
||||
assert calls.count("/events/events/") == 1
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_authentik_unknown_embedded_outpost_is_warning_not_high(monkeypatch):
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
|
||||
previous = _set_authentik_config(outpost_url=None)
|
||||
try:
|
||||
monkeypatch.setattr(AuthentikConnector, "applications", lambda self: {"success": True, "items": AuthentikConnector._map_applications(_applications())}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "providers", lambda self: {"success": True, "items": []}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "proxy_providers", lambda self: {"success": True, "items": []}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "oauth2_providers", lambda self: {"success": True, "items": []}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "outposts", lambda self: {"success": True, "items": [{"id": "embedded", "name": "Embedded Outpost", "health": "unknown"}]}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "outpost_health", lambda self: {"success": True, "available": False, "embeddedExpected": True, "status": "warning", "items": [{"id": "embedded", "name": "Embedded Outpost", "health": "unknown"}]}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "events", lambda self, limit=25: {"success": True, "items": []}, raising=True)
|
||||
|
||||
warnings = AuthentikConnector().security_warnings()
|
||||
embedded = next(item for item in warnings["items"] if item["type"] == "embedded_outpost_unavailable")
|
||||
|
||||
assert embedded["severity"] == "warning"
|
||||
finally:
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_authentik_numeric_pagination_reuses_collection_path(monkeypatch):
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
|
||||
previous = _set_authentik_config()
|
||||
calls = []
|
||||
|
||||
def request(self, path, params=None):
|
||||
calls.append((path, params))
|
||||
page = int((params or {}).get("page", 1))
|
||||
return {
|
||||
"results": [{"pk": f"app-{page}"}],
|
||||
"pagination": {"next": page + 1 if page < 3 else 0},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(AuthentikConnector, "_request", request, raising=True)
|
||||
try:
|
||||
items = AuthentikConnector()._paged("/core/applications/", {"page_size": 20})
|
||||
|
||||
assert [item["pk"] for item in items] == ["app-1", "app-2", "app-3"]
|
||||
assert calls == [
|
||||
("/core/applications/", {"page_size": 20}),
|
||||
("/core/applications/", {"page_size": 20, "page": 2}),
|
||||
("/core/applications/", {"page_size": 20, "page": 3}),
|
||||
]
|
||||
finally:
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_authentik_outpost_reachability_is_not_mislabeled_as_health(monkeypatch):
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
|
||||
previous = _set_authentik_config(outpost_url="https://outpost.example.test")
|
||||
try:
|
||||
monkeypatch.setattr(
|
||||
AuthentikConnector,
|
||||
"outposts",
|
||||
lambda self: {
|
||||
"success": True,
|
||||
"items": [{"id": "embedded", "name": "Embedded Outpost", "health": "unknown"}],
|
||||
},
|
||||
raising=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
AuthentikConnector,
|
||||
"_probe_outpost",
|
||||
lambda self, config: {"status": "reachable", "reachable": True, "statusCode": 302},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
result = AuthentikConnector().outpost_health()
|
||||
|
||||
assert result["available"] is False
|
||||
assert result["reachable"] is True
|
||||
assert result["status"] == "reachable_unverified"
|
||||
assert result["telemetryStatus"] == "unverified"
|
||||
assert result["message"] == "outpost_reachable_health_unverified"
|
||||
finally:
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_authentik_outpost_probe_never_forwards_api_token(monkeypatch):
|
||||
from backend.app.connectors import authentik
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
|
||||
previous = _set_authentik_config(token="SENSITIVE_AUTHENTIK_TOKEN")
|
||||
captured = {}
|
||||
|
||||
class FakeResponse:
|
||||
status_code = 401
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, **kwargs):
|
||||
captured["client"] = kwargs
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def get(self, url, headers):
|
||||
captured["url"] = url
|
||||
captured["headers"] = headers
|
||||
return FakeResponse()
|
||||
|
||||
monkeypatch.setattr(authentik.httpx, "Client", FakeClient)
|
||||
try:
|
||||
config = AuthentikConnector()._config()
|
||||
result = AuthentikConnector._probe_outpost(config)
|
||||
finally:
|
||||
_restore_config(previous)
|
||||
|
||||
assert result == {"status": "reachable", "reachable": True, "statusCode": 401}
|
||||
assert "Authorization" not in captured["headers"]
|
||||
assert "SENSITIVE_AUTHENTIK_TOKEN" not in str(captured)
|
||||
|
||||
|
||||
def test_authentik_routes_are_read_only_and_shape_is_consistent(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
previous = _set_authentik_config()
|
||||
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
from backend.app.main import app
|
||||
|
||||
monkeypatch.setattr(AuthentikConnector, "_request", lambda self, path, params=None: {"version": "2026.1.0"}, raising=True)
|
||||
monkeypatch.setattr(AuthentikConnector, "_paged", lambda self, path, params=None: [], raising=True)
|
||||
|
||||
with TestClient(app) as client:
|
||||
for path in [
|
||||
"/api/apps/authentik/health",
|
||||
"/api/apps/authentik/system",
|
||||
"/api/apps/authentik/applications",
|
||||
"/api/apps/authentik/providers",
|
||||
"/api/apps/authentik/providers/proxy",
|
||||
"/api/apps/authentik/providers/oauth2",
|
||||
"/api/apps/authentik/outposts",
|
||||
"/api/apps/authentik/outposts/health",
|
||||
"/api/apps/authentik/flows",
|
||||
"/api/apps/authentik/policies",
|
||||
"/api/apps/authentik/events",
|
||||
"/api/apps/authentik/auth-coverage",
|
||||
"/api/apps/authentik/security-warnings",
|
||||
]:
|
||||
response = client.get(path)
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert set(["success", "status", "message", "data"]).issubset(payload.keys())
|
||||
assert client.post("/api/apps/authentik/applications").status_code == 405
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_authentik_policy_blocks_mutations_and_unknowns_fail_closed(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
|
||||
from backend.app.main import app
|
||||
|
||||
with TestClient(app) as client:
|
||||
for action in [
|
||||
"authentik.create_application",
|
||||
"authentik.update_application",
|
||||
"authentik.delete_application",
|
||||
"authentik.create_provider",
|
||||
"authentik.delete_provider",
|
||||
"authentik.create_outpost",
|
||||
"authentik.delete_policy",
|
||||
"authentik.create_user",
|
||||
"authentik.delete_group",
|
||||
"authentik.rotate_token",
|
||||
"authentik.force_sync",
|
||||
"authentik.mass_edit",
|
||||
"authentik.unknown_mutation",
|
||||
]:
|
||||
payload = client.post("/api/policies/evaluate", json={"action": action, "target": "authentik", "source": "manual"}).json()
|
||||
assert payload["decision"] == "blocked"
|
||||
events_voice = client.post("/api/policies/evaluate", json={"action": "authentik.events", "target": "authentik", "source": "voice"}).json()
|
||||
assert events_voice["decision"] == "approval_required"
|
||||
|
||||
|
||||
def test_authentik_inventory_classification_and_candidate(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
|
||||
from backend.app.main import app
|
||||
from backend.app.services.app_classifier import classify_container
|
||||
|
||||
result = classify_container({"name": "authentik-server", "image": "ghcr.io/goauthentik/server:latest", "ports": ["9000:9000"]})
|
||||
|
||||
assert result["app_type"] == "authentik"
|
||||
assert result["category"] == "auth"
|
||||
assert result["subcategory"] == "identity_provider"
|
||||
assert result["connector"] == "authentik"
|
||||
assert result["criticality"] == "critical"
|
||||
assert result["policy_profile"] == "auth_provider"
|
||||
assert result["protected"] is True
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.test_connection", lambda self: {"status": "connected"}, raising=True)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.portainer.PortainerConnector.list_containers",
|
||||
lambda self: {"status": "ok", "containers": [{"id": "auth-container", "name": "authentik-worker", "image": "ghcr.io/goauthentik/proxy", "state": "running", "status": "running", "ports": ["9000:9000"]}]},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
client.post("/api/inventory/discover")
|
||||
candidates = client.get("/api/connectors/authentik/candidates").json()
|
||||
|
||||
assert len(candidates) == 1
|
||||
assert candidates[0]["connectorId"] == "authentik"
|
||||
assert candidates[0]["appType"] == "authentik"
|
||||
assert candidates[0]["missingSecrets"] == ["AUTHENTIK_URL", "AUTHENTIK_TOKEN"]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_controller_promoter_action_is_fixed_and_registered():
|
||||
registry = (ROOT / "backend/app/services/action_registry.py").read_text(encoding="utf-8")
|
||||
execution = (ROOT / "backend/app/services/action_execution.py").read_text(encoding="utf-8")
|
||||
assert "'controller.artifact.promote'" in registry
|
||||
assert "return 'tower/deployment-controller'" in registry
|
||||
assert 'action == "controller.artifact.promote"' in execution
|
||||
assert 'target.strip().lower() != "tower/deployment-controller"' in execution
|
||||
|
||||
|
||||
def test_controller_promoter_helper_spec_is_fixed_and_bounded():
|
||||
from backend.app.services.controller_artifact_promoter import (
|
||||
CONTROLLER_VERSION,
|
||||
HELPER_NAME,
|
||||
helper_container_spec,
|
||||
)
|
||||
|
||||
revision = "a" * 40
|
||||
image_id = "sha256:" + "b" * 64
|
||||
spec = helper_container_spec(revision=revision, image_id=image_id)
|
||||
|
||||
assert HELPER_NAME == "itworx-controller-artifact-promoter"
|
||||
assert re.fullmatch(r"2\.1\.\d+", CONTROLLER_VERSION)
|
||||
assert spec["Image"] == image_id
|
||||
assert spec["HostConfig"]["Privileged"] is False
|
||||
assert spec["HostConfig"]["NetworkMode"] == "host"
|
||||
assert spec["HostConfig"]["SecurityOpt"] == ["no-new-privileges:true"]
|
||||
assert set(spec["HostConfig"]["Binds"]) == {
|
||||
"/mnt/cache:/mnt/cache",
|
||||
"/mnt/user:/mnt/user",
|
||||
"/srv/example/appdata/gitea/git/repositories:/gitea-repositories:ro",
|
||||
"/var/run/docker.sock:/var/run/docker.sock",
|
||||
}
|
||||
assert spec["Cmd"][:2] == ["/bin/bash", "-lc"]
|
||||
command = spec["Cmd"][2]
|
||||
assert "scripts/upgrade_deployment_controller.sh" in command
|
||||
assert "UPGRADE_DEPLOYMENT_CONTROLLER_V2" in command
|
||||
assert revision in command
|
||||
assert CONTROLLER_VERSION in command
|
||||
assert "/gitea-repositories/jens/unraid-appops-gateway.git" in command
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"revision,image_id",
|
||||
[
|
||||
("main", "sha256:" + "b" * 64),
|
||||
("a" * 39, "sha256:" + "b" * 64),
|
||||
("a" * 40, "ubuntu:latest"),
|
||||
("a" * 40, "sha256:" + "z" * 64),
|
||||
],
|
||||
)
|
||||
def test_controller_promoter_rejects_mutable_or_invalid_identity(revision: str, image_id: str):
|
||||
from backend.app.services.controller_artifact_promoter import helper_container_spec
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
helper_container_spec(revision=revision, image_id=image_id)
|
||||
|
||||
|
||||
def test_controller_promoter_uses_only_protected_main_and_push_gate():
|
||||
source = (ROOT / "backend/app/services/controller_artifact_promoter.py").read_text(encoding="utf-8")
|
||||
assert 'branch = "main"' in source
|
||||
assert 'REQUIRED_PUSH_CONTEXT = "Managed validation / full (push)"' in source
|
||||
assert "controller_promoter_main_not_validated" in source
|
||||
assert "caller_revision" not in source
|
||||
|
||||
|
||||
def test_deployment_broker_exposes_bounded_controller_health():
|
||||
source = (ROOT / "backend/app/services/deployment_broker.py").read_text(encoding="utf-8")
|
||||
assert "def controller_health(self)" in source
|
||||
assert 'f"{self.base_url}/health"' in source
|
||||
assert "_MAX_CONTROLLER_HEALTH_BYTES" in source
|
||||
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "import_controller_bootstrap.py"
|
||||
SPEC = importlib.util.spec_from_file_location("controller_bootstrap_import_host", SCRIPT)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
host_import = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(host_import)
|
||||
|
||||
|
||||
def test_reads_root_private_bootstrap_without_newline(tmp_path, monkeypatch):
|
||||
path = tmp_path / "bootstrap"
|
||||
path.write_bytes(b"x" * 64 + b"\n")
|
||||
path.chmod(0o600)
|
||||
monkeypatch.setattr(host_import.os, "geteuid", lambda: 0, raising=False)
|
||||
assert host_import.read_bootstrap(path) == b"x" * 64
|
||||
path.write_bytes(b"short\n")
|
||||
with pytest.raises(ValueError):
|
||||
host_import.read_bootstrap(path)
|
||||
|
||||
|
||||
def test_bootstrap_passes_only_on_stdin(monkeypatch):
|
||||
observed = {}
|
||||
|
||||
def fake_run(argv, *, input, capture_output, check, timeout):
|
||||
observed.update({"argv": argv, "input": input, "capture_output": capture_output})
|
||||
|
||||
class Result:
|
||||
returncode = 0
|
||||
stdout = json.dumps({"status": "imported", "secretRef": "ENROLLMENT_BOOTSTRAP"}).encode()
|
||||
|
||||
return Result()
|
||||
|
||||
monkeypatch.setattr(host_import.subprocess, "run", fake_run)
|
||||
assert host_import.import_bootstrap(b"sensitive-test-value") == "imported"
|
||||
assert observed["input"] == b"sensitive-test-value"
|
||||
assert "sensitive-test-value" not in " ".join(observed["argv"])
|
||||
assert observed["capture_output"] is True
|
||||
@@ -0,0 +1,255 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
HELPER_PATH = ROOT / "scripts" / "reconcile_controller_state.py"
|
||||
UPGRADE_PATH = ROOT / "scripts" / "upgrade_deployment_controller.sh"
|
||||
WRAPPER_PATH = ROOT / "scripts" / "upgrade_deployment_controller.ps1"
|
||||
|
||||
spec = importlib.util.spec_from_file_location("reconcile_controller_state", HELPER_PATH)
|
||||
assert spec is not None and spec.loader is not None
|
||||
reconciliation = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(reconciliation)
|
||||
|
||||
|
||||
def _roots(tmp_path: Path) -> tuple[Path, Path]:
|
||||
legacy = tmp_path / "legacy-state"
|
||||
cache = tmp_path / "cache-state"
|
||||
legacy.mkdir()
|
||||
cache.mkdir()
|
||||
return legacy, cache
|
||||
|
||||
|
||||
def test_reconciliation_migrates_only_missing_legacy_state_with_metadata(tmp_path: Path):
|
||||
legacy, cache = _roots(tmp_path)
|
||||
(legacy / "nested").mkdir(mode=0o750)
|
||||
source = legacy / "nested" / "receipt.json"
|
||||
source.write_bytes(b'{"revision":"abc"}\n')
|
||||
source.chmod(0o640)
|
||||
identical = legacy / "existing.json"
|
||||
identical.write_bytes(b'{"status":"healthy"}\n')
|
||||
(cache / "existing.json").write_bytes(identical.read_bytes())
|
||||
|
||||
result = reconciliation.reconcile_state_roots(legacy, cache)
|
||||
|
||||
migrated = cache / "nested" / "receipt.json"
|
||||
assert migrated.read_bytes() == source.read_bytes()
|
||||
assert stat.S_IMODE(migrated.stat().st_mode) == stat.S_IMODE(source.stat().st_mode)
|
||||
assert result == {"legacyEntries": 3, "cacheEntries": 1, "migratedDirectories": 1, "migratedFiles": 1}
|
||||
|
||||
|
||||
def test_reconciliation_detects_every_conflict_before_copying_any_missing_file(tmp_path: Path):
|
||||
legacy, cache = _roots(tmp_path)
|
||||
(legacy / "a-missing.json").write_text("legacy-only", encoding="utf-8")
|
||||
(legacy / "z-conflict.json").write_text("legacy", encoding="utf-8")
|
||||
(cache / "z-conflict.json").write_text("cache", encoding="utf-8")
|
||||
|
||||
with pytest.raises(reconciliation.StateConflictError, match="z-conflict.json"):
|
||||
reconciliation.reconcile_state_roots(legacy, cache)
|
||||
|
||||
assert not (cache / "a-missing.json").exists()
|
||||
assert (cache / "z-conflict.json").read_text(encoding="utf-8") == "cache"
|
||||
|
||||
|
||||
def test_reconciliation_rejects_symlinks_without_mutating_cache(tmp_path: Path):
|
||||
if not hasattr(os, "symlink"):
|
||||
pytest.skip("symlinks are unavailable")
|
||||
legacy, cache = _roots(tmp_path)
|
||||
target = tmp_path / "outside.json"
|
||||
target.write_text("outside", encoding="utf-8")
|
||||
try:
|
||||
(legacy / "unsafe.json").symlink_to(target)
|
||||
except OSError:
|
||||
pytest.skip("the test process cannot create symlinks")
|
||||
|
||||
with pytest.raises(reconciliation.UnsafeStateError, match="unsafe.json"):
|
||||
reconciliation.reconcile_state_roots(legacy, cache)
|
||||
|
||||
assert list(cache.iterdir()) == []
|
||||
|
||||
|
||||
def test_reconciliation_allows_cache_only_state_and_keeps_it_unchanged(tmp_path: Path):
|
||||
legacy, cache = _roots(tmp_path)
|
||||
(legacy / "shared.json").write_text("shared", encoding="utf-8")
|
||||
(cache / "shared.json").write_text("shared", encoding="utf-8")
|
||||
cache_only = cache / "new-controller.json"
|
||||
cache_only.write_text("cache-only", encoding="utf-8")
|
||||
|
||||
result = reconciliation.reconcile_state_roots(legacy, cache)
|
||||
|
||||
assert cache_only.read_text(encoding="utf-8") == "cache-only"
|
||||
assert result["migratedDirectories"] == 0
|
||||
assert result["migratedFiles"] == 0
|
||||
|
||||
|
||||
def test_reconciliation_rejects_metadata_conflict_before_copying_missing_state(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
legacy, cache = _roots(tmp_path)
|
||||
(legacy / "missing.json").write_text("legacy-only", encoding="utf-8")
|
||||
(legacy / "shared.json").write_text("shared", encoding="utf-8")
|
||||
(cache / "shared.json").write_text("shared", encoding="utf-8")
|
||||
real_inventory = reconciliation._inventory
|
||||
|
||||
def inventory_with_conflicting_cache_metadata(root: Path):
|
||||
entries = real_inventory(root)
|
||||
if root == cache:
|
||||
entries["shared.json"] = {
|
||||
**entries["shared.json"],
|
||||
"mode": int(entries["shared.json"]["mode"]) ^ 0o020,
|
||||
"uid": int(entries["shared.json"]["uid"]) + 1,
|
||||
"gid": int(entries["shared.json"]["gid"]) + 1,
|
||||
}
|
||||
return entries
|
||||
|
||||
monkeypatch.setattr(reconciliation, "_inventory", inventory_with_conflicting_cache_metadata)
|
||||
|
||||
with pytest.raises(reconciliation.StateConflictError, match="shared.json"):
|
||||
reconciliation.reconcile_state_roots(legacy, cache)
|
||||
|
||||
assert not (cache / "missing.json").exists()
|
||||
|
||||
|
||||
def test_reconciliation_rejects_legacy_inode_replacement_and_rolls_back(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
legacy, cache = _roots(tmp_path)
|
||||
source = legacy / "receipt.json"
|
||||
source.write_bytes(b'{"revision":"abc"}\n')
|
||||
real_create_file = reconciliation._create_file
|
||||
|
||||
def replace_source_before_copy(path: Path, payload: bytes, metadata: dict[str, int | str]):
|
||||
replacement = source.with_suffix(".replacement")
|
||||
replacement.write_bytes(source.read_bytes())
|
||||
os.replace(replacement, source)
|
||||
real_create_file(path, payload, metadata)
|
||||
|
||||
monkeypatch.setattr(reconciliation, "_create_file", replace_source_before_copy)
|
||||
|
||||
with pytest.raises(reconciliation.UnsafeStateError, match="changed during migration"):
|
||||
reconciliation.reconcile_state_roots(legacy, cache)
|
||||
|
||||
assert list(cache.iterdir()) == []
|
||||
|
||||
|
||||
def test_reconciliation_rolls_back_every_created_entry_after_partial_failure(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
legacy, cache = _roots(tmp_path)
|
||||
(legacy / "nested").mkdir()
|
||||
(legacy / "nested" / "a.json").write_text("first", encoding="utf-8")
|
||||
(legacy / "nested" / "b.json").write_text("second", encoding="utf-8")
|
||||
real_create_file = reconciliation._create_file
|
||||
created_files = 0
|
||||
|
||||
def fail_after_first_file(path: Path, payload: bytes, metadata: dict[str, int | str]):
|
||||
nonlocal created_files
|
||||
if created_files == 1:
|
||||
raise OSError("injected second-file failure")
|
||||
real_create_file(path, payload, metadata)
|
||||
created_files += 1
|
||||
|
||||
monkeypatch.setattr(reconciliation, "_create_file", fail_after_first_file)
|
||||
|
||||
with pytest.raises(OSError, match="injected second-file failure"):
|
||||
reconciliation.reconcile_state_roots(legacy, cache)
|
||||
|
||||
assert list(cache.iterdir()) == []
|
||||
|
||||
|
||||
def test_successful_reconciliation_can_be_rolled_back_before_controller_swap(tmp_path: Path):
|
||||
legacy, cache = _roots(tmp_path)
|
||||
preexisting = cache / "cache-only.json"
|
||||
preexisting.write_text("keep", encoding="utf-8")
|
||||
(legacy / "nested").mkdir()
|
||||
(legacy / "nested" / "receipt.json").write_text("migrate", encoding="utf-8")
|
||||
journal = tmp_path / "migration-journal.json"
|
||||
|
||||
reconciliation.reconcile_state_roots(legacy, cache, journal_path=journal)
|
||||
|
||||
assert (cache / "nested" / "receipt.json").read_text(encoding="utf-8") == "migrate"
|
||||
reconciliation.rollback_reconciliation(cache, journal)
|
||||
|
||||
assert preexisting.read_text(encoding="utf-8") == "keep"
|
||||
assert not (cache / "nested").exists()
|
||||
assert not journal.exists()
|
||||
|
||||
|
||||
def test_upgrade_stops_old_writer_before_locking_cache_and_reconciles_while_stopped():
|
||||
script = UPGRADE_PATH.read_text(encoding="utf-8")
|
||||
|
||||
cache_lock = 'flock -x "$cache_lock_fd"'
|
||||
stop = 'writers_stopped=1\ndocker compose --project-directory "$ROOT" -f "$COMPOSE_FILE" stop "$COMPOSE_SERVICE"'
|
||||
reconcile = 'python3 "${stage}/reconcile_controller_state.py" \\\n --legacy-root'
|
||||
backup = 'cp -a -- "$CONTROLLER_DIR" "$backup/controller"'
|
||||
swap = 'mv -- "$CONTROLLER_DIR" "$previous"'
|
||||
|
||||
assert '/srv/example/appdata/gitea-deploy-runners/deploy.lock' not in script
|
||||
assert script.index(stop) < script.index(cache_lock)
|
||||
assert script.index(stop) < script.index(backup)
|
||||
assert script.index(cache_lock) < script.index(backup)
|
||||
assert script.index(backup) < script.index(reconcile)
|
||||
assert script.index(reconcile) < script.index(swap)
|
||||
assert 'flock -u "$cache_lock_fd"' not in script
|
||||
|
||||
|
||||
def test_upgrade_retains_and_validates_optional_private_tls_before_stopping_controller():
|
||||
script = UPGRADE_PATH.read_text(encoding="utf-8")
|
||||
|
||||
stage_copy = 'cp -a -- "$CONTROLLER_DIR" "${stage}/controller"'
|
||||
tls_gate = (
|
||||
'if [[ -e "${stage}/controller/private-tls.json" '
|
||||
'|| -L "${stage}/controller/private-tls.json" ]]; then\n'
|
||||
' python3 "${stage}/controller/status_server.py" --check-private-tls-config\n'
|
||||
'fi'
|
||||
)
|
||||
stop = (
|
||||
'writers_stopped=1\ndocker compose --project-directory "$ROOT" '
|
||||
'-f "$COMPOSE_FILE" stop "$COMPOSE_SERVICE"'
|
||||
)
|
||||
|
||||
assert stage_copy in script
|
||||
assert script.index(stage_copy) < script.index(tls_gate)
|
||||
assert script.index(tls_gate) < script.index(stop)
|
||||
assert 'cp -a -- "$CONTROLLER_DIR" "$backup/controller"' in script
|
||||
|
||||
|
||||
def test_upgrade_rolls_back_state_before_restoring_or_starting_the_old_writer():
|
||||
script = UPGRADE_PATH.read_text(encoding="utf-8")
|
||||
restore = script[script.index("restore_previous() {") : script.index("cleanup() {")]
|
||||
|
||||
stop = 'docker compose --project-directory "$ROOT" -f "$COMPOSE_FILE" stop "$COMPOSE_SERVICE"'
|
||||
rollback = '--rollback'
|
||||
restore_controller = 'mv -- "$previous" "$CONTROLLER_DIR"'
|
||||
start = 'docker compose --project-directory "$ROOT" -f "$COMPOSE_FILE" up -d --force-recreate "$COMPOSE_SERVICE"'
|
||||
|
||||
assert restore.index(stop) < restore.index(rollback)
|
||||
assert restore.index(rollback) < restore.index(restore_controller)
|
||||
assert restore.index(restore_controller) < restore.index(start)
|
||||
assert "the old writer will remain stopped" in script
|
||||
reconcile = 'python3 "${stage}/reconcile_controller_state.py" \\\n --legacy-root'
|
||||
assert script.index("reconciliation_active=1", script.index("writers_stopped=1")) < script.index(reconcile)
|
||||
assert '[[ -f "$migration_journal" && ! -L "$migration_journal" ]]' in restore
|
||||
assert '[[ -e "$migration_journal" || -L "$migration_journal" ]]' in restore
|
||||
|
||||
|
||||
def test_wrapper_binds_reconciliation_helper_to_the_immutable_revision():
|
||||
wrapper = WRAPPER_PATH.read_text(encoding="utf-8")
|
||||
|
||||
assert "scripts/reconcile_controller_state.py" in wrapper
|
||||
assert "git archive" in wrapper
|
||||
|
||||
|
||||
def test_wrapper_archives_the_fixed_operator_handoff_required_by_root_upgrade():
|
||||
wrapper = WRAPPER_PATH.read_text(encoding="utf-8")
|
||||
upgrade = UPGRADE_PATH.read_text(encoding="utf-8")
|
||||
archive_line = next(line for line in wrapper.splitlines() if "git archive" in line)
|
||||
|
||||
assert "scripts/run_repopulse_adoption.py" in archive_line
|
||||
assert '"scripts/run_repopulse_adoption.py"' in upgrade
|
||||
@@ -0,0 +1,405 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "deluge_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
from backend.app.main import app
|
||||
|
||||
previous_apis = dict(settings.apis)
|
||||
for key in [
|
||||
"DELUGE_URL",
|
||||
"DELUGE_PASSWORD",
|
||||
"DELUGE_VERIFY_TLS",
|
||||
"DELUGE_TIMEOUT_SECONDS",
|
||||
"DELUGE_DEFAULT_HOST_ID",
|
||||
"DELUGE_WEB_PATH",
|
||||
]:
|
||||
settings.apis[key] = None
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
settings.apis = previous_apis
|
||||
|
||||
|
||||
def _set_deluge_config(url: str | None = "http://deluge:8112", password: str | None = "DELUGE_SECRET_PASSWORD"):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = dict(settings.apis)
|
||||
settings.apis["DELUGE_URL"] = url
|
||||
settings.apis["DELUGE_PASSWORD"] = password
|
||||
settings.apis["DELUGE_VERIFY_TLS"] = False
|
||||
settings.apis["DELUGE_TIMEOUT_SECONDS"] = "10"
|
||||
settings.apis["DELUGE_WEB_PATH"] = "/json"
|
||||
return previous
|
||||
|
||||
|
||||
def _restore_config(previous):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous)
|
||||
|
||||
|
||||
def _torrent_payload():
|
||||
return {
|
||||
"active-id": {
|
||||
"name": "Active Linux ISO",
|
||||
"state": "Downloading",
|
||||
"progress": 42.5,
|
||||
"download_payload_rate": 2048,
|
||||
"upload_payload_rate": 512,
|
||||
"eta": 120,
|
||||
"ratio": 0.4,
|
||||
"total_size": 1000000,
|
||||
"tracker_host": "tracker.example",
|
||||
"label": "linux",
|
||||
"save_path": "/srv/example/downloads/linux",
|
||||
"is_finished": False,
|
||||
},
|
||||
"paused-id": {
|
||||
"name": "Paused Media",
|
||||
"state": "Paused",
|
||||
"progress": 12,
|
||||
"download_payload_rate": 0,
|
||||
"upload_payload_rate": 0,
|
||||
"eta": 0,
|
||||
"ratio": 0,
|
||||
"total_size": 2000000,
|
||||
"tracker_host": "tracker.example",
|
||||
"label": "tv",
|
||||
"save_path": "/srv/example/downloads/tv",
|
||||
"is_finished": False,
|
||||
},
|
||||
"error-id": {
|
||||
"name": "Broken Torrent",
|
||||
"state": "Error",
|
||||
"progress": 50,
|
||||
"download_payload_rate": 0,
|
||||
"upload_payload_rate": 0,
|
||||
"eta": 0,
|
||||
"ratio": 0.1,
|
||||
"total_size": 3000000,
|
||||
"tracker_host": "bad.example",
|
||||
"label": "",
|
||||
"save_path": "/srv/example/downloads/broken",
|
||||
"error": "Tracker failed",
|
||||
"is_finished": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_deluge_connector_missing_config_reports_required_env(api_client):
|
||||
response = api_client.post("/api/connectors/deluge/test")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "missing_config"
|
||||
assert payload["missing_config"] is True
|
||||
assert payload["required_env_vars"] == ["DELUGE_URL", "DELUGE_PASSWORD"]
|
||||
assert payload["secretRef"] == "DELUGE_PASSWORD"
|
||||
|
||||
|
||||
def test_deluge_health_success_normalizes_response(monkeypatch):
|
||||
from backend.app.connectors.deluge import DelugeConnector
|
||||
|
||||
previous = _set_deluge_config()
|
||||
|
||||
monkeypatch.setattr(DelugeConnector, "_login", lambda self: True, raising=True)
|
||||
monkeypatch.setattr(
|
||||
DelugeConnector,
|
||||
"_rpc",
|
||||
lambda self, method, params=None, authed=True: {
|
||||
"web.connected": True,
|
||||
"daemon.connected": True,
|
||||
}.get(method, "2.1.1"),
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = DelugeConnector().health()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "connected"
|
||||
assert payload["webConnected"] is True
|
||||
assert payload["daemonConnected"] is True
|
||||
assert payload["version"] == "2.1.1"
|
||||
assert payload["url"] == "http://deluge:8112"
|
||||
assert "DELUGE_SECRET_PASSWORD" not in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_deluge_auth_failure_redacts_password(monkeypatch):
|
||||
from backend.app.connectors.deluge import DelugeConnector
|
||||
|
||||
previous = _set_deluge_config()
|
||||
|
||||
def fail_login(self):
|
||||
raise RuntimeError("auth_failed password=DELUGE_SECRET_PASSWORD")
|
||||
|
||||
monkeypatch.setattr(DelugeConnector, "_login", fail_login, raising=True)
|
||||
|
||||
payload = DelugeConnector().health()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "auth_failed"
|
||||
assert "DELUGE_SECRET_PASSWORD" not in str(payload)
|
||||
assert "[REDACTED]" in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_deluge_health_uses_session_fallback_when_optional_methods_unknown(monkeypatch):
|
||||
from backend.app.connectors.deluge import DelugeConnector
|
||||
|
||||
previous = _set_deluge_config()
|
||||
monkeypatch.setattr(DelugeConnector, "_login", lambda self: True, raising=True)
|
||||
|
||||
def fake_rpc(self, method, params=None, authed=True):
|
||||
if method in {"web.connected", "daemon.connected", "daemon.get_version"}:
|
||||
raise RuntimeError("Unknown method")
|
||||
if method == "core.get_session_status":
|
||||
return {"num_torrents": 57}
|
||||
raise AssertionError(f"unexpected method {method}")
|
||||
|
||||
monkeypatch.setattr(DelugeConnector, "_rpc", fake_rpc, raising=True)
|
||||
|
||||
payload = DelugeConnector().health()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "connected"
|
||||
assert payload["message"] == "Deluge connector ready."
|
||||
assert payload["diagnostics"]["healthFallback"] == "core.get_session_status"
|
||||
assert payload["diagnostics"]["optionalMethodWarnings"]
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_deluge_session_status_normalizes_rates_and_counts(monkeypatch):
|
||||
from backend.app.connectors.deluge import DelugeConnector
|
||||
|
||||
previous = _set_deluge_config()
|
||||
monkeypatch.setattr(DelugeConnector, "_login", lambda self: True, raising=True)
|
||||
|
||||
def fake_rpc(self, method, params=None, authed=True):
|
||||
if method == "web.connected":
|
||||
return True
|
||||
if method == "core.get_session_status":
|
||||
return {"download_rate": 4096, "upload_rate": 1024, "num_torrents": 3}
|
||||
if method == "web.update_ui":
|
||||
return {"torrents": _torrent_payload()}
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(DelugeConnector, "_rpc", fake_rpc, raising=True)
|
||||
|
||||
payload = DelugeConnector().session_status()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["connected"] is True
|
||||
assert payload["downloadRate"] == 4096
|
||||
assert payload["uploadRate"] == 1024
|
||||
assert payload["numTorrents"] == 3
|
||||
assert payload["active"] == 1
|
||||
assert payload["paused"] == 1
|
||||
assert payload["stalled"] == 1
|
||||
assert payload["errors"] == 1
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_deluge_torrent_status_and_filters(monkeypatch):
|
||||
from backend.app.connectors.deluge import DelugeConnector
|
||||
|
||||
previous = _set_deluge_config()
|
||||
monkeypatch.setattr(DelugeConnector, "_login", lambda self: True, raising=True)
|
||||
monkeypatch.setattr(
|
||||
DelugeConnector,
|
||||
"_rpc",
|
||||
lambda self, method, params=None, authed=True: {"torrents": _torrent_payload()} if method == "web.update_ui" else True,
|
||||
raising=True,
|
||||
)
|
||||
|
||||
torrents = DelugeConnector().torrent_status()
|
||||
active = DelugeConnector().active_downloads()
|
||||
stalled = DelugeConnector().stalled_downloads()
|
||||
paused = DelugeConnector().paused_downloads()
|
||||
|
||||
assert torrents["success"] is True
|
||||
assert torrents["total"] == 3
|
||||
assert torrents["items"][0]["id"] == "active-id"
|
||||
assert torrents["items"][0]["category"] == "linux"
|
||||
assert torrents["items"][0]["savePath"] == ".../linux"
|
||||
assert "DELUGE_SECRET_PASSWORD" not in str(torrents)
|
||||
assert active["total"] == 1
|
||||
assert active["items"][0]["id"] == "active-id"
|
||||
assert stalled["total"] == 1
|
||||
assert stalled["items"][0]["id"] == "error-id"
|
||||
assert paused["total"] == 1
|
||||
assert paused["items"][0]["id"] == "paused-id"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_deluge_speed_status_uses_session_rates(monkeypatch):
|
||||
from backend.app.connectors.deluge import DelugeConnector
|
||||
|
||||
previous = _set_deluge_config()
|
||||
monkeypatch.setattr(
|
||||
DelugeConnector,
|
||||
"session_status",
|
||||
lambda self: {"success": True, "connected": True, "downloadRate": 100, "uploadRate": 25, "numTorrents": 2, "active": 1, "paused": 0, "stalled": 0, "errors": 0},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = DelugeConnector().speed_status()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["downloadRate"] == 100
|
||||
assert payload["uploadRate"] == 25
|
||||
assert payload["active"] == 1
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_deluge_pause_torrent_allowed_creates_job_and_audit(api_client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.deluge.DelugeConnector.pause_torrent",
|
||||
lambda self, torrent_id: {"success": True, "status": "ok", "message": "torrent_paused", "data": {"torrentId": torrent_id}},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
response = api_client.post("/api/apps/deluge/torrents/active-id/pause", json={"source": "manual"})
|
||||
payload = response.json()
|
||||
|
||||
assert response.status_code == 200
|
||||
assert payload["success"] is True
|
||||
assert payload["decision"] == "allowed"
|
||||
assert payload["jobId"].startswith("job-")
|
||||
assert payload["auditId"].startswith("audit-")
|
||||
assert payload["data"]["torrentId"] == "active-id"
|
||||
|
||||
|
||||
def test_deluge_resume_torrent_voice_requires_approval_without_connector_call(api_client, monkeypatch):
|
||||
def fail_resume(*_args, **_kwargs):
|
||||
raise AssertionError("connector should not run before approval")
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.deluge.DelugeConnector.resume_torrent", fail_resume, raising=True)
|
||||
|
||||
response = api_client.post("/api/apps/deluge/torrents/paused-id/resume", json={"source": "voice"})
|
||||
payload = response.json()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["decision"] == "approval_required"
|
||||
assert payload["approvalId"].startswith("appr-")
|
||||
assert payload["auditId"].startswith("audit-")
|
||||
|
||||
|
||||
def test_deluge_pause_all_resume_all_require_approval_or_block_without_connector_call(api_client, monkeypatch):
|
||||
def fail_bulk(*_args, **_kwargs):
|
||||
raise AssertionError("bulk connector call should wait for approval")
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.deluge.DelugeConnector.pause_all", fail_bulk, raising=True)
|
||||
monkeypatch.setattr("backend.app.connectors.deluge.DelugeConnector.resume_all", fail_bulk, raising=True)
|
||||
|
||||
manual_pause = api_client.post("/api/apps/deluge/pause-all", json={"source": "manual"}).json()
|
||||
manual_resume = api_client.post("/api/apps/deluge/resume-all", json={"source": "manual"}).json()
|
||||
voice_pause = api_client.post("/api/apps/deluge/pause-all", json={"source": "voice"}).json()
|
||||
voice_resume = api_client.post("/api/apps/deluge/resume-all", json={"source": "voice"}).json()
|
||||
|
||||
assert manual_pause["decision"] == "approval_required"
|
||||
assert manual_resume["decision"] == "approval_required"
|
||||
assert voice_pause["decision"] == "blocked"
|
||||
assert voice_resume["decision"] == "blocked"
|
||||
|
||||
|
||||
def test_deluge_pause_all_executes_only_after_separate_approval_step(api_client, monkeypatch):
|
||||
calls = []
|
||||
|
||||
def pause_all(_self):
|
||||
calls.append('pause')
|
||||
return {'success': True, 'status': 'ok', 'message': 'all_torrents_paused'}
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.deluge.DelugeConnector.pause_all", pause_all, raising=True)
|
||||
|
||||
requested = api_client.post("/api/apps/deluge/pause-all", json={"source": "manual"}).json()
|
||||
assert calls == []
|
||||
|
||||
approved = api_client.patch(f"/api/approvals/{requested['approvalId']}", json={"status": "approved"}).json()
|
||||
assert approved['status'] == 'approved'
|
||||
assert calls == []
|
||||
|
||||
executed = api_client.post(f"/api/approvals/{requested['approvalId']}/execute").json()
|
||||
assert executed['success'] is True
|
||||
assert executed['approvalStatus'] == 'executed'
|
||||
assert executed['jobId'].startswith('job-')
|
||||
assert executed['auditId'].startswith('audit-')
|
||||
assert calls == ['pause']
|
||||
|
||||
approvals = api_client.get('/api/approvals').json()
|
||||
approval = next(item for item in approvals if item['id'] == requested['approvalId'])
|
||||
assert approval['status'] == 'executed'
|
||||
assert approval['job_id'] == executed['jobId']
|
||||
|
||||
|
||||
def test_deluge_destructive_actions_are_blocked(api_client):
|
||||
for action in [
|
||||
"deluge.remove_torrent",
|
||||
"deluge.remove_torrent_data",
|
||||
"deluge.delete_data",
|
||||
"deluge.force_recheck",
|
||||
"deluge.move_storage",
|
||||
"deluge.add_torrent",
|
||||
"deluge.add_magnet",
|
||||
"deluge.mass_edit",
|
||||
"deluge.set_options",
|
||||
"deluge.unknown_mutation",
|
||||
]:
|
||||
payload = api_client.post("/api/policies/evaluate", json={"action": action, "target": "deluge", "source": "manual"}).json()
|
||||
assert payload["decision"] == "blocked"
|
||||
|
||||
|
||||
def test_deluge_inventory_classification_and_candidate(api_client, monkeypatch):
|
||||
from backend.app.services.app_classifier import classify_container
|
||||
|
||||
result = classify_container({"name": "binhex-delugevpn", "image": "binhex/arch-delugevpn:latest", "ports": ["8112:8112"]})
|
||||
|
||||
assert result["app_type"] == "deluge"
|
||||
assert result["category"] == "downloads"
|
||||
assert result["subcategory"] == "torrent_client"
|
||||
assert result["connector"] == "deluge"
|
||||
assert result["policy_profile"] == "download_client"
|
||||
assert result["criticality"] == "medium"
|
||||
assert result["healthcheck_suggestion"] == {"port": 8112}
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.test_connection", lambda self: {"status": "connected"}, raising=True)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.portainer.PortainerConnector.list_containers",
|
||||
lambda self: {"status": "ok", "containers": [{"id": "deluge-container", "name": "binhex-delugevpn", "image": "binhex/arch-delugevpn", "state": "running", "status": "running", "ports": ["8112:8112"]}]},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
api_client.post("/api/inventory/discover")
|
||||
candidates = api_client.get("/api/connectors/deluge/candidates").json()
|
||||
|
||||
assert len(candidates) == 1
|
||||
assert candidates[0]["connectorId"] == "deluge"
|
||||
assert candidates[0]["appType"] == "deluge"
|
||||
assert candidates[0]["missingSecrets"] == ["DELUGE_URL", "DELUGE_PASSWORD"]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,400 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
from backend.app.db.models import Policy
|
||||
from backend.app.schemas.deployment import DeploymentPlanV1
|
||||
from backend.app.services.deployment_control_plane import DeploymentControlPlane
|
||||
from backend.app.services.secret_store import set_secret
|
||||
|
||||
|
||||
def _plan() -> DeploymentPlanV1:
|
||||
return DeploymentPlanV1.model_validate(
|
||||
{
|
||||
"application_id": "itworx-mcp-hub",
|
||||
"repository": "Jens/ITWorx_MCP_Hub",
|
||||
"revision": "a" * 40,
|
||||
"release_version": "1.0.0",
|
||||
"image_repository": "itworx-mcp-hub",
|
||||
"image_tag": "1.0.0",
|
||||
"topology": {
|
||||
"compose_project": "itworx-mcp-hub",
|
||||
"containers": ["itworx-mcp-hub"],
|
||||
"proxy_network": "itworx-npm",
|
||||
"internal_port": 8080,
|
||||
},
|
||||
"proxy": {"hostname": "mcp-test.example.com"},
|
||||
"oidc": None,
|
||||
"broker_secret_ref": "MCP_HUB_TEST_DEPLOYMENT_BROKER",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _Broker:
|
||||
def __init__(self, rollback_status: str = "rolled_back") -> None:
|
||||
self.rollback_status = rollback_status
|
||||
self.rollback_calls: list[str] = []
|
||||
|
||||
def preflight(self, plan, *, secret):
|
||||
return {"status": "preflight_ok"}
|
||||
|
||||
def apply(self, plan, *, secret, idempotency_key):
|
||||
return {"status": "deployed", "request": "a" * 64}
|
||||
|
||||
def rollback(self, plan, *, secret, idempotency_key):
|
||||
self.rollback_calls.append(idempotency_key)
|
||||
return {"status": self.rollback_status, "request": "b" * 64}
|
||||
|
||||
def status(self, request_id, *, secret):
|
||||
return {"status": "rolled_back", "request": request_id}
|
||||
|
||||
|
||||
class _Portainer:
|
||||
def list_containers(self):
|
||||
return {"status": "ok", "containers": []}
|
||||
|
||||
|
||||
class _FailingNpm:
|
||||
def proxy_hosts(self):
|
||||
return {"success": True, "items": []}
|
||||
|
||||
def ensure_managed_proxy_host(self, **kwargs):
|
||||
return {"success": False, "status": "failed"}
|
||||
|
||||
|
||||
class _Unraid:
|
||||
def get_share_summary(self):
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
class _Authentik:
|
||||
def test_connection(self):
|
||||
return {"status": "connected"}
|
||||
|
||||
def applications(self):
|
||||
return {"success": True, "items": []}
|
||||
|
||||
|
||||
def _session(tmp_path) -> Session:
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'auto-rollback.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
return Session(engine)
|
||||
|
||||
|
||||
def _policy(action: str) -> Policy:
|
||||
return Policy(
|
||||
id=f"allow-mcp-hub-{action.replace('.', '-')}",
|
||||
action=action,
|
||||
target="itworx-mcp-hub",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Exact-target no-regression deployment test policy.",
|
||||
)
|
||||
|
||||
|
||||
def _plane(broker: _Broker) -> DeploymentControlPlane:
|
||||
return DeploymentControlPlane(
|
||||
broker=broker,
|
||||
portainer=_Portainer(),
|
||||
npm=_FailingNpm(),
|
||||
authentik=_Authentik(),
|
||||
unraid=_Unraid(),
|
||||
)
|
||||
|
||||
|
||||
def _configure(session: Session, *, rollback_policy: bool = True) -> None:
|
||||
set_secret(session, "MCP_HUB_TEST_DEPLOYMENT_BROKER", "x" * 32)
|
||||
session.add(_policy("deployment.apply"))
|
||||
if rollback_policy:
|
||||
session.add(_policy("deployment.rollback"))
|
||||
session.commit()
|
||||
|
||||
|
||||
def test_default_compose_file_survives_durable_plan_roundtrip():
|
||||
plan = _plan()
|
||||
restored = DeploymentPlanV1.model_validate(plan.model_dump(mode="json"))
|
||||
assert restored.topology.compose_file == ".itworx/deployment.yml"
|
||||
|
||||
|
||||
def test_post_apply_verification_failure_auto_rolls_back_with_exact_policy(tmp_path):
|
||||
broker = _Broker("rolled_back")
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0001",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "rolled_back"
|
||||
assert result["stage"] == "auto_rollback_verified"
|
||||
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
||||
assert result["evidence"]["autoRollback"]["status"] == "rolled_back"
|
||||
assert broker.rollback_calls == [f"auto-rollback:{result['id']}"]
|
||||
|
||||
|
||||
def test_post_apply_verification_failure_does_not_rollback_without_exact_policy(tmp_path):
|
||||
broker = _Broker("rolled_back")
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session, rollback_policy=False)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0002",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "external_integration_failed"
|
||||
assert result["evidence"]["autoRollback"]["status"] == "blocked"
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
|
||||
def test_post_apply_verification_failure_keeps_failure_when_controller_rejects_rollback(tmp_path):
|
||||
broker = _Broker("rejected")
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0003",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "auto_rollback_failed"
|
||||
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
||||
assert result["evidence"]["autoRollback"]["status"] == "rejected"
|
||||
assert len(broker.rollback_calls) == 1
|
||||
|
||||
|
||||
def test_queued_auto_rollback_is_polled_and_finishes_as_auto_rollback(tmp_path):
|
||||
broker = _Broker("accepted")
|
||||
plane = _plane(broker)
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
initial = plane.execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0004",
|
||||
operation="apply",
|
||||
)
|
||||
assert initial["status"] == "rolling_back"
|
||||
assert initial["stage"] == "auto_rollback_queued"
|
||||
|
||||
finished = plane.refresh(session, initial["id"])
|
||||
|
||||
assert finished["status"] == "rolled_back"
|
||||
assert finished["stage"] == "auto_rollback_verified"
|
||||
assert finished["evidence"]["autoRollback"]["status"] == "rolled_back"
|
||||
|
||||
|
||||
def test_successful_post_apply_verification_never_invokes_auto_rollback(tmp_path):
|
||||
class _HealthyNpm(_FailingNpm):
|
||||
def ensure_managed_proxy_host(self, **kwargs):
|
||||
return {"success": True, "status": "created", "hostId": 42}
|
||||
|
||||
broker = _Broker("rolled_back")
|
||||
plane = DeploymentControlPlane(
|
||||
broker=broker,
|
||||
portainer=_Portainer(),
|
||||
npm=_HealthyNpm(),
|
||||
authentik=_Authentik(),
|
||||
unraid=_Unraid(),
|
||||
)
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = plane.execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0005",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "succeeded"
|
||||
assert result["stage"] == "controller_and_edge_verified"
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
def test_auto_rollback_is_scoped_to_exact_mcp_hub_target(tmp_path):
|
||||
broker = _Broker("rolled_back")
|
||||
plan = _plan().model_copy(update={"repository": "jens/other-repository"})
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=plan,
|
||||
idempotency_key="mcp-hub-apply-0006",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "external_integration_failed"
|
||||
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
||||
assert "autoRollback" not in result["evidence"]
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
def test_auto_rollback_rejects_same_repository_with_different_application(tmp_path):
|
||||
broker = _Broker("rolled_back")
|
||||
plan = _plan().model_copy(
|
||||
update={
|
||||
"application_id": "itworx-mcp-hub-copy",
|
||||
"topology": _plan().topology.model_copy(
|
||||
update={
|
||||
"compose_project": "itworx-mcp-hub-copy",
|
||||
"containers": ("itworx-mcp-hub-copy",),
|
||||
}
|
||||
),
|
||||
}
|
||||
)
|
||||
with _session(tmp_path) as session:
|
||||
set_secret(session, "MCP_HUB_TEST_DEPLOYMENT_BROKER", "x" * 32)
|
||||
session.add(
|
||||
Policy(
|
||||
id="allow-copy-apply",
|
||||
action="deployment.apply",
|
||||
target="itworx-mcp-hub-copy",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Test-only exact target.",
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
Policy(
|
||||
id="allow-copy-rollback",
|
||||
action="deployment.rollback",
|
||||
target="itworx-mcp-hub-copy",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Test-only exact target.",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=plan,
|
||||
idempotency_key="mcp-hub-apply-0007",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "external_integration_failed"
|
||||
assert result["evidence"]["postApplyVerification"]["status"] == "failed"
|
||||
assert "autoRollback" not in result["evidence"]
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
def test_queued_auto_rollback_rejects_deployed_status_as_rollback_proof(tmp_path):
|
||||
class _AmbiguousRollbackBroker(_Broker):
|
||||
def status(self, request_id, *, secret):
|
||||
return {"status": "deployed", "request": request_id}
|
||||
|
||||
broker = _AmbiguousRollbackBroker("accepted")
|
||||
plane = _plane(broker)
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
initial = plane.execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0008",
|
||||
operation="apply",
|
||||
)
|
||||
assert initial["status"] == "rolling_back"
|
||||
|
||||
finished = plane.refresh(session, initial["id"])
|
||||
|
||||
assert finished["status"] == "failed"
|
||||
assert finished["stage"] == "auto_rollback_failed"
|
||||
assert finished["evidence"]["autoRollback"]["status"] == "accepted"
|
||||
assert "deployed state" in finished["result"]
|
||||
|
||||
def test_manual_rollback_requires_rolled_back_terminal_proof(tmp_path):
|
||||
broker = _Broker("deployed")
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-rollback-0009",
|
||||
operation="rollback",
|
||||
)
|
||||
|
||||
assert result["status"] == "rollback_failed"
|
||||
assert result["stage"] == "controller_rejected"
|
||||
|
||||
|
||||
def test_apply_rejects_rolled_back_terminal_status(tmp_path):
|
||||
class _RolledBackApplyBroker(_Broker):
|
||||
def apply(self, plan, *, secret, idempotency_key):
|
||||
return {"status": "rolled_back", "request": "c" * 64}
|
||||
|
||||
broker = _RolledBackApplyBroker()
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0010",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "controller_rejected"
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
def test_auto_rollback_refuses_duplicate_exact_rollback_policies(tmp_path):
|
||||
broker = _Broker("rolled_back")
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
session.add(
|
||||
Policy(
|
||||
id="duplicate-rollback-policy",
|
||||
action="deployment.rollback",
|
||||
target="itworx-mcp-hub",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Duplicate exact rollback policy.",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0011",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "external_integration_failed"
|
||||
assert result["evidence"]["autoRollback"] == {
|
||||
"status": "blocked",
|
||||
"reason": "exact-target-policy",
|
||||
}
|
||||
assert broker.rollback_calls == []
|
||||
|
||||
def test_auto_rollback_rejects_unpollable_accepted_response(tmp_path):
|
||||
class _MissingRequestBroker(_Broker):
|
||||
def rollback(self, plan, *, secret, idempotency_key):
|
||||
self.rollback_calls.append(idempotency_key)
|
||||
return {"status": "accepted"}
|
||||
|
||||
broker = _MissingRequestBroker()
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(broker).execute(
|
||||
session,
|
||||
plan=_plan(),
|
||||
idempotency_key="mcp-hub-apply-0012",
|
||||
operation="apply",
|
||||
)
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["stage"] == "auto_rollback_failed"
|
||||
assert result["evidence"]["autoRollback"] == {
|
||||
"status": "accepted",
|
||||
"reason": "missing-controller-request",
|
||||
}
|
||||
assert len(broker.rollback_calls) == 1
|
||||
@@ -0,0 +1,85 @@
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_canonical_compose_keeps_runtime_data_outside_the_source_checkout():
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
dockerignore = (ROOT / ".dockerignore").read_text(encoding="utf-8").splitlines()
|
||||
gitignore = (ROOT / ".gitignore").read_text(encoding="utf-8").splitlines()
|
||||
|
||||
assert "${APPOPS_DATA_PATH:-./data}:/data" in compose
|
||||
assert "APPOPS_SECRET_KEY_FILE: /data/secret-key" in compose
|
||||
assert "- .:/data" not in compose
|
||||
assert "data" in dockerignore
|
||||
assert "/data/" in gitignore
|
||||
|
||||
|
||||
def test_canonical_compose_uses_explicit_publish_host_and_persistent_voice_token():
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "Dockerfile").read_text(encoding="utf-8")
|
||||
entrypoint = (ROOT / "docker" / "entrypoint.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "${APPOPS_PUBLISH_HOST:-127.0.0.1}:1216:1216" in compose
|
||||
assert "APP_OPS_VOICE_TOKEN_FILE=/data/voice-token" in dockerfile
|
||||
assert 'generate_service_token "true" "${APP_OPS_VOICE_TOKEN:-}"' in entrypoint
|
||||
assert '"${APP_OPS_VOICE_TOKEN_FILE:-${DATA_DIR}/voice-token}"' in entrypoint
|
||||
|
||||
|
||||
def test_canonical_compose_keeps_reverse_proxy_network_after_redeploy():
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert "- appops_proxy" in compose
|
||||
assert "external: true" in compose
|
||||
assert "name: ${APPOPS_PROXY_NETWORK:-itworx-mcp-hub-npm}" in compose
|
||||
|
||||
|
||||
def test_canonical_compose_forwards_trusted_operator_proxy_configuration():
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert 'APPOPS_OPERATOR_PROXY_AUTH: "${APPOPS_OPERATOR_PROXY_AUTH:-false}"' in compose
|
||||
assert 'APPOPS_OPERATOR_PROXY_TRUSTED_NETWORKS: "${APPOPS_OPERATOR_PROXY_TRUSTED_NETWORKS:-}"' in compose
|
||||
assert 'APPOPS_OPERATOR_PROXY_USERS: "${APPOPS_OPERATOR_PROXY_USERS:-}"' in compose
|
||||
assert 'APPOPS_OPERATOR_PROXY_GROUPS: "${APPOPS_OPERATOR_PROXY_GROUPS:-appops-operators}"' in compose
|
||||
|
||||
|
||||
def test_canonical_compose_mounts_only_the_file_based_gitea_observer_secret():
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
|
||||
assert "GITEA_URL: https://gitea.example.com" in compose
|
||||
assert "GITEA_ADMIN_TOKEN_FILE: /data/gitea-admin-token" in compose
|
||||
assert "GITEA_ADMIN_TOKEN:" not in compose
|
||||
assert (
|
||||
"GITEA_RUNNER_TARGETS_JSON: '{}'"
|
||||
in compose
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_image_label_matches_the_app_version():
|
||||
dockerfile = (ROOT / "Dockerfile").read_text(encoding="utf-8")
|
||||
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||
version = (ROOT / "backend/app/core/version.py").read_text(encoding="utf-8")
|
||||
|
||||
expected_provenance = {
|
||||
"APPOPS_BUILD_ID": "org.opencontainers.image.version",
|
||||
"APPOPS_SOURCE_REVISION": "org.opencontainers.image.revision",
|
||||
"APPOPS_SOURCE_REPOSITORY": "org.opencontainers.image.source",
|
||||
"APPOPS_SOURCE_TREE": "org.opencontainers.image.source-tree",
|
||||
"APPOPS_BUILD_DATE": "org.opencontainers.image.created",
|
||||
}
|
||||
for build_argument, image_label in expected_provenance.items():
|
||||
assert f"ARG {build_argument}=" in dockerfile
|
||||
assert f'{image_label}="${{{build_argument}}}"' in dockerfile
|
||||
assert f'{build_argument}: "${{{build_argument}:-' in compose
|
||||
|
||||
assert 'io.itworx.app.version="1.8.0"' in dockerfile
|
||||
assert 'io.itworx.app.operator-auth="http-only-session"' in dockerfile
|
||||
assert 'APP_VERSION = "1.8.0"' in version
|
||||
|
||||
|
||||
def test_unraid_upgrade_can_restore_voice_token_without_logging_it():
|
||||
script = (ROOT / "scripts" / "upgrade_single_1216.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "for env_key in \\\n APP_OPS_VOICE_TOKEN \\" in script
|
||||
assert 'printf \'%s=%s\\n\' "$env_key" "$env_value" >> "$ENV_FILE"' in script
|
||||
assert 'log "${env_value}"' not in script
|
||||
@@ -0,0 +1,345 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
from backend.app.db.models import Policy
|
||||
from backend.app.schemas.deployment import (
|
||||
DeploymentPlanV1,
|
||||
validate_compose_runtime_contract,
|
||||
)
|
||||
from backend.app.services.deployment_control_plane import DeploymentControlPlane
|
||||
from backend.app.services.secret_store import set_secret
|
||||
|
||||
|
||||
def _plan(**overrides):
|
||||
payload = {
|
||||
"application_id": "wsbc-portal",
|
||||
"repository": "Jens/Windows-Support-Bundle-Collector",
|
||||
"revision": "a" * 40,
|
||||
"release_version": "0.23.0",
|
||||
"image_repository": "wsbc-portal",
|
||||
"image_tag": "0.23.0",
|
||||
"topology": {
|
||||
"compose_project": "wsbc-portal",
|
||||
"containers": ["wsbc-portal"],
|
||||
"proxy_network": "itworx-npm",
|
||||
"internal_port": 8080,
|
||||
},
|
||||
"proxy": {"hostname": "support.example.com"},
|
||||
"oidc": {
|
||||
"application_slug": "wsbc-portal",
|
||||
"client_secret_ref": "WSBC_PORTAL_OIDC_CLIENT_SECRET",
|
||||
"roles": {
|
||||
"administrator_group": "Administrator",
|
||||
"support_agent_group": "SupportAgent",
|
||||
"read_only_group": "ReadOnly",
|
||||
},
|
||||
},
|
||||
"broker_secret_ref": "WSBC_PORTAL_DEPLOYMENT_BROKER",
|
||||
}
|
||||
payload.update(overrides)
|
||||
return DeploymentPlanV1.model_validate(payload)
|
||||
|
||||
|
||||
class _Broker:
|
||||
def preflight(self, plan, *, secret):
|
||||
assert secret == "a" * 32
|
||||
return {"status": "preflight_ok"}
|
||||
|
||||
def apply(self, plan, *, secret, idempotency_key):
|
||||
assert secret == "a" * 32
|
||||
return {"status": "accepted", "request": "b" * 64}
|
||||
|
||||
def rollback(self, plan, *, secret, idempotency_key):
|
||||
return {"status": "accepted", "request": "c" * 64}
|
||||
|
||||
def status(self, request_id, *, secret):
|
||||
return {"status": "deployed", "request": request_id}
|
||||
|
||||
|
||||
class _Portainer:
|
||||
def list_containers(self):
|
||||
return {"status": "ok", "containers": []}
|
||||
|
||||
|
||||
class _Npm:
|
||||
def proxy_hosts(self):
|
||||
return {"success": True, "items": []}
|
||||
|
||||
def ensure_managed_proxy_host(self, **kwargs):
|
||||
assert kwargs["forward_host"] == "wsbc-portal"
|
||||
return {"success": True, "status": "created", "hostId": 42}
|
||||
|
||||
|
||||
class _Authentik:
|
||||
def test_connection(self):
|
||||
return {"status": "connected"}
|
||||
|
||||
def applications(self):
|
||||
return {"success": True, "items": []}
|
||||
|
||||
def ensure_deployment_oidc_application(self, **kwargs):
|
||||
assert kwargs["application_slug"] == "wsbc-portal"
|
||||
assert kwargs["client_secret"] == "b" * 32
|
||||
return {"success": True, "status": "created", "applicationId": "app-1"}
|
||||
|
||||
|
||||
class _Unraid:
|
||||
def get_share_summary(self):
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
def _session(tmp_path):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'deployment.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
return Session(engine)
|
||||
|
||||
|
||||
def _plane():
|
||||
return DeploymentControlPlane(
|
||||
broker=_Broker(),
|
||||
portainer=_Portainer(),
|
||||
npm=_Npm(),
|
||||
authentik=_Authentik(),
|
||||
unraid=_Unraid(),
|
||||
)
|
||||
|
||||
|
||||
def test_plan_rejects_latest_tag_and_raw_secret_like_fields():
|
||||
raw = _plan().model_dump()
|
||||
raw["image_tag"] = "latest"
|
||||
try:
|
||||
DeploymentPlanV1.model_validate(raw)
|
||||
except ValueError as exc:
|
||||
assert "non-latest" in str(exc)
|
||||
else: # pragma: no cover - assertion clarity
|
||||
raise AssertionError("latest tag must be rejected")
|
||||
|
||||
raw = _plan().model_dump()
|
||||
raw["deployment_secret"] = "not-allowed"
|
||||
try:
|
||||
DeploymentPlanV1.model_validate(raw)
|
||||
except ValueError as exc:
|
||||
assert "Extra inputs" in str(exc)
|
||||
else: # pragma: no cover - assertion clarity
|
||||
raise AssertionError("raw secret fields must be rejected")
|
||||
|
||||
raw = _plan().model_dump()
|
||||
raw["proxy"]["force_ssl"] = False
|
||||
try:
|
||||
DeploymentPlanV1.model_validate(raw)
|
||||
except ValueError as exc:
|
||||
assert "force HTTPS" in str(exc)
|
||||
else: # pragma: no cover - assertion clarity
|
||||
raise AssertionError("managed proxy must force HTTPS")
|
||||
|
||||
|
||||
def test_preflight_fails_closed_without_exact_target_policy(tmp_path):
|
||||
plan = _plan(oidc=None)
|
||||
with _session(tmp_path) as session:
|
||||
set_secret(session, "WSBC_PORTAL_DEPLOYMENT_BROKER", "a" * 32)
|
||||
result = _plane().execute(
|
||||
session,
|
||||
plan=plan,
|
||||
idempotency_key="preflight-without-policy-0001",
|
||||
operation="preflight",
|
||||
)
|
||||
|
||||
assert result["status"] == "preflight_failed"
|
||||
assert "deployment_policy" in result["preflight"]["blockers"]
|
||||
|
||||
|
||||
def test_preflight_rejects_wildcard_deployment_policy(tmp_path):
|
||||
plan = _plan(oidc=None)
|
||||
with _session(tmp_path) as session:
|
||||
set_secret(session, "WSBC_PORTAL_DEPLOYMENT_BROKER", "a" * 32)
|
||||
session.add(Policy(
|
||||
id="wildcard-deploy",
|
||||
action="deployment.apply",
|
||||
target="*",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Wildcard must not authorize a production deployment.",
|
||||
))
|
||||
session.commit()
|
||||
result = _plane().execute(
|
||||
session,
|
||||
plan=plan,
|
||||
idempotency_key="preflight-wildcard-policy-0001",
|
||||
operation="preflight",
|
||||
)
|
||||
|
||||
assert result["status"] == "preflight_failed"
|
||||
assert "deployment_policy" in result["preflight"]["blockers"]
|
||||
|
||||
|
||||
def test_compose_contract_rejects_docker_socket_and_unowned_mounts():
|
||||
plan = _plan(oidc=None)
|
||||
unsafe = """
|
||||
services:
|
||||
wsbc-portal:
|
||||
image: wsbc-portal:0.23.0
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
networks: [itworx-npm]
|
||||
networks:
|
||||
itworx-npm:
|
||||
external: true
|
||||
"""
|
||||
try:
|
||||
validate_compose_runtime_contract(plan, unsafe)
|
||||
except ValueError as exc:
|
||||
assert "docker socket" in str(exc)
|
||||
else: # pragma: no cover - assertion clarity
|
||||
raise AssertionError("docker socket mount must be rejected")
|
||||
|
||||
|
||||
def test_compose_contract_rejects_path_traversal_outside_owned_appdata():
|
||||
plan = _plan(oidc=None)
|
||||
unsafe = """
|
||||
services:
|
||||
wsbc-portal:
|
||||
image: wsbc-portal:0.23.0
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
volumes:
|
||||
- /srv/example/appdata/wsbc-portal/../../other-app:/data
|
||||
networks: [itworx-npm]
|
||||
networks:
|
||||
itworx-npm:
|
||||
external: true
|
||||
"""
|
||||
try:
|
||||
validate_compose_runtime_contract(plan, unsafe)
|
||||
except ValueError as exc:
|
||||
assert "normalized" in str(exc)
|
||||
else: # pragma: no cover - assertion clarity
|
||||
raise AssertionError("path traversal mount must be rejected")
|
||||
|
||||
|
||||
def test_compose_contract_requires_an_owned_persistent_bind_mount():
|
||||
plan = _plan(oidc=None)
|
||||
unsafe = """
|
||||
services:
|
||||
wsbc-portal:
|
||||
image: wsbc-portal:0.23.0
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
networks: [itworx-npm]
|
||||
networks:
|
||||
itworx-npm:
|
||||
external: true
|
||||
"""
|
||||
try:
|
||||
validate_compose_runtime_contract(plan, unsafe)
|
||||
except ValueError as exc:
|
||||
assert "persistent" in str(exc)
|
||||
else: # pragma: no cover - assertion clarity
|
||||
raise AssertionError("persistent bind mount must be required")
|
||||
|
||||
|
||||
def test_preflight_and_apply_are_durable_and_idempotent(tmp_path):
|
||||
plan = _plan()
|
||||
with _session(tmp_path) as session:
|
||||
set_secret(session, "WSBC_PORTAL_DEPLOYMENT_BROKER", "a" * 32)
|
||||
set_secret(session, "WSBC_PORTAL_OIDC_CLIENT_SECRET", "b" * 32)
|
||||
session.add(Policy(
|
||||
id="allow-wsbc-deploy",
|
||||
action="deployment.apply",
|
||||
target="wsbc-portal",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Test exact-target deployment policy.",
|
||||
))
|
||||
session.commit()
|
||||
plane = _plane()
|
||||
result = plane.execute(session, plan=plan, idempotency_key="apply-wsbc-0001", operation="apply")
|
||||
|
||||
assert not result["preflight"]["blockers"], result["preflight"]
|
||||
assert result["status"] == "applying", result
|
||||
assert result["brokerRequestId"] == "b" * 64
|
||||
replay = plane.execute(session, plan=plan, idempotency_key="apply-wsbc-0001", operation="apply")
|
||||
assert replay["idempotentReplay"] is True
|
||||
assert replay["id"] == result["id"]
|
||||
|
||||
|
||||
def test_verified_controller_apply_completes_only_after_typed_oidc_and_proxy(tmp_path):
|
||||
class DirectBroker(_Broker):
|
||||
def apply(self, plan, *, secret, idempotency_key):
|
||||
return {"status": "deployed", "request": "b" * 64}
|
||||
|
||||
plan = _plan()
|
||||
with _session(tmp_path) as session:
|
||||
set_secret(session, "WSBC_PORTAL_DEPLOYMENT_BROKER", "a" * 32)
|
||||
set_secret(session, "WSBC_PORTAL_OIDC_CLIENT_SECRET", "b" * 32)
|
||||
session.add(Policy(
|
||||
id="allow-wsbc-deploy-direct",
|
||||
action="deployment.apply",
|
||||
target="wsbc-portal",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Test exact-target deployment policy.",
|
||||
))
|
||||
session.commit()
|
||||
plane = DeploymentControlPlane(
|
||||
broker=DirectBroker(),
|
||||
portainer=_Portainer(),
|
||||
npm=_Npm(),
|
||||
authentik=_Authentik(),
|
||||
unraid=_Unraid(),
|
||||
)
|
||||
result = plane.execute(session, plan=plan, idempotency_key="apply-wsbc-direct-0001", operation="apply")
|
||||
|
||||
assert result["status"] == "succeeded"
|
||||
assert result["stage"] == "controller_and_edge_verified"
|
||||
assert result["evidence"]["externalIntegration"]["oidc"]["status"] == "created"
|
||||
assert result["evidence"]["externalIntegration"]["proxy"]["status"] == "created"
|
||||
|
||||
|
||||
def test_deployment_evidence_uses_allowlists_and_never_persists_connector_secrets(tmp_path):
|
||||
class DirectBroker(_Broker):
|
||||
def apply(self, plan, *, secret, idempotency_key):
|
||||
return {"status": "deployed", "request": "b" * 64, "token": "broker-secret"}
|
||||
|
||||
class SecretEchoNpm(_Npm):
|
||||
def ensure_managed_proxy_host(self, **kwargs):
|
||||
return {"success": True, "status": "created", "hostId": 42, "meta": {"token": "npm-secret"}}
|
||||
|
||||
class SecretEchoAuthentik(_Authentik):
|
||||
def ensure_deployment_oidc_application(self, **kwargs):
|
||||
return {"success": True, "status": "created", "applicationId": "app-1", "client_secret_value": "oidc-secret"}
|
||||
|
||||
plan = _plan()
|
||||
with _session(tmp_path) as session:
|
||||
set_secret(session, "WSBC_PORTAL_DEPLOYMENT_BROKER", "a" * 32)
|
||||
set_secret(session, "WSBC_PORTAL_OIDC_CLIENT_SECRET", "b" * 32)
|
||||
session.add(Policy(
|
||||
id="allow-wsbc-deploy-evidence",
|
||||
action="deployment.apply",
|
||||
target="wsbc-portal",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Test exact-target deployment policy.",
|
||||
))
|
||||
session.commit()
|
||||
result = DeploymentControlPlane(
|
||||
broker=DirectBroker(),
|
||||
portainer=_Portainer(),
|
||||
npm=SecretEchoNpm(),
|
||||
authentik=SecretEchoAuthentik(),
|
||||
unraid=_Unraid(),
|
||||
).execute(session, plan=plan, idempotency_key="apply-wsbc-evidence-0001", operation="apply")
|
||||
|
||||
assert result["status"] == "succeeded"
|
||||
assert "broker-secret" not in str(result)
|
||||
assert "npm-secret" not in str(result)
|
||||
assert "oidc-secret" not in str(result)
|
||||
@@ -0,0 +1,54 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_upgrade_replaces_the_canonical_container_without_versioned_copies():
|
||||
script = (ROOT / 'scripts' / 'upgrade_single_1216.sh').read_text(encoding='utf-8')
|
||||
|
||||
assert 'assert_single_canonical_container' in script
|
||||
assert 'docker rm "$SOURCE_CONTAINER_NAME"' in script
|
||||
assert 'run_canonical_container "$TARGET_IMAGE"' in script
|
||||
assert 'run_canonical_container "$CURRENT_IMAGE_ID"' in script
|
||||
assert 'unraid-appops-gateway:rollback' in script
|
||||
assert 'docker rename' not in script
|
||||
assert 'OLD_CONTAINER' not in script
|
||||
assert '${CONTAINER_NAME}-pre-' not in script
|
||||
assert '${CONTAINER_NAME}-failed-' not in script
|
||||
assert 'CONTAINER_NAME="${CONTAINER_NAME:-Unraid-AppOps-Gateway}"' in script
|
||||
assert '--hostname unraid-appops-gateway' in script
|
||||
assert 'net.unraid.docker.icon' in script
|
||||
assert 'net.unraid.docker.webui' in script
|
||||
assert 'urlopen(req, timeout=30)' in script
|
||||
assert 'previous-container-name.txt' in script
|
||||
assert 'umask 077' in script
|
||||
assert 'trap \'rm -f "$ENV_FILE" "$ENV_FILE_RAW"\' EXIT' in script
|
||||
assert 'value[key] = $0' in script
|
||||
assert 'for (position = 1; position <= count; position++)' in script
|
||||
assert 'for (index = 1;' not in script
|
||||
assert '/^APPOPS_ADMIN_TOKEN=/d' in script
|
||||
assert '/^VITE_APPOPS_ADMIN_TOKEN=/d' in script
|
||||
assert '/^VITE_APPOPS_VOICE_TOKEN=/d' in script
|
||||
assert '/^APPOPS_PORT=/d' in script
|
||||
assert '/^APPOPS_FRONTEND_ORIGIN=/d' in script
|
||||
assert 'attempt ${attempt}/3 failed; retrying' in script
|
||||
assert 'if ! check_endpoint "$path"' in script
|
||||
assert 'return 1' in script
|
||||
|
||||
|
||||
def test_upgrade_retention_is_scoped_to_appops_timestamped_backups_and_images():
|
||||
script = (ROOT / 'scripts' / 'upgrade_single_1216.sh').read_text(encoding='utf-8')
|
||||
|
||||
assert 'BACKUP_RETENTION="${BACKUP_RETENTION:-20}"' in script
|
||||
assert '^[0-9]{8}T[0-9]{6}Z$' in script
|
||||
assert 'rm -rf -- "$candidate"' in script
|
||||
assert "docker image ls unraid-appops-gateway" in script
|
||||
assert 'unraid-appops-gateway:production|unraid-appops-gateway:rollback' in script
|
||||
|
||||
|
||||
def test_powershell_wrapper_requires_explicit_execute_confirmation():
|
||||
script = (ROOT / 'scripts' / 'upgrade_single_1216.ps1').read_text(encoding='utf-8')
|
||||
|
||||
assert 'throw "Execute requires -Confirm UPGRADE_APPOPS_1216"' in script
|
||||
assert '$Confirm = "UPGRADE_APPOPS_1216"' not in script
|
||||
@@ -0,0 +1,239 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from backend.app.services.deployment_target_registry import DeploymentTargetRegistry
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakePortainer:
|
||||
def __init__(self, exit_code=0):
|
||||
self.exit_code = exit_code
|
||||
self.calls = []
|
||||
|
||||
def get_container_status(self, name):
|
||||
assert name == "gitea-deploy-control"
|
||||
return {
|
||||
"status": "ok",
|
||||
"container": {
|
||||
"id": "controller-id",
|
||||
"name": "gitea-deploy-control",
|
||||
"state": "running",
|
||||
},
|
||||
}
|
||||
|
||||
def _config(self):
|
||||
return SimpleNamespace(endpoint_id="3")
|
||||
|
||||
def _request(self, method, path, **kwargs):
|
||||
self.calls.append((method, path, kwargs))
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return FakeResponse({"Id": "exec-1"})
|
||||
if path.endswith("/exec/exec-1/start"):
|
||||
return FakeResponse({})
|
||||
if path.endswith("/exec/exec-1/json"):
|
||||
return FakeResponse({"Running": False, "ExitCode": self.exit_code})
|
||||
raise AssertionError(path)
|
||||
|
||||
|
||||
def test_rejects_non_jens_repository_without_portainer_access():
|
||||
fake = FakePortainer()
|
||||
result = DeploymentTargetRegistry(fake).register_single_container(
|
||||
"other/CertScope", "a" * 40
|
||||
)
|
||||
assert result["status"] == "invalid"
|
||||
assert fake.calls == []
|
||||
|
||||
|
||||
def test_rejects_non_sha_without_portainer_access():
|
||||
fake = FakePortainer()
|
||||
result = DeploymentTargetRegistry(fake).register_single_container(
|
||||
"Jens/CertScope", "main"
|
||||
)
|
||||
assert result["status"] == "invalid"
|
||||
assert fake.calls == []
|
||||
|
||||
|
||||
def test_derives_bounded_single_container_topology_and_uses_fixed_exec():
|
||||
fake = FakePortainer()
|
||||
revision = "b" * 40
|
||||
result = DeploymentTargetRegistry(fake).register_single_container(
|
||||
"Jens/CertScope", revision
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"status": "registered",
|
||||
"repository": "jens/certscope",
|
||||
"revision": revision,
|
||||
"branch": "main",
|
||||
"slug": "certscope",
|
||||
"compose_file": "docker-compose.yml",
|
||||
"compose_project": "certscope",
|
||||
"container": "certscope",
|
||||
"containers": ["certscope"],
|
||||
}
|
||||
create = fake.calls[0]
|
||||
assert create[0] == "POST"
|
||||
assert create[1].endswith("/containers/controller-id/exec")
|
||||
command = create[2]["json"]["Cmd"]
|
||||
assert command[:2] == ["python3", "-c"]
|
||||
assert command[3:6] == ["jens/certscope", revision, "main"]
|
||||
assert '"compose_files":["docker-compose.yml"]' in command[6]
|
||||
assert '"source_dir":"/mnt/cache/appdata/gitea-deploy-runners/sources/certscope"' in command[6]
|
||||
assert '"containers":["certscope"]' in command[6]
|
||||
script = command[2]
|
||||
assert '/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json' in script
|
||||
assert '/srv/example/appdata/gitea-deploy-runners/deploy.lock' not in script
|
||||
assert '/mnt/cache/appdata/gitea-deploy-runners/deploy.lock' in script
|
||||
assert 'os.path.samefile(path, host_path)' in script
|
||||
assert 'refs/heads/{branch}' in script
|
||||
assert create[2]["json"]["AttachStdout"] is False
|
||||
assert create[2]["json"]["AttachStderr"] is False
|
||||
|
||||
|
||||
def test_supports_master_branch_and_bounded_multi_container_topology():
|
||||
fake = FakePortainer()
|
||||
revision = "e" * 40
|
||||
result = DeploymentTargetRegistry(fake).register_compose_target(
|
||||
"Jens/Mediarr",
|
||||
revision,
|
||||
branch="master",
|
||||
compose_file="compose.yaml",
|
||||
compose_project="mediarr",
|
||||
containers=["mediarr-api-1", "mediarr-web-1"],
|
||||
)
|
||||
assert result["status"] == "registered"
|
||||
assert result["branch"] == "master"
|
||||
assert result["compose_file"] == "compose.yaml"
|
||||
assert result["container"] is None
|
||||
assert result["containers"] == ["mediarr-api-1", "mediarr-web-1"]
|
||||
command = fake.calls[0][2]["json"]["Cmd"]
|
||||
assert command[3:6] == ["jens/mediarr", revision, "master"]
|
||||
|
||||
|
||||
def test_rejects_branch_traversal_and_unsafe_compose_path_before_portainer_access():
|
||||
fake = FakePortainer()
|
||||
result = DeploymentTargetRegistry(fake).register_compose_target(
|
||||
"Jens/Mediarr",
|
||||
"e" * 40,
|
||||
branch="../main",
|
||||
compose_file="../compose.yml",
|
||||
containers=["mediarr"],
|
||||
)
|
||||
assert result["status"] == "invalid"
|
||||
assert fake.calls == []
|
||||
|
||||
|
||||
def test_reports_existing_conflicting_target_without_exposing_exec_output():
|
||||
fake = FakePortainer(exit_code=23)
|
||||
result = DeploymentTargetRegistry(fake).register_single_container(
|
||||
"jens/eventlens", "c" * 40
|
||||
)
|
||||
assert result == {
|
||||
"status": "conflict",
|
||||
"reason": "repository already has a different deployment target",
|
||||
}
|
||||
|
||||
|
||||
def test_reports_bounded_controller_preflight_and_mutation_reasons_without_exec_output():
|
||||
expected = {
|
||||
20: "deployment projects registry is unavailable or unsafe",
|
||||
21: "deployment projects registry is not a regular file",
|
||||
22: "deployment projects registry does not contain an object",
|
||||
24: "mounted bare repository is unavailable",
|
||||
25: "exact revision cannot be resolved in the mounted bare repository",
|
||||
26: "default branch cannot be resolved in the mounted bare repository",
|
||||
27: "exact revision is not an ancestor of the default branch in the mounted bare repository",
|
||||
28: "configured Compose file is absent at the exact revision",
|
||||
29: "deployment registry lock file cannot be opened",
|
||||
30: "deployment registry lock cannot be acquired",
|
||||
31: "deployment projects registry cannot be read as JSON",
|
||||
32: "deployment projects registry backup cannot be created",
|
||||
33: "deployment projects registry replacement file cannot be durably written",
|
||||
34: "deployment projects registry replacement cannot be installed atomically",
|
||||
35: "deployment projects registry directory durability sync failed after replacement",
|
||||
36: "deployment projects registry host alias is unavailable or does not match the controller registry",
|
||||
37: "controller project branch does not match the requested default branch",
|
||||
38: "controller project Compose contract is invalid",
|
||||
}
|
||||
for exit_code, reason in expected.items():
|
||||
fake = FakePortainer(exit_code=exit_code)
|
||||
result = DeploymentTargetRegistry(fake).register_single_container(
|
||||
"jens/eventlens", "d" * 40
|
||||
)
|
||||
assert result == {"status": "validation_failed", "reason": reason}
|
||||
|
||||
|
||||
def test_existing_target_may_preserve_only_bounded_root_owned_runtime_inputs():
|
||||
fake = FakePortainer()
|
||||
revision = "f" * 40
|
||||
result = DeploymentTargetRegistry(fake).register_compose_target(
|
||||
"Jens/Mediarr",
|
||||
revision,
|
||||
branch="master",
|
||||
compose_file="docker-compose.unraid.yml",
|
||||
compose_project="mediarr",
|
||||
containers=["mediarr"],
|
||||
verify_oci_revision=True,
|
||||
)
|
||||
assert result["status"] == "registered"
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
assert '"env_source"' in script
|
||||
assert '"environment"' in script
|
||||
assert '"gitea_repository_id"' in script
|
||||
assert '"gitea_webhook_secret_file"' in script
|
||||
assert '"validation"' in script
|
||||
assert "extra_keys.issubset(allowed_extra_keys)" in script
|
||||
assert 'env_source.startswith("/srv/example/appdata/")' in script
|
||||
assert "not isinstance(environment, dict) or len(environment) > 64" in script
|
||||
assert 'expected_secret_file = repository.replace("/", "__")' in script
|
||||
assert '"Managed validation / full (push)"' in script
|
||||
assert '"timeout_seconds": 3600 if repository == "jens/mediarr" else 1800' in script
|
||||
|
||||
|
||||
def test_existing_target_compatibility_still_rejects_unknown_or_behavioral_metadata():
|
||||
fake = FakePortainer()
|
||||
DeploymentTargetRegistry(fake).register_single_container(
|
||||
"Jens/EventLens", "a" * 40
|
||||
)
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
line = next(
|
||||
value for value in script.splitlines()
|
||||
if "allowed_extra_keys =" in value
|
||||
)
|
||||
assert '"deploy_script"' not in line
|
||||
assert '"manual"' not in line
|
||||
assert '"live_probe"' not in line
|
||||
|
||||
|
||||
|
||||
def test_runnerless_validation_compatibility_is_fixed_push_only():
|
||||
fake = FakePortainer()
|
||||
DeploymentTargetRegistry(fake).register_single_container(
|
||||
"Jens/EventLens", "b" * 40
|
||||
)
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
assert '"validation_gate_version": 1' in script
|
||||
assert '"required": True' in script
|
||||
assert '"contexts": ["Managed validation / full (push)"]' in script
|
||||
assert '"timeout_seconds": 3600 if repository == "jens/mediarr" else 1800' in script
|
||||
assert '"poll_interval_seconds": 5' in script
|
||||
assert '"max_poll_interval_seconds": 30' in script
|
||||
assert "if validation != expected_validation" in script
|
||||
|
||||
def test_runnerless_compatibility_requires_the_complete_metadata_triplet():
|
||||
fake = FakePortainer()
|
||||
DeploymentTargetRegistry(fake).register_single_container(
|
||||
"Jens/EventLens", "c" * 40
|
||||
)
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
assert 'runnerless_keys = {' in script
|
||||
assert '"gitea_repository_id"' in script
|
||||
assert '"gitea_webhook_secret_file"' in script
|
||||
assert '"validation"' in script
|
||||
assert "present_runnerless_keys and present_runnerless_keys != runnerless_keys" in script
|
||||
@@ -0,0 +1,32 @@
|
||||
from backend.app.services.app_classifier import classify_container
|
||||
|
||||
|
||||
def test_dockdeck_image_has_first_class_inventory_metadata():
|
||||
result = classify_container(
|
||||
{
|
||||
'name': 'web-portal',
|
||||
'image': 'dockdeck:local',
|
||||
'ports': ['1218:3000'],
|
||||
}
|
||||
)
|
||||
|
||||
assert result['app_type'] == 'dockdeck'
|
||||
assert result['suggested_app_id'] == 'dockdeck'
|
||||
assert result['category'] == 'infrastructure'
|
||||
assert result['subcategory'] == 'service_portal'
|
||||
assert result['criticality'] == 'medium'
|
||||
assert result['connector'] == 'none'
|
||||
assert result['policy_profile'] == 'internal_service'
|
||||
assert result['protected'] is False
|
||||
assert result['privacy_sensitive'] is True
|
||||
assert result['healthcheck_suggestion'] == {'port': 1218}
|
||||
assert result['voice_suggestion'] == 'disabled'
|
||||
assert result['confidence'] >= 0.8
|
||||
assert result['import_status'] == 'discovered'
|
||||
|
||||
|
||||
def test_dockdeck_name_match_does_not_require_local_image_tag():
|
||||
result = classify_container({'name': 'DockDeck', 'image': 'custom/build'})
|
||||
|
||||
assert result['app_type'] == 'dockdeck'
|
||||
assert result['category'] == 'infrastructure'
|
||||
@@ -0,0 +1,453 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from sqlmodel import Session, SQLModel, create_engine, select
|
||||
|
||||
from backend.app.db.models import Enrollment, EnrollmentJob, Policy
|
||||
from backend.app.schemas.enrollment import EnrollmentPlanV1, EnrollmentTrustConfigV1
|
||||
from backend.app.services import enrollment_plane as enrollment_plane_module
|
||||
from backend.app.services.enrollment_plane import EnrollmentPlane
|
||||
from backend.app.services.secret_store import get_secret, set_secret
|
||||
|
||||
|
||||
def _plan(**overrides):
|
||||
value = {
|
||||
"application_id": "wsbc-portal",
|
||||
"display_name": "Windows Support Bundle Collector",
|
||||
"repository": "Jens/Windows-Support-Bundle-Collector",
|
||||
"revision": "a" * 40,
|
||||
"profile": "internal-webapp-oidc",
|
||||
"target": {"target_identity": "unraid-controller", "container_name": "wsbc-portal"},
|
||||
"storage": {"runtime_uid": 1654, "runtime_gid": 1654},
|
||||
"network": {"docker_network": "itworx-npm", "internal_port": 8080},
|
||||
"artifact": {"release_version": "0.23.0", "image_repository": "wsbc-portal", "image_tag": "0.23.0"},
|
||||
"exposure": {"hostname": "wsbc.example.com", "upstream_service": "wsbc-portal"},
|
||||
"authentication": {
|
||||
"mode": "native-oidc",
|
||||
"redirect_path": "/signin-oidc",
|
||||
"logout_path": "/signout-callback-oidc",
|
||||
"roles": {"administrator": "Administrator", "support_agent": "SupportAgent", "read_only": "ReadOnly"},
|
||||
},
|
||||
"secrets": {"oidc_client_secret_required": True},
|
||||
}
|
||||
value.update(overrides)
|
||||
return EnrollmentPlanV1.model_validate(value)
|
||||
|
||||
|
||||
def _repopulse_plan(**overrides):
|
||||
value = {
|
||||
"application_id": "repopulse",
|
||||
"display_name": "RepoPulse",
|
||||
"environment": "production",
|
||||
"repository": "Jens/RepoPulse",
|
||||
"revision": "c" * 40,
|
||||
"profile": "repopulse-authenticated",
|
||||
"target": {"target_identity": "unraid-controller", "container_name": "repopulse"},
|
||||
"storage": {"runtime_uid": 10001, "runtime_gid": 10001},
|
||||
"network": {"docker_network": "itworx-mcp-hub-npm", "internal_port": 8000},
|
||||
"artifact": {
|
||||
"release_version": "1.5.0",
|
||||
"image_repository": "repopulse-managed",
|
||||
"image_tag": "production",
|
||||
"health_path": "/health",
|
||||
},
|
||||
"exposure": {"hostname": "repopulse.example.com", "upstream_service": "repopulse"},
|
||||
"secrets": {"additional_secret_refs": ["REPOPULSE_GITEA_TOKEN"]},
|
||||
}
|
||||
value.update(overrides)
|
||||
return EnrollmentPlanV1.model_validate(value)
|
||||
|
||||
|
||||
def _trust():
|
||||
return EnrollmentTrustConfigV1.model_validate({
|
||||
"controller_identity": "unraid-controller",
|
||||
"bootstrap_secret_ref": "ENROLLMENT_BOOTSTRAP",
|
||||
"approved_networks": ["itworx-npm", "itworx-mcp-hub-npm"],
|
||||
"allowed_domain_suffixes": ["example.com"],
|
||||
"autonomous_profiles": ["internal-webapp-oidc", "internal-webapp", "internal-service", "repopulse-authenticated"],
|
||||
"repopulse_proxy_access_list_id": 42,
|
||||
"oidc_issuer": "https://auth.example.com/application/o/wsbc-portal",
|
||||
"administrator_group": "Administrator",
|
||||
"support_agent_group": "SupportAgent",
|
||||
"read_only_group": "ReadOnly",
|
||||
})
|
||||
|
||||
|
||||
class _Broker:
|
||||
def is_configured(self):
|
||||
return True
|
||||
|
||||
def enrollment_capabilities(self, *, secret):
|
||||
assert secret == "b" * 64
|
||||
return {"status": "ready"}
|
||||
|
||||
def enrollment_preflight(self, enrollment, *, secret):
|
||||
assert secret == "b" * 64
|
||||
assert "target_credential" not in enrollment
|
||||
return {"status": "preflight_ok"}
|
||||
|
||||
def enroll_target(self, enrollment, *, secret, idempotency_key):
|
||||
assert secret == "b" * 64
|
||||
assert len(enrollment["target_credential"]) >= 48
|
||||
assert enrollment["application_id"] == "wsbc-portal"
|
||||
return {"status": "enrolled", "targetId": enrollment["enrollment_id"], "applicationId": "wsbc-portal"}
|
||||
|
||||
def configure_enrollment_runtime(self, runtime, *, secret):
|
||||
assert len(secret) >= 48
|
||||
assert set(runtime["values"]) == {"APPOPS_PUBLIC_URL", "OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "OIDC_ISSUER", "OIDC_ROLE_CLAIM"}
|
||||
return {"status": "configured", "enrollmentId": runtime["enrollment_id"], "applicationId": "wsbc-portal"}
|
||||
|
||||
|
||||
class _Portainer:
|
||||
def list_containers(self):
|
||||
return {"status": "ok", "containers": []}
|
||||
|
||||
|
||||
class _CollisionPortainer(_Portainer):
|
||||
def list_containers(self):
|
||||
return {"status": "ok", "containers": [{"name": "wsbc-portal"}]}
|
||||
|
||||
|
||||
class _Npm:
|
||||
def is_configured(self):
|
||||
return True
|
||||
|
||||
def proxy_hosts(self):
|
||||
return {"success": True, "items": []}
|
||||
|
||||
def ensure_managed_proxy_host(self, **kwargs):
|
||||
assert kwargs["hostname"] == "wsbc.example.com"
|
||||
assert kwargs["certificate_mode"] == "managed"
|
||||
return {"success": True, "status": "created", "hostId": 10, "hostname": kwargs["hostname"]}
|
||||
|
||||
|
||||
class _RepoPulseNpm(_Npm):
|
||||
def __init__(self, *, access_list_ok=True):
|
||||
self.access_list_ok = access_list_ok
|
||||
self.validated_ids = []
|
||||
self.created_host = None
|
||||
|
||||
def validate_managed_access_list(self, access_list_id):
|
||||
self.validated_ids.append(access_list_id)
|
||||
return {
|
||||
"success": self.access_list_ok,
|
||||
"status": "verified" if self.access_list_ok else "blocked",
|
||||
"message": "required_access_list_verified" if self.access_list_ok else "required_access_list_unavailable",
|
||||
**({"accessListId": access_list_id} if self.access_list_ok else {}),
|
||||
}
|
||||
|
||||
def ensure_managed_proxy_host(self, **kwargs):
|
||||
assert kwargs["hostname"] == "repopulse.example.com"
|
||||
assert kwargs["forward_host"] == "repopulse"
|
||||
assert kwargs["forward_port"] == 8000
|
||||
assert kwargs["access_list_id"] == 42
|
||||
self.created_host = dict(kwargs)
|
||||
return {"success": True, "status": "created", "hostId": 10, "hostname": kwargs["hostname"]}
|
||||
|
||||
|
||||
class _RepoPulseBroker(_Broker):
|
||||
def __init__(self):
|
||||
self.runtime_values = None
|
||||
|
||||
def enroll_target(self, enrollment, *, secret, idempotency_key):
|
||||
assert secret == "b" * 64
|
||||
assert enrollment["application_id"] == "repopulse"
|
||||
assert enrollment["repository"] == "jens/repopulse"
|
||||
assert enrollment["profile"] == "repopulse-authenticated"
|
||||
assert enrollment["network"] == "itworx-mcp-hub-npm"
|
||||
assert enrollment["runtime_uid"] == enrollment["runtime_gid"] == 10001
|
||||
assert enrollment["image_repository"] == "repopulse-managed"
|
||||
assert enrollment["image_tag"] == "production"
|
||||
return {"status": "enrolled", "targetId": enrollment["enrollment_id"], "applicationId": "repopulse"}
|
||||
|
||||
def configure_enrollment_runtime(self, runtime, *, secret):
|
||||
assert len(secret) >= 48
|
||||
self.runtime_values = dict(runtime["values"])
|
||||
assert set(self.runtime_values) == {
|
||||
"REPOPULSE_GITEA_URL", "REPOPULSE_GITEA_TOKEN", "REPOPULSE_MAINTENANCE_TOKEN",
|
||||
}
|
||||
return {"status": "configured", "enrollmentId": runtime["enrollment_id"], "applicationId": "repopulse"}
|
||||
|
||||
|
||||
class _Authentik:
|
||||
def is_configured(self):
|
||||
return True
|
||||
|
||||
def test_connection(self):
|
||||
return {"status": "connected"}
|
||||
|
||||
def applications(self):
|
||||
return {"success": True, "items": []}
|
||||
|
||||
def groups(self):
|
||||
return {"success": True, "items": [{"name": "Administrator"}, {"name": "SupportAgent"}, {"name": "ReadOnly"}]}
|
||||
|
||||
def deployment_oidc_template_status(self, *, role_claim):
|
||||
assert role_claim == "roles"
|
||||
return {"status": "ready", "missing": []}
|
||||
|
||||
def ensure_deployment_oidc_application(self, **kwargs):
|
||||
assert kwargs["application_slug"] == "wsbc-portal"
|
||||
assert kwargs["client_secret"]
|
||||
return {"success": True, "status": "created", "applicationId": "application-1", "providerId": "provider-1"}
|
||||
|
||||
|
||||
def _session(tmp_path):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'enrollment.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
return Session(engine)
|
||||
|
||||
|
||||
def _plane(portainer=None, broker=None, npm=None, authentik=None):
|
||||
return EnrollmentPlane(
|
||||
broker=broker or _Broker(),
|
||||
portainer=portainer or _Portainer(),
|
||||
npm=npm or _Npm(),
|
||||
authentik=authentik or _Authentik(),
|
||||
)
|
||||
|
||||
|
||||
def _configure(session):
|
||||
set_secret(session, "ENROLLMENT_BOOTSTRAP", "b" * 64)
|
||||
EnrollmentPlane.set_trust_config(session, _trust())
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _production_grade_secret_store(monkeypatch):
|
||||
"""Plane unit tests exercise enrollment behavior with a managed key store."""
|
||||
monkeypatch.setattr(
|
||||
enrollment_plane_module,
|
||||
"secret_storage_status",
|
||||
lambda: {"status": "ready", "productionReady": True},
|
||||
)
|
||||
|
||||
|
||||
def test_enrollment_schema_rejects_plaintext_secrets_and_untrusted_topology():
|
||||
raw = _plan().model_dump()
|
||||
raw["target"]["command"] = "id"
|
||||
try:
|
||||
EnrollmentPlanV1.model_validate(raw)
|
||||
except ValueError as exc:
|
||||
assert "Extra inputs" in str(exc)
|
||||
else: # pragma: no cover - assertion clarity
|
||||
raise AssertionError("arbitrary target command must be rejected")
|
||||
|
||||
raw = _plan().model_dump()
|
||||
raw["storage"]["appdata_subpath"] = "../etc"
|
||||
try:
|
||||
EnrollmentPlanV1.model_validate(raw)
|
||||
except ValueError as exc:
|
||||
assert "subpath" in str(exc)
|
||||
else: # pragma: no cover - assertion clarity
|
||||
raise AssertionError("appdata traversal must be rejected")
|
||||
|
||||
raw = _plan().model_dump()
|
||||
raw["exposure"]["hostname"] = "*.example.com"
|
||||
try:
|
||||
EnrollmentPlanV1.model_validate(raw)
|
||||
except ValueError as exc:
|
||||
assert "hostname" in str(exc)
|
||||
else: # pragma: no cover - assertion clarity
|
||||
raise AssertionError("wildcard hostname must be rejected")
|
||||
|
||||
|
||||
def test_repopulse_profile_accepts_only_its_fixed_authenticated_target():
|
||||
plan = _repopulse_plan()
|
||||
assert plan.application_id == "repopulse"
|
||||
assert plan.repository == "jens/repopulse"
|
||||
assert plan.network.docker_network == "itworx-mcp-hub-npm"
|
||||
assert plan.secrets.additional_secret_refs == ("REPOPULSE_GITEA_TOKEN",)
|
||||
|
||||
for field, value in (
|
||||
("application_id", "another-app"),
|
||||
("repository", "Jens/another-app"),
|
||||
("network", {"docker_network": "other-network", "internal_port": 8000}),
|
||||
("artifact", {"release_version": "1.5.0", "image_repository": "repopulse-managed", "image_tag": "latest", "health_path": "/health"}),
|
||||
("exposure", {"hostname": "other.example.com", "upstream_service": "repopulse"}),
|
||||
("secrets", {"additional_secret_refs": ["OTHER_TOKEN"]}),
|
||||
):
|
||||
raw = _repopulse_plan().model_dump(mode="json")
|
||||
raw[field] = value
|
||||
with pytest.raises(ValueError):
|
||||
EnrollmentPlanV1.model_validate(raw)
|
||||
|
||||
|
||||
def test_repopulse_preflight_requires_existing_secret_and_trusted_npm_access_list(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
npm = _RepoPulseNpm(access_list_ok=False)
|
||||
plane = _plane(npm=npm)
|
||||
|
||||
missing_secret = plane.preflight(session, _repopulse_plan())
|
||||
assert missing_secret["status"] == "blocked"
|
||||
assert "repopulse_gitea_token_unavailable" in missing_secret["blockers"]
|
||||
assert npm.validated_ids == [42]
|
||||
|
||||
set_secret(session, "REPOPULSE_GITEA_TOKEN", "existing-encrypted-gitea-token")
|
||||
bad_access_list = plane.preflight(session, _repopulse_plan())
|
||||
assert bad_access_list["status"] == "blocked"
|
||||
assert "repopulse_access_list_unavailable" in bad_access_list["blockers"]
|
||||
assert session.exec(select(Enrollment)).all() == []
|
||||
assert session.exec(select(EnrollmentJob)).all() == []
|
||||
|
||||
|
||||
def test_repopulse_enrollment_seals_exact_runtime_values_without_exposing_secrets(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
gitea_token = "existing-encrypted-gitea-token"
|
||||
set_secret(session, "REPOPULSE_GITEA_TOKEN", gitea_token)
|
||||
npm = _RepoPulseNpm()
|
||||
broker = _RepoPulseBroker()
|
||||
result = _plane(broker=broker, npm=npm).enroll(
|
||||
session, _repopulse_plan(), idempotency_key="enroll-repopulse-0001",
|
||||
)
|
||||
|
||||
assert result["enrollment"]["status"] == "ready"
|
||||
assert npm.validated_ids == [42]
|
||||
assert npm.created_host["access_list_id"] == 42
|
||||
assert broker.runtime_values["REPOPULSE_GITEA_URL"] == "https://gitea.example.com"
|
||||
assert broker.runtime_values["REPOPULSE_GITEA_TOKEN"] == gitea_token
|
||||
maintenance_token = get_secret(session, "ENROLL_REPOPULSE_MAINTENANCE_TOKEN")
|
||||
assert maintenance_token and broker.runtime_values["REPOPULSE_MAINTENANCE_TOKEN"] == maintenance_token
|
||||
assert gitea_token not in str(result)
|
||||
assert maintenance_token not in str(result)
|
||||
assert gitea_token not in str(result["job"]["evidence"])
|
||||
assert maintenance_token not in str(result["job"]["evidence"])
|
||||
|
||||
|
||||
def test_preflight_is_read_only_and_rejects_existing_container(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(_CollisionPortainer()).preflight(session, _plan())
|
||||
assert result["status"] == "blocked"
|
||||
assert "unmanaged_resource_collision" in result["blockers"]
|
||||
assert session.exec(select(Enrollment)).all() == []
|
||||
assert session.exec(select(EnrollmentJob)).all() == []
|
||||
|
||||
|
||||
def test_preflight_rejects_the_development_secret_fallback(tmp_path, monkeypatch):
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
monkeypatch.setattr(
|
||||
enrollment_plane_module,
|
||||
"secret_storage_status",
|
||||
lambda: {"status": "ready", "productionReady": False},
|
||||
)
|
||||
result = _plane().preflight(session, _plan())
|
||||
assert result["status"] == "blocked"
|
||||
assert "secret_store" in result["blockers"]
|
||||
assert session.exec(select(Enrollment)).all() == []
|
||||
|
||||
|
||||
def test_capabilities_require_a_live_bootstrap_controller_contract(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane().capabilities(session)
|
||||
assert result["targetEnrollment"]["status"] == "ready"
|
||||
assert result["controller"]["status"] == "ready"
|
||||
|
||||
|
||||
def test_enrollment_creates_write_only_credentials_exact_policies_and_is_idempotent(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
plane = _plane()
|
||||
result = plane.enroll(session, _plan(), idempotency_key="enroll-wsbc-portal-0001")
|
||||
assert result["enrollment"]["status"] == "ready"
|
||||
assert result["job"]["status"] == "ready"
|
||||
assert result["enrollment"]["targetSecretRef"] == "ENROLL_WSBC_PORTAL_TARGET_HMAC"
|
||||
assert result["enrollment"]["oidcSecretRef"] == "ENROLL_WSBC_PORTAL_OIDC_CLIENT_SECRET"
|
||||
target_secret = get_secret(session, "ENROLL_WSBC_PORTAL_TARGET_HMAC")
|
||||
oidc_secret = get_secret(session, "ENROLL_WSBC_PORTAL_OIDC_CLIENT_SECRET")
|
||||
assert target_secret and oidc_secret
|
||||
assert target_secret not in str(result)
|
||||
assert oidc_secret not in str(result)
|
||||
policies = session.exec(select(Policy).where(Policy.target == "wsbc-portal")).all()
|
||||
assert {(policy.action, policy.target, policy.manual, policy.voice) for policy in policies} == {
|
||||
("deployment.apply", "wsbc-portal", "allowed", "blocked"),
|
||||
("deployment.rollback", "wsbc-portal", "allowed", "blocked"),
|
||||
}
|
||||
assert all(policy.provenance_json["enrollmentId"] == result["enrollment"]["id"] for policy in policies)
|
||||
replay = plane.enroll(session, _plan(), idempotency_key="enroll-wsbc-portal-0001")
|
||||
assert replay["idempotentReplay"] is True
|
||||
assert replay["enrollment"]["id"] == result["enrollment"]["id"]
|
||||
assert len(session.exec(select(Enrollment)).all()) == 1
|
||||
assert len(session.exec(select(EnrollmentJob)).all()) == 1
|
||||
|
||||
|
||||
def test_interrupted_external_enrollment_requires_explicit_recovery(tmp_path):
|
||||
class InterruptedBroker(_Broker):
|
||||
def enroll_target(self, _enrollment, *, secret, idempotency_key):
|
||||
assert secret == "b" * 64
|
||||
assert idempotency_key == "enroll-wsbc-interrupted-0001"
|
||||
raise RuntimeError("controller connection interrupted after request dispatch")
|
||||
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
plane = _plane(broker=InterruptedBroker())
|
||||
with pytest.raises(RuntimeError, match="interrupted"):
|
||||
plane.enroll(session, _plan(), idempotency_key="enroll-wsbc-interrupted-0001")
|
||||
|
||||
job = session.exec(select(EnrollmentJob)).one()
|
||||
assert job.status == "running"
|
||||
assert job.stage == "registering_target"
|
||||
assert job.evidence_json["intent"]["controller"]["applicationId"] == "wsbc-portal"
|
||||
assert "target_credential" not in str(job.evidence_json)
|
||||
|
||||
recovered = plane.enroll(session, _plan(), idempotency_key="enroll-wsbc-interrupted-0001")
|
||||
assert recovered["recoveryRequired"] is True
|
||||
assert recovered["job"]["status"] == "recovery_required"
|
||||
assert recovered["enrollment"]["status"] == "recovery_required"
|
||||
|
||||
|
||||
def test_preflight_blocks_hostname_outside_root_trust_boundary(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
raw = _plan().model_dump()
|
||||
raw["exposure"]["hostname"] = "wsbc.example.net"
|
||||
result = _plane().preflight(session, EnrollmentPlanV1.model_validate(raw))
|
||||
assert result["status"] == "blocked"
|
||||
assert "hostname_not_allowed" in result["blockers"]
|
||||
|
||||
|
||||
def test_preflight_rejects_an_oidc_role_model_outside_root_trust(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
raw = _plan().model_dump()
|
||||
raw["authentication"]["roles"]["administrator"] = "TenantAdmin"
|
||||
result = _plane().preflight(session, EnrollmentPlanV1.model_validate(raw))
|
||||
assert result["status"] == "blocked"
|
||||
assert "oidc_role_model" in result["blockers"]
|
||||
|
||||
|
||||
def test_failed_identity_provisioning_compensates_only_created_resources(tmp_path):
|
||||
class CompensatingBroker(_Broker):
|
||||
def unenroll_target(self, target, *, secret):
|
||||
assert target["application_id"] == "wsbc-portal"
|
||||
assert secret == "b" * 64
|
||||
return {"status": "rolled_back"}
|
||||
|
||||
class CompensatingNpm(_Npm):
|
||||
def remove_managed_proxy_host(self, *, host_id, ownership_id):
|
||||
assert host_id == 10
|
||||
assert ownership_id.startswith("enr-")
|
||||
return {"success": True, "status": "removed"}
|
||||
|
||||
class FailingAuthentik(_Authentik):
|
||||
def ensure_deployment_oidc_application(self, **_kwargs):
|
||||
return {"success": False, "status": "failed"}
|
||||
|
||||
with _session(tmp_path) as session:
|
||||
_configure(session)
|
||||
result = _plane(
|
||||
broker=CompensatingBroker(),
|
||||
npm=CompensatingNpm(),
|
||||
authentik=FailingAuthentik(),
|
||||
).enroll(session, _plan(), idempotency_key="enroll-wsbc-compensate-0001")
|
||||
assert result["enrollment"]["status"] == "failed"
|
||||
assert result["job"]["evidence"]["compensation"] == {
|
||||
"completed": True,
|
||||
"outcomes": {"proxy": "removed", "controller": "rolled_back"},
|
||||
}
|
||||
assert get_secret(session, "ENROLL_WSBC_PORTAL_TARGET_HMAC") is None
|
||||
assert get_secret(session, "ENROLL_WSBC_PORTAL_OIDC_CLIENT_SECRET") is None
|
||||
assert session.exec(select(Policy).where(Policy.target == "wsbc-portal")).all() == []
|
||||
@@ -0,0 +1,326 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
from backend.app.connectors import gitea_actions
|
||||
from backend.app.connectors.gitea_actions import GiteaActionsConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "appops_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
from backend.app.main import app
|
||||
|
||||
engine = create_engine(
|
||||
f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False}
|
||||
)
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
|
||||
def _configure(monkeypatch):
|
||||
monkeypatch.setitem(settings.apis, "GITEA_URL", "https://gitea.test")
|
||||
monkeypatch.setitem(settings.apis, "GITEA_ADMIN_TOKEN", "fixture-secret")
|
||||
monkeypatch.setitem(
|
||||
settings.apis,
|
||||
"GITEA_RUNNER_TARGETS_JSON",
|
||||
'{"unraid-deploy":"itworx-security-runner"}',
|
||||
)
|
||||
monkeypatch.setattr(gitea_actions, "_CACHE", None)
|
||||
|
||||
|
||||
def test_runner_snapshot_recommends_only_approval_gated_restart_for_stalled_queue(
|
||||
monkeypatch,
|
||||
):
|
||||
_configure(monkeypatch)
|
||||
connector = GiteaActionsConnector()
|
||||
created_at = (datetime.now(UTC) - timedelta(minutes=12)).isoformat()
|
||||
|
||||
def fake_request(path, params=None):
|
||||
if path.endswith("/runners"):
|
||||
return {
|
||||
"total_count": 1,
|
||||
"runners": [
|
||||
{
|
||||
"id": 7,
|
||||
"name": "deploy-runner",
|
||||
"status": "offline",
|
||||
"busy": False,
|
||||
"disabled": False,
|
||||
"labels": [{"name": "unraid-deploy"}],
|
||||
}
|
||||
],
|
||||
}
|
||||
if params.get("status") == "queued":
|
||||
return {
|
||||
"total_count": 1,
|
||||
"jobs": [
|
||||
{
|
||||
"id": 8,
|
||||
"status": "queued",
|
||||
"created_at": created_at,
|
||||
"labels": ["unraid-deploy"],
|
||||
}
|
||||
],
|
||||
}
|
||||
return {"total_count": 0, "jobs": []}
|
||||
|
||||
monkeypatch.setattr(connector, "_request", fake_request)
|
||||
result = connector.snapshot(refresh=True)
|
||||
|
||||
assert result["severity"] == "critical"
|
||||
assert result["queue"]["queued"] == 1
|
||||
assert result["queue"]["oldestQueuedAgeSeconds"] >= 600
|
||||
assert result["queue"]["sampleTruncated"] is False
|
||||
assert result["recommendedAction"] == {
|
||||
"intent": "container.restart",
|
||||
"target": "itworx-security-runner",
|
||||
"requiresApproval": True,
|
||||
"autoExecute": False,
|
||||
"reason": "Gitea Actions queue is stalled without a matching online runner.",
|
||||
}
|
||||
assert "fixture-secret" not in repr(result)
|
||||
|
||||
|
||||
def test_runner_snapshot_is_cached_and_healthy_when_queue_is_clear(monkeypatch):
|
||||
_configure(monkeypatch)
|
||||
connector = GiteaActionsConnector()
|
||||
calls = 0
|
||||
|
||||
def fake_request(path, params=None):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if path.endswith("/runners"):
|
||||
return {
|
||||
"total_count": 1,
|
||||
"runners": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "shared",
|
||||
"status": "online",
|
||||
"labels": ["ubuntu-latest"],
|
||||
}
|
||||
],
|
||||
}
|
||||
return {"total_count": 0, "jobs": []}
|
||||
|
||||
monkeypatch.setattr(connector, "_request", fake_request)
|
||||
first = connector.snapshot(refresh=True)
|
||||
second = connector.snapshot()
|
||||
|
||||
assert first["severity"] == "healthy"
|
||||
assert first["recommendedAction"] is None
|
||||
assert second["cache"]["hit"] is True
|
||||
assert calls == 4
|
||||
|
||||
|
||||
def test_long_running_shared_job_is_critical_without_ambiguous_restart(monkeypatch):
|
||||
_configure(monkeypatch)
|
||||
monkeypatch.setitem(settings.apis, "GITEA_RUNNER_TARGETS_JSON", "{}")
|
||||
connector = GiteaActionsConnector()
|
||||
created_at = (datetime.now(UTC) - timedelta(hours=3)).isoformat()
|
||||
|
||||
def fake_request(path, params=None):
|
||||
if path.endswith("/runners"):
|
||||
return {
|
||||
"total_count": 1,
|
||||
"runners": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "shared",
|
||||
"status": "online",
|
||||
"busy": True,
|
||||
"labels": ["ubuntu-latest"],
|
||||
}
|
||||
],
|
||||
}
|
||||
if params.get("status") == "in_progress":
|
||||
return {
|
||||
"total_count": 1,
|
||||
"jobs": [
|
||||
{
|
||||
"id": 9,
|
||||
"status": "in_progress",
|
||||
"created_at": created_at,
|
||||
"labels": ["ubuntu-latest"],
|
||||
}
|
||||
],
|
||||
}
|
||||
return {"total_count": 0, "jobs": []}
|
||||
|
||||
monkeypatch.setattr(connector, "_request", fake_request)
|
||||
result = connector.snapshot(refresh=True)
|
||||
|
||||
assert result["severity"] == "critical"
|
||||
assert result["queue"]["oldestInProgressAgeSeconds"] >= 7_200
|
||||
assert result["recommendedAction"] is None
|
||||
|
||||
|
||||
def test_recently_started_old_queued_job_is_not_reported_as_stale(monkeypatch):
|
||||
_configure(monkeypatch)
|
||||
connector = GiteaActionsConnector()
|
||||
created_at = (datetime.now(UTC) - timedelta(hours=3)).isoformat()
|
||||
started_at = (datetime.now(UTC) - timedelta(seconds=30)).isoformat()
|
||||
|
||||
def fake_request(path, params=None):
|
||||
if path.endswith("/runners"):
|
||||
return {
|
||||
"total_count": 1,
|
||||
"runners": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "shared",
|
||||
"status": "online",
|
||||
"busy": True,
|
||||
"labels": ["ubuntu-latest"],
|
||||
}
|
||||
],
|
||||
}
|
||||
if params.get("status") == "in_progress":
|
||||
return {
|
||||
"total_count": 1,
|
||||
"jobs": [
|
||||
{
|
||||
"id": 9,
|
||||
"status": "in_progress",
|
||||
"created_at": created_at,
|
||||
"started_at": started_at,
|
||||
"labels": ["ubuntu-latest"],
|
||||
}
|
||||
],
|
||||
}
|
||||
return {"total_count": 0, "jobs": []}
|
||||
|
||||
monkeypatch.setattr(connector, "_request", fake_request)
|
||||
result = connector.snapshot(refresh=True)
|
||||
|
||||
assert result["severity"] == "healthy"
|
||||
assert 0 <= result["queue"]["oldestInProgressAgeSeconds"] < 120
|
||||
assert result["queue"]["staleActiveRunnerLabels"] == []
|
||||
|
||||
|
||||
def test_long_running_job_recommends_only_exact_mapped_runner(monkeypatch):
|
||||
_configure(monkeypatch)
|
||||
connector = GiteaActionsConnector()
|
||||
created_at = (datetime.now(UTC) - timedelta(hours=3)).isoformat()
|
||||
|
||||
def fake_request(path, params=None):
|
||||
if path.endswith("/runners"):
|
||||
return {
|
||||
"total_count": 1,
|
||||
"runners": [
|
||||
{
|
||||
"id": 7,
|
||||
"name": "deploy-runner",
|
||||
"status": "online",
|
||||
"busy": True,
|
||||
"labels": ["unraid-deploy"],
|
||||
}
|
||||
],
|
||||
}
|
||||
if params.get("status") == "in_progress":
|
||||
return {
|
||||
"total_count": 1,
|
||||
"jobs": [
|
||||
{
|
||||
"id": 10,
|
||||
"status": "in_progress",
|
||||
"created_at": created_at,
|
||||
"labels": ["unraid-deploy"],
|
||||
}
|
||||
],
|
||||
}
|
||||
return {"total_count": 0, "jobs": []}
|
||||
|
||||
monkeypatch.setattr(connector, "_request", fake_request)
|
||||
result = connector.snapshot(refresh=True)
|
||||
|
||||
assert result["recommendedAction"] == {
|
||||
"intent": "container.restart",
|
||||
"target": "itworx-security-runner",
|
||||
"requiresApproval": True,
|
||||
"autoExecute": False,
|
||||
"reason": "A Gitea Actions job exceeded the two-hour guardrail on an explicitly mapped runner.",
|
||||
}
|
||||
|
||||
|
||||
def test_runner_status_endpoint_degrades_when_not_configured(api_client, monkeypatch):
|
||||
monkeypatch.setitem(settings.apis, "GITEA_URL", None)
|
||||
monkeypatch.setitem(settings.apis, "GITEA_ADMIN_TOKEN", None)
|
||||
monkeypatch.setattr(gitea_actions, "_CACHE", None)
|
||||
|
||||
response = api_client.get("/api/gitea/actions/status")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "missing_config"
|
||||
assert response.json()["recommendedAction"] is None
|
||||
|
||||
|
||||
def test_default_docker_target_falls_back_to_v1_capability(monkeypatch):
|
||||
_configure(monkeypatch)
|
||||
monkeypatch.setitem(settings.apis, "GITEA_RUNNER_TARGETS_JSON", "{}")
|
||||
|
||||
target = GiteaActionsConnector._target_for_label(
|
||||
"docker-validation",
|
||||
[
|
||||
{
|
||||
"name": "itworx-validation-runner-2",
|
||||
"labels": ["linux-validation", "docker-validation"],
|
||||
}
|
||||
],
|
||||
GiteaActionsConnector._restart_targets(),
|
||||
)
|
||||
|
||||
assert target == "itworx-validation-runner-2"
|
||||
|
||||
|
||||
def test_default_docker_target_prefers_v2_specialized_runner(monkeypatch):
|
||||
_configure(monkeypatch)
|
||||
monkeypatch.setitem(settings.apis, "GITEA_RUNNER_TARGETS_JSON", "{}")
|
||||
|
||||
target = GiteaActionsConnector._target_for_label(
|
||||
"docker-validation",
|
||||
[
|
||||
{
|
||||
"name": "itworx-validation-runner-2",
|
||||
"labels": ["linux-validation"],
|
||||
},
|
||||
{
|
||||
"name": "itworx-docker-runner",
|
||||
"labels": ["docker-validation"],
|
||||
},
|
||||
],
|
||||
GiteaActionsConnector._restart_targets(),
|
||||
)
|
||||
|
||||
assert target == "itworx-docker-runner"
|
||||
|
||||
|
||||
def test_explicit_runner_target_mapping_remains_authoritative(monkeypatch):
|
||||
_configure(monkeypatch)
|
||||
monkeypatch.setitem(
|
||||
settings.apis,
|
||||
"GITEA_RUNNER_TARGETS_JSON",
|
||||
'{"docker-validation":"operator-selected-container"}',
|
||||
)
|
||||
|
||||
target = GiteaActionsConnector._target_for_label(
|
||||
"docker-validation",
|
||||
[{"name": "some-registration", "labels": ["docker-validation"]}],
|
||||
GiteaActionsConnector._restart_targets(),
|
||||
)
|
||||
|
||||
assert target == "operator-selected-container"
|
||||
@@ -0,0 +1,395 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.pool import StaticPool
|
||||
from sqlmodel import Session, SQLModel, create_engine, select
|
||||
|
||||
from backend.app.db.models import Policy
|
||||
from backend.app.services.gitea_project_auto_registration import (
|
||||
GiteaProjectAutoRegistration,
|
||||
)
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class FakeTargetRegistry:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def register_compose_target(self, repository, revision, **kwargs):
|
||||
self.calls.append((repository, revision, kwargs))
|
||||
return {
|
||||
"status": "registered",
|
||||
"repository": repository,
|
||||
"revision": revision,
|
||||
}
|
||||
|
||||
|
||||
class FakeProjectSync(GiteaProjectAutoRegistration):
|
||||
def __init__(self, repositories, contracts, *, target_registry):
|
||||
super().__init__(target_registry=target_registry)
|
||||
self.repositories = repositories
|
||||
self.contracts = contracts
|
||||
self.branch_calls = []
|
||||
self.contract_calls = []
|
||||
|
||||
def _client(self):
|
||||
return FakeClient()
|
||||
|
||||
def _repositories(self, client):
|
||||
return list(self.repositories)
|
||||
|
||||
def _branch_head(self, client, *, repository_name, branch):
|
||||
self.branch_calls.append((repository_name, branch))
|
||||
return (repository_name[0].lower() if repository_name else "a") * 40
|
||||
|
||||
def _compose_contract(self, client, *, repository_name, revision):
|
||||
self.contract_calls.append((repository_name, revision))
|
||||
return self.contracts.get(repository_name)
|
||||
|
||||
|
||||
class FakeComposeInspection(GiteaProjectAutoRegistration):
|
||||
def __init__(self, root_names, files):
|
||||
super().__init__(target_registry=FakeTargetRegistry())
|
||||
self.root_names = list(root_names)
|
||||
self.files = dict(files)
|
||||
|
||||
def _contents(
|
||||
self,
|
||||
client,
|
||||
*,
|
||||
repository_name,
|
||||
revision,
|
||||
path="",
|
||||
allow_not_found=False,
|
||||
):
|
||||
if not path:
|
||||
return [{"name": name} for name in self.root_names]
|
||||
raise AssertionError(f"unexpected direct contents read: {path}")
|
||||
|
||||
def _read_text(
|
||||
self,
|
||||
client,
|
||||
*,
|
||||
repository_name,
|
||||
revision,
|
||||
path,
|
||||
allow_not_found=False,
|
||||
):
|
||||
value = self.files.get(path)
|
||||
if value is None and not allow_not_found:
|
||||
raise AssertionError(f"missing fake file: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def make_session():
|
||||
engine = create_engine(
|
||||
"sqlite://",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
return Session(engine)
|
||||
|
||||
|
||||
def repo(name, *, branch="main", updated="2026-09-16T12:00:00Z", archived=False, empty=False):
|
||||
return {
|
||||
"name": name,
|
||||
"full_name": f"Jens/{name}",
|
||||
"default_branch": branch,
|
||||
"updated_at": updated,
|
||||
"archived": archived,
|
||||
"empty": empty,
|
||||
"private": True,
|
||||
}
|
||||
|
||||
|
||||
def deployable_contract():
|
||||
return {
|
||||
"applicationId": "deployable",
|
||||
"composeFile": "docker-compose.yml",
|
||||
"composeProject": "deployable",
|
||||
"containers": ["deployable"],
|
||||
"serviceNames": ["deployable"],
|
||||
"healthTimeout": 180,
|
||||
"verifyOciRevision": False,
|
||||
}
|
||||
|
||||
|
||||
def test_repository_names_normalize_to_deployment_plan_application_ids():
|
||||
assert GiteaProjectAutoRegistration._application_id("ITWorx_MCP_Hub") == "itworx-mcp-hub"
|
||||
assert GiteaProjectAutoRegistration._application_id("Media.Codec_Dashboard") == "media-codec-dashboard"
|
||||
|
||||
|
||||
def test_single_service_compose_can_be_inferred_without_descriptor():
|
||||
service = FakeComposeInspection(
|
||||
["docker-compose.yml"],
|
||||
{
|
||||
"docker-compose.yml": "services:\n my-app:\n image: example/app:1\n",
|
||||
},
|
||||
)
|
||||
contract = service._compose_contract(
|
||||
FakeClient(), repository_name="My_App", revision="a" * 40
|
||||
)
|
||||
assert contract is not None
|
||||
assert contract["applicationId"] == "my-app"
|
||||
assert contract["composeProject"] == "my-app"
|
||||
assert contract["serviceNames"] == ["my-app"]
|
||||
assert contract["containers"] == ["my-app-my-app-1"]
|
||||
|
||||
|
||||
def test_ambiguous_multi_service_compose_stays_known_but_unconfigured():
|
||||
service = FakeComposeInspection(
|
||||
["compose.yml"],
|
||||
{
|
||||
"compose.yml": "services:\n api:\n image: example/api:1\n worker:\n image: example/worker:1\n",
|
||||
},
|
||||
)
|
||||
assert (
|
||||
service._compose_contract(
|
||||
FakeClient(), repository_name="DifferentName", revision="b" * 40
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_descriptor_can_select_primary_service_for_multi_service_project():
|
||||
descriptor = {
|
||||
"apiVersion": "itworx.deployment-target/v1",
|
||||
"applicationId": "api",
|
||||
"composeFile": "compose.yml",
|
||||
}
|
||||
service = FakeComposeInspection(
|
||||
[".itworx", "compose.yml"],
|
||||
{
|
||||
".itworx/deployment-target.json": json.dumps(descriptor),
|
||||
"compose.yml": "services:\n api:\n image: example/api:1\n worker:\n image: example/worker:1\n",
|
||||
},
|
||||
)
|
||||
contract = service._compose_contract(
|
||||
FakeClient(), repository_name="MultiService", revision="c" * 40
|
||||
)
|
||||
assert contract is not None
|
||||
assert contract["applicationId"] == "api"
|
||||
assert contract["composeProject"] == "api"
|
||||
assert contract["serviceNames"] == ["api", "worker"]
|
||||
assert contract["containers"] == ["api-api-1", "api-worker-1"]
|
||||
|
||||
|
||||
def test_descriptor_cannot_diverge_compose_project_from_application_id():
|
||||
with pytest.raises(ValueError, match="composeProject must equal applicationId"):
|
||||
GiteaProjectAutoRegistration._safe_descriptor(
|
||||
{
|
||||
"apiVersion": "itworx.deployment-target/v1",
|
||||
"applicationId": "api",
|
||||
"composeProject": "different",
|
||||
},
|
||||
"Example",
|
||||
)
|
||||
|
||||
|
||||
def test_sync_catalogs_every_repo_but_only_activates_proven_compose_targets():
|
||||
registry = FakeTargetRegistry()
|
||||
contract = deployable_contract()
|
||||
contract["containers"] = ["deployable-api-1", "deployable-web-1"]
|
||||
contract["serviceNames"] = ["deployable", "worker"]
|
||||
service = FakeProjectSync(
|
||||
[repo("Deployable"), repo("DocsOnly"), repo("Legacy", branch="master")],
|
||||
{
|
||||
"Deployable": contract,
|
||||
"DocsOnly": None,
|
||||
"Legacy": None,
|
||||
},
|
||||
target_registry=registry,
|
||||
)
|
||||
|
||||
with make_session() as session:
|
||||
result = service.sync(session)
|
||||
assert result["status"] == "ok"
|
||||
assert result["summary"]["total"] == 3
|
||||
assert result["summary"]["registered"] == 1
|
||||
assert result["summary"]["knownUnconfigured"] == 2
|
||||
assert len(registry.calls) == 1
|
||||
repository, revision, kwargs = registry.calls[0]
|
||||
assert repository == "jens/deployable"
|
||||
assert len(revision) == 40
|
||||
assert kwargs["branch"] == "main"
|
||||
assert kwargs["containers"] == ["deployable-api-1", "deployable-web-1"]
|
||||
|
||||
policies = session.exec(
|
||||
select(Policy).where(Policy.target == "deployable")
|
||||
).all()
|
||||
assert {item.action for item in policies} == {
|
||||
"deployment.apply",
|
||||
"deployment.rollback",
|
||||
"deployment.runnerless.provision",
|
||||
}
|
||||
assert all(item.manual == "allowed" for item in policies)
|
||||
assert all(item.voice == "blocked" for item in policies)
|
||||
|
||||
|
||||
def test_unchanged_catalog_entries_do_not_repeat_registration_work():
|
||||
registry = FakeTargetRegistry()
|
||||
repositories = [repo("Deployable")]
|
||||
service = FakeProjectSync(
|
||||
repositories,
|
||||
{"Deployable": deployable_contract()},
|
||||
target_registry=registry,
|
||||
)
|
||||
|
||||
with make_session() as session:
|
||||
first = service.sync(session)
|
||||
assert first["summary"]["registered"] == 1
|
||||
assert len(registry.calls) == 1
|
||||
first_branch_calls = len(service.branch_calls)
|
||||
first_contract_calls = len(service.contract_calls)
|
||||
|
||||
second = service.sync(session)
|
||||
assert second["summary"]["registered"] == 1
|
||||
assert second["summary"]["changedThisSync"] == 0
|
||||
assert len(registry.calls) == 1
|
||||
assert len(service.branch_calls) == first_branch_calls
|
||||
assert len(service.contract_calls) == first_contract_calls
|
||||
|
||||
|
||||
def test_existing_exact_policy_is_not_repurposed_when_it_blocks_deployments():
|
||||
registry = FakeTargetRegistry()
|
||||
service = FakeProjectSync(
|
||||
[repo("Deployable")],
|
||||
{"Deployable": deployable_contract()},
|
||||
target_registry=registry,
|
||||
)
|
||||
|
||||
with make_session() as session:
|
||||
session.add(
|
||||
Policy(
|
||||
id="operator-owned-policy",
|
||||
action="deployment.apply",
|
||||
target="deployable",
|
||||
voice="blocked",
|
||||
manual="blocked",
|
||||
risk="high",
|
||||
reason="operator-owned denial",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
result = service.sync(session)
|
||||
project = result["projects"][0]
|
||||
assert project["state"] == "policy_conflict"
|
||||
preserved = session.get(Policy, "operator-owned-policy")
|
||||
assert preserved is not None
|
||||
assert preserved.manual == "blocked"
|
||||
policies = session.exec(select(Policy).where(Policy.target == "deployable")).all()
|
||||
assert [item.id for item in policies] == ["operator-owned-policy"]
|
||||
|
||||
|
||||
def test_rollback_policy_conflict_does_not_stage_or_commit_apply_policy():
|
||||
registry = FakeTargetRegistry()
|
||||
service = FakeProjectSync(
|
||||
[repo("Deployable")],
|
||||
{"Deployable": deployable_contract()},
|
||||
target_registry=registry,
|
||||
)
|
||||
|
||||
with make_session() as session:
|
||||
session.add(
|
||||
Policy(
|
||||
id="operator-owned-rollback-denial",
|
||||
action="deployment.rollback",
|
||||
target="deployable",
|
||||
voice="blocked",
|
||||
manual="blocked",
|
||||
risk="high",
|
||||
reason="operator-owned rollback denial",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
result = service.sync(session)
|
||||
assert result["projects"][0]["state"] == "policy_conflict"
|
||||
policies = session.exec(select(Policy).where(Policy.target == "deployable")).all()
|
||||
assert [(item.id, item.action, item.manual) for item in policies] == [
|
||||
("operator-owned-rollback-denial", "deployment.rollback", "blocked")
|
||||
]
|
||||
|
||||
def test_unchanged_controller_conflict_is_retried_and_can_self_heal():
|
||||
class ConflictThenRegisteredRegistry(FakeTargetRegistry):
|
||||
def register_compose_target(self, repository, revision, **kwargs):
|
||||
self.calls.append((repository, revision, kwargs))
|
||||
if len(self.calls) == 1:
|
||||
return {
|
||||
"status": "conflict",
|
||||
"repository": repository,
|
||||
"revision": revision,
|
||||
"reason": "deployment target already exists with different topology",
|
||||
}
|
||||
return {
|
||||
"status": "registered",
|
||||
"repository": repository,
|
||||
"revision": revision,
|
||||
}
|
||||
|
||||
registry = ConflictThenRegisteredRegistry()
|
||||
service = FakeProjectSync(
|
||||
[repo("Deployable")],
|
||||
{"Deployable": deployable_contract()},
|
||||
target_registry=registry,
|
||||
)
|
||||
|
||||
with make_session() as session:
|
||||
first = service.sync(session)
|
||||
assert first["projects"][0]["state"] == "controller_conflict"
|
||||
assert len(registry.calls) == 1
|
||||
|
||||
second = service.sync(session)
|
||||
assert second["projects"][0]["state"] == "registered"
|
||||
assert second["summary"]["changedThisSync"] == 1
|
||||
assert len(registry.calls) == 2
|
||||
assert len(second["projects"][0]["policyIds"]) == 3
|
||||
|
||||
|
||||
def test_unchanged_policy_conflict_is_retried_after_operator_owned_policy_is_repaired():
|
||||
registry = FakeTargetRegistry()
|
||||
service = FakeProjectSync(
|
||||
[repo("Deployable")],
|
||||
{"Deployable": deployable_contract()},
|
||||
target_registry=registry,
|
||||
)
|
||||
|
||||
with make_session() as session:
|
||||
policy = Policy(
|
||||
id="operator-owned-policy",
|
||||
action="deployment.apply",
|
||||
target="deployable",
|
||||
voice="blocked",
|
||||
manual="blocked",
|
||||
risk="high",
|
||||
reason="operator-owned denial",
|
||||
)
|
||||
session.add(policy)
|
||||
session.commit()
|
||||
|
||||
first = service.sync(session)
|
||||
assert first["projects"][0]["state"] == "policy_conflict"
|
||||
assert len(registry.calls) == 1
|
||||
|
||||
policy.manual = "allowed"
|
||||
session.add(policy)
|
||||
session.commit()
|
||||
|
||||
second = service.sync(session)
|
||||
assert second["projects"][0]["state"] == "registered"
|
||||
assert second["summary"]["changedThisSync"] == 1
|
||||
assert len(registry.calls) == 2
|
||||
assert {
|
||||
item.action
|
||||
for item in session.exec(select(Policy).where(Policy.target == "deployable")).all()
|
||||
} == {"deployment.apply", "deployment.rollback", "deployment.runnerless.provision"}
|
||||
@@ -0,0 +1,217 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "appops_voice_export.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
monkeypatch.setenv("HOME_ASSISTANT_TOKEN", "test-ha-notify-token")
|
||||
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
from backend.app.main import app
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
|
||||
def _voice_headers():
|
||||
return {"Authorization": "Bearer test-voice-token"}
|
||||
|
||||
|
||||
def test_home_assistant_export_contains_safe_yaml_without_tokens(api_client):
|
||||
response = api_client.get("/api/voice/home-assistant/export", headers=_voice_headers())
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
yaml = payload["yaml"]
|
||||
assert "rest_command:" in yaml
|
||||
assert "script:" in yaml
|
||||
assert "appops_voice_authorization" in yaml
|
||||
assert "Authorization: !secret appops_voice_authorization" in yaml
|
||||
assert "test-voice-token" not in yaml
|
||||
assert "test-ha-notify-token" not in yaml
|
||||
assert "HOME_ASSISTANT_TOKEN" not in yaml
|
||||
assert "http://testserver/api/voice/actions/setup/status" in yaml
|
||||
assert "response_variable: appops_result" in yaml
|
||||
assert "appops_base_url" not in yaml
|
||||
assert len(payload["setupGuide"]) == 6
|
||||
|
||||
|
||||
def test_home_assistant_export_does_not_include_blocked_actions(api_client):
|
||||
payload = api_client.get("/api/voice/home-assistant/export", headers=_voice_headers()).json()
|
||||
text = str(payload)
|
||||
|
||||
assert "unraid.mover.start" not in text
|
||||
assert "volume.delete" not in text
|
||||
assert "stack.redeploy" not in text
|
||||
assert "authentik/restart" not in text
|
||||
assert "pause-all" not in text
|
||||
|
||||
|
||||
def test_voice_commands_expose_full_command_contract(api_client):
|
||||
response = api_client.get("/api/voice/commands", headers=_voice_headers())
|
||||
|
||||
assert response.status_code == 200
|
||||
commands = response.json()
|
||||
assert len(commands) >= 23
|
||||
required = {
|
||||
"id",
|
||||
"label",
|
||||
"spokenPhraseNl",
|
||||
"category",
|
||||
"endpoint",
|
||||
"method",
|
||||
"payload",
|
||||
"risk",
|
||||
"policyDecision",
|
||||
"requiresApproval",
|
||||
"enabledByDefault",
|
||||
"homeAssistantRestCommandName",
|
||||
"homeAssistantScriptName",
|
||||
"description",
|
||||
"safetyNotes",
|
||||
}
|
||||
assert required <= set(commands[0])
|
||||
assert any(command["id"] == "container.widefrog.restart" for command in commands)
|
||||
assert any(command["id"] == "unraid.vms" for command in commands)
|
||||
assert any(command["id"] == "security.exposure" for command in commands)
|
||||
assert next(command for command in commands if command["id"] == "unraid.storage")["policyDecision"] == "allowed"
|
||||
assert all(command["policyDecision"] != "blocked" for command in commands)
|
||||
|
||||
|
||||
def test_voice_export_endpoints_require_voice_token(api_client):
|
||||
missing = api_client.get("/api/voice/home-assistant/export")
|
||||
wrong = api_client.get(
|
||||
"/api/voice/home-assistant/export",
|
||||
headers={"Authorization": "Bearer wrong-token"},
|
||||
)
|
||||
|
||||
assert missing.status_code == 401
|
||||
assert wrong.status_code == 403
|
||||
|
||||
|
||||
def test_operator_voice_catalog_is_read_only_and_does_not_require_browser_secret(api_client):
|
||||
commands = api_client.get('/api/operator/voice/commands')
|
||||
export = api_client.get('/api/operator/voice/home-assistant/export')
|
||||
dry_run = api_client.post('/api/operator/voice/test-intent', json={'input': 'controleer appops setup status'})
|
||||
|
||||
assert commands.status_code == 200
|
||||
assert len(commands.json()) >= 11
|
||||
assert export.status_code == 200
|
||||
assert 'test-voice-token' not in str(export.json())
|
||||
assert dry_run.status_code == 200
|
||||
assert dry_run.json()['executed'] is False
|
||||
assert api_client.post('/api/voice/actions/setup/status').status_code == 401
|
||||
|
||||
|
||||
def test_operator_voice_readiness_uses_live_home_assistant_state(api_client, monkeypatch):
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.services import home_assistant_voice_readiness
|
||||
|
||||
previous = dict(settings.apis)
|
||||
settings.apis["HOME_ASSISTANT_URL"] = "http://home-assistant:8123"
|
||||
settings.apis["HOME_ASSISTANT_TOKEN"] = "runtime-token-not-returned"
|
||||
|
||||
def fake_get_json(_url, _token, path, _verify_tls):
|
||||
if path == "/api/config":
|
||||
return {
|
||||
"components": [
|
||||
"conversation",
|
||||
"google_generative_ai_conversation",
|
||||
]
|
||||
}, "ok", 200
|
||||
return [
|
||||
{"entity_id": "script.appops_check_setup"},
|
||||
{"entity_id": "script.appops_check_unraid"},
|
||||
], "ok", 200
|
||||
|
||||
monkeypatch.setattr(home_assistant_voice_readiness, "_get_json", fake_get_json)
|
||||
try:
|
||||
response = api_client.get("/api/operator/voice/home-assistant/readiness")
|
||||
finally:
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["apiReachable"] is True
|
||||
assert payload["operational"] is True
|
||||
assert payload["coreScriptAvailable"] is True
|
||||
assert payload["conversationAvailable"] is True
|
||||
assert payload["geminiConversationAvailable"] is True
|
||||
assert payload["detectedScriptCount"] == 2
|
||||
assert payload["expectedScriptCount"] >= 20
|
||||
assert payload["llmExposureStatus"] == "manual_verification_required"
|
||||
assert "runtime-token-not-returned" not in str(payload)
|
||||
|
||||
|
||||
def test_operator_voice_readiness_reports_missing_connector_without_network(api_client, monkeypatch):
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.services import home_assistant_voice_readiness
|
||||
|
||||
previous = dict(settings.apis)
|
||||
settings.apis["HOME_ASSISTANT_URL"] = None
|
||||
settings.apis["HOME_ASSISTANT_TOKEN"] = None
|
||||
monkeypatch.setattr(
|
||||
home_assistant_voice_readiness,
|
||||
"_get_json",
|
||||
lambda *_args: (_ for _ in ()).throw(AssertionError("missing config must not call Home Assistant")),
|
||||
)
|
||||
try:
|
||||
response = api_client.get("/api/operator/voice/home-assistant/readiness")
|
||||
finally:
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["status"] == "missing_config"
|
||||
assert payload["apiReachable"] is False
|
||||
assert set(payload["missing"]) == {"HOME_ASSISTANT_URL", "HOME_ASSISTANT_TOKEN"}
|
||||
|
||||
|
||||
def test_test_intent_is_dry_run_and_unknown_does_not_execute(api_client):
|
||||
before_jobs = api_client.get("/api/jobs").json()
|
||||
|
||||
response = api_client.post(
|
||||
"/api/voice/test-intent",
|
||||
headers=_voice_headers(),
|
||||
json={"input": "zet alles stuk aub"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["status"] == "unknown"
|
||||
assert payload["executed"] is False
|
||||
assert "jobId" not in payload
|
||||
assert api_client.get("/api/jobs").json() == before_jobs
|
||||
|
||||
|
||||
def test_dry_run_does_not_mutate_or_create_approval(api_client):
|
||||
before_jobs = api_client.get("/api/jobs").json()
|
||||
before_approvals = api_client.get("/api/approvals").json()
|
||||
|
||||
response = api_client.post(
|
||||
"/api/voice/commands/container.widefrog.restart/dry-run",
|
||||
headers=_voice_headers(),
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["executed"] is False
|
||||
assert payload["command"]["id"] == "container.widefrog.restart"
|
||||
assert payload["policyPreview"]["decision"] == "allowed"
|
||||
assert api_client.get("/api/jobs").json() == before_jobs
|
||||
assert api_client.get("/api/approvals").json() == before_approvals
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,164 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "appops_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.main import app
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
|
||||
def test_statistics_summary_uses_live_audit_job_and_unraid_data(api_client, monkeypatch):
|
||||
from backend.app.connectors.unraid import UnraidConnector
|
||||
from backend.app.api import routes_observability
|
||||
|
||||
monkeypatch.setattr(
|
||||
UnraidConnector,
|
||||
"get_system_metrics",
|
||||
lambda self: {"status": "ok", "cpuUsage": 37, "ramUsage": 64},
|
||||
raising=True,
|
||||
)
|
||||
routes_observability._STATISTICS_HISTORY.clear()
|
||||
|
||||
now = datetime.now(UTC).replace(tzinfo=None)
|
||||
|
||||
api_client.post(
|
||||
"/api/actions/execute",
|
||||
json={
|
||||
"intent": "volume.delete",
|
||||
"target": "postgres-data",
|
||||
"source": "manual",
|
||||
"risk": "critical",
|
||||
"plan": [],
|
||||
},
|
||||
)
|
||||
api_client.post(
|
||||
"/api/actions/execute",
|
||||
json={
|
||||
"intent": "container.restart",
|
||||
"target": "portainer",
|
||||
"source": "voice",
|
||||
"risk": "high",
|
||||
"plan": [],
|
||||
},
|
||||
)
|
||||
|
||||
response = api_client.get("/api/statistics/summary")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["cpuHistory"][-1]["value"] == 37
|
||||
assert payload["ramHistory"][-1]["value"] == 64
|
||||
assert payload["blockedActions"] >= 1
|
||||
assert payload["voiceActionCount"] >= 1
|
||||
assert all(point["time"] != "00:00" for point in payload["cpuHistory"])
|
||||
assert payload["generatedAt"] >= (now - timedelta(minutes=1)).isoformat()
|
||||
|
||||
|
||||
def test_statistics_summary_keeps_partial_unraid_metrics_explicit(api_client, monkeypatch):
|
||||
from backend.app.connectors.unraid import UnraidConnector
|
||||
from backend.app.api import routes_observability
|
||||
|
||||
monkeypatch.setattr(
|
||||
UnraidConnector,
|
||||
"get_system_metrics",
|
||||
lambda self: {
|
||||
"status": "partial",
|
||||
"cpuUsage": None,
|
||||
"ramUsage": 92,
|
||||
"reason": "cpu_unavailable",
|
||||
},
|
||||
raising=True,
|
||||
)
|
||||
routes_observability._STATISTICS_HISTORY.clear()
|
||||
|
||||
response = api_client.get("/api/statistics/summary")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["sources"]["unraid"] == "partial"
|
||||
assert payload["cpuHistory"][-1]["value"] is None
|
||||
assert payload["ramHistory"][-1]["value"] == 92
|
||||
assert payload["sourceWarnings"][0]["source"] == "unraid"
|
||||
assert payload["sourceWarnings"][0]["surface"] == "metrics"
|
||||
|
||||
|
||||
def test_storage_summary_exposes_cache_pool_inventory(api_client, monkeypatch):
|
||||
from backend.app.connectors.portainer import PortainerConnector
|
||||
from backend.app.connectors.unraid import UnraidConnector
|
||||
|
||||
monkeypatch.setattr(
|
||||
UnraidConnector,
|
||||
"get_cache_status",
|
||||
lambda self: {
|
||||
"status": "ok",
|
||||
"cacheUsage": 88,
|
||||
"pools": [
|
||||
{
|
||||
"name": "cache",
|
||||
"status": "DISK_OK",
|
||||
"healthy": True,
|
||||
"type": "CACHE",
|
||||
"usedGiB": 88,
|
||||
"sizeGiB": 100,
|
||||
"freeGiB": 12,
|
||||
"usagePercent": 88,
|
||||
}
|
||||
],
|
||||
},
|
||||
raising=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
UnraidConnector,
|
||||
"get_share_summary",
|
||||
lambda self: {"status": "ok", "shares": [{"name": "appdata", "usedGiB": 88, "sizeGiB": 100, "critical": True}]},
|
||||
raising=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
UnraidConnector,
|
||||
"get_disk_warnings",
|
||||
lambda self: {
|
||||
"status": "ok",
|
||||
"disks": [{"name": "disk1", "status": "DISK_OK", "healthy": True, "type": "DATA", "usedGiB": 50, "sizeGiB": 100, "freeGiB": 50}],
|
||||
"diskWarnings": 0,
|
||||
},
|
||||
raising=True,
|
||||
)
|
||||
monkeypatch.setattr(PortainerConnector, "list_containers", lambda self: {"status": "ok", "containers": []}, raising=True)
|
||||
|
||||
response = api_client.get("/api/storage/summary")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["cacheUsage"] == 88
|
||||
assert payload["storagePools"] == [
|
||||
{
|
||||
"name": "cache",
|
||||
"status": "DISK_OK",
|
||||
"healthy": True,
|
||||
"type": "CACHE",
|
||||
"usedGiB": 88,
|
||||
"sizeGiB": 100,
|
||||
"freeGiB": 12,
|
||||
"usagePercent": 88,
|
||||
}
|
||||
]
|
||||
assert payload["disks"][0]["name"] == "disk1"
|
||||
@@ -0,0 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "managed-validation.yml"
|
||||
|
||||
|
||||
def test_managed_validation_uses_internal_gitea_checkout_path() -> None:
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
assert "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" in workflow
|
||||
assert "github-server-url: http://gitea-runner-internal:3000" in workflow
|
||||
@@ -0,0 +1,18 @@
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW = ROOT / ".gitea" / "workflows" / "managed-validation.yml"
|
||||
|
||||
|
||||
def test_managed_validation_keeps_parallel_lanes_and_full_gate():
|
||||
text = WORKFLOW.read_text(encoding="utf-8")
|
||||
for job in ("preflight:", "backend:", "frontend:", "security:", "polyglot:", "full:"):
|
||||
assert f" {job}" in text
|
||||
assert "needs: [preflight, backend, frontend, security, polyglot]" in text
|
||||
assert "All managed validation lanes passed." in text
|
||||
|
||||
|
||||
def test_preflight_preserves_docs_only_fast_path():
|
||||
text = WORKFLOW.read_text(encoding="utf-8")
|
||||
assert "runtime_change" in text
|
||||
assert "*.md|*.mdx|docs/*" in text
|
||||
@@ -0,0 +1,282 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from backend.app.services.mcp_hub_deployment_reconciliation import (
|
||||
McpHubDeploymentReconciliation,
|
||||
)
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakePortainer:
|
||||
def __init__(self, exit_code=0):
|
||||
self.exit_code = exit_code
|
||||
self.calls = []
|
||||
|
||||
def get_container_status(self, name):
|
||||
assert name == "gitea-deploy-control"
|
||||
return {
|
||||
"status": "ok",
|
||||
"container": {
|
||||
"id": "controller-id",
|
||||
"name": "gitea-deploy-control",
|
||||
"state": "running",
|
||||
},
|
||||
}
|
||||
|
||||
def _config(self):
|
||||
return SimpleNamespace(endpoint_id="3")
|
||||
|
||||
def _request(self, method, path, **kwargs):
|
||||
self.calls.append((method, path, kwargs))
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return FakeResponse({"Id": "exec-1"})
|
||||
if path.endswith("/exec/exec-1/start"):
|
||||
return FakeResponse({})
|
||||
if path.endswith("/exec/exec-1/json"):
|
||||
return FakeResponse({"Running": False, "ExitCode": self.exit_code})
|
||||
raise AssertionError(path)
|
||||
|
||||
|
||||
def test_reconciliation_is_fixed_to_mcp_hub_and_reconciles_only_verified_failed_forward_state():
|
||||
fake = FakePortainer()
|
||||
result = McpHubDeploymentReconciliation(fake).run()
|
||||
|
||||
assert result == {
|
||||
"status": "reconciled",
|
||||
"repository": "jens/itworx_mcp_hub",
|
||||
"composeProject": "itworx-mcp-hub",
|
||||
}
|
||||
|
||||
create = fake.calls[0]
|
||||
command = create[2]["json"]["Cmd"]
|
||||
assert command[:2] == ["python3", "-c"]
|
||||
assert command[3:] == ["jens/itworx_mcp_hub", "itworx-mcp-hub"]
|
||||
assert create[2]["json"]["AttachStdout"] is False
|
||||
assert create[2]["json"]["AttachStderr"] is False
|
||||
|
||||
script = command[2]
|
||||
assert "/opt/gitea-deploy/projects.json" in script
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json" in script
|
||||
assert "os.path.samefile(registry_path, host_registry_path)" in script
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script
|
||||
assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/deploy.lock" in script
|
||||
assert 'project.get("branch") != "main"' in script
|
||||
assert 'project.get("compose_project") != expected_compose_project' in script
|
||||
assert 'recorded_status != "failed"' in script
|
||||
assert 'recorded_live_status not in {"healthy", "unknown"}' in script
|
||||
assert 'attempt_status != "failed"' in script
|
||||
assert 'attempt_revision == recorded_live' in script
|
||||
assert 'git_run("merge-base", "--is-ancestor", recorded_live, attempt_revision)' in script
|
||||
assert 'git_run("merge-base", "--is-ancestor", attempt_revision, main_revision)' in script
|
||||
|
||||
|
||||
def test_unknown_live_status_is_accepted_only_for_a_proven_snapshot_stage_failure_without_rollback_error():
|
||||
fake = FakePortainer()
|
||||
McpHubDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert 'last_attempt_message = str(last_attempt.get("message") or "")' in script
|
||||
assert "snapshot_failure_prefixes = (" in script
|
||||
assert 'recorded_live_status == "unknown"' in script
|
||||
assert 'last_attempt_message.startswith(snapshot_failure_prefixes)' in script
|
||||
assert '"; rollback error:" in last_attempt_message' in script
|
||||
assert '"; source rollback error:" in last_attempt_message' in script
|
||||
assert "raise SystemExit(55)" in script
|
||||
|
||||
|
||||
def test_reconciliation_proves_complete_live_compose_topology_and_preserves_actual_image_ids():
|
||||
fake = FakePortainer()
|
||||
McpHubDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert 'expected_containers = project.get("containers")' in script
|
||||
assert "set(project_names) != set(expected_containers)" in script
|
||||
assert 'labels.get("com.docker.compose.project") != expected_compose_project' in script
|
||||
assert 'not runtime_state.get("Running")' in script
|
||||
assert 'health in {"starting", "unhealthy"}' in script
|
||||
assert 'image_id = str(container.get("Image") or "")' in script
|
||||
assert 'artifacts = {"containers": container_artifacts, "compose_sha256": ""}' in script
|
||||
assert '"live_legacy_provenance": True' in script
|
||||
assert '"live_artifacts": artifacts' in script
|
||||
assert '"live_revision": recorded_live' in script
|
||||
|
||||
|
||||
def test_reconciliation_recognizes_only_the_fixed_mcp_hub_oci_source_as_owned():
|
||||
fake = FakePortainer()
|
||||
McpHubDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert 'owned_image_source = "https://gitea.example.com/jens/itworx-mcp-hub"' in script
|
||||
assert "if image_source == owned_image_source:" in script
|
||||
assert 'git_run("merge-base", "--is-ancestor", recorded_live, image_revision)' in script
|
||||
assert 'git_run("merge-base", "--is-ancestor", image_revision, main_revision)' in script
|
||||
assert 'image_source.startswith("https://gitea.example.com/")' not in script
|
||||
assert 'image_source == repository' not in script
|
||||
assert '"live_revision": main_revision' not in script
|
||||
assert '"live_revision": attempt_revision' not in script
|
||||
|
||||
|
||||
def test_owned_image_ancestry_can_fail_closed_to_the_canonical_gitea_store():
|
||||
fake = FakePortainer()
|
||||
McpHubDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert 'canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories")' in script
|
||||
assert 'canonical_bare_repo = canonical_repository_root / owner / f"{name}.git"' in script
|
||||
assert "def canonical_git_run(*args):" in script
|
||||
assert "def canonical_image_ancestry_is_proven(" in script
|
||||
assert 'resolved_recorded = canonical_git_run("rev-parse", f"{recorded_revision}^{{commit}}")' in script
|
||||
assert 'resolved_image = canonical_git_run("rev-parse", f"{image_revision}^{{commit}}")' in script
|
||||
assert 'resolved_main = canonical_git_run("rev-parse", "refs/heads/main")' in script
|
||||
assert 'resolved_main.stdout.strip() == expected_main' in script
|
||||
assert '"merge-base", "--is-ancestor", recorded_revision, image_revision' in script
|
||||
assert '"merge-base", "--is-ancestor", image_revision, expected_main' in script
|
||||
assert "if not mounted_ancestry_proven and not canonical_image_ancestry_is_proven(" in script
|
||||
assert "canonical_bare_repo.is_symlink()" in script
|
||||
|
||||
|
||||
def test_unchanged_recorded_image_ids_do_not_reinterpret_historical_oci_revision_labels():
|
||||
fake = FakePortainer()
|
||||
McpHubDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
change_probe = "image_changed = image_id != recorded_image_id"
|
||||
provenance_gate = "if image_changed:"
|
||||
compat_probe = "contextforge_compat_image = main_declares_contextforge_compat_image(name, image_ref)"
|
||||
dataplane_probe = "main_declares_sessionless_dataplane_image(name, image_ref)"
|
||||
owned_probe = "elif image_source == owned_image_source:"
|
||||
external_probe = "elif not main_declares_external_image(image_ref):"
|
||||
|
||||
assert change_probe in script
|
||||
assert provenance_gate in script
|
||||
assert compat_probe in script
|
||||
assert dataplane_probe in script
|
||||
assert "not already_reconciled" in script
|
||||
assert owned_probe in script
|
||||
assert external_probe in script
|
||||
assert script.index(change_probe) < script.index(provenance_gate)
|
||||
assert script.index(provenance_gate) < script.index(compat_probe)
|
||||
assert script.index(compat_probe) < script.index(dataplane_probe)
|
||||
assert script.index(dataplane_probe) < script.index(owned_probe)
|
||||
assert script.index(owned_probe) < script.index(external_probe)
|
||||
assert "if image_id != recorded_image_id or container_id != recorded_container_id:" in script
|
||||
|
||||
|
||||
def test_external_image_drift_requires_an_immutable_digest_declared_in_reviewed_main_compose():
|
||||
fake = FakePortainer()
|
||||
McpHubDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert "def main_declares_external_image(image_ref):" in script
|
||||
assert "@sha256:" in script
|
||||
assert 'git_run("show", f"{main_revision}:{compose_path}")' in script
|
||||
assert '"infrastructure/compose/docker-compose.yml"' in script
|
||||
assert '"infrastructure/compose/docker-compose.prod.yml"' in script
|
||||
assert 'line.strip() in {f"image: {image_ref}", f\'image: "{image_ref}"\', f"image: \'{image_ref}\'"}' in script
|
||||
assert "main_declares_external_image(image_ref)" in script
|
||||
assert "elif not main_declares_external_image(image_ref):" in script
|
||||
|
||||
|
||||
def test_contextforge_compat_image_drift_requires_exact_reviewed_main_contract():
|
||||
fake = FakePortainer()
|
||||
McpHubDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert 'container_name != "itworx-mcp-hub-contextforge-1"' in script
|
||||
assert 'f"{main_revision}:infrastructure/images/contextforge/compat-image.env"' in script
|
||||
assert 'f"{main_revision}:scripts/build_contextforge_compat_image.sh"' in script
|
||||
assert '"CONTEXTFORGE_UPSTREAM_COMMIT"' in script
|
||||
assert '"CONTEXTFORGE_ALEMBIC_HEAD"' in script
|
||||
assert '"CONTEXTFORGE_MCP_SDK"' in script
|
||||
assert '"CONTEXTFORGE_LOCAL_IMAGE"' in script
|
||||
assert 're.fullmatch(r"[0-9a-f]{40}", upstream_commit)' in script
|
||||
assert 're.fullmatch(r"[0-9a-f]{12}", alembic_head)' in script
|
||||
assert 'local_image != f"itworx-mcp-hub/contextforge-schema:{alembic_head}"' in script
|
||||
assert 'git -C "$SRC" fetch --depth 1 origin "$CONTEXTFORGE_UPSTREAM_COMMIT"' in script
|
||||
assert 'git -C "$SRC" checkout --detach FETCH_HEAD' in script
|
||||
assert 'MCP_REQUIREMENT="mcp==${CONTEXTFORGE_MCP_SDK}"' in script
|
||||
assert 'alembic -c mcpgateway/alembic.ini heads' in script
|
||||
assert 'docker tag "$STAGED_IMAGE" "$CONTEXTFORGE_LOCAL_IMAGE"' in script
|
||||
assert 'image_ref.startswith("itworx-mcp-hub/contextforge-schema:")' not in script
|
||||
|
||||
|
||||
compat_probe = "contextforge_compat_image = main_declares_contextforge_compat_image(name, image_ref)"
|
||||
owned_probe = "elif image_source == owned_image_source:"
|
||||
assert compat_probe in script
|
||||
assert "if contextforge_compat_image or sessionless_dataplane_image:" in script
|
||||
assert owned_probe in script
|
||||
assert script.index(compat_probe) < script.index(owned_probe)
|
||||
|
||||
|
||||
def test_sessionless_dataplane_baseline_is_one_time_and_commit_bound():
|
||||
fake = FakePortainer()
|
||||
McpHubDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert 'container_name != "itworx-mcp-hub-dataplane-router-1"' in script
|
||||
assert 'image_ref != "itworx-mcp-hub/dataplane:sessionless-final"' in script
|
||||
assert 'sessionless_commit = "b519a8c4cf17cd9df00a575acfe096838aab7789"' in script
|
||||
assert 'f"{sessionless_commit}:services/dataplane_router/router.py"' in script
|
||||
assert 'f"{sessionless_commit}:tests/test_dataplane_session_headers.py"' in script
|
||||
assert 'f"{main_revision}:evidence/mcp-session-boundary-20260916/SUMMARY.md"' in script
|
||||
assert 'MCP_TRANSPORT_STATE_HEADERS = frozenset({b"mcp-session-id"})' in script
|
||||
assert 'del response.headers["mcp-session-id"]' in script
|
||||
assert '"2025-11-25"' in script
|
||||
assert '"2026-07-28"' in script
|
||||
assert '"permanently enforce stateless public transport"' in script
|
||||
assert (
|
||||
'git_run("merge-base", "--is-ancestor", sessionless_commit, main_revision)'
|
||||
in script
|
||||
)
|
||||
assert "not already_reconciled" in script
|
||||
|
||||
|
||||
def test_already_reconciled_state_is_reverified_before_idempotent_success():
|
||||
fake = FakePortainer()
|
||||
McpHubDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert "already_reconciled = (" in script
|
||||
assert script.index("already_reconciled = (") < script.index("project_names =")
|
||||
assert script.index("container_artifacts = {}") < script.index("if already_reconciled:")
|
||||
assert 'print("already-reconciled", flush=True)' in script
|
||||
assert "if drift_found:" in script
|
||||
|
||||
|
||||
def test_reconciliation_refuses_unknown_runtime_topology():
|
||||
result = McpHubDeploymentReconciliation(FakePortainer(exit_code=47)).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "live MCP Hub Compose project differs from the controller target",
|
||||
}
|
||||
|
||||
|
||||
def test_reconciliation_refuses_unproven_runtime_revision_drift():
|
||||
result = McpHubDeploymentReconciliation(FakePortainer(exit_code=51)).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "live MCP Hub image revision or external immutable image is not proven by the recorded-live to main chain",
|
||||
}
|
||||
|
||||
|
||||
def test_reconciliation_refuses_state_that_is_not_the_known_failed_forward_shape():
|
||||
result = McpHubDeploymentReconciliation(FakePortainer(exit_code=52)).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "controller state drift is not the exact known recoverable MCP Hub state",
|
||||
}
|
||||
|
||||
|
||||
def test_reconciliation_refuses_unknown_state_without_snapshot_stage_proof():
|
||||
result = McpHubDeploymentReconciliation(FakePortainer(exit_code=55)).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "unknown MCP Hub live status was not produced by a proven recoverable snapshot-stage failure",
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlmodel import Session, SQLModel, create_engine, select
|
||||
|
||||
from backend.app.db.models import Policy
|
||||
from backend.app.services.discovery_scheduler import (
|
||||
_reconcile_mcp_hub_exact_policies,
|
||||
)
|
||||
|
||||
|
||||
def _session(tmp_path) -> Session:
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'mcp-hub-policies.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
return Session(engine)
|
||||
|
||||
|
||||
def test_reconciliation_creates_only_exact_apply_and_rollback_policies(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
result = _reconcile_mcp_hub_exact_policies(session)
|
||||
rows = session.exec(
|
||||
select(Policy).where(Policy.target == "itworx-mcp-hub")
|
||||
).all()
|
||||
|
||||
assert result["status"] == "ready"
|
||||
assert result["created"] == 2
|
||||
assert {row.action for row in rows} == {
|
||||
"deployment.apply",
|
||||
"deployment.rollback",
|
||||
}
|
||||
assert all(row.manual == "allowed" for row in rows)
|
||||
assert all(row.voice == "blocked" for row in rows)
|
||||
assert all(
|
||||
row.provenance_json.get("createdBy") == "mcp-hub-state-reconciliation/v1"
|
||||
for row in rows
|
||||
)
|
||||
|
||||
|
||||
def test_reconciliation_is_idempotent(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
first = _reconcile_mcp_hub_exact_policies(session)
|
||||
second = _reconcile_mcp_hub_exact_policies(session)
|
||||
rows = session.exec(
|
||||
select(Policy).where(Policy.target == "itworx-mcp-hub")
|
||||
).all()
|
||||
|
||||
assert first["created"] == 2
|
||||
assert second["status"] == "ready"
|
||||
assert second["created"] == 0
|
||||
assert len(rows) == 2
|
||||
|
||||
|
||||
def test_reconciliation_never_overwrites_blocked_exact_policy(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
session.add(
|
||||
Policy(
|
||||
id="existing-block",
|
||||
action="deployment.apply",
|
||||
target="itworx-mcp-hub",
|
||||
voice="blocked",
|
||||
manual="blocked",
|
||||
risk="high",
|
||||
reason="Explicit local block.",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
result = _reconcile_mcp_hub_exact_policies(session)
|
||||
rows = session.exec(
|
||||
select(Policy).where(Policy.target == "itworx-mcp-hub")
|
||||
).all()
|
||||
|
||||
assert result == {
|
||||
"status": "conflict",
|
||||
"action": "deployment.apply",
|
||||
"created": 0,
|
||||
"policyIds": [],
|
||||
}
|
||||
assert len(rows) == 1
|
||||
assert rows[0].manual == "blocked"
|
||||
|
||||
def test_reconciliation_refuses_weakened_allowed_exact_policy(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
session.add(
|
||||
Policy(
|
||||
id="existing-weak",
|
||||
action="deployment.apply",
|
||||
target="itworx-mcp-hub",
|
||||
voice="allowed",
|
||||
manual="allowed",
|
||||
risk="medium",
|
||||
reason="Unsafe test policy.",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
result = _reconcile_mcp_hub_exact_policies(session)
|
||||
rows = session.exec(
|
||||
select(Policy).where(Policy.target == "itworx-mcp-hub")
|
||||
).all()
|
||||
|
||||
assert result == {
|
||||
"status": "conflict",
|
||||
"action": "deployment.apply",
|
||||
"created": 0,
|
||||
"policyIds": [],
|
||||
}
|
||||
assert len(rows) == 1
|
||||
assert rows[0].voice == "allowed"
|
||||
assert rows[0].risk == "medium"
|
||||
|
||||
def test_reconciliation_refuses_duplicate_exact_policies(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
for policy_id in ("duplicate-a", "duplicate-b"):
|
||||
session.add(
|
||||
Policy(
|
||||
id=policy_id,
|
||||
action="deployment.rollback",
|
||||
target="itworx-mcp-hub",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="high",
|
||||
reason="Duplicate exact policy.",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
result = _reconcile_mcp_hub_exact_policies(session)
|
||||
rows = session.exec(
|
||||
select(Policy).where(
|
||||
(Policy.action == "deployment.rollback")
|
||||
& (Policy.target == "itworx-mcp-hub")
|
||||
)
|
||||
).all()
|
||||
|
||||
assert result == {
|
||||
"status": "conflict",
|
||||
"action": "deployment.rollback",
|
||||
"created": 0,
|
||||
"policyIds": [],
|
||||
}
|
||||
assert len(rows) == 2
|
||||
|
||||
def test_reconciliation_refuses_deterministic_policy_id_collision(tmp_path):
|
||||
with _session(tmp_path) as session:
|
||||
session.add(
|
||||
Policy(
|
||||
id="mcp-hub-reconciled-apply",
|
||||
action="container.restart",
|
||||
target="unrelated-container",
|
||||
voice="blocked",
|
||||
manual="allowed",
|
||||
risk="medium",
|
||||
reason="Unrelated existing policy ID.",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
result = _reconcile_mcp_hub_exact_policies(session)
|
||||
|
||||
assert result == {
|
||||
"status": "conflict",
|
||||
"action": "deployment.apply",
|
||||
"created": 0,
|
||||
"policyIds": [],
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,190 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from backend.app.services.media_codec_dashboard_deployment_reconciliation import (
|
||||
MediaCodecDashboardDeploymentReconciliation,
|
||||
)
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakePortainer:
|
||||
def __init__(self, exit_code=0):
|
||||
self.exit_code = exit_code
|
||||
self.calls = []
|
||||
|
||||
def get_container_status(self, name):
|
||||
assert name == "gitea-deploy-control"
|
||||
return {
|
||||
"status": "ok",
|
||||
"container": {
|
||||
"id": "controller-id",
|
||||
"name": "gitea-deploy-control",
|
||||
"state": "running",
|
||||
},
|
||||
}
|
||||
|
||||
def _config(self):
|
||||
return SimpleNamespace(endpoint_id="3")
|
||||
|
||||
def _request(self, method, path, **kwargs):
|
||||
self.calls.append((method, path, kwargs))
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return FakeResponse({"Id": "exec-1"})
|
||||
if path.endswith("/exec/exec-1/start"):
|
||||
return FakeResponse({})
|
||||
if path.endswith("/exec/exec-1/json"):
|
||||
return FakeResponse({"Running": False, "ExitCode": self.exit_code})
|
||||
raise AssertionError(path)
|
||||
|
||||
|
||||
def test_embedded_reconciliation_script_is_valid_python():
|
||||
fake = FakePortainer()
|
||||
MediaCodecDashboardDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
compile(script, "media-codec-dashboard-reconcile", "exec")
|
||||
|
||||
|
||||
def test_reconciliation_derives_the_verified_live_revision_from_runtime_evidence():
|
||||
fake = FakePortainer()
|
||||
result = MediaCodecDashboardDeploymentReconciliation(fake).run()
|
||||
|
||||
assert result == {
|
||||
"status": "reconciled",
|
||||
"repository": "jens/media-codec-dashboard",
|
||||
"container": "Media-Codec-Dashboard",
|
||||
}
|
||||
|
||||
create = fake.calls[0]
|
||||
command = create[2]["json"]["Cmd"]
|
||||
assert command[:2] == ["python3", "-c"]
|
||||
assert command[3:] == [
|
||||
"jens/media-codec-dashboard",
|
||||
"Media-Codec-Dashboard",
|
||||
]
|
||||
script = command[2]
|
||||
# The controller registry must be the canonical physical cache bind, not
|
||||
# the user-share alias that can resolve to a different inode.
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json" in script
|
||||
assert "/srv/example/appdata/gitea-deploy-runners/controller/projects.json" not in script
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script
|
||||
assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/deploy.lock" in script
|
||||
assert "os.path.samefile(registry_path, host_registry_path)" in script
|
||||
assert '"/app/data": "/srv/example/appdata/media-codec-dashboard/data"' in script
|
||||
assert '"/app/data/quality-runtime": "/srv/example/appdata/tdarr/logs/media-codec-dashboard-quality-audit"' in script
|
||||
assert '"/run/secrets": "/srv/example/appdata/media-codec-dashboard/secrets"' in script
|
||||
assert '"/tdarr-output": "/srv/example/Media"' in script
|
||||
assert '"/tdarr-db"' not in script
|
||||
|
||||
# Never freeze a one-time recovery helper to a historical deployment SHA.
|
||||
assert "971df2cc305c1b447b4feb255d4e9fda871315c3" not in script
|
||||
assert "adf53604739d4247193a7b5bfeb8ad8981d9d359" not in script
|
||||
assert "live_revision, stale_revision" not in script
|
||||
|
||||
# The immutable revision must be proven independently by both the running
|
||||
# container label and the immutable image label, then resolved in Gitea.
|
||||
assert 'container_revision = str(labels.get("org.opencontainers.image.revision") or "")' in script
|
||||
assert 'image_revision = str(image_labels.get("org.opencontainers.image.revision") or "")' in script
|
||||
assert "container_revision != image_revision" in script
|
||||
assert 're.fullmatch(r"[0-9a-f]{40}", live_revision)' in script
|
||||
assert 'git_run("rev-parse", f"{live_revision}^{{commit}}")' in script
|
||||
assert 'git_run("merge-base", "--is-ancestor", live_revision, main_revision)' in script
|
||||
|
||||
# The stale controller revision is read from durable state. Recovery is
|
||||
# allowed only when that recorded live revision is a real older ancestor of
|
||||
# the currently running immutable image, never for arbitrary drift.
|
||||
assert 'recorded_live = str(state.get("live_revision") or state.get("revision") or "")' in script
|
||||
assert 're.fullmatch(r"[0-9a-f]{40}", recorded_live)' in script
|
||||
assert 'git_run("merge-base", "--is-ancestor", recorded_live, live_revision)' in script
|
||||
assert 'recorded_live == live_revision' in script
|
||||
assert '"live_legacy_provenance": True' in script
|
||||
assert create[2]["json"]["AttachStdout"] is False
|
||||
assert create[2]["json"]["AttachStderr"] is False
|
||||
|
||||
|
||||
def test_reconciliation_distinguishes_failed_attempt_from_recorded_live_revision():
|
||||
fake = FakePortainer()
|
||||
MediaCodecDashboardDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
# Controller schema v3 keeps the real live revision while top-level status
|
||||
# describes the most recent deployment attempt. A failed forward attempt is
|
||||
# therefore recoverable when the independently verified runtime still equals
|
||||
# recorded_live and the failed attempt is a distinct valid descendant.
|
||||
assert 'recorded_live_status = str(state.get("live_status") or "")' in script
|
||||
assert 'last_attempt = dict(state.get("last_attempt") or {})' in script
|
||||
assert 'attempt_revision = str(last_attempt.get("revision") or "")' in script
|
||||
assert 'attempt_status = str(last_attempt.get("status") or "")' in script
|
||||
assert "current_live_failed_baseline = (" in script
|
||||
assert 'recorded_status == "failed"' in script
|
||||
assert 'recorded_live_status == "healthy" or current_live_unknown_snapshot_baseline' in script
|
||||
assert 'git_run("merge-base", "--is-ancestor", live_revision, attempt_revision)' in script
|
||||
assert 'git_run("merge-base", "--is-ancestor", attempt_revision, main_revision)' in script
|
||||
|
||||
|
||||
def test_reconciliation_accepts_exact_current_legacy_snapshot_failure_only_with_matching_artifacts():
|
||||
fake = FakePortainer()
|
||||
MediaCodecDashboardDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert "current_live_unknown_snapshot_baseline = (" in script
|
||||
assert 'recorded_live_status == "unknown"' in script
|
||||
assert 'state.get("live_legacy_provenance") is True' in script
|
||||
assert 'attempt.get("message") == "Existing production containers require a valid healthy live deployment state"' in script
|
||||
assert 'attempt.get("updated_at_epoch") == state.get("updated_at_epoch")' in script
|
||||
assert 'current_receipt == {}' in script
|
||||
assert 'set(current_artifacts) == {"compose_sha256", "containers"}' in script
|
||||
assert 'set(current_artifact_containers) == {expected_container}' in script
|
||||
assert 'all(current_stored_artifact.get(field) == container_artifact.get(field) for field in current_immutable_fields)' in script
|
||||
assert 'current_live_failed_baseline = (' in script
|
||||
assert 'recorded_live_status == "healthy" or current_live_unknown_snapshot_baseline' in script
|
||||
|
||||
|
||||
def test_reconciliation_accepts_only_an_internally_consistent_stale_success_baseline():
|
||||
fake = FakePortainer()
|
||||
MediaCodecDashboardDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
# A previously successful controller baseline may legitimately lag behind
|
||||
# an independently verified newer immutable runtime. Accept that drift only
|
||||
# when the old state is internally self-consistent: healthy live status and
|
||||
# a successful last attempt for exactly the recorded old live revision.
|
||||
assert "stale_success_baseline = (" in script
|
||||
assert 'recorded_status == "success"' in script
|
||||
assert 'recorded_live_status == "healthy"' in script
|
||||
assert 'attempt_status == "success"' in script
|
||||
assert "attempt_revision == recorded_live" in script
|
||||
assert 'stale_failed_attempt = recorded_status == "failed"' in script
|
||||
assert "if not (stale_success_baseline or stale_failed_attempt):" in script
|
||||
assert 'git_run("merge-base", "--is-ancestor", recorded_live, live_revision)' in script
|
||||
|
||||
|
||||
def test_reconciliation_refuses_wrong_live_mount_topology():
|
||||
result = MediaCodecDashboardDeploymentReconciliation(FakePortainer(exit_code=48)).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "live Media Codec Dashboard mount topology differs from the verified runtime",
|
||||
}
|
||||
|
||||
|
||||
def test_reconciliation_refuses_unknown_state_drift():
|
||||
result = MediaCodecDashboardDeploymentReconciliation(FakePortainer(exit_code=52)).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "controller state drift is not the exact known recoverable state",
|
||||
}
|
||||
|
||||
|
||||
def test_reconciliation_refuses_wrong_image_revision_label():
|
||||
result = MediaCodecDashboardDeploymentReconciliation(FakePortainer(exit_code=51)).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "live Media Codec Dashboard image revision label does not match the verified runtime",
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
import ast
|
||||
import copy
|
||||
import re
|
||||
|
||||
from backend.app.services.media_codec_dashboard_deployment_reconciliation import (
|
||||
_RECONCILE_SCRIPT,
|
||||
MediaCodecDashboardDeploymentReconciliation,
|
||||
)
|
||||
from backend.tests.test_media_codec_dashboard_deployment_reconciliation import (
|
||||
FakePortainer,
|
||||
)
|
||||
|
||||
|
||||
def _load_reconcile_script_function(name: str):
|
||||
tree = ast.parse(_RECONCILE_SCRIPT)
|
||||
function = next(
|
||||
(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name),
|
||||
None,
|
||||
)
|
||||
assert function is not None, f"reconciliation script function {name} is missing"
|
||||
namespace = {"re": re}
|
||||
exec(compile(ast.Module(body=[function], type_ignores=[]), "<reconcile-test>", "exec"), namespace) # noqa: S102
|
||||
return namespace[name]
|
||||
|
||||
|
||||
def test_full_receipt_snapshot_recovery_requires_exact_failed_state():
|
||||
classify = _load_reconcile_script_function("_is_current_verified_receipt_snapshot_baseline")
|
||||
live = "a" * 40
|
||||
failed = "b" * 40
|
||||
artifact = {"image_id": "sha256:" + "c" * 64}
|
||||
state = {
|
||||
"schema_version": 3,
|
||||
"repository": "jens/media-codec-dashboard",
|
||||
"revision": live,
|
||||
"live_revision": live,
|
||||
"status": "failed",
|
||||
"live_status": "unknown",
|
||||
"live_legacy_provenance": False,
|
||||
"updated_at_epoch": 123,
|
||||
"last_attempt": {
|
||||
"revision": failed,
|
||||
"status": "failed",
|
||||
"message": "Existing production containers require a valid healthy live deployment state",
|
||||
"updated_at_epoch": 123,
|
||||
},
|
||||
"live_artifacts": {
|
||||
"compose_sha256": "d" * 64,
|
||||
"containers": {"Media-Codec-Dashboard": artifact},
|
||||
},
|
||||
"live_receipt": {
|
||||
"schema": "gitea-deploy-receipt.v1",
|
||||
"repository": "jens/media-codec-dashboard",
|
||||
"revision": live,
|
||||
"live": None,
|
||||
"containers": {"Media-Codec-Dashboard": artifact},
|
||||
},
|
||||
}
|
||||
assert classify(state, "jens/media-codec-dashboard", "Media-Codec-Dashboard", live, failed)
|
||||
for path, value in (
|
||||
(("live_status",), "healthy"),
|
||||
(("live_legacy_provenance",), True),
|
||||
(("last_attempt", "message"), "unrelated failure"),
|
||||
(("last_attempt", "updated_at_epoch"), 122),
|
||||
(("live_artifacts", "compose_sha256"), ""),
|
||||
(("live_receipt", "containers"), {}),
|
||||
):
|
||||
changed = copy.deepcopy(state)
|
||||
target = changed
|
||||
for key in path[:-1]:
|
||||
target = target[key]
|
||||
target[path[-1]] = value
|
||||
assert not classify(changed, "jens/media-codec-dashboard", "Media-Codec-Dashboard", live, failed)
|
||||
|
||||
|
||||
def test_reconciliation_accepts_only_the_receipt_proven_legacy_state_ahead_of_runtime():
|
||||
classify = _load_reconcile_script_function("_is_exact_legacy_state_ahead_baseline")
|
||||
recorded = "b" * 40
|
||||
live = "9" * 40
|
||||
attempt = "d" * 40
|
||||
artifact = {
|
||||
"container_id": "f" * 64,
|
||||
"image_id": "sha256:" + "c" * 64,
|
||||
"image_ref": "media-codec-dashboard-media-codec-dashboard",
|
||||
"container_revision": live,
|
||||
"image_revision": live,
|
||||
"image_source": "",
|
||||
"image_tree": "",
|
||||
"image_version": "",
|
||||
"image_created": "",
|
||||
}
|
||||
state = {
|
||||
"schema_version": 3,
|
||||
"repository": "jens/media-codec-dashboard",
|
||||
"revision": recorded,
|
||||
"live_revision": recorded,
|
||||
"live_status": "unknown",
|
||||
"live_legacy_provenance": False,
|
||||
"status": "failed",
|
||||
"updated_at_epoch": 30,
|
||||
"last_attempt": {
|
||||
"revision": attempt,
|
||||
"status": "failed",
|
||||
"message": "Existing production containers require a valid healthy live deployment state",
|
||||
"updated_at_epoch": 30,
|
||||
"metadata": {
|
||||
"id": attempt[:12],
|
||||
"repository": "jens/media-codec-dashboard",
|
||||
"revision": attempt,
|
||||
"tree": "1" * 40,
|
||||
"treeDigest": "2" * 64,
|
||||
"builtAt": "2026-09-20T11:17:29.051Z",
|
||||
},
|
||||
},
|
||||
"live_artifacts": {
|
||||
"compose_sha256": "3" * 64,
|
||||
"containers": {"Media-Codec-Dashboard": artifact},
|
||||
},
|
||||
"live_receipt": {
|
||||
"schema": "gitea-deploy-receipt.v1",
|
||||
"repository": "jens/media-codec-dashboard",
|
||||
"revision": recorded,
|
||||
"tree": "4" * 40,
|
||||
"treeDigest": "5" * 64,
|
||||
"build": {
|
||||
"id": recorded[:12],
|
||||
"repository": "jens/media-codec-dashboard",
|
||||
"revision": recorded,
|
||||
"tree": "4" * 40,
|
||||
"treeDigest": "5" * 64,
|
||||
"builtAt": "2026-09-16T23:43:44.387Z",
|
||||
},
|
||||
"containers": {"Media-Codec-Dashboard": artifact},
|
||||
"live": None,
|
||||
},
|
||||
}
|
||||
git_facts = {
|
||||
"recorded_to_live": 1,
|
||||
"canonical_recorded_to_live": 1,
|
||||
"live_to_recorded": 0,
|
||||
"canonical_live_to_recorded": 0,
|
||||
"mounted_recorded_main": 0,
|
||||
"canonical_recorded_main": 0,
|
||||
"canonical_live_main": 0,
|
||||
"recorded_to_attempt": 0,
|
||||
"canonical_recorded_to_attempt": 0,
|
||||
"attempt_to_main": 0,
|
||||
"canonical_attempt_to_main": 0,
|
||||
}
|
||||
|
||||
assert classify(
|
||||
state,
|
||||
"jens/media-codec-dashboard",
|
||||
"Media-Codec-Dashboard",
|
||||
recorded,
|
||||
live,
|
||||
artifact,
|
||||
git_facts,
|
||||
) is True
|
||||
|
||||
recreated_container_state = copy.deepcopy(state)
|
||||
recreated_container_state["live_artifacts"]["containers"]["Media-Codec-Dashboard"][
|
||||
"container_id"
|
||||
] = "e" * 64
|
||||
recreated_container_state["live_receipt"]["containers"]["Media-Codec-Dashboard"][
|
||||
"container_id"
|
||||
] = "e" * 64
|
||||
assert classify(
|
||||
recreated_container_state,
|
||||
"jens/media-codec-dashboard",
|
||||
"Media-Codec-Dashboard",
|
||||
recorded,
|
||||
live,
|
||||
artifact,
|
||||
git_facts,
|
||||
) is True
|
||||
|
||||
wrong_artifact_state = copy.deepcopy(state)
|
||||
wrong_artifact_state["live_artifacts"]["containers"]["Media-Codec-Dashboard"] = {
|
||||
**artifact,
|
||||
"container_id": "e" * 64,
|
||||
}
|
||||
assert classify(
|
||||
wrong_artifact_state,
|
||||
"jens/media-codec-dashboard",
|
||||
"Media-Codec-Dashboard",
|
||||
recorded,
|
||||
live,
|
||||
artifact,
|
||||
git_facts,
|
||||
) is False
|
||||
|
||||
for field, changed_value in {
|
||||
"image_id": "sha256:" + "a" * 64,
|
||||
"image_ref": "changed-image-reference",
|
||||
"container_revision": "8" * 40,
|
||||
"image_revision": "8" * 40,
|
||||
"image_source": "changed-source",
|
||||
"image_tree": "changed-tree",
|
||||
"image_version": "changed-version",
|
||||
"image_created": "changed-created",
|
||||
}.items():
|
||||
changed_immutable_state = copy.deepcopy(recreated_container_state)
|
||||
changed_immutable_state["live_artifacts"]["containers"]["Media-Codec-Dashboard"][
|
||||
field
|
||||
] = changed_value
|
||||
changed_immutable_state["live_receipt"]["containers"]["Media-Codec-Dashboard"][
|
||||
field
|
||||
] = changed_value
|
||||
assert classify(
|
||||
changed_immutable_state,
|
||||
"jens/media-codec-dashboard",
|
||||
"Media-Codec-Dashboard",
|
||||
recorded,
|
||||
live,
|
||||
artifact,
|
||||
git_facts,
|
||||
) is False
|
||||
|
||||
malformed_stored_container_id = copy.deepcopy(recreated_container_state)
|
||||
malformed_stored_container_id["live_artifacts"]["containers"]["Media-Codec-Dashboard"][
|
||||
"container_id"
|
||||
] = "not-a-container-id"
|
||||
malformed_stored_container_id["live_receipt"]["containers"]["Media-Codec-Dashboard"][
|
||||
"container_id"
|
||||
] = "not-a-container-id"
|
||||
assert classify(
|
||||
malformed_stored_container_id,
|
||||
"jens/media-codec-dashboard",
|
||||
"Media-Codec-Dashboard",
|
||||
recorded,
|
||||
live,
|
||||
artifact,
|
||||
git_facts,
|
||||
) is False
|
||||
|
||||
malformed_live_artifact = {**artifact, "container_id": "not-a-container-id"}
|
||||
assert classify(
|
||||
recreated_container_state,
|
||||
"jens/media-codec-dashboard",
|
||||
"Media-Codec-Dashboard",
|
||||
recorded,
|
||||
live,
|
||||
malformed_live_artifact,
|
||||
git_facts,
|
||||
) is False
|
||||
|
||||
wrong_failure_state = copy.deepcopy(state)
|
||||
wrong_failure_state["last_attempt"]["message"] = "different failure"
|
||||
assert classify(
|
||||
wrong_failure_state,
|
||||
"jens/media-codec-dashboard",
|
||||
"Media-Codec-Dashboard",
|
||||
recorded,
|
||||
live,
|
||||
artifact,
|
||||
git_facts,
|
||||
) is False
|
||||
|
||||
wrong_ancestry = {**git_facts, "live_to_recorded": 1}
|
||||
assert classify(
|
||||
state,
|
||||
"jens/media-codec-dashboard",
|
||||
"Media-Codec-Dashboard",
|
||||
recorded,
|
||||
live,
|
||||
artifact,
|
||||
wrong_ancestry,
|
||||
) is False
|
||||
|
||||
|
||||
def test_reconciliation_reports_specific_state_drift_failure_boundaries():
|
||||
expected = {
|
||||
55: "controller state for the current live revision is internally inconsistent",
|
||||
56: "failed deployment attempt is not a descendant of the live revision",
|
||||
57: "failed deployment attempt is not an ancestor of main",
|
||||
58: "recorded live deployment revision is invalid",
|
||||
59: "stale controller state metadata is not a supported recoverable baseline",
|
||||
60: "recorded live deployment revision is not an ancestor of the verified runtime",
|
||||
61: "mounted deployment repository ancestry differs from canonical Gitea repository",
|
||||
}
|
||||
|
||||
for exit_code, reason in expected.items():
|
||||
result = MediaCodecDashboardDeploymentReconciliation(FakePortainer(exit_code=exit_code)).run()
|
||||
assert result == {"status": "validation_failed", "reason": reason}
|
||||
|
||||
fake = FakePortainer()
|
||||
MediaCodecDashboardDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
for exit_code in expected:
|
||||
assert f"raise SystemExit({exit_code})" in script
|
||||
|
||||
assert 'canonical_repository_root = Path("/mnt/cache/appdata/gitea/git/repositories")' in script
|
||||
assert "def canonical_git_run(*args):" in script
|
||||
assert 'canonical_recorded_to_live = canonical_git_run("merge-base", "--is-ancestor", recorded_live, live_revision)' in script
|
||||
|
||||
|
||||
def test_reconciliation_falls_back_only_to_positive_canonical_gitea_ancestry():
|
||||
fake = FakePortainer()
|
||||
MediaCodecDashboardDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert "live_revision = image_revision" in script
|
||||
assert 'recorded_live = str(state.get("live_revision") or state.get("revision") or "")' in script
|
||||
|
||||
assert 'recorded_to_live = git_run("merge-base", "--is-ancestor", recorded_live, live_revision)' in script
|
||||
assert "if recorded_to_live.returncode != 0:" in script
|
||||
|
||||
assert 'canonical_recorded = canonical_git_run("rev-parse", f"{recorded_live}^{{commit}}")' in script
|
||||
assert 'canonical_live = canonical_git_run("rev-parse", f"{live_revision}^{{commit}}")' in script
|
||||
assert 'canonical_main = canonical_git_run("rev-parse", "refs/heads/main")' in script
|
||||
assert "canonical_main.stdout.strip() == main_revision" in script
|
||||
|
||||
assert 'canonical_recorded_to_live = canonical_git_run("merge-base", "--is-ancestor", recorded_live, live_revision)' in script
|
||||
assert 'canonical_recorded_main = canonical_git_run("merge-base", "--is-ancestor", recorded_live, main_revision)' in script
|
||||
assert 'canonical_live_main = canonical_git_run("merge-base", "--is-ancestor", live_revision, main_revision)' in script
|
||||
assert "canonical_ancestry_fallback = (" in script
|
||||
assert "canonical_recorded_to_live.returncode == 0" in script
|
||||
assert "canonical_recorded_main.returncode == 0" in script
|
||||
assert "canonical_live_main.returncode == 0" in script
|
||||
assert "if not canonical_ancestry_fallback:" in script
|
||||
|
||||
# Positive canonical ancestry remains the normal fallback. A genuine
|
||||
# history transition requires the stricter cross-repository proof below.
|
||||
assert "history_transition_baseline = (" in script
|
||||
assert 'recorded_updated_at = int(state.get("updated_at_epoch") or 0)' in script
|
||||
assert "canonical_merge_base_value == mounted_merge_base_value" in script
|
||||
assert "recorded_commit_epoch < live_commit_epoch" in script
|
||||
assert "0 <= live_distance < recorded_distance" in script
|
||||
assert "971df2cc305c1b447b4feb255d4e9fda871315c3" not in script
|
||||
assert "adf53604739d4247193a7b5bfeb8ad8981d9d359" not in script
|
||||
|
||||
|
||||
def test_reconciliation_requires_cross_repository_proof_for_divergent_history_transition():
|
||||
fake = FakePortainer()
|
||||
MediaCodecDashboardDeploymentReconciliation(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
compile(script, "<media-codec-dashboard-reconciliation>", "exec")
|
||||
|
||||
assert "history_transition_baseline = (" in script
|
||||
assert "stale_success_baseline" in script
|
||||
assert "recorded_to_live.returncode == 1" in script
|
||||
assert "canonical_recorded_to_live.returncode == 1" in script
|
||||
assert "live_to_recorded.returncode == 1" in script
|
||||
assert "canonical_live_to_recorded.returncode == 1" in script
|
||||
assert "mounted_recorded_main.returncode == 0" in script
|
||||
assert "canonical_recorded_main.returncode == 0" in script
|
||||
assert "canonical_live_main.returncode == 0" in script
|
||||
assert "canonical_merge_base_value == mounted_merge_base_value" in script
|
||||
assert "mounted_merge_base_value not in {recorded_live, live_revision}" in script
|
||||
assert "canonical_recorded_commit_epoch == recorded_commit_epoch" in script
|
||||
assert "canonical_live_commit_epoch == live_commit_epoch" in script
|
||||
assert "recorded_commit_epoch <= recorded_updated_at" in script
|
||||
assert "recorded_updated_at < live_commit_epoch" in script
|
||||
assert "canonical_recorded_distance_value == recorded_distance" in script
|
||||
assert "canonical_live_distance_value == live_distance" in script
|
||||
assert "0 <= live_distance < recorded_distance" in script
|
||||
|
||||
|
||||
def test_current_live_unknown_recovery_is_runtime_proven_not_error_text_bound():
|
||||
script = _RECONCILE_SCRIPT
|
||||
historical_message = (
|
||||
'attempt.get("message") == '
|
||||
'"Existing production containers require a valid healthy live deployment state"'
|
||||
)
|
||||
|
||||
# The full-receipt path requires the exact snapshot error; the older
|
||||
# legacy snapshot path remains based on immutable runtime evidence.
|
||||
assert script.count(historical_message) == 2
|
||||
current_live_block = script[
|
||||
script.index("current_live_unknown_snapshot_baseline = ("):
|
||||
script.index("current_live_failed_baseline = (")
|
||||
]
|
||||
assert historical_message not in current_live_block
|
||||
|
||||
# Recovery still requires exact immutable runtime evidence, exact candidate
|
||||
# metadata and the same atomic state timestamp.
|
||||
assert 'current_artifacts.get("compose_sha256") == ""' in current_live_block
|
||||
assert "current_receipt == {}" in current_live_block
|
||||
assert "current_stored_artifact.get(field) == container_artifact.get(field)" in current_live_block
|
||||
assert 'set(current_attempt_metadata) == {"builtAt", "id", "repository", "revision", "tree", "treeDigest"}' in current_live_block
|
||||
assert 'attempt.get("updated_at_epoch") == state.get("updated_at_epoch")' in current_live_block
|
||||
@@ -0,0 +1,117 @@
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
from backend.app.services import action_execution
|
||||
from backend.app.services.action_execution import approval_execution_supported
|
||||
from backend.app.services.action_registry import (
|
||||
ACTION_RISK,
|
||||
ACTIONS,
|
||||
action_default_target,
|
||||
)
|
||||
|
||||
|
||||
def test_media_codec_dashboard_reconciliation_is_fixed_high_risk_typed_action():
|
||||
action = "deployment.target.reconcile_media_codec_dashboard"
|
||||
assert action in ACTIONS
|
||||
assert ACTION_RISK[action] == "high"
|
||||
assert action_default_target(action) == "jens/media-codec-dashboard"
|
||||
assert approval_execution_supported(action) is True
|
||||
|
||||
|
||||
def test_media_codec_dashboard_reconciliation_rejects_any_other_target():
|
||||
result = action_execution.execute_governed_action(
|
||||
None, # type: ignore[arg-type]
|
||||
action="deployment.target.reconcile_media_codec_dashboard",
|
||||
target="jens/not-media-codec-dashboard",
|
||||
source="chatgpt-mcp",
|
||||
risk="high",
|
||||
reason="Regression test for exact-target enforcement.",
|
||||
original_policy_decision="allowed",
|
||||
)
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["result"] == "blocked"
|
||||
assert result["message"] == (
|
||||
"Media Codec Dashboard reconciliation accepts only "
|
||||
"jens/media-codec-dashboard."
|
||||
)
|
||||
|
||||
|
||||
def test_media_codec_dashboard_reconciliation_dispatches_fixed_executor(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'dashboard-reconciliation.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
class FakeReconciliation:
|
||||
def run(self):
|
||||
return {
|
||||
"status": "reconciled",
|
||||
"repository": "jens/media-codec-dashboard",
|
||||
"container": "Media-Codec-Dashboard",
|
||||
}
|
||||
|
||||
import backend.app.services.media_codec_dashboard_deployment_reconciliation as module
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"MediaCodecDashboardDeploymentReconciliation",
|
||||
FakeReconciliation,
|
||||
)
|
||||
|
||||
with Session(engine) as session:
|
||||
result = action_execution.execute_governed_action(
|
||||
session,
|
||||
action="deployment.target.reconcile_media_codec_dashboard",
|
||||
target="jens/media-codec-dashboard",
|
||||
source="chatgpt-mcp",
|
||||
risk="high",
|
||||
reason="Re-attest only the bounded dashboard controller state.",
|
||||
original_policy_decision="allowed",
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["status"] == "success"
|
||||
assert result["result"] == "success"
|
||||
assert result["policyDecision"] == "allowed"
|
||||
assert result["originalPolicyDecision"] == "allowed"
|
||||
assert result["jobId"].startswith("job-")
|
||||
assert result["auditId"].startswith("audit-")
|
||||
|
||||
|
||||
def test_media_codec_dashboard_reconciliation_propagates_fail_closed_result(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'dashboard-failure.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
class FakeReconciliation:
|
||||
def run(self):
|
||||
return {
|
||||
"status": "validation_failed",
|
||||
"reason": "controller state for the current live revision is internally inconsistent",
|
||||
}
|
||||
|
||||
import backend.app.services.media_codec_dashboard_deployment_reconciliation as module
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"MediaCodecDashboardDeploymentReconciliation",
|
||||
FakeReconciliation,
|
||||
)
|
||||
|
||||
with Session(engine) as session:
|
||||
result = action_execution.execute_governed_action(
|
||||
session,
|
||||
action="deployment.target.reconcile_media_codec_dashboard",
|
||||
target="jens/media-codec-dashboard",
|
||||
source="chatgpt-mcp",
|
||||
risk="high",
|
||||
reason="Verify fail-closed propagation.",
|
||||
original_policy_decision="allowed",
|
||||
)
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["status"] == "failed"
|
||||
assert result["message"] == (
|
||||
"controller state for the current live revision is internally inconsistent"
|
||||
)
|
||||
@@ -0,0 +1,386 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "media_pipeline_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
from backend.app.main import app
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
|
||||
def _healthy_payloads():
|
||||
return {
|
||||
"sonarr_health": {"success": True, "status": "connected"},
|
||||
"sonarr_wanted": {"success": True, "status": "connected", "total": 3, "items": [{"title": "The Boys"}]},
|
||||
"sonarr_queue": {"success": True, "status": "connected", "total": 1, "downloading": 1, "queued": 0, "failed": 0, "stalled": 0, "items": [{"title": "The Boys", "status": "downloading"}]},
|
||||
"prowlarr_health": {"success": True, "status": "connected"},
|
||||
"prowlarr_indexers": {"success": True, "status": "ok", "total": 4, "enabled": 4, "failing": 0, "items": []},
|
||||
"prowlarr_apps": {"success": True, "status": "ok", "total": 2, "items": [{"name": "Sonarr", "status": "healthy"}, {"name": "Radarr", "status": "healthy"}]},
|
||||
"prowlarr_sync": {"success": True, "status": "ok", "applicationsConfigured": 2, "indexersConfigured": 4, "indexersEnabled": 4, "indexersFailing": 0, "healthWarnings": 0, "healthErrors": 0},
|
||||
"deluge_health": {"success": True, "status": "connected"},
|
||||
"deluge_session": {"success": True, "status": "ok", "connected": True, "downloadRate": 1_200_000, "uploadRate": 200_000, "numTorrents": 2, "active": 1, "paused": 1, "stalled": 0, "errors": 0},
|
||||
"deluge_active": {"success": True, "status": "ok", "total": 1, "items": [{"id": "active-1", "name": "The Boys", "state": "Downloading", "progress": 55}]},
|
||||
"deluge_stalled": {"success": True, "status": "ok", "total": 0, "items": []},
|
||||
"deluge_paused": {"success": True, "status": "ok", "total": 1, "items": [{"id": "paused-1", "name": "Paused", "state": "Paused"}]},
|
||||
"deluge_speeds": {"success": True, "status": "ok", "downloadRate": 1_200_000, "uploadRate": 200_000, "active": 1, "paused": 1, "stalled": 0, "errors": 0},
|
||||
"plex_health": {"success": True, "status": "connected"},
|
||||
"plex_libraries": {"success": True, "status": "ok", "total": 2, "libraries": [{"id": "1", "title": "Series", "type": "show", "itemCount": 200}]},
|
||||
"plex_sessions": {"success": True, "status": "ok", "total": 1, "sessions": [{"id": "plex-1", "title": "The Boys", "user": "Jens", "videoDecision": "directplay"}]},
|
||||
"plex_recent": {"success": True, "status": "ok", "total": 1, "items": [{"title": "The Boys", "library": "Series", "type": "episode"}]},
|
||||
"plex_streams": {"success": True, "status": "ok", "total": 1, "directPlay": 1, "directStream": 0, "transcoding": 0, "bandwidth": 4000, "users": 1},
|
||||
"tautulli_health": {"success": True, "status": "connected"},
|
||||
"tautulli_activity": {"success": True, "status": "ok", "streamCount": 1, "sessions": [{"sessionKey": "t1", "title": "The Boys", "user": "Jens", "ipAddress": "192.168.10.x", "transcodeDecision": "directplay"}]},
|
||||
"tautulli_history": {"success": True, "status": "ok", "total": 1, "items": [{"rowId": "h1", "title": "The Boys", "user": "Jens", "duration": 3600}]},
|
||||
"tautulli_stats": {"success": True, "status": "ok", "totalPlays": 10, "totalDuration": 36000, "topUsers": [{"name": "Jens", "plays": 10, "duration": 36000}], "topLibraries": [], "topPlatforms": [], "playsByDay": []},
|
||||
"tautulli_streams": {"success": True, "status": "ok", "activeStreams": 1, "directPlay": 1, "directStream": 0, "transcoding": 0, "lanStreams": 1, "wanStreams": 0, "bandwidth": 4000},
|
||||
"tautulli_libraries": {"success": True, "status": "ok", "total": 1, "items": [{"sectionId": "1", "name": "Series", "type": "show", "count": 200}]},
|
||||
"widefrog_health": {"success": True, "status": "connected"},
|
||||
"widefrog_queue": {"success": True, "status": "ok", "total": 1, "active": 1, "queued": 0, "completed": 0, "failed": 0, "items": [{"id": "wf-1", "title": "Process media"}]},
|
||||
"widefrog_failed": {"success": True, "status": "ok", "failed": 0, "items": []},
|
||||
"widefrog_activity": {"success": True, "status": "ok", "recent": [{"message": "Processed media"}], "last_event": {"message": "Processed media"}},
|
||||
"widefrog_downloads": {"success": True, "status": "ok", "active": 1, "processing": 1, "items": [{"title": "Import"}]},
|
||||
}
|
||||
|
||||
|
||||
def _patch_pipeline(monkeypatch, payloads=None):
|
||||
payloads = payloads or _healthy_payloads()
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.sonarr.SonarrConnector.health", lambda self: payloads["sonarr_health"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.sonarr.SonarrConnector.wanted_missing", lambda self: payloads["sonarr_wanted"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.sonarr.SonarrConnector.queue_status", lambda self: payloads["sonarr_queue"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.prowlarr.ProwlarrConnector.health", lambda self: payloads["prowlarr_health"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.prowlarr.ProwlarrConnector.indexer_status", lambda self: payloads["prowlarr_indexers"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.prowlarr.ProwlarrConnector.application_status", lambda self: payloads["prowlarr_apps"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.prowlarr.ProwlarrConnector.sync_status", lambda self: payloads["prowlarr_sync"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.deluge.DelugeConnector.health", lambda self: payloads["deluge_health"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.deluge.DelugeConnector.session_status", lambda self: payloads["deluge_session"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.deluge.DelugeConnector.active_downloads", lambda self: payloads["deluge_active"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.deluge.DelugeConnector.stalled_downloads", lambda self: payloads["deluge_stalled"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.deluge.DelugeConnector.paused_downloads", lambda self: payloads["deluge_paused"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.deluge.DelugeConnector.speed_status", lambda self: payloads["deluge_speeds"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.plex.PlexConnector.health", lambda self: payloads["plex_health"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.plex.PlexConnector.library_status", lambda self: payloads["plex_libraries"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.plex.PlexConnector.active_sessions", lambda self: payloads["plex_sessions"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.plex.PlexConnector.recent_media", lambda self: payloads["plex_recent"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.plex.PlexConnector.stream_summary", lambda self: payloads["plex_streams"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.tautulli.TautulliConnector.health", lambda self: payloads["tautulli_health"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.tautulli.TautulliConnector.activity", lambda self: payloads["tautulli_activity"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.tautulli.TautulliConnector.history", lambda self: payloads["tautulli_history"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.tautulli.TautulliConnector.watch_stats", lambda self: payloads["tautulli_stats"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.tautulli.TautulliConnector.stream_summary", lambda self: payloads["tautulli_streams"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.tautulli.TautulliConnector.libraries", lambda self: payloads["tautulli_libraries"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.widefrog.WidefrogConnector.health", lambda self: payloads["widefrog_health"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.widefrog.WidefrogConnector.queue_status", lambda self: payloads["widefrog_queue"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.widefrog.WidefrogConnector.failed_jobs", lambda self: payloads["widefrog_failed"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.widefrog.WidefrogConnector.activity", lambda self: payloads["widefrog_activity"], raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.widefrog.WidefrogConnector.download_status", lambda self: payloads["widefrog_downloads"], raising=False)
|
||||
return payloads
|
||||
|
||||
|
||||
def test_media_pipeline_summary_healthy(api_client, monkeypatch):
|
||||
_patch_pipeline(monkeypatch)
|
||||
|
||||
payload = api_client.get("/api/media-pipeline/summary").json()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "healthy"
|
||||
assert payload["connectors"]["sonarr"]["status"] == "connected"
|
||||
assert payload["counts"]["wantedSeries"] == 3
|
||||
assert payload["counts"]["activeDownloads"] == 1
|
||||
assert payload["counts"]["plexLibraries"] == 2
|
||||
assert payload["counts"]["activeStreams"] == 1
|
||||
assert payload["counts"]["indexers"] == 4
|
||||
assert payload["risk"]["level"] == "low"
|
||||
|
||||
|
||||
def test_media_pipeline_reuses_snapshot_for_burst_reads(api_client, monkeypatch):
|
||||
from backend.app.services import media_pipeline_service
|
||||
|
||||
payloads = _patch_pipeline(monkeypatch)
|
||||
calls = {"sonarr_health": 0}
|
||||
|
||||
def sonarr_health(_self):
|
||||
calls["sonarr_health"] += 1
|
||||
return payloads["sonarr_health"]
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.sonarr.SonarrConnector.health", sonarr_health, raising=False)
|
||||
media_pipeline_service._SNAPSHOT_CACHE.clear()
|
||||
|
||||
summary_payload = api_client.get("/api/media-pipeline/summary").json()
|
||||
issues_payload = api_client.get("/api/media-pipeline/issues").json()
|
||||
|
||||
assert summary_payload["success"] is True
|
||||
assert issues_payload["success"] is True
|
||||
assert calls["sonarr_health"] == 1
|
||||
|
||||
|
||||
def test_media_pipeline_summary_reuses_persistent_cache_after_snapshot_loss(api_client, monkeypatch):
|
||||
from backend.app.services import media_pipeline_service
|
||||
|
||||
_patch_pipeline(monkeypatch)
|
||||
media_pipeline_service._SNAPSHOT_CACHE.clear()
|
||||
|
||||
first = api_client.get("/api/media-pipeline/summary").json()
|
||||
media_pipeline_service._SNAPSHOT_CACHE.clear()
|
||||
|
||||
def fail_build_snapshot():
|
||||
raise AssertionError("persistent media summary cache should avoid cold connector reads")
|
||||
|
||||
monkeypatch.setattr(media_pipeline_service, "_build_snapshot", fail_build_snapshot)
|
||||
|
||||
second = api_client.get("/api/media-pipeline/summary").json()
|
||||
|
||||
assert second["status"] == first["status"]
|
||||
assert second["counts"] == first["counts"]
|
||||
assert second["connectors"] == first["connectors"]
|
||||
|
||||
|
||||
def test_media_pipeline_snapshot_cache_prevents_concurrent_stampede(api_client, monkeypatch):
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from time import sleep
|
||||
from backend.app.services import media_pipeline_service
|
||||
|
||||
payloads = _patch_pipeline(monkeypatch)
|
||||
calls = {"sonarr_health": 0}
|
||||
|
||||
def sonarr_health(_self):
|
||||
calls["sonarr_health"] += 1
|
||||
sleep(0.05)
|
||||
return payloads["sonarr_health"]
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.sonarr.SonarrConnector.health", sonarr_health, raising=False)
|
||||
media_pipeline_service._SNAPSHOT_CACHE.clear()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
results = list(executor.map(lambda _index: media_pipeline_service._snapshot_cached(), range(4)))
|
||||
|
||||
assert all(result["sonarrHealth"]["success"] is True for result in results)
|
||||
assert calls["sonarr_health"] == 1
|
||||
|
||||
|
||||
def test_media_pipeline_cold_snapshot_runs_connector_reads_in_parallel(api_client, monkeypatch):
|
||||
from threading import Barrier
|
||||
from backend.app.services import media_pipeline_service
|
||||
|
||||
payloads = _patch_pipeline(monkeypatch)
|
||||
barrier = Barrier(3, timeout=1)
|
||||
|
||||
def wait_and_return(key):
|
||||
def _inner(_self):
|
||||
barrier.wait()
|
||||
return payloads[key]
|
||||
|
||||
return _inner
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.sonarr.SonarrConnector.health", wait_and_return("sonarr_health"), raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.sonarr.SonarrConnector.wanted_missing", wait_and_return("sonarr_wanted"), raising=False)
|
||||
monkeypatch.setattr("backend.app.connectors.sonarr.SonarrConnector.queue_status", wait_and_return("sonarr_queue"), raising=False)
|
||||
media_pipeline_service._SNAPSHOT_CACHE.clear()
|
||||
|
||||
payload = api_client.get("/api/media-pipeline/summary").json()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["connectors"]["sonarr"]["status"] == "connected"
|
||||
assert payload["counts"]["wantedSeries"] == 3
|
||||
|
||||
|
||||
def test_media_pipeline_summary_partial_when_connector_missing_config(api_client, monkeypatch):
|
||||
payloads = _healthy_payloads()
|
||||
payloads["prowlarr_health"] = {"success": False, "status": "missing_config", "message": "missing_config"}
|
||||
payloads["prowlarr_indexers"] = {"success": False, "status": "missing_config", "total": 0, "enabled": 0, "failing": 0, "items": []}
|
||||
payloads["prowlarr_sync"] = {"success": False, "status": "missing_config", "applicationsConfigured": 0, "indexersConfigured": 0, "indexersEnabled": 0, "indexersFailing": 0, "healthWarnings": 0, "healthErrors": 0}
|
||||
_patch_pipeline(monkeypatch, payloads)
|
||||
|
||||
payload = api_client.get("/api/media-pipeline/summary").json()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] in {"partial", "missing_config"}
|
||||
assert payload["connectors"]["prowlarr"]["status"] == "missing_config"
|
||||
assert "prowlarr" in " ".join(payload["risk"]["reasons"]).lower()
|
||||
|
||||
|
||||
def test_media_pipeline_flow_returns_nodes_and_edges(api_client, monkeypatch):
|
||||
_patch_pipeline(monkeypatch)
|
||||
|
||||
payload = api_client.get("/api/media-pipeline/flow").json()
|
||||
|
||||
node_ids = {node["id"] for node in payload["nodes"]}
|
||||
assert {"sonarr", "prowlarr", "deluge", "widefrog", "plex", "tautulli"}.issubset(node_ids)
|
||||
assert {"from": "sonarr", "to": "prowlarr", "status": "healthy", "label": "indexer sync"} in payload["edges"]
|
||||
assert not any(edge["from"] == "deluge" and edge["to"] == "widefrog" for edge in payload["edges"])
|
||||
assert {"from": "widefrog", "to": "plex", "status": "healthy", "label": "media processing"} in payload["edges"]
|
||||
action_ids = [action["actionId"] for node in payload["nodes"] for action in node.get("actions", [])]
|
||||
assert not any("delete" in action or "remove" in action for action in action_ids)
|
||||
|
||||
|
||||
def test_media_pipeline_issues_detect_cross_connector_problems(api_client, monkeypatch):
|
||||
payloads = _healthy_payloads()
|
||||
payloads["prowlarr_indexers"] = {"success": True, "status": "ok", "total": 3, "enabled": 2, "failing": 1, "items": [{"name": "Indexer", "status": "error"}]}
|
||||
payloads["deluge_stalled"] = {"success": True, "status": "ok", "total": 1, "items": [{"id": "stalled-1", "name": "The Boys", "state": "Error"}]}
|
||||
payloads["widefrog_failed"] = {"success": True, "status": "ok", "failed": 2, "items": [{"id": "wf-fail", "title": "Import failed"}]}
|
||||
payloads["tautulli_health"] = {"success": False, "status": "missing_config", "message": "missing_config"}
|
||||
_patch_pipeline(monkeypatch, payloads)
|
||||
|
||||
payload = api_client.get("/api/media-pipeline/issues").json()
|
||||
titles = " ".join(issue["title"] for issue in payload["issues"]).lower()
|
||||
|
||||
assert "indexer" in titles
|
||||
assert "stalled" in titles
|
||||
assert "widefrog" in titles
|
||||
assert "tautulli" in titles
|
||||
actions = [action for issue in payload["issues"] for action in issue["suggestedActions"]]
|
||||
assert actions
|
||||
assert all(action["policyDecision"] in {"allowed", "approval_required", "blocked"} for action in actions)
|
||||
assert all(action.get("link") for action in actions)
|
||||
assert not any(any(blocked in action["actionId"] for blocked in ["delete", "remove", "kill_session", "terminate"]) for action in actions)
|
||||
|
||||
|
||||
def test_deluge_stalled_issue_includes_source_and_timestamp(api_client, monkeypatch):
|
||||
payloads = _healthy_payloads()
|
||||
payloads["deluge_stalled"] = {"success": True, "status": "ok", "total": 1, "items": [{"id": "stalled-1", "name": "The Boys", "state": "Downloading"}]}
|
||||
_patch_pipeline(monkeypatch, payloads)
|
||||
|
||||
payload = api_client.get("/api/media-pipeline/issues").json()
|
||||
stalled = next(issue for issue in payload["issues"] if issue["id"] == "deluge-stalled")
|
||||
|
||||
assert stalled["source"] == "deluge.stalled_downloads"
|
||||
assert stalled["observedAt"]
|
||||
assert any(str(item).startswith("source=deluge.stalled_downloads") for item in stalled["evidence"])
|
||||
assert any(str(item).startswith("observedAt=") for item in stalled["evidence"])
|
||||
|
||||
|
||||
def test_media_pipeline_issues_handles_review_apps_without_connector_type(api_client, monkeypatch):
|
||||
from backend.app.db.models import App
|
||||
from backend.app.db.session import engine
|
||||
from sqlmodel import Session
|
||||
|
||||
_patch_pipeline(monkeypatch)
|
||||
with Session(engine) as session:
|
||||
session.add(App(id="media-review-app", name="Media Review App", category="media", needs_review=True))
|
||||
session.commit()
|
||||
|
||||
response = api_client.get("/api/media-pipeline/issues")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
review_issue = next(issue for issue in payload["issues"] if issue["id"] == "app-needs-review-media-review-app")
|
||||
assert review_issue["affectedConnectors"] == ["media-review-app"]
|
||||
assert review_issue["suggestedActions"][0]["target"] == "media-review-app"
|
||||
assert review_issue["suggestedActions"][0]["link"] == "/apps/media-review-app"
|
||||
|
||||
|
||||
def test_media_pipeline_downloads_combines_deluge_and_widefrog(api_client, monkeypatch):
|
||||
payloads = _healthy_payloads()
|
||||
payloads["deluge_stalled"] = {"success": True, "status": "ok", "total": 1, "items": [{"id": "stalled-1", "name": "The Boys", "state": "Error"}]}
|
||||
payloads["widefrog_failed"] = {"success": True, "status": "ok", "failed": 1, "items": [{"id": "wf-fail", "title": "Import failed"}]}
|
||||
_patch_pipeline(monkeypatch, payloads)
|
||||
|
||||
payload = api_client.get("/api/media-pipeline/downloads").json()
|
||||
|
||||
assert payload["active"]["total"] == 1
|
||||
assert payload["stalled"]["total"] == 1
|
||||
assert payload["failedWidefrogJobs"]["failed"] == 1
|
||||
action_ids = [action["actionId"] for action in payload["suggestedActions"]]
|
||||
assert "deluge.resume_torrent" in action_ids
|
||||
assert "widefrog.retry_failed" in action_ids
|
||||
assert {action["target"]: action["link"] for action in payload["suggestedActions"] if action["target"] in {"deluge", "widefrog"}} == {
|
||||
"deluge": "/apps/deluge",
|
||||
"widefrog": "/apps/widefrog",
|
||||
}
|
||||
assert not any("remove" in action or "delete" in action for action in action_ids)
|
||||
|
||||
|
||||
def test_media_pipeline_library_combines_plex_tautulli_widefrog(api_client, monkeypatch):
|
||||
_patch_pipeline(monkeypatch)
|
||||
|
||||
payload = api_client.get("/api/media-pipeline/library").json()
|
||||
|
||||
assert payload["plexLibraries"]["total"] == 2
|
||||
assert payload["tautulliStats"]["totalPlays"] == 10
|
||||
assert payload["widefrogQueue"]["total"] == 1
|
||||
action_ids = [action["actionId"] for action in payload["suggestedActions"]]
|
||||
assert "plex.scan_library" in action_ids
|
||||
assert "plex.scan_all_libraries" in action_ids
|
||||
assert "widefrog.scan_library" in action_ids
|
||||
assert all(action.get("link") in {"/apps/plex", "/apps/widefrog"} for action in payload["suggestedActions"])
|
||||
assert not any("empty_trash" in action or "kill_session" in action for action in action_ids)
|
||||
|
||||
|
||||
def test_media_pipeline_activity_marks_privacy_sensitive_data(api_client, monkeypatch):
|
||||
_patch_pipeline(monkeypatch)
|
||||
|
||||
payload = api_client.get("/api/media-pipeline/activity").json()
|
||||
|
||||
assert payload["privacy"]["sensitive"] is True
|
||||
assert "tautulli.history" in payload["privacy"]["actions"]
|
||||
assert payload["tautulliActivity"]["streamCount"] == 1
|
||||
assert payload["plexSessions"]["total"] == 1
|
||||
assert "192.168.10.x" in str(payload)
|
||||
|
||||
|
||||
def test_media_pipeline_recommendations_are_generated_from_issues(api_client, monkeypatch):
|
||||
payloads = _healthy_payloads()
|
||||
payloads["prowlarr_indexers"] = {"success": True, "status": "ok", "total": 3, "enabled": 2, "failing": 1, "items": []}
|
||||
payloads["plex_streams"] = {"success": True, "status": "ok", "total": 3, "directPlay": 0, "directStream": 0, "transcoding": 3, "bandwidth": 12000, "users": 3}
|
||||
_patch_pipeline(monkeypatch, payloads)
|
||||
|
||||
payload = api_client.get("/api/media-pipeline/recommendations").json()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["recommendations"]
|
||||
assert all("confidence" in item for item in payload["recommendations"])
|
||||
assert any("transcode" in item["title"].lower() or "indexer" in item["title"].lower() for item in payload["recommendations"])
|
||||
|
||||
|
||||
def test_media_pipeline_check_is_read_only_and_audited(api_client, monkeypatch):
|
||||
_patch_pipeline(monkeypatch)
|
||||
|
||||
payload = api_client.post("/api/media-pipeline/check").json()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["summary"]["status"] == "healthy"
|
||||
assert payload["auditId"].startswith("audit-")
|
||||
assert payload["jobId"].startswith("job-")
|
||||
|
||||
|
||||
def test_media_pipeline_diagnose_returns_evidence_without_execution(api_client, monkeypatch):
|
||||
payloads = _healthy_payloads()
|
||||
payloads["sonarr_wanted"] = {"success": True, "status": "connected", "total": 1, "items": [{"title": "The Boys"}]}
|
||||
payloads["deluge_active"] = {"success": True, "status": "ok", "total": 0, "items": []}
|
||||
payloads["prowlarr_indexers"] = {"success": True, "status": "ok", "total": 3, "enabled": 2, "failing": 1, "items": []}
|
||||
_patch_pipeline(monkeypatch, payloads)
|
||||
|
||||
payload = api_client.post("/api/media-pipeline/diagnose", json={"query": "waarom downloadt The Boys niet?", "scope": "downloads"}).json()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["diagnosis"]["likelyCauses"]
|
||||
assert payload["diagnosis"]["evidence"]
|
||||
assert payload["diagnosis"]["executedActions"] == []
|
||||
assert not any("delete" in action["actionId"] or "remove" in action["actionId"] for action in payload["diagnosis"]["suggestedActions"])
|
||||
|
||||
|
||||
def test_media_pipeline_diagnose_policy_is_read_only_allowed(api_client):
|
||||
payload = api_client.post(
|
||||
"/api/policies/evaluate",
|
||||
json={"action": "media_pipeline.diagnose", "target": "media-pipeline", "source": "manual"},
|
||||
).json()
|
||||
|
||||
assert payload["decision"] == "allowed"
|
||||
assert payload["risk"] == "low"
|
||||
@@ -0,0 +1,68 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[2] / "scripts/reconcile_mediarr_validation_window.py"
|
||||
SPEC = importlib.util.spec_from_file_location("mediarr_validation_window", SCRIPT)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
window = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(window)
|
||||
|
||||
|
||||
def _projects(timeout: int = 1800) -> dict:
|
||||
return {
|
||||
"jens/mediarr": {
|
||||
"branch": "master",
|
||||
"slug": "mediarr",
|
||||
"source_dir": "/srv/example/appdata/mediarr/source",
|
||||
"compose_files": ["docker-compose.unraid.yml"],
|
||||
"compose_project": "mediarr",
|
||||
"containers": ["mediarr"],
|
||||
"gitea_repository_id": 42,
|
||||
"gitea_webhook_secret_file": "jens__mediarr",
|
||||
"validation": {**window.EXPECTED_VALIDATION, "timeout_seconds": timeout},
|
||||
},
|
||||
"jens/other": {"validation": {"timeout_seconds": 1800}},
|
||||
}
|
||||
|
||||
|
||||
def test_accepts_only_exact_old_or_new_mediarr_policy() -> None:
|
||||
assert window._validate(_projects()) == 1800
|
||||
assert window._validate(_projects(3600)) == 3600
|
||||
for mutate in (
|
||||
lambda value: value["jens/mediarr"]["validation"].update(timeout_seconds=7200),
|
||||
lambda value: value["jens/mediarr"].update(containers=["other"]),
|
||||
lambda value: value["jens/mediarr"]["validation"].update(required=False),
|
||||
):
|
||||
value = _projects()
|
||||
mutate(value)
|
||||
with pytest.raises(RuntimeError):
|
||||
window._validate(value)
|
||||
|
||||
|
||||
def test_refuses_active_mediarr_delivery_and_ignores_other_projects(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(window, "INBOX", tmp_path)
|
||||
(tmp_path / "other.json").write_text(
|
||||
json.dumps({"repository": "jens/other", "status": "deploying"}), encoding="utf-8"
|
||||
)
|
||||
window._no_active_delivery()
|
||||
(tmp_path / "mediarr.json").write_text(
|
||||
json.dumps({"repository": "jens/mediarr", "status": "awaiting_validation"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="active delivery"):
|
||||
window._no_active_delivery()
|
||||
|
||||
|
||||
def test_refuses_invalid_delivery_record(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(window, "INBOX", tmp_path)
|
||||
(tmp_path / "broken.json").write_text("{", encoding="utf-8")
|
||||
with pytest.raises(RuntimeError, match="invalid"):
|
||||
window._no_active_delivery()
|
||||
@@ -0,0 +1,927 @@
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
def _reset_db(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "npm_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
|
||||
def _set_npm_config(
|
||||
url: str | None = "https://npm.example.test",
|
||||
identity: str | None = "admin@example.test",
|
||||
secret: str | None = "NPM_SECRET_VALUE",
|
||||
):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = dict(settings.apis)
|
||||
settings.apis["NPM_URL"] = url
|
||||
settings.apis["NPM_IDENTITY"] = identity
|
||||
settings.apis["NPM_SECRET"] = secret
|
||||
settings.apis["NPM_VERIFY_TLS"] = False
|
||||
settings.apis["NPM_TIMEOUT_SECONDS"] = "10"
|
||||
return previous
|
||||
|
||||
|
||||
def _restore_config(previous):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous)
|
||||
|
||||
|
||||
def _proxy_hosts():
|
||||
return [
|
||||
{
|
||||
"id": 1,
|
||||
"domain_names": ["media.example.com"],
|
||||
"forward_host": "plex",
|
||||
"forward_port": 32400,
|
||||
"forward_scheme": "http",
|
||||
"enabled": 1,
|
||||
"ssl_forced": 1,
|
||||
"http2_support": 1,
|
||||
"hsts_enabled": 0,
|
||||
"certificate_id": 10,
|
||||
"access_list_id": 2,
|
||||
"block_exploits": 1,
|
||||
"caching_enabled": 0,
|
||||
"allow_websocket_upgrade": 1,
|
||||
"locations": [{"path": "/"}],
|
||||
"meta": {"letsencrypt_agree": True},
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"domain_names": ["open.example.com"],
|
||||
"forward_host": "unknown-app",
|
||||
"forward_port": 8080,
|
||||
"forward_scheme": "http",
|
||||
"enabled": True,
|
||||
"ssl_forced": False,
|
||||
"http2_support": False,
|
||||
"hsts_enabled": False,
|
||||
"certificate_id": None,
|
||||
"access_list_id": 0,
|
||||
"block_exploits": False,
|
||||
"caching_enabled": False,
|
||||
"allow_websocket_upgrade": False,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _certificates():
|
||||
return [
|
||||
{
|
||||
"id": 10,
|
||||
"nice_name": "media wildcard",
|
||||
"domain_names": ["media.example.com"],
|
||||
"provider": "letsencrypt",
|
||||
"expires_on": "2099-01-01T00:00:00Z",
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"nice_name": "soon",
|
||||
"domain_names": ["soon.example.com"],
|
||||
"provider": "letsencrypt",
|
||||
"expires_on": "2000-01-01T00:00:00Z",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _access_lists():
|
||||
return [
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Authentik forward auth",
|
||||
"auth_type": "authentik",
|
||||
"satisfy_any": False,
|
||||
"items": [{"username": "redacted"}],
|
||||
"clients": [{"address": "198.18.10.0/24"}],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_npm_connector_missing_config_reports_required_env(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
previous = _set_npm_config(url=None, identity=None, secret=None)
|
||||
|
||||
from backend.app.main import app
|
||||
|
||||
with TestClient(app) as client:
|
||||
payload = client.post("/api/connectors/nginx_proxy_manager/test").json()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "missing_config"
|
||||
assert payload["missing_config"] is True
|
||||
assert payload["required_env_vars"] == ["NPM_URL", "NPM_IDENTITY", "NPM_SECRET"]
|
||||
assert payload["secretRef"] == "NPM_SECRET"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_npm_health_success_and_auth_failure_redact_secret(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
|
||||
previous = _set_npm_config()
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, "_request", lambda self, method, path, **kwargs: _proxy_hosts() if path == "/nginx/proxy-hosts" else [], raising=True)
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, "_login", lambda self: "runtime-token", raising=True)
|
||||
|
||||
healthy = NginxProxyManagerConnector().health()
|
||||
|
||||
assert healthy["success"] is True
|
||||
assert healthy["status"] == "connected"
|
||||
assert healthy["proxyHostCount"] == 2
|
||||
assert healthy["url"] == "https://npm.example.test"
|
||||
assert "NPM_SECRET_VALUE" not in str(healthy)
|
||||
|
||||
def fail_login(self):
|
||||
raise RuntimeError("auth_failed password=NPM_SECRET_VALUE")
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, "_login", fail_login, raising=True)
|
||||
failed = NginxProxyManagerConnector().health()
|
||||
|
||||
assert failed["success"] is False
|
||||
assert failed["status"] == "auth_failed"
|
||||
assert "NPM_SECRET_VALUE" not in str(failed)
|
||||
assert "[REDACTED]" in str(failed)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_typed_proxy_host_refuses_takeover_and_creates_only_hardened_plan_target(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = _set_npm_config()
|
||||
settings.apis['NPM_LETSENCRYPT_EMAIL'] = 'ops@example.test'
|
||||
calls = []
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
calls.append((method, path, kwargs))
|
||||
if method == 'GET' and path == '/nginx/proxy-hosts':
|
||||
return []
|
||||
if method == 'POST' and path == '/nginx/proxy-hosts':
|
||||
return {'id': 42}
|
||||
raise AssertionError((method, path))
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, '_request', request, raising=True)
|
||||
result = NginxProxyManagerConnector().ensure_managed_proxy_host(
|
||||
hostname='support.itworx.test',
|
||||
forward_host='wsbc-portal',
|
||||
forward_port=8080,
|
||||
certificate_mode='managed',
|
||||
force_ssl=True,
|
||||
http2=True,
|
||||
upload_limit_mb=256,
|
||||
read_timeout_seconds=180,
|
||||
)
|
||||
|
||||
assert result == {
|
||||
'success': True,
|
||||
'status': 'created',
|
||||
'message': 'proxy_host_created_from_typed_plan',
|
||||
'hostId': 42,
|
||||
'hostname': 'support.itworx.test',
|
||||
}
|
||||
payload = calls[-1][2]['json']
|
||||
assert payload['forward_host'] == 'wsbc-portal'
|
||||
assert payload['forward_port'] == 8080
|
||||
assert payload['ssl_forced'] is True
|
||||
assert payload['http2_support'] is True
|
||||
assert payload['caching_enabled'] is False
|
||||
assert payload['allow_websocket_upgrade'] is False
|
||||
assert payload['block_exploits'] is True
|
||||
assert payload['advanced_config'] == 'client_max_body_size 256m;\nproxy_read_timeout 180s;\nproxy_send_timeout 180s;'
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_managed_proxy_requires_and_verifies_requested_access_list(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = _set_npm_config()
|
||||
settings.apis['NPM_LETSENCRYPT_EMAIL'] = 'ops@example.test'
|
||||
calls = []
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
calls.append((method, path, kwargs))
|
||||
if path == '/nginx/access-lists':
|
||||
return [{'id': 1, 'name': 'Jens', 'satisfy_any': False, 'items': [{'id': 7}]}]
|
||||
if method == 'GET' and path == '/nginx/proxy-hosts':
|
||||
if len([call for call in calls if call[:2] == ('GET', path)]) == 1:
|
||||
return []
|
||||
return [{
|
||||
'id': 42, 'domain_names': ['repopulse.itworx.test'],
|
||||
'forward_host': 'repopulse', 'forward_port': 8000,
|
||||
'access_list_id': 1,
|
||||
}]
|
||||
if method == 'POST' and path == '/nginx/proxy-hosts':
|
||||
return {'id': 42}
|
||||
raise AssertionError((method, path))
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, '_request', request, raising=True)
|
||||
result = NginxProxyManagerConnector().ensure_managed_proxy_host(
|
||||
hostname='repopulse.itworx.test', forward_host='repopulse', forward_port=8000,
|
||||
certificate_mode='managed', force_ssl=True, http2=True,
|
||||
upload_limit_mb=128, read_timeout_seconds=120, access_list_id=1,
|
||||
)
|
||||
|
||||
assert result['status'] == 'created'
|
||||
assert next(call[2]['json'] for call in calls if call[:2] == ('POST', '/nginx/proxy-hosts'))['access_list_id'] == 1
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_managed_proxy_blocks_missing_or_bypassable_access_list(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
|
||||
previous = _set_npm_config()
|
||||
lists = []
|
||||
calls = []
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
calls.append((method, path))
|
||||
if path == '/nginx/access-lists':
|
||||
return lists
|
||||
raise AssertionError('Proxy host must not be inspected or created')
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, '_request', request, raising=True)
|
||||
arguments = {
|
||||
'hostname': 'repopulse.itworx.test', 'forward_host': 'repopulse', 'forward_port': 8000,
|
||||
'certificate_mode': 'managed', 'force_ssl': True, 'http2': True,
|
||||
'upload_limit_mb': 128, 'read_timeout_seconds': 120, 'access_list_id': 1,
|
||||
}
|
||||
assert NginxProxyManagerConnector().ensure_managed_proxy_host(**arguments)['status'] == 'blocked'
|
||||
lists.append({'id': 1, 'name': 'Jens', 'satisfy_any': True, 'items': [{'id': 7}]})
|
||||
assert NginxProxyManagerConnector().ensure_managed_proxy_host(**arguments)['status'] == 'blocked'
|
||||
lists[0]['satisfy_any'] = False
|
||||
lists[0]['items'] = []
|
||||
assert NginxProxyManagerConnector().ensure_managed_proxy_host(**arguments)['status'] == 'blocked'
|
||||
lists[0]['items'] = [{'id': 7}]
|
||||
del lists[0]['satisfy_any']
|
||||
assert NginxProxyManagerConnector().ensure_managed_proxy_host(**arguments)['status'] == 'blocked'
|
||||
assert all(path == '/nginx/access-lists' for _, path in calls)
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_preflight_access_list_verification_is_read_only_and_redacted(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
|
||||
previous = _set_npm_config()
|
||||
calls = []
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
calls.append((method, path, kwargs))
|
||||
return [{
|
||||
'id': 42, 'name': 'RepoPulse operators', 'satisfy_any': False,
|
||||
'items': [{'username': 'private-user', 'password': 'private-password'}],
|
||||
'clients': [],
|
||||
}]
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, '_request', request, raising=True)
|
||||
result = NginxProxyManagerConnector().validate_managed_access_list(42)
|
||||
|
||||
assert result == {
|
||||
'success': True, 'status': 'verified',
|
||||
'message': 'required_access_list_verified', 'accessListId': 42,
|
||||
}
|
||||
assert calls == [('GET', '/nginx/access-lists', {
|
||||
'params': {'limit': 1000, 'expand': 'items,clients'},
|
||||
})]
|
||||
assert 'private-user' not in str(result)
|
||||
assert 'private-password' not in str(result)
|
||||
|
||||
|
||||
def test_managed_proxy_rejects_access_list_lost_after_creation(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = _set_npm_config()
|
||||
settings.apis['NPM_LETSENCRYPT_EMAIL'] = 'ops@example.test'
|
||||
host_reads = 0
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
nonlocal host_reads
|
||||
if path == '/nginx/access-lists':
|
||||
return [{'id': 1, 'satisfy_any': False, 'items': [{'id': 7}]}]
|
||||
if method == 'GET' and path == '/nginx/proxy-hosts':
|
||||
host_reads += 1
|
||||
return [] if host_reads == 1 else [{
|
||||
'id': 42, 'domain_names': ['repopulse.itworx.test'],
|
||||
'forward_host': 'repopulse', 'forward_port': 8000,
|
||||
'access_list_id': 0,
|
||||
}]
|
||||
if method == 'POST' and path == '/nginx/proxy-hosts':
|
||||
return {'id': 42}
|
||||
raise AssertionError((method, path))
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, '_request', request, raising=True)
|
||||
result = NginxProxyManagerConnector().ensure_managed_proxy_host(
|
||||
hostname='repopulse.itworx.test', forward_host='repopulse', forward_port=8000,
|
||||
certificate_mode='managed', force_ssl=True, http2=True,
|
||||
upload_limit_mb=128, read_timeout_seconds=120, access_list_id=1,
|
||||
)
|
||||
assert result['status'] == 'ownership_unverified'
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_enrollment_proxy_creation_requires_persisted_ownership_marker(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = _set_npm_config()
|
||||
settings.apis['NPM_LETSENCRYPT_EMAIL'] = 'ops@example.test'
|
||||
requests = []
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
requests.append((method, path, kwargs))
|
||||
if method == 'POST':
|
||||
return {'id': 42}
|
||||
if len([item for item in requests if item[0] == 'GET']) == 1:
|
||||
return []
|
||||
return [{
|
||||
'id': 42,
|
||||
'domain_names': ['support.itworx.test'],
|
||||
'forward_host': 'wsbc-portal',
|
||||
'forward_port': 8080,
|
||||
'meta': {'appops_enrollment_id': 'enr-0123456789abcdef0123'},
|
||||
}]
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, '_request', request, raising=True)
|
||||
result = NginxProxyManagerConnector().ensure_managed_proxy_host(
|
||||
hostname='support.itworx.test',
|
||||
forward_host='wsbc-portal',
|
||||
forward_port=8080,
|
||||
certificate_mode='managed',
|
||||
force_ssl=True,
|
||||
http2=True,
|
||||
upload_limit_mb=256,
|
||||
read_timeout_seconds=180,
|
||||
ownership_id='enr-0123456789abcdef0123',
|
||||
)
|
||||
|
||||
assert result['success'] is True
|
||||
assert result['status'] == 'created'
|
||||
assert len([item for item in requests if item[0] == 'GET']) == 2
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_enrollment_proxy_creation_stops_when_ownership_is_not_persisted(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = _set_npm_config()
|
||||
settings.apis['NPM_LETSENCRYPT_EMAIL'] = 'ops@example.test'
|
||||
calls = {'get': 0}
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
if method == 'POST':
|
||||
return {'id': 42}
|
||||
calls['get'] += 1
|
||||
return [] if calls['get'] == 1 else [{
|
||||
'id': 42,
|
||||
'domain_names': ['support.itworx.test'],
|
||||
'forward_host': 'wsbc-portal',
|
||||
'forward_port': 8080,
|
||||
'meta': {},
|
||||
}]
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, '_request', request, raising=True)
|
||||
result = NginxProxyManagerConnector().ensure_managed_proxy_host(
|
||||
hostname='support.itworx.test',
|
||||
forward_host='wsbc-portal',
|
||||
forward_port=8080,
|
||||
certificate_mode='managed',
|
||||
force_ssl=True,
|
||||
http2=True,
|
||||
upload_limit_mb=256,
|
||||
read_timeout_seconds=180,
|
||||
ownership_id='enr-0123456789abcdef0123',
|
||||
)
|
||||
|
||||
assert result == {
|
||||
'success': False,
|
||||
'status': 'ownership_unverified',
|
||||
'message': 'managed_proxy_creation_ownership_not_verified',
|
||||
'hostId': 42,
|
||||
}
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_typed_proxy_host_does_not_take_over_existing_hostname(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
|
||||
previous = _set_npm_config()
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
'_request',
|
||||
lambda self, method, path, **kwargs: [{
|
||||
'id': 12,
|
||||
'domain_names': ['support.itworx.test'],
|
||||
'forward_host': 'other-portal',
|
||||
'forward_port': 8080,
|
||||
}],
|
||||
raising=True,
|
||||
)
|
||||
result = NginxProxyManagerConnector().ensure_managed_proxy_host(
|
||||
hostname='support.itworx.test',
|
||||
forward_host='wsbc-portal',
|
||||
forward_port=8080,
|
||||
certificate_mode='existing',
|
||||
force_ssl=True,
|
||||
http2=True,
|
||||
upload_limit_mb=256,
|
||||
read_timeout_seconds=180,
|
||||
)
|
||||
|
||||
assert result == {
|
||||
'success': False,
|
||||
'status': 'conflict',
|
||||
'message': 'hostname_owned_by_different_target',
|
||||
}
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_typed_proxy_host_rejects_insecure_existing_target(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
|
||||
previous = _set_npm_config()
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
'_request',
|
||||
lambda self, method, path, **kwargs: [{
|
||||
'id': 12,
|
||||
'domain_names': ['support.itworx.test'],
|
||||
'forward_host': 'wsbc-portal',
|
||||
'forward_port': 8080,
|
||||
'ssl_forced': False,
|
||||
'certificate_id': None,
|
||||
}],
|
||||
raising=True,
|
||||
)
|
||||
result = NginxProxyManagerConnector().ensure_managed_proxy_host(
|
||||
hostname='support.itworx.test',
|
||||
forward_host='wsbc-portal',
|
||||
forward_port=8080,
|
||||
certificate_mode='existing',
|
||||
force_ssl=True,
|
||||
http2=True,
|
||||
upload_limit_mb=256,
|
||||
read_timeout_seconds=180,
|
||||
)
|
||||
|
||||
assert result == {
|
||||
'success': False,
|
||||
'status': 'conflict',
|
||||
'message': 'existing_proxy_host_does_not_meet_managed_security_contract',
|
||||
}
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_npm_proxy_hosts_certificates_and_access_lists_normalize(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
|
||||
previous = _set_npm_config()
|
||||
calls = []
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
calls.append((path, kwargs))
|
||||
return {
|
||||
"/nginx/proxy-hosts": _proxy_hosts(),
|
||||
"/nginx/certificates": _certificates(),
|
||||
"/nginx/access-lists": _access_lists(),
|
||||
}[path]
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, "_login", lambda self: "runtime-token", raising=True)
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
"_request",
|
||||
request,
|
||||
raising=True,
|
||||
)
|
||||
|
||||
hosts = NginxProxyManagerConnector().proxy_hosts()
|
||||
certs = NginxProxyManagerConnector().certificates()
|
||||
lists = NginxProxyManagerConnector().access_lists()
|
||||
|
||||
assert hosts["success"] is True
|
||||
assert hosts["items"][0]["domainNames"] == ["media.example.com"]
|
||||
assert hosts["items"][0]["forwardHost"] == "plex"
|
||||
assert hosts["items"][0]["forwardPort"] == 32400
|
||||
assert hosts["items"][0]["sslForced"] is True
|
||||
assert hosts["items"][0]["accessListId"] == 2
|
||||
assert hosts["items"][0]["locationsCount"] == 1
|
||||
assert certs["items"][0]["provider"] == "letsencrypt"
|
||||
assert certs["items"][1]["isExpired"] is True
|
||||
assert certs["items"][1]["isExpiringSoon"] is True
|
||||
assert lists["items"][0]["name"] == "Authentik forward auth"
|
||||
assert lists["items"][0]["itemsCount"] == 1
|
||||
assert all(kwargs.get("params") == {"limit": 100} for _, kwargs in calls)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_npm_exposure_and_security_warnings(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
|
||||
previous = _set_npm_config()
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, "_login", lambda self: "runtime-token", raising=True)
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
"_request",
|
||||
lambda self, method, path, **kwargs: {
|
||||
"/nginx/proxy-hosts": _proxy_hosts(),
|
||||
"/nginx/certificates": _certificates(),
|
||||
"/nginx/access-lists": _access_lists(),
|
||||
}[path],
|
||||
raising=True,
|
||||
)
|
||||
|
||||
summary = NginxProxyManagerConnector().exposure_summary()
|
||||
warnings = NginxProxyManagerConnector().security_warnings()
|
||||
|
||||
assert summary["totalProxyHosts"] == 2
|
||||
assert summary["enabledProxyHosts"] == 2
|
||||
assert summary["hostsWithoutCertificate"] == 1
|
||||
assert summary["hostsWithoutAccessList"] == 1
|
||||
assert summary["hostsWithoutSslForce"] == 1
|
||||
assert any(item["type"] == "host_without_certificate" for item in warnings["items"])
|
||||
assert any(item["type"] == "public_host_without_access_list" for item in warnings["items"])
|
||||
assert any(item["type"] == "ssl_not_forced" for item in warnings["items"])
|
||||
expiring = NginxProxyManagerConnector._build_warnings(
|
||||
[
|
||||
{
|
||||
"id": 3,
|
||||
"domainNames": ["soon.example.com"],
|
||||
"enabled": True,
|
||||
"sslForced": True,
|
||||
"hstsEnabled": True,
|
||||
"certificateId": 11,
|
||||
"accessListId": 2,
|
||||
"forwardScheme": "https",
|
||||
"forwardHost": "soon",
|
||||
"forwardPort": 443,
|
||||
}
|
||||
],
|
||||
{11: {"isExpiringSoon": True, "expiresInDays": 30, "expiresOn": "2026-08-01T00:00:00Z"}},
|
||||
)
|
||||
expiring_warning = next(item for item in expiring if item["type"] == "certificate_expiring_soon")
|
||||
assert expiring_warning["evidence"][0] == "soon.example.com"
|
||||
assert expiring_warning["evidence"][1] == "30 days"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_npm_reuses_read_results_within_connector_instance(monkeypatch):
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
|
||||
previous = _set_npm_config()
|
||||
calls = []
|
||||
|
||||
def request(self, method, path, **kwargs):
|
||||
calls.append(path)
|
||||
return {
|
||||
"/nginx/proxy-hosts": _proxy_hosts(),
|
||||
"/nginx/certificates": _certificates(),
|
||||
"/nginx/access-lists": _access_lists(),
|
||||
}[path]
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, "_login", lambda self: "runtime-token", raising=True)
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, "_request", request, raising=True)
|
||||
|
||||
connector = NginxProxyManagerConnector()
|
||||
summary = connector.exposure_summary()
|
||||
hosts = connector.proxy_hosts()
|
||||
warnings = connector.security_warnings()
|
||||
|
||||
assert summary["success"] is True
|
||||
assert hosts["total"] == 2
|
||||
assert warnings["total"] >= 1
|
||||
assert calls.count("/nginx/proxy-hosts") == 1
|
||||
assert calls.count("/nginx/certificates") == 1
|
||||
assert calls.count("/nginx/access-lists") == 1
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_npm_routes_are_read_only_and_shape_is_consistent(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
previous = _set_npm_config()
|
||||
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from backend.app.main import app
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, "_login", lambda self: "runtime-token", raising=True)
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
"_request",
|
||||
lambda self, method, path, **kwargs: {
|
||||
"/nginx/proxy-hosts": _proxy_hosts(),
|
||||
"/nginx/certificates": _certificates(),
|
||||
"/nginx/access-lists": _access_lists(),
|
||||
}[path],
|
||||
raising=True,
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
for path in [
|
||||
"/api/apps/nginx-proxy-manager/health",
|
||||
"/api/apps/nginx-proxy-manager/proxy-hosts",
|
||||
"/api/apps/nginx-proxy-manager/certificates",
|
||||
"/api/apps/nginx-proxy-manager/certificates/expiring",
|
||||
"/api/apps/nginx-proxy-manager/access-lists",
|
||||
"/api/apps/nginx-proxy-manager/exposure-summary",
|
||||
"/api/apps/nginx-proxy-manager/host-reachability",
|
||||
"/api/apps/nginx-proxy-manager/security-warnings",
|
||||
"/api/apps/npm/health",
|
||||
]:
|
||||
response = client.get(path)
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert {"success", "status", "message", "data"}.issubset(payload.keys())
|
||||
assert client.post("/api/apps/nginx-proxy-manager/proxy-hosts").status_code == 405
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_network_exposure_uses_npm_metadata_without_demo_hosts(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
previous = _set_npm_config()
|
||||
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from backend.app.main import app
|
||||
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, "_login", lambda self: "runtime-token", raising=True)
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
"_request",
|
||||
lambda self, method, path, **kwargs: {
|
||||
"/nginx/proxy-hosts": _proxy_hosts(),
|
||||
"/nginx/certificates": _certificates(),
|
||||
"/nginx/access-lists": _access_lists(),
|
||||
}[path],
|
||||
raising=True,
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
payload = client.get("/api/network/exposure").json()
|
||||
|
||||
assert [item["host"] for item in payload] == ["media.example.com", "open.example.com"]
|
||||
assert all(item["source"] == "reverse_proxy" for item in payload)
|
||||
assert "unraid.example.net" not in str(payload)
|
||||
assert payload[0]["protected"] is True
|
||||
assert payload[0]["risk"] == "low"
|
||||
assert payload[1]["protected"] is False
|
||||
assert payload[1]["risk"] == "high"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_network_exposure_uses_authentik_coverage_for_protection(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
previous = _set_npm_config()
|
||||
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from backend.app.main import app
|
||||
|
||||
hosts = [
|
||||
{
|
||||
"id": 10,
|
||||
"domain_names": ["protected.example.com"],
|
||||
"forward_host": "protected-app",
|
||||
"forward_port": 8080,
|
||||
"forward_scheme": "http",
|
||||
"enabled": True,
|
||||
"ssl_forced": True,
|
||||
"hsts_enabled": True,
|
||||
"certificate_id": 10,
|
||||
"access_list_id": 0,
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"domain_names": ["open.example.com"],
|
||||
"forward_host": "open-app",
|
||||
"forward_port": 8081,
|
||||
"forward_scheme": "http",
|
||||
"enabled": True,
|
||||
"ssl_forced": True,
|
||||
"hsts_enabled": True,
|
||||
"certificate_id": 10,
|
||||
"access_list_id": 0,
|
||||
},
|
||||
]
|
||||
monkeypatch.setattr(NginxProxyManagerConnector, "_login", lambda self: "runtime-token", raising=True)
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
"_request",
|
||||
lambda self, method, path, **kwargs: {
|
||||
"/nginx/proxy-hosts": hosts,
|
||||
"/nginx/certificates": _certificates(),
|
||||
"/nginx/access-lists": [],
|
||||
}[path],
|
||||
raising=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
AuthentikConnector,
|
||||
"auth_coverage",
|
||||
lambda self: {
|
||||
"success": True,
|
||||
"status": "ok",
|
||||
"publicHosts": [
|
||||
{"hostId": 10, "coverageStatus": "protected", "authentikMatches": ["protected.example.com"]},
|
||||
{"hostId": 11, "coverageStatus": "unprotected", "authentikMatches": []},
|
||||
],
|
||||
},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
payload = client.get("/api/network/exposure").json()
|
||||
|
||||
protected = next(item for item in payload if item["host"] == "protected.example.com")
|
||||
open_host = next(item for item in payload if item["host"] == "open.example.com")
|
||||
assert protected["protected"] is True
|
||||
assert protected["protectionSource"] == "authentik"
|
||||
assert protected["authCoverageStatus"] == "protected"
|
||||
assert protected["risk"] == "low"
|
||||
assert open_host["protected"] is False
|
||||
assert open_host["protectionSource"] == "none"
|
||||
assert open_host["risk"] == "high"
|
||||
assert "No NPM access list or Authentik match" in open_host["riskReasons"]
|
||||
assert open_host["externalChangeRequired"] is True
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_network_exposure_reads_npm_and_authentik_in_parallel(tmp_path, monkeypatch):
|
||||
from threading import Barrier
|
||||
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
previous = _set_npm_config()
|
||||
|
||||
from backend.app.connectors.authentik import AuthentikConnector
|
||||
from backend.app.connectors.nginx_proxy_manager import NginxProxyManagerConnector
|
||||
from backend.app.main import app
|
||||
|
||||
barrier = Barrier(3, timeout=1)
|
||||
|
||||
def wait_and_return(payload):
|
||||
def _inner(self):
|
||||
barrier.wait()
|
||||
return payload
|
||||
|
||||
return _inner
|
||||
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
"proxy_hosts",
|
||||
wait_and_return({"success": True, "status": "ok", "items": [{"id": 10, "domainNames": ["protected.example.com"], "forwardPort": 8080, "forwardScheme": "http", "enabled": True, "sslForced": True, "certificateId": 10, "accessListId": 0}]}),
|
||||
raising=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
NginxProxyManagerConnector,
|
||||
"certificates",
|
||||
wait_and_return({"success": True, "status": "ok", "items": [{"id": 10, "isExpired": False, "isExpiringSoon": False}]}),
|
||||
raising=True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
AuthentikConnector,
|
||||
"auth_coverage",
|
||||
wait_and_return({"success": True, "status": "ok", "publicHosts": [{"hostId": 10, "coverageStatus": "protected", "authentikMatches": ["protected.example.com"]}]}),
|
||||
raising=True,
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
payload = client.get("/api/network/exposure").json()
|
||||
|
||||
assert payload[0]["protected"] is True
|
||||
assert payload[0]["protectionSource"] == "authentik"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_network_exposure_reuses_persistent_cache_after_live_context_loss(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
previous = _set_npm_config()
|
||||
|
||||
from backend.app.api import routes_observability
|
||||
from backend.app.main import app
|
||||
|
||||
monkeypatch.setattr(
|
||||
routes_observability,
|
||||
"_build_network_exposure",
|
||||
lambda: [
|
||||
{
|
||||
"source": "reverse_proxy",
|
||||
"host": "cached.example.com",
|
||||
"port": 443,
|
||||
"protocol": "https",
|
||||
"protected": True,
|
||||
"protectionSource": "authentik",
|
||||
"authCoverageStatus": "protected",
|
||||
"authentikMatches": ["cached.example.com"],
|
||||
"appId": "plex",
|
||||
"risk": "low",
|
||||
}
|
||||
],
|
||||
raising=True,
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
first = client.get("/api/network/exposure").json()
|
||||
|
||||
def fail_live_build():
|
||||
raise AssertionError("persistent network exposure cache should avoid live connector reads")
|
||||
|
||||
monkeypatch.setattr(routes_observability, "_build_network_exposure", fail_live_build, raising=True)
|
||||
|
||||
with TestClient(app) as client:
|
||||
second = client.get("/api/network/exposure").json()
|
||||
|
||||
assert second == first
|
||||
assert second[0]["host"] == "cached.example.com"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_network_exposure_missing_npm_config_does_not_infer_public_hosts(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
previous = _set_npm_config(url=None, identity=None, secret=None)
|
||||
|
||||
from backend.app.main import app
|
||||
|
||||
with TestClient(app) as client:
|
||||
payload = client.get("/api/network/exposure").json()
|
||||
|
||||
assert payload == []
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_npm_policy_blocks_mutations_and_unknowns_fail_closed(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
|
||||
from backend.app.main import app
|
||||
|
||||
with TestClient(app) as client:
|
||||
for action in [
|
||||
"npm.delete_proxy_host",
|
||||
"npm.update_proxy_host",
|
||||
"npm.renew_certificate",
|
||||
"npm.modify_access_list",
|
||||
"npm.create_user",
|
||||
"npm.mass_edit",
|
||||
"npm.unknown_mutation",
|
||||
]:
|
||||
payload = client.post("/api/policies/evaluate", json={"action": action, "target": "nginx_proxy_manager", "source": "manual"}).json()
|
||||
assert payload["decision"] == "blocked"
|
||||
read = client.post("/api/policies/evaluate", json={"action": "npm.exposure_summary", "target": "nginx_proxy_manager", "source": "voice"}).json()
|
||||
assert read["decision"] == "allowed"
|
||||
|
||||
|
||||
def test_npm_inventory_classification_and_candidate(tmp_path, monkeypatch):
|
||||
_reset_db(tmp_path, monkeypatch)
|
||||
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.main import app
|
||||
from backend.app.services.app_classifier import classify_container
|
||||
|
||||
for key in list(settings.apis):
|
||||
monkeypatch.delitem(settings.apis, key)
|
||||
|
||||
result = classify_container({"name": "nginx-proxy-manager", "image": "jc21/nginx-proxy-manager:latest", "ports": ["81:81"]})
|
||||
|
||||
assert result["app_type"] == "nginx_proxy_manager"
|
||||
assert result["category"] == "network"
|
||||
assert result["subcategory"] == "reverse_proxy"
|
||||
assert result["connector"] == "nginx_proxy_manager"
|
||||
assert result["criticality"] == "critical"
|
||||
assert result["protected"] is True
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.test_connection", lambda self: {"status": "connected"}, raising=True)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.portainer.PortainerConnector.list_containers",
|
||||
lambda self: {"status": "ok", "containers": [{"id": "npm-container", "name": "nginx-proxy-manager", "image": "jc21/nginx-proxy-manager", "state": "running", "status": "running", "ports": ["81:81"]}]},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
client.post("/api/inventory/discover")
|
||||
candidates = client.get("/api/connectors/nginx_proxy_manager/candidates").json()
|
||||
|
||||
assert len(candidates) == 1
|
||||
assert candidates[0]["connectorId"] == "nginx_proxy_manager"
|
||||
assert candidates[0]["appType"] == "nginx_proxy_manager"
|
||||
assert candidates[0]["missingSecrets"] == ["NPM_URL", "NPM_IDENTITY", "NPM_SECRET"]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,35 @@
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
|
||||
def test_safety_signature_ignores_identical_discovery_history(tmp_path):
|
||||
from backend.app.db.models import DiscoveredContainer, DiscoveryRun
|
||||
from backend.app.services import inventory_discovery, safety_center_service
|
||||
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'safety-signature.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
with Session(engine) as session:
|
||||
session.add(DiscoveryRun(id="disc-1", status="success", containers_found=1))
|
||||
session.add(DiscoveredContainer(
|
||||
id="dc-1", discovery_run_id="disc-1", container_id="container-1",
|
||||
name="AppOps", image="appops:test", state="running", status="running",
|
||||
import_status="managed", matched_app_id="appops",
|
||||
))
|
||||
session.commit()
|
||||
inventory_discovery.invalidate_discovery_cache()
|
||||
with Session(engine) as session:
|
||||
first = safety_center_service._source_signature(session)
|
||||
|
||||
with Session(engine) as session:
|
||||
session.add(DiscoveryRun(id="disc-2", status="success", containers_found=1))
|
||||
session.add(DiscoveredContainer(
|
||||
id="dc-2", discovery_run_id="disc-2", container_id="container-1",
|
||||
name="AppOps", image="appops:test", state="running", status="running",
|
||||
import_status="managed", matched_app_id="appops",
|
||||
))
|
||||
session.commit()
|
||||
inventory_discovery.invalidate_discovery_cache()
|
||||
with Session(engine) as session:
|
||||
second = safety_center_service._source_signature(session)
|
||||
|
||||
assert second == first
|
||||
@@ -0,0 +1,345 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "plex_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
from backend.app.main import app
|
||||
|
||||
previous_apis = dict(settings.apis)
|
||||
for key in ["PLEX_URL", "PLEX_TOKEN", "PLEX_VERIFY_TLS", "PLEX_TIMEOUT_SECONDS", "PLEX_DEFAULT_LIBRARY_ID", "PLEX_MACHINE_IDENTIFIER"]:
|
||||
settings.apis[key] = None
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
settings.apis = previous_apis
|
||||
|
||||
|
||||
def _set_plex_config(url: str | None = "http://plex:32400", token: str | None = "PLEX_SECRET_TOKEN"):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = dict(settings.apis)
|
||||
settings.apis["PLEX_URL"] = url
|
||||
settings.apis["PLEX_TOKEN"] = token
|
||||
settings.apis["PLEX_VERIFY_TLS"] = False
|
||||
settings.apis["PLEX_TIMEOUT_SECONDS"] = "10"
|
||||
settings.apis["PLEX_DEFAULT_LIBRARY_ID"] = None
|
||||
settings.apis["PLEX_MACHINE_IDENTIFIER"] = None
|
||||
return previous
|
||||
|
||||
|
||||
def _restore_config(previous):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous)
|
||||
|
||||
|
||||
def _server_payload():
|
||||
return {
|
||||
"MediaContainer": {
|
||||
"friendlyName": "Widefrog Plex",
|
||||
"machineIdentifier": "machine-123456789",
|
||||
"version": "1.40.2",
|
||||
"platform": "Linux",
|
||||
"updatedAt": 1700000000,
|
||||
"myPlex": True,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _libraries_payload():
|
||||
return {
|
||||
"MediaContainer": {
|
||||
"Directory": [
|
||||
{"key": "1", "title": "Movies", "type": "movie", "agent": "tv.plex.agents.movie", "scanner": "Plex Movie", "Location": [{"path": "/srv/example/media/movies"}]},
|
||||
{"key": "2", "title": "Shows", "type": "show", "agent": "tv.plex.agents.series", "scanner": "Plex Series", "Location": [{"path": "/srv/example/media/tv"}]},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _sessions_payload():
|
||||
return {
|
||||
"MediaContainer": {
|
||||
"size": 2,
|
||||
"Metadata": [
|
||||
{
|
||||
"sessionKey": "11",
|
||||
"title": "Direct Movie",
|
||||
"type": "movie",
|
||||
"User": {"title": "Jens"},
|
||||
"Player": {"title": "Apple TV", "state": "playing", "product": "Plex", "platform": "tvOS", "address": "198.18.10.55"},
|
||||
"Session": {"bandwidth": 12000},
|
||||
"Media": [{"Part": [{"decision": "directplay"}], "videoDecision": "directplay", "audioDecision": "directplay"}],
|
||||
"viewOffset": 120000,
|
||||
"duration": 240000,
|
||||
},
|
||||
{
|
||||
"sessionKey": "12",
|
||||
"title": "Transcoded Episode",
|
||||
"type": "episode",
|
||||
"User": {"title": "Remote"},
|
||||
"Player": {"title": "Chrome", "state": "playing", "product": "Plex Web", "platform": "Windows", "address": "203.0.113.9"},
|
||||
"Session": {"bandwidth": 4500},
|
||||
"TranscodeSession": {"videoDecision": "transcode", "audioDecision": "copy"},
|
||||
"Media": [{"videoDecision": "transcode", "audioDecision": "copy"}],
|
||||
"viewOffset": 50000,
|
||||
"duration": 100000,
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _recent_payload():
|
||||
return {
|
||||
"MediaContainer": {
|
||||
"Metadata": [
|
||||
{"title": "Arrival", "type": "movie", "year": 2016, "librarySectionTitle": "Movies", "addedAt": 1700000000, "thumb": "/library/metadata/1/thumb", "summary": "A quiet first contact story."},
|
||||
{"title": "Pilot", "grandparentTitle": "Example Show", "type": "episode", "year": 2024, "librarySectionTitle": "Shows", "addedAt": 1700000500},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_plex_connector_missing_config_reports_required_env(api_client):
|
||||
response = api_client.post("/api/connectors/plex/test")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "missing_config"
|
||||
assert payload["missing_config"] is True
|
||||
assert payload["required_env_vars"] == ["PLEX_URL", "PLEX_TOKEN"]
|
||||
assert payload["secretRef"] == "PLEX_TOKEN"
|
||||
|
||||
|
||||
def test_plex_health_success_normalizes_response(monkeypatch):
|
||||
from backend.app.connectors.plex import PlexConnector
|
||||
|
||||
previous = _set_plex_config()
|
||||
monkeypatch.setattr(PlexConnector, "_request", lambda self, path, params=None: _server_payload(), raising=True)
|
||||
|
||||
payload = PlexConnector().health()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "connected"
|
||||
assert payload["friendlyName"] == "Widefrog Plex"
|
||||
assert payload["version"] == "1.40.2"
|
||||
assert payload["machineIdentifier"] == "machine-123456789"
|
||||
assert payload["url"] == "http://plex:32400"
|
||||
assert "PLEX_SECRET_TOKEN" not in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_plex_auth_failure_redacts_token(monkeypatch):
|
||||
from backend.app.connectors.plex import PlexConnector
|
||||
|
||||
previous = _set_plex_config()
|
||||
|
||||
def fail_request(self, path, params=None):
|
||||
raise RuntimeError("auth_failed token=PLEX_SECRET_TOKEN")
|
||||
|
||||
monkeypatch.setattr(PlexConnector, "_request", fail_request, raising=True)
|
||||
|
||||
payload = PlexConnector().health()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "auth_failed"
|
||||
assert "PLEX_SECRET_TOKEN" not in str(payload)
|
||||
assert "[REDACTED]" in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_plex_library_status_normalizes_sections(monkeypatch):
|
||||
from backend.app.connectors.plex import PlexConnector
|
||||
|
||||
previous = _set_plex_config()
|
||||
|
||||
def fake_request(self, path, params=None):
|
||||
if path == "/library/sections":
|
||||
return _libraries_payload()
|
||||
if path == "/library/sections/1/all":
|
||||
return {"MediaContainer": {"size": 14}}
|
||||
if path == "/library/sections/2/all":
|
||||
return {"MediaContainer": {"size": 8}}
|
||||
return _server_payload()
|
||||
|
||||
monkeypatch.setattr(PlexConnector, "_request", fake_request, raising=True)
|
||||
|
||||
payload = PlexConnector().library_status()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["total"] == 2
|
||||
assert payload["items"][0]["id"] == "1"
|
||||
assert payload["items"][0]["type"] == "movie"
|
||||
assert payload["items"][0]["count"] == 14
|
||||
assert payload["items"][0]["locations"] == [".../movies"]
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_plex_sessions_and_stream_summary(monkeypatch):
|
||||
from backend.app.connectors.plex import PlexConnector
|
||||
|
||||
previous = _set_plex_config()
|
||||
monkeypatch.setattr(PlexConnector, "_request", lambda self, path, params=None: _sessions_payload(), raising=True)
|
||||
|
||||
sessions = PlexConnector().active_sessions()
|
||||
summary = PlexConnector().stream_summary()
|
||||
|
||||
assert sessions["success"] is True
|
||||
assert sessions["total"] == 2
|
||||
assert sessions["items"][0]["user"] == "Jens"
|
||||
assert sessions["items"][0]["ip"] == "192.168.10.x"
|
||||
assert sessions["items"][1]["transcodeDecision"] == "transcode"
|
||||
assert summary["activeStreams"] == 2
|
||||
assert summary["directPlay"] == 1
|
||||
assert summary["transcoding"] == 1
|
||||
assert summary["users"] == 2
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_plex_recent_media_normalizes_items(monkeypatch):
|
||||
from backend.app.connectors.plex import PlexConnector
|
||||
|
||||
previous = _set_plex_config()
|
||||
monkeypatch.setattr(PlexConnector, "_request", lambda self, path, params=None: _recent_payload(), raising=True)
|
||||
|
||||
payload = PlexConnector().recent_media()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["total"] == 2
|
||||
assert payload["items"][0]["title"] == "Arrival"
|
||||
assert payload["items"][1]["title"] == "Example Show - Pilot"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_plex_scan_library_allowed_creates_job_and_audit(api_client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.plex.PlexConnector.scan_library",
|
||||
lambda self, library_id: {"success": True, "status": "ok", "message": "scan_started", "data": {"libraryId": library_id}},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
response = api_client.post("/api/apps/plex/libraries/1/scan", json={"source": "manual"})
|
||||
payload = response.json()
|
||||
|
||||
assert response.status_code == 200
|
||||
assert payload["success"] is True
|
||||
assert payload["decision"] == "allowed"
|
||||
assert payload["jobId"].startswith("job-")
|
||||
assert payload["auditId"].startswith("audit-")
|
||||
assert payload["data"]["libraryId"] == "1"
|
||||
|
||||
|
||||
def test_plex_scan_library_voice_requires_approval_without_connector_call(api_client, monkeypatch):
|
||||
def fail_scan(*_args, **_kwargs):
|
||||
raise AssertionError("connector should not run before approval")
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.plex.PlexConnector.scan_library", fail_scan, raising=True)
|
||||
|
||||
payload = api_client.post("/api/apps/plex/libraries/1/scan", json={"source": "voice"}).json()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["decision"] == "approval_required"
|
||||
assert payload["approvalId"].startswith("appr-")
|
||||
assert payload["auditId"].startswith("audit-")
|
||||
|
||||
|
||||
def test_plex_scan_all_requires_approval_or_blocks_without_connector_call(api_client, monkeypatch):
|
||||
def fail_scan_all(*_args, **_kwargs):
|
||||
raise AssertionError("scan all should wait for approval")
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.plex.PlexConnector.scan_all_libraries", fail_scan_all, raising=True)
|
||||
|
||||
manual = api_client.post("/api/apps/plex/libraries/scan-all", json={"source": "manual"}).json()
|
||||
voice = api_client.post("/api/apps/plex/libraries/scan-all", json={"source": "voice"}).json()
|
||||
|
||||
assert manual["decision"] == "approval_required"
|
||||
assert voice["decision"] == "blocked"
|
||||
|
||||
|
||||
def test_plex_scan_all_executes_after_approval(api_client, monkeypatch):
|
||||
calls = []
|
||||
|
||||
def scan_all(_self):
|
||||
calls.append('scan')
|
||||
return {'success': True, 'status': 'accepted', 'message': 'scan_all_started'}
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.plex.PlexConnector.scan_all_libraries", scan_all, raising=True)
|
||||
|
||||
requested = api_client.post("/api/apps/plex/libraries/scan-all", json={"source": "manual"}).json()
|
||||
api_client.patch(f"/api/approvals/{requested['approvalId']}", json={"status": "approved"})
|
||||
executed = api_client.post(f"/api/approvals/{requested['approvalId']}/execute").json()
|
||||
|
||||
assert executed['success'] is True
|
||||
assert executed['approvalStatus'] == 'executed'
|
||||
assert calls == ['scan']
|
||||
|
||||
|
||||
def test_plex_destructive_actions_are_blocked(api_client):
|
||||
for action in [
|
||||
"plex.delete_media",
|
||||
"plex.delete_library",
|
||||
"plex.empty_trash",
|
||||
"plex.clean_bundles",
|
||||
"plex.optimize_database",
|
||||
"plex.kill_session",
|
||||
"plex.user_management",
|
||||
"plex.force_refresh_all_metadata",
|
||||
"plex.mass_edit",
|
||||
"plex.unknown_mutation",
|
||||
]:
|
||||
payload = api_client.post("/api/policies/evaluate", json={"action": action, "target": "plex", "source": "manual"}).json()
|
||||
assert payload["decision"] == "blocked"
|
||||
|
||||
|
||||
def test_plex_inventory_classification_and_candidate(api_client, monkeypatch):
|
||||
from backend.app.services.app_classifier import classify_container
|
||||
|
||||
result = classify_container({"name": "binhex-plexpass", "image": "binhex/arch-plexpass:latest", "ports": ["32400:32400"]})
|
||||
|
||||
assert result["app_type"] == "plex"
|
||||
assert result["category"] == "media"
|
||||
assert result["subcategory"] == "media_server"
|
||||
assert result["connector"] == "plex"
|
||||
assert result["policy_profile"] == "media_server"
|
||||
assert result["healthcheck_suggestion"] == {"port": 32400}
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.test_connection", lambda self: {"status": "connected"}, raising=True)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.portainer.PortainerConnector.list_containers",
|
||||
lambda self: {"status": "ok", "containers": [{"id": "plex-container", "name": "binhex-plexpass", "image": "binhex/arch-plexpass", "state": "running", "status": "running", "ports": ["32400:32400"]}]},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
api_client.post("/api/inventory/discover")
|
||||
candidates = api_client.get("/api/connectors/plex/candidates").json()
|
||||
|
||||
assert len(candidates) == 1
|
||||
assert candidates[0]["connectorId"] == "plex"
|
||||
assert candidates[0]["appType"] == "plex"
|
||||
assert candidates[0]["missingSecrets"] == ["PLEX_URL", "PLEX_TOKEN"]
|
||||
@@ -0,0 +1,41 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from backend.app.connectors.portainer import PortainerConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
|
||||
def test_restart_request_allows_docker_grace_period_without_shortening_container_timeout(monkeypatch):
|
||||
previous = dict(settings.apis)
|
||||
settings.apis.update({
|
||||
"PORTAINER_URL": "https://portainer.example.test",
|
||||
"PORTAINER_API_KEY": "test-key",
|
||||
"PORTAINER_ENDPOINT_ID": "9",
|
||||
})
|
||||
connector = PortainerConnector()
|
||||
monkeypatch.setattr(
|
||||
connector,
|
||||
"list_containers",
|
||||
lambda: {
|
||||
"status": "ok",
|
||||
"containers": [{"id": "runner-id", "name": "itworx-validation-runner-2", "appId": "itworx-validation-runner-2"}],
|
||||
},
|
||||
)
|
||||
calls: list[tuple[str, str, dict]] = []
|
||||
|
||||
def request(method: str, path: str, **kwargs):
|
||||
calls.append((method, path, kwargs))
|
||||
return SimpleNamespace(status_code=204)
|
||||
|
||||
monkeypatch.setattr(connector, "_request", request)
|
||||
try:
|
||||
result = connector.restart_container("itworx-validation-runner-2")
|
||||
finally:
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert calls == [(
|
||||
"POST",
|
||||
"/api/endpoints/9/docker/containers/runner-id/restart",
|
||||
{"timeout": 30.0},
|
||||
)]
|
||||
@@ -0,0 +1,156 @@
|
||||
from backend.app.services.action_execution import approval_execution_supported
|
||||
from backend.app.services.action_registry import ACTION_RISK, ACTIONS, action_default_target
|
||||
from backend.app.services.portfolio_deployment_enrollment import (
|
||||
_ENROLL_SCRIPT,
|
||||
_INSPECT_SCRIPT,
|
||||
_LEGACY_REVISION,
|
||||
_LEGACY_SINGLE_PROJECT,
|
||||
_PROJECT,
|
||||
_RECOVERY_FAILED_REVISION,
|
||||
_RECOVERY_FAILURE_MESSAGE,
|
||||
_RECOVERY_LIVE_REVISION,
|
||||
_REPOSITORY,
|
||||
PortfolioDeploymentEnrollment,
|
||||
)
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, *, payload=None, text=""):
|
||||
self._payload = payload
|
||||
self.text = text
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class _InspectPortainer:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def _request(self, method, path, **kwargs):
|
||||
self.calls.append((method, path, kwargs))
|
||||
if path.endswith("/exec"):
|
||||
return _Response(payload={"Id": "exec-safe-inspect"})
|
||||
if path.endswith("/start"):
|
||||
return _Response(
|
||||
text='{"branch":"main","compose_files":["docker-compose.yml"],"containers":["Portfolio","Portfolio-api"],"present":true}'
|
||||
)
|
||||
raise AssertionError(f"Unexpected Portainer path: {path}")
|
||||
|
||||
|
||||
def test_portfolio_enrollment_contract_is_fixed_and_paired():
|
||||
assert _REPOSITORY == "jens/portfolio"
|
||||
assert _LEGACY_REVISION == "e59bfb3484531883e5486ee9e10b663ed0f6764f"
|
||||
assert _PROJECT["branch"] == "main"
|
||||
assert _PROJECT["exact_source"] is True
|
||||
assert _PROJECT["env_source"] == "/srv/example/appdata/Portfolio/.env"
|
||||
assert _PROJECT["compose_files"] == ["docker-compose.yml"]
|
||||
assert _PROJECT["compose_project"] == "portfolio"
|
||||
assert _PROJECT["containers"] == ["Portfolio", "Portfolio-api"]
|
||||
assert _PROJECT["verify_oci_revision"] is False
|
||||
assert _PROJECT["environment"] == {
|
||||
"GIT_COMMIT": "{revision}",
|
||||
"BUILD_ID": "build-{short_revision}",
|
||||
}
|
||||
|
||||
|
||||
def test_portfolio_enrollment_registry_alias_matches_controller_cache_bind_source():
|
||||
assert 'registry_path = Path("/opt/gitea-deploy/projects.json")' in _ENROLL_SCRIPT
|
||||
assert (
|
||||
'host_registry_path = Path("/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json")'
|
||||
in _ENROLL_SCRIPT
|
||||
)
|
||||
assert "os.path.samefile(registry_path, host_registry_path)" in _ENROLL_SCRIPT
|
||||
assert '/srv/example/appdata/gitea-deploy-runners/deploy.lock' not in _ENROLL_SCRIPT
|
||||
assert '/mnt/cache/appdata/gitea-deploy-runners/deploy.lock' in _ENROLL_SCRIPT
|
||||
|
||||
|
||||
def test_portfolio_enrollment_uses_project_then_global_lock_order_without_child_deadlock():
|
||||
project_lock = "fcntl.flock(project_lock, fcntl.LOCK_EX)"
|
||||
cache_lock = "fcntl.flock(lock, fcntl.LOCK_EX)"
|
||||
|
||||
assert _ENROLL_SCRIPT.index(project_lock) < _ENROLL_SCRIPT.index(cache_lock)
|
||||
assert "controller.bootstrap_legacy_state(repository, project, legacy_revision)" in _ENROLL_SCRIPT
|
||||
assert '["python3", "/opt/gitea-deploy/deploy.py", "bootstrap-legacy"' not in _ENROLL_SCRIPT
|
||||
|
||||
|
||||
def test_portfolio_enrollment_only_migrates_known_legacy_single_target():
|
||||
assert _LEGACY_SINGLE_PROJECT["containers"] == ["portfolio"]
|
||||
assert _LEGACY_SINGLE_PROJECT["compose_project"] == "portfolio"
|
||||
assert _LEGACY_SINGLE_PROJECT["compose_files"] == ["docker-compose.yml"]
|
||||
assert "env_source" not in _LEGACY_SINGLE_PROJECT
|
||||
|
||||
|
||||
def test_portfolio_target_registration_is_a_typed_high_risk_action():
|
||||
action = "deployment.target.register_portfolio"
|
||||
assert action in ACTIONS
|
||||
assert ACTION_RISK[action] == "high"
|
||||
assert action_default_target(action) == "jens/portfolio"
|
||||
assert approval_execution_supported(action) is True
|
||||
|
||||
|
||||
def test_portfolio_target_inspection_returns_only_sanitized_json_from_controller():
|
||||
portainer = _InspectPortainer()
|
||||
service = PortfolioDeploymentEnrollment(portainer=portainer)
|
||||
|
||||
result = service._inspect_current_target(endpoint="1", container_id="controller-id")
|
||||
|
||||
assert result == {
|
||||
"branch": "main",
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"containers": ["Portfolio", "Portfolio-api"],
|
||||
"present": True,
|
||||
}
|
||||
assert portainer.calls[0][2]["json"]["AttachStdout"] is True
|
||||
assert portainer.calls[0][2]["json"]["Tty"] is True
|
||||
assert portainer.calls[1][2]["json"] == {"Detach": False, "Tty": True}
|
||||
|
||||
|
||||
def test_portfolio_target_inspection_exposes_only_bounded_state_diagnostics():
|
||||
assert (
|
||||
'state_path = Path("/mnt/cache/appdata/gitea-deploy-runners/state/portfolio.json")'
|
||||
in _INSPECT_SCRIPT
|
||||
)
|
||||
for allowed in (
|
||||
'"schema_version"',
|
||||
'"live_revision"',
|
||||
'"live_status"',
|
||||
'"live_legacy_provenance"',
|
||||
'"last_attempt"',
|
||||
'"live_artifact_containers"',
|
||||
):
|
||||
assert allowed in _INSPECT_SCRIPT
|
||||
assert 'safe_attempt["message"]' in _INSPECT_SCRIPT
|
||||
assert "[:768]" in _INSPECT_SCRIPT
|
||||
assert 'state.get("live_receipt")' not in _INSPECT_SCRIPT
|
||||
assert 'state.get("rollback_snapshot")' not in _INSPECT_SCRIPT
|
||||
|
||||
|
||||
def test_portfolio_runtime_reconciliation_is_pinned_to_observed_failure():
|
||||
assert _RECOVERY_LIVE_REVISION == "52c6f2fc74a1e39ad288f1c5e987a9cd5b072d3e"
|
||||
assert _RECOVERY_FAILED_REVISION == "b7955d90b63860ffff8e0bd84be123aaba632043"
|
||||
assert (
|
||||
_RECOVERY_FAILURE_MESSAGE
|
||||
== "Existing production containers require a valid healthy live deployment state"
|
||||
)
|
||||
assert 'existing_state.get("live_status") == "unknown"' in _ENROLL_SCRIPT
|
||||
assert 'existing_state.get("live_legacy_provenance") is False' in _ENROLL_SCRIPT
|
||||
assert 'last_attempt.get("revision") == recovery_failed_revision' in _ENROLL_SCRIPT
|
||||
assert 'last_attempt.get("message") == recovery_failure_message' in _ENROLL_SCRIPT
|
||||
assert 'isinstance(live_receipt, dict)' in _ENROLL_SCRIPT
|
||||
assert 'bool(live_receipt)' in _ENROLL_SCRIPT
|
||||
assert 'set(artifact_containers) == set(project["containers"])' in _ENROLL_SCRIPT
|
||||
|
||||
|
||||
def test_portfolio_runtime_reconciliation_proves_exact_recorded_artifacts():
|
||||
assert 'revision_label != recovery_live_revision' in _ENROLL_SCRIPT
|
||||
assert 'build_label != f"build-{recovery_live_revision[:12]}"' in _ENROLL_SCRIPT
|
||||
assert 'artifact.get("image_id")' in _ENROLL_SCRIPT
|
||||
assert 'artifact.get("container_id")' in _ENROLL_SCRIPT
|
||||
assert 'str(container.get("Image") or "")' in _ENROLL_SCRIPT
|
||||
assert 'str(container.get("Id") or "")' in _ENROLL_SCRIPT
|
||||
assert 'reconciled_state = dict(existing_state)' in _ENROLL_SCRIPT
|
||||
assert 'reconciled_state["live_status"] = "healthy"' in _ENROLL_SCRIPT
|
||||
assert 'reconciled_state["status"] = "success"' in _ENROLL_SCRIPT
|
||||
assert 'bool(verify.get("live_receipt"))' in _ENROLL_SCRIPT
|
||||
assert "runtime-state-reconciled" in _ENROLL_SCRIPT
|
||||
@@ -0,0 +1,62 @@
|
||||
from pathlib import Path
|
||||
|
||||
from backend.app.services.portfolio_deployment_recovery import (
|
||||
_FAILED_REVISION,
|
||||
_FAILURE_MESSAGE,
|
||||
_LIVE_REVISION,
|
||||
_REATTEST_SCRIPT,
|
||||
_REPOSITORY,
|
||||
)
|
||||
|
||||
|
||||
def test_portfolio_recovery_is_pinned_to_exact_observed_incident():
|
||||
assert _REPOSITORY == "jens/portfolio"
|
||||
assert _LIVE_REVISION == "52c6f2fc74a1e39ad288f1c5e987a9cd5b072d3e"
|
||||
assert _FAILED_REVISION == "b7955d90b63860ffff8e0bd84be123aaba632043"
|
||||
assert _FAILURE_MESSAGE == "Existing production containers require a valid healthy live deployment state"
|
||||
assert 'main.stdout.strip() != failed_revision' in _REATTEST_SCRIPT
|
||||
assert 'state.get("live_status") == "unknown"' in _REATTEST_SCRIPT
|
||||
assert 'attempt.get("revision") == failed_revision' in _REATTEST_SCRIPT
|
||||
assert 'attempt.get("message") == failure_message' in _REATTEST_SCRIPT
|
||||
assert '/mnt/cache/appdata/gitea-deploy-runners/state' in _REATTEST_SCRIPT
|
||||
assert '/srv/example/appdata/gitea-deploy-runners/deploy.lock' not in _REATTEST_SCRIPT
|
||||
assert '/mnt/cache/appdata/gitea-deploy-runners/deploy.lock' in _REATTEST_SCRIPT
|
||||
|
||||
|
||||
def test_portfolio_recovery_reuses_controller_receipt_verifiers():
|
||||
assert "controller.validate_recorded_receipt" in _REATTEST_SCRIPT
|
||||
assert "controller.verify_deployment" in _REATTEST_SCRIPT
|
||||
assert "expected_image_ids=expected_image_ids" in _REATTEST_SCRIPT
|
||||
assert "controller.deployment_artifacts" in _REATTEST_SCRIPT
|
||||
assert "controller.verified_current_receipt" in _REATTEST_SCRIPT
|
||||
|
||||
|
||||
def test_portfolio_recovery_allows_only_ephemeral_container_identity_change():
|
||||
assert 'if key != "container_id"' in _REATTEST_SCRIPT
|
||||
assert "container_id_changed = False" in _REATTEST_SCRIPT
|
||||
assert "if not container_id_changed" in _REATTEST_SCRIPT
|
||||
assert 'artifacts.get("compose_sha256") != observed_artifacts.get("compose_sha256")' in _REATTEST_SCRIPT
|
||||
assert 'repaired["live_receipt"] = observed_receipt' in _REATTEST_SCRIPT
|
||||
assert 'repaired["live_artifacts"] = observed_artifacts' in _REATTEST_SCRIPT
|
||||
|
||||
|
||||
def test_portfolio_recovery_has_no_application_runtime_mutation_commands():
|
||||
forbidden = (
|
||||
"docker restart",
|
||||
"docker stop",
|
||||
"docker rm",
|
||||
"docker compose up",
|
||||
"docker compose down",
|
||||
"subprocess.run([\"docker\", \"restart\"",
|
||||
"subprocess.run([\"docker\", \"rm\"",
|
||||
)
|
||||
assert all(value not in _REATTEST_SCRIPT for value in forbidden)
|
||||
assert "shutil.copy2(state_path, backup)" in _REATTEST_SCRIPT
|
||||
assert "os.fsync" in _REATTEST_SCRIPT
|
||||
|
||||
|
||||
def test_portfolio_action_execution_accepts_reconciled_status():
|
||||
source = Path("backend/app/services/action_execution.py").read_text(encoding="utf-8")
|
||||
assert "from .portfolio_deployment_recovery import PortfolioDeploymentRecovery" in source
|
||||
assert "PortfolioDeploymentRecovery().run()" in source
|
||||
assert 'status in {"enrolled", "already-enrolled", "reconciled"}' in source
|
||||
@@ -0,0 +1,206 @@
|
||||
import importlib
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from test_privatemuse_f84_adoption import (
|
||||
_install_fixture,
|
||||
_post_adoption_evidence,
|
||||
_run_adoption,
|
||||
adoption,
|
||||
diagnosis,
|
||||
)
|
||||
|
||||
|
||||
def _setup(monkeypatch, tmp_path):
|
||||
module_path = Path(adoption.__file__).with_name("finalize_privatemuse_adoption_health.py")
|
||||
assert module_path.exists(), "The approved baseline needs a verified health finalization command"
|
||||
finalizer = importlib.import_module("finalize_privatemuse_adoption_health")
|
||||
state_path, archive, *_rest, controller = _install_fixture(monkeypatch, tmp_path)
|
||||
state = json.loads(state_path.read_text())
|
||||
state["live_status"] = "unknown"
|
||||
state_path.write_bytes(adoption._json_bytes(state))
|
||||
_run_adoption()
|
||||
state = json.loads(state_path.read_text())
|
||||
state.pop("f84_runtime_adoption")
|
||||
state["message"] = finalizer.FAILURE
|
||||
state["last_attempt"] = {
|
||||
"revision": finalizer.REVISION,
|
||||
"status": "failed",
|
||||
"message": finalizer.FAILURE,
|
||||
"metadata": {"revision": finalizer.REVISION},
|
||||
}
|
||||
state["updated_at_epoch"] += 1
|
||||
state_path.write_bytes(adoption._json_bytes(state))
|
||||
monkeypatch.setattr(finalizer, "JOURNAL", tmp_path / "health-finalization")
|
||||
monkeypatch.setattr(finalizer, "_main_revision", lambda *_: finalizer.REVISION)
|
||||
monkeypatch.setattr(diagnosis, "collect_live_evidence_under_lock", lambda **_: _post_adoption_evidence())
|
||||
return finalizer, state_path, archive, controller
|
||||
|
||||
|
||||
def _run(module):
|
||||
return module.finalize(approval_reference=adoption._APPROVAL_REFERENCE, apply=True)
|
||||
|
||||
|
||||
def test_finalization_restores_usable_health_without_rewriting_failed_attempt(monkeypatch, tmp_path):
|
||||
module, path, archive, _ = _setup(monkeypatch, tmp_path)
|
||||
before = path.read_bytes()
|
||||
old_archive = {p.name: p.read_bytes() for p in archive.iterdir()}
|
||||
result = _run(module)
|
||||
after = json.loads(path.read_bytes())
|
||||
expected = json.loads(before)
|
||||
expected["live_status"] = "healthy"
|
||||
assert after == expected
|
||||
assert result["status"] == "finalized"
|
||||
assert (module.JOURNAL / "before.json").read_bytes() == before
|
||||
assert {p.name: p.read_bytes() for p in archive.iterdir()} == old_archive
|
||||
assert _run(module)["status"] == "already_finalized"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["main", "attempt", "receipt", "configuration", "archive"])
|
||||
def test_unproven_baseline_never_changes_state(monkeypatch, tmp_path, change):
|
||||
module, path, archive, controller = _setup(monkeypatch, tmp_path)
|
||||
if change == "main":
|
||||
monkeypatch.setattr(module, "_main_revision", lambda *_: "f" * 40)
|
||||
elif change == "attempt":
|
||||
state = json.loads(path.read_bytes())
|
||||
state["last_attempt"]["message"] = "A different failure"
|
||||
path.write_bytes(adoption._json_bytes(state))
|
||||
elif change == "receipt":
|
||||
controller.current_receipt["containers"]["privatemuse-backend-1"]["container_id"] = "f" * 64
|
||||
elif change == "configuration":
|
||||
evidence = _post_adoption_evidence()
|
||||
evidence["backend_runtime_mount_network_port_and_env_match_current_compose"] = False
|
||||
monkeypatch.setattr(diagnosis, "collect_live_evidence_under_lock", lambda **_: evidence)
|
||||
else:
|
||||
(archive / "approval-audit.json").write_text("{}")
|
||||
before = path.read_bytes()
|
||||
with pytest.raises(adoption.AdoptionError):
|
||||
_run(module)
|
||||
assert path.read_bytes() == before
|
||||
|
||||
|
||||
def test_failure_after_write_restores_exact_state(monkeypatch, tmp_path):
|
||||
module, path, _, _ = _setup(monkeypatch, tmp_path)
|
||||
before = path.read_bytes()
|
||||
real_verify = module._verify_runtime
|
||||
calls = 0
|
||||
|
||||
def fail_post(*args):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 3:
|
||||
raise adoption.AdoptionError("runtime changed after write")
|
||||
return real_verify(*args)
|
||||
|
||||
monkeypatch.setattr(module, "_verify_runtime", fail_post)
|
||||
with pytest.raises(adoption.AdoptionError, match="runtime changed"):
|
||||
_run(module)
|
||||
assert path.read_bytes() == before
|
||||
|
||||
|
||||
def test_concurrent_state_is_preserved(monkeypatch, tmp_path):
|
||||
module, path, _, _ = _setup(monkeypatch, tmp_path)
|
||||
original_verify = module._verify_runtime
|
||||
calls = 0
|
||||
outsider = b'{"other":"operator state"}\n'
|
||||
|
||||
def change_post(*args):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 3:
|
||||
path.write_bytes(outsider)
|
||||
raise adoption.AdoptionError("concurrent update")
|
||||
return original_verify(*args)
|
||||
|
||||
monkeypatch.setattr(module, "_verify_runtime", change_post)
|
||||
with pytest.raises(adoption.AdoptionError, match="diverged"):
|
||||
_run(module)
|
||||
assert path.read_bytes() == outsider
|
||||
|
||||
|
||||
def test_journal_tampering_and_missing_approval_fail_closed(monkeypatch, tmp_path):
|
||||
module, path, _, _ = _setup(monkeypatch, tmp_path)
|
||||
before = path.read_bytes()
|
||||
with pytest.raises(adoption.AdoptionError):
|
||||
module.finalize(approval_reference="other", apply=True)
|
||||
assert path.read_bytes() == before
|
||||
_run(module)
|
||||
healthy = path.read_bytes()
|
||||
(module.JOURNAL / "after.json").write_text("{}")
|
||||
with pytest.raises(adoption.AdoptionError):
|
||||
_run(module)
|
||||
assert path.read_bytes() == healthy
|
||||
|
||||
|
||||
def test_main_change_after_write_restores_unknown_state(monkeypatch, tmp_path):
|
||||
module, path, _, _ = _setup(monkeypatch, tmp_path)
|
||||
before = path.read_bytes()
|
||||
calls = 0
|
||||
|
||||
def moving_main(*_):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return module.REVISION if calls < 5 else "f" * 40
|
||||
|
||||
monkeypatch.setattr(module, "_main_revision", moving_main)
|
||||
with pytest.raises(adoption.AdoptionError, match="main changed"):
|
||||
_run(module)
|
||||
assert path.read_bytes() == before
|
||||
|
||||
|
||||
def test_canonical_bare_repository_needs_no_origin(monkeypatch, tmp_path):
|
||||
import finalize_privatemuse_adoption_health as module
|
||||
|
||||
repo = tmp_path / "canonical.git"
|
||||
subprocess.run(["git", "init", "--bare", str(repo)], check=True, capture_output=True)
|
||||
tree = subprocess.check_output(["git", f"--git-dir={repo}", "mktree"], input=b"").decode().strip()
|
||||
revision = subprocess.check_output([
|
||||
"git", f"--git-dir={repo}", "-c", "user.name=Test", "-c", "user.email=test@example.invalid",
|
||||
"commit-tree", tree, "-m", "baseline",
|
||||
]).decode().strip()
|
||||
subprocess.run(["git", f"--git-dir={repo}", "update-ref", "refs/heads/main", revision], check=True)
|
||||
monkeypatch.setattr(module, "REVISION", revision)
|
||||
|
||||
class Controller:
|
||||
def repository_path(self, *_):
|
||||
return repo
|
||||
|
||||
assert module._main_revision(Controller(), {}) == revision
|
||||
subprocess.run(["git", f"--git-dir={repo}", "update-ref", "-d", "refs/heads/main"], check=True)
|
||||
with pytest.raises(adoption.AdoptionError):
|
||||
module._main_revision(Controller(), {})
|
||||
|
||||
|
||||
def test_finalized_baseline_passes_real_rollback_capture_gate(monkeypatch, tmp_path):
|
||||
from test_deploy_broker import BROKER_ROOT, _load_module
|
||||
|
||||
deploy = _load_module("privatemuse_health_consumer_test", BROKER_ROOT / "deploy.py")
|
||||
|
||||
module, path, _, fake = _setup(monkeypatch, tmp_path)
|
||||
receipt = fake.current_receipt
|
||||
names = list(receipt["containers"])
|
||||
project = {"slug": "privatemuse", "compose_project": "privatemuse", "containers": names,
|
||||
"_repository_name": "jens/privatemuse"}
|
||||
monkeypatch.setattr(deploy, "read_state", lambda _: json.loads(path.read_bytes()))
|
||||
monkeypatch.setattr(deploy, "compose_project_container_names", lambda _: set(names))
|
||||
monkeypatch.setattr(deploy, "inspect_container", lambda name: {
|
||||
"Id": receipt["containers"][name]["container_id"],
|
||||
"Image": receipt["containers"][name]["image_id"],
|
||||
"Config": {"Image": receipt["containers"][name]["image_ref"],
|
||||
"Labels": {"com.docker.compose.project": "privatemuse"}},
|
||||
"State": {"Running": True},
|
||||
})
|
||||
monkeypatch.setattr(deploy, "inspect_image", lambda _: {"present": True})
|
||||
monkeypatch.setattr(deploy, "verify_deployment", lambda *_, **__: receipt)
|
||||
monkeypatch.setattr(deploy, "run", lambda _: None)
|
||||
monkeypatch.setattr(deploy, "cleanup_rollback_image_tags", lambda *_: None)
|
||||
with pytest.raises(deploy.DeployError, match="valid healthy live deployment state"):
|
||||
deploy.capture_compose_rollback(project)
|
||||
_run(module)
|
||||
snapshot = deploy.capture_compose_rollback(project)
|
||||
assert snapshot["revision"] == diagnosis._LIVE_REVISION
|
||||
assert snapshot["receipt"] == receipt
|
||||
assert snapshot["compose_containers"] == sorted(names)
|
||||
assert json.loads(path.read_bytes())["status"] == "failed"
|
||||
@@ -0,0 +1,265 @@
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services.privatemuse_artifact_installer import (
|
||||
ArtifactInstallError,
|
||||
install,
|
||||
preflight,
|
||||
validate_request,
|
||||
)
|
||||
|
||||
|
||||
def request(tmp_path: Path, data: bytes = b"model"):
|
||||
source = tmp_path / "source.safetensors"
|
||||
source.write_bytes(data)
|
||||
return {
|
||||
"artifact_type": "lora",
|
||||
"source_url": "https://models.example.invalid/model.safetensors",
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"filename": "candidate.safetensors",
|
||||
"source_id": "civitai:model/version",
|
||||
"license": "upstream terms verified",
|
||||
}
|
||||
|
||||
|
||||
def test_validation_is_fail_closed():
|
||||
payload = {
|
||||
"artifact_type": "lora",
|
||||
"source_url": "http://example.test/x",
|
||||
"sha256": "0" * 64,
|
||||
"filename": "x.safetensors",
|
||||
"source_id": "v1",
|
||||
"license": "x",
|
||||
}
|
||||
with pytest.raises(ArtifactInstallError, match="HTTPS"):
|
||||
validate_request(payload)
|
||||
payload["source_url"] = "https://example.test/x"
|
||||
payload["filename"] = "../x.safetensors"
|
||||
with pytest.raises(ArtifactInstallError, match="filename"):
|
||||
validate_request(payload)
|
||||
|
||||
|
||||
def test_installer_hash_pins_and_writes_receipt(tmp_path, monkeypatch):
|
||||
data = b"verified model bytes"
|
||||
payload = request(tmp_path, data)
|
||||
|
||||
class Response:
|
||||
headers = {}
|
||||
|
||||
def __init__(self):
|
||||
self.data = data
|
||||
self.headers = {"Content-Length": str(len(self.data))}
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
def geturl(self):
|
||||
return payload["source_url"]
|
||||
|
||||
def read(self, n):
|
||||
out = self.data[:n]
|
||||
self.data = self.data[n:]
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(
|
||||
"urllib.request.urlopen",
|
||||
lambda *args, **kwargs: Response(),
|
||||
)
|
||||
result = install(
|
||||
payload,
|
||||
model_root=tmp_path / "models",
|
||||
receipt_root=tmp_path / "receipts",
|
||||
)
|
||||
assert result["status"] == "installed"
|
||||
assert (
|
||||
tmp_path / "models" / "loras" / "candidate.safetensors"
|
||||
).read_bytes() == data
|
||||
assert (tmp_path / "receipts" / f'{payload["sha256"]}.json').exists()
|
||||
again = install(payload, model_root=tmp_path / "models", receipt_root=tmp_path / "receipts")
|
||||
assert again["status"] == "already_installed"
|
||||
assert again["receipt"] == result["receipt"]
|
||||
|
||||
|
||||
def test_installer_rejects_digest_mismatch(tmp_path, monkeypatch):
|
||||
payload = request(tmp_path, b"expected")
|
||||
|
||||
class Response:
|
||||
headers = {}
|
||||
data = b"different"
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
def geturl(self):
|
||||
return payload["source_url"]
|
||||
|
||||
def read(self, n):
|
||||
out = self.data[:n]
|
||||
self.data = self.data[n:]
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(
|
||||
"urllib.request.urlopen",
|
||||
lambda *args, **kwargs: Response(),
|
||||
)
|
||||
with pytest.raises(ArtifactInstallError, match="SHA-256"):
|
||||
install(
|
||||
payload,
|
||||
model_root=tmp_path / "models",
|
||||
receipt_root=tmp_path / "receipts",
|
||||
)
|
||||
assert not (
|
||||
tmp_path / "models" / "loras" / "candidate.safetensors"
|
||||
).exists()
|
||||
assert not list((tmp_path / "models" / "loras").glob("*.part"))
|
||||
|
||||
|
||||
def test_preflight_requires_persistent_mount_and_detects_changed_destination(tmp_path, monkeypatch):
|
||||
payload = request(tmp_path)
|
||||
root = tmp_path / "models"
|
||||
(root / "loras").mkdir(parents=True)
|
||||
with pytest.raises(ArtifactInstallError, match="persistent mount"):
|
||||
preflight(payload, model_root=root)
|
||||
monkeypatch.setattr("os.path.ismount", lambda path: path == root)
|
||||
first = preflight(payload, model_root=root)
|
||||
assert first["already_installed"] is False
|
||||
(root / "loras" / payload["filename"]).write_bytes(b"model")
|
||||
second = preflight(payload, model_root=root)
|
||||
assert second["already_installed"] is True
|
||||
assert second["fingerprint"] != first["fingerprint"]
|
||||
|
||||
|
||||
def test_preflight_rejects_unwritable_model_directory(tmp_path, monkeypatch):
|
||||
payload = request(tmp_path)
|
||||
root = tmp_path / "models"
|
||||
(root / "loras").mkdir(parents=True)
|
||||
monkeypatch.setattr("os.path.ismount", lambda path: path == root)
|
||||
monkeypatch.setattr("os.access", lambda path, mode: False)
|
||||
with pytest.raises(ArtifactInstallError, match="not writable"):
|
||||
preflight(payload, model_root=root)
|
||||
|
||||
|
||||
def test_installer_never_follows_destination_symlink(tmp_path):
|
||||
payload = request(tmp_path)
|
||||
outside = tmp_path / "outside.safetensors"
|
||||
outside.write_bytes(b"model")
|
||||
destination_dir = tmp_path / "models" / "loras"
|
||||
destination_dir.mkdir(parents=True)
|
||||
try:
|
||||
(destination_dir / payload["filename"]).symlink_to(outside)
|
||||
except OSError:
|
||||
pytest.skip("creating symlinks is not permitted on this host")
|
||||
with pytest.raises(ArtifactInstallError, match="symlinks"):
|
||||
install(payload, model_root=tmp_path / "models", receipt_root=tmp_path / "receipts")
|
||||
assert outside.read_bytes() == b"model"
|
||||
|
||||
|
||||
def test_racing_destination_is_never_overwritten(tmp_path, monkeypatch):
|
||||
payload = request(tmp_path)
|
||||
|
||||
class Response:
|
||||
headers = {"Content-Length": "5"}
|
||||
|
||||
def __enter__(self):
|
||||
self.data = b"model"
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
def geturl(self):
|
||||
return payload["source_url"]
|
||||
|
||||
def read(self, size):
|
||||
data, self.data = self.data, b""
|
||||
return data
|
||||
|
||||
monkeypatch.setattr("urllib.request.urlopen", lambda *args, **kwargs: Response())
|
||||
|
||||
def race_link(source, destination):
|
||||
Path(destination).write_bytes(b"other")
|
||||
raise FileExistsError
|
||||
|
||||
monkeypatch.setattr("os.link", race_link)
|
||||
with pytest.raises(ArtifactInstallError, match="different digest"):
|
||||
install(payload, model_root=tmp_path / "models", receipt_root=tmp_path / "receipts")
|
||||
assert (tmp_path / "models" / "loras" / payload["filename"]).read_bytes() == b"other"
|
||||
assert not list((tmp_path / "models" / "loras").glob("*.part"))
|
||||
|
||||
|
||||
def test_gguf_requires_exact_type_and_extension():
|
||||
payload = {
|
||||
"artifact_type": "gguf",
|
||||
"source_url": "https://models.example.invalid/qwen.gguf",
|
||||
"sha256": "a" * 64,
|
||||
"filename": "qwen-image-2.1-Q4_K_M.gguf",
|
||||
"source_id": "hf:qwen-image-2.1:q4km",
|
||||
"license": "qwen-research",
|
||||
}
|
||||
assert validate_request(payload)["artifact_type"] == "gguf"
|
||||
|
||||
wrong_type = {**payload, "artifact_type": "lora"}
|
||||
with pytest.raises(ArtifactInstallError, match="safetensors"):
|
||||
validate_request(wrong_type)
|
||||
|
||||
wrong_extension = {**payload, "filename": "qwen-image-2.1-Q4_K_M.safetensors"}
|
||||
with pytest.raises(ArtifactInstallError, match="gguf"):
|
||||
validate_request(wrong_extension)
|
||||
|
||||
|
||||
def test_gguf_installs_only_into_diffusion_models(tmp_path, monkeypatch):
|
||||
data = b"gguf model bytes"
|
||||
payload = {
|
||||
"artifact_type": "gguf",
|
||||
"source_url": "https://models.example.invalid/qwen.gguf",
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"filename": "qwen-image-2.1-Q4_K_M.gguf",
|
||||
"source_id": "hf:qwen-image-2.1:q4km",
|
||||
"license": "qwen-research",
|
||||
}
|
||||
|
||||
class Response:
|
||||
def __init__(self):
|
||||
self.data = data
|
||||
self.headers = {"Content-Length": str(len(data))}
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
def geturl(self):
|
||||
return payload["source_url"]
|
||||
|
||||
def read(self, n):
|
||||
out = self.data[:n]
|
||||
self.data = self.data[n:]
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(
|
||||
"urllib.request.urlopen",
|
||||
lambda *args, **kwargs: Response(),
|
||||
)
|
||||
result = install(
|
||||
payload,
|
||||
model_root=tmp_path / "models",
|
||||
receipt_root=tmp_path / "receipts",
|
||||
)
|
||||
expected = (
|
||||
tmp_path
|
||||
/ "models"
|
||||
/ "diffusion_models"
|
||||
/ "qwen-image-2.1-Q4_K_M.gguf"
|
||||
)
|
||||
assert result["status"] == "installed"
|
||||
assert expected.read_bytes() == data
|
||||
assert not (tmp_path / "models" / "loras" / payload["filename"]).exists()
|
||||
@@ -0,0 +1,249 @@
|
||||
from sqlmodel import Session, SQLModel, create_engine, select
|
||||
|
||||
from backend.app.api import routes_jobs
|
||||
from backend.app.db.models import AppSetting, AuditEvent, JobRun
|
||||
from backend.app.services import action_execution, mcp_operations
|
||||
from backend.app.services import privatemuse_deployment_recovery as recovery
|
||||
from backend.app.services.action_execution import approval_execution_supported
|
||||
from backend.app.services.action_registry import (
|
||||
ACTION_RISK,
|
||||
ACTIONS,
|
||||
action_default_target,
|
||||
)
|
||||
|
||||
|
||||
def test_privatemuse_target_registration_is_a_typed_high_risk_action():
|
||||
action = "deployment.target.register_privatemuse"
|
||||
assert action in ACTIONS
|
||||
assert ACTION_RISK[action] == "high"
|
||||
assert action_default_target(action) == "jens/privatemuse"
|
||||
assert approval_execution_supported(action) is True
|
||||
|
||||
|
||||
def test_privatemuse_target_registration_rejects_any_other_target():
|
||||
result = action_execution.execute_governed_action(
|
||||
None, # type: ignore[arg-type]
|
||||
action="deployment.target.register_privatemuse",
|
||||
target="jens/not-privatemuse",
|
||||
source="chatgpt-mcp",
|
||||
risk="high",
|
||||
reason="Regression test for exact-target enforcement.",
|
||||
original_policy_decision="allowed",
|
||||
)
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["result"] == "blocked"
|
||||
assert result["policyDecision"] == "blocked"
|
||||
assert result["message"] == "PrivateMuse target registration accepts only jens/privatemuse."
|
||||
|
||||
|
||||
def test_privatemuse_target_registration_dispatches_only_fixed_executor(monkeypatch):
|
||||
observed = {}
|
||||
|
||||
def fake_executor(session, **kwargs):
|
||||
observed["session"] = session
|
||||
observed.update(kwargs)
|
||||
return {
|
||||
"success": True,
|
||||
"status": "success",
|
||||
"result": "success",
|
||||
"message": "enrolled",
|
||||
"jobId": "job-private-muse",
|
||||
"auditId": "audit-private-muse",
|
||||
"policyDecision": "allowed",
|
||||
"originalPolicyDecision": kwargs["original_policy_decision"],
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
action_execution,
|
||||
"_execute_privatemuse_target_enrollment",
|
||||
fake_executor,
|
||||
)
|
||||
|
||||
sentinel_session = object()
|
||||
result = action_execution.execute_governed_action(
|
||||
sentinel_session, # type: ignore[arg-type]
|
||||
action="deployment.target.register_privatemuse",
|
||||
target="JENS/PrivateMuse",
|
||||
source="chatgpt-mcp",
|
||||
risk="high",
|
||||
reason="Enroll the fixed PrivateMuse deployment target.",
|
||||
original_policy_decision="allowed",
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert observed == {
|
||||
"session": sentinel_session,
|
||||
"source": "chatgpt-mcp",
|
||||
"risk": "high",
|
||||
"reason": "Enroll the fixed PrivateMuse deployment target.",
|
||||
"original_policy_decision": "allowed",
|
||||
}
|
||||
|
||||
|
||||
def test_privatemuse_unknown_outcome_keeps_job_running_and_retains_exec_evidence(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'unknown-action.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
def uncertain_run(self, *, session, job_id):
|
||||
return {
|
||||
"status": "outcome_unknown",
|
||||
"reason": "Detached recovery outcome is unknown; do not retry.",
|
||||
"execution_id": "exec-action-unknown",
|
||||
"original_job_id": job_id,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(recovery.PrivateMuseDeploymentRecovery, "run", uncertain_run)
|
||||
|
||||
with Session(engine) as session:
|
||||
result = action_execution.execute_governed_action(
|
||||
session,
|
||||
action="deployment.target.register_privatemuse",
|
||||
target="jens/privatemuse",
|
||||
source="chatgpt-mcp",
|
||||
risk="high",
|
||||
reason="Exercise durable unknown-outcome reporting.",
|
||||
original_policy_decision="allowed",
|
||||
)
|
||||
job = session.get(JobRun, result["jobId"])
|
||||
audit = session.get(AuditEvent, result["auditId"])
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["status"] == "outcome_unknown"
|
||||
assert result["result"] == "outcome_unknown"
|
||||
assert result["executionId"] == "exec-action-unknown"
|
||||
assert job is not None
|
||||
assert job.status == "running"
|
||||
assert "exec-action-unknown" in str(job.result_summary)
|
||||
assert audit is not None
|
||||
assert audit.result == "outcome_unknown"
|
||||
assert audit.after_state["executionId"] == "exec-action-unknown"
|
||||
assert audit.after_state["originalJobId"] == result["jobId"]
|
||||
|
||||
|
||||
def test_unknown_privatemuse_recovery_job_cannot_be_cancelled(tmp_path):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'cancel-guard.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
session.add(
|
||||
JobRun(
|
||||
id="job-unknown-recovery",
|
||||
type="deployment.target.register_privatemuse",
|
||||
status="running",
|
||||
source="chatgpt-mcp",
|
||||
action_id="deployment.target.register_privatemuse",
|
||||
target="jens/privatemuse",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
result_summary="outcome_unknown; executionId=exec-cancel-guard",
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
AppSetting(
|
||||
key=recovery._RECOVERY_STATE_SETTING,
|
||||
value_json={
|
||||
"state": "outcome_unknown",
|
||||
"executionId": "exec-cancel-guard",
|
||||
"controllerId": "controller-id",
|
||||
"jobId": "job-unknown-recovery",
|
||||
},
|
||||
is_secret=False,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
result = mcp_operations.cancel_operation_job(
|
||||
session, "job-unknown-recovery"
|
||||
)
|
||||
route_result = routes_jobs.cancel("job-unknown-recovery", session=session)
|
||||
persisted = session.exec(
|
||||
select(JobRun).where(JobRun.id == "job-unknown-recovery")
|
||||
).one()
|
||||
|
||||
assert result == {
|
||||
"jobId": "job-unknown-recovery",
|
||||
"status": "running",
|
||||
"cancelled": False,
|
||||
"changed": False,
|
||||
"reason": (
|
||||
"This job owns an unresolved detached PrivateMuse recovery execution; "
|
||||
"reconcile that execution before changing the job record."
|
||||
),
|
||||
}
|
||||
assert route_result == {
|
||||
"success": False,
|
||||
"message": (
|
||||
"This job owns an unresolved detached PrivateMuse recovery execution; "
|
||||
"reconcile that execution before changing the job record."
|
||||
),
|
||||
"id": "job-unknown-recovery",
|
||||
"status": "running",
|
||||
}
|
||||
assert persisted.status == "running"
|
||||
|
||||
|
||||
def test_privatemuse_job_is_cancellation_guarded_at_admission(tmp_path):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'admission-cancel.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
_state, admitted = recovery.PrivateMuseDeploymentRecovery.admit_job(
|
||||
session,
|
||||
JobRun(
|
||||
id="job-admitted-recovery",
|
||||
type="deployment.target.register_privatemuse",
|
||||
status="queued",
|
||||
source="chatgpt-mcp",
|
||||
action_id="deployment.target.register_privatemuse",
|
||||
target="jens/privatemuse",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
),
|
||||
)
|
||||
result = mcp_operations.cancel_operation_job(
|
||||
session, "job-admitted-recovery"
|
||||
)
|
||||
|
||||
assert admitted is True
|
||||
assert result["cancelled"] is False
|
||||
assert result["changed"] is False
|
||||
assert result["status"] == "queued"
|
||||
assert "unresolved detached PrivateMuse recovery" in result["reason"]
|
||||
|
||||
|
||||
def test_privatemuse_cancellation_guard_applies_only_to_the_owner(tmp_path):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'owner-cancel.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
recovery.PrivateMuseDeploymentRecovery.admit_job(
|
||||
session,
|
||||
JobRun(
|
||||
id="job-owner",
|
||||
type="deployment.target.register_privatemuse",
|
||||
status="queued",
|
||||
source="chatgpt-mcp",
|
||||
action_id="deployment.target.register_privatemuse",
|
||||
target="jens/privatemuse",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
),
|
||||
)
|
||||
unrelated = JobRun(
|
||||
id="job-unrelated",
|
||||
type="deployment.target.register_privatemuse",
|
||||
status="running",
|
||||
source="chatgpt-mcp",
|
||||
action_id="deployment.target.register_privatemuse",
|
||||
target="jens/privatemuse",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
)
|
||||
session.add(unrelated)
|
||||
session.commit()
|
||||
|
||||
result = mcp_operations.cancel_operation_job(session, unrelated.id)
|
||||
|
||||
assert result["cancelled"] is True
|
||||
assert result["changed"] is True
|
||||
assert result["status"] == "cancelled"
|
||||
@@ -0,0 +1,189 @@
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
from backend.app.services.privatemuse_deployment_enrollment import (
|
||||
PrivateMuseDeploymentEnrollment,
|
||||
)
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakePortainer:
|
||||
def __init__(self, exit_code=0):
|
||||
self.exit_code = exit_code
|
||||
self.calls = []
|
||||
|
||||
def get_container_status(self, name):
|
||||
assert name == "gitea-deploy-control"
|
||||
return {
|
||||
"status": "ok",
|
||||
"container": {
|
||||
"id": "controller-id",
|
||||
"name": "gitea-deploy-control",
|
||||
"state": "running",
|
||||
},
|
||||
}
|
||||
|
||||
def _config(self):
|
||||
return SimpleNamespace(endpoint_id="3")
|
||||
|
||||
def _request(self, method, path, **kwargs):
|
||||
self.calls.append((method, path, kwargs))
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return FakeResponse({"Id": "exec-1"})
|
||||
if path.endswith("/exec/exec-1/start"):
|
||||
return FakeResponse({})
|
||||
if path.endswith("/exec/exec-1/json"):
|
||||
return FakeResponse({"Running": False, "ExitCode": self.exit_code})
|
||||
raise AssertionError(path)
|
||||
|
||||
|
||||
def test_enrollment_uses_fixed_private_muse_contract_and_controller_exec():
|
||||
fake = FakePortainer()
|
||||
result = PrivateMuseDeploymentEnrollment(fake).run()
|
||||
|
||||
assert result["status"] == "enrolled"
|
||||
assert result["repository"] == "jens/privatemuse"
|
||||
assert result["legacy_revision"] == "715f7d152d35745521a2e014e334a1509bdce6a2"
|
||||
assert result["containers"] == [
|
||||
"privatemuse-backend-1",
|
||||
"privatemuse-frontend-1",
|
||||
"privatemuse-audio-enhancement-1",
|
||||
]
|
||||
|
||||
create = fake.calls[0]
|
||||
assert create[0] == "POST"
|
||||
assert create[1].endswith("/containers/controller-id/exec")
|
||||
command = create[2]["json"]["Cmd"]
|
||||
assert command[:2] == ["python3", "-c"]
|
||||
assert command[3] == "jens/privatemuse"
|
||||
assert command[4] == "715f7d152d35745521a2e014e334a1509bdce6a2"
|
||||
assert '"exact_source":true' in command[5]
|
||||
assert '"env_source":"/srv/example/appdata/privatemuse/.env"' in command[5]
|
||||
assert '"compose_files":["docker-compose.yml","docker-compose.unraid.yml"]' in command[5]
|
||||
assert '"privatemuse-audio-enhancement-1"' in command[5]
|
||||
canonical_payload = json.loads(command[5])
|
||||
assert "gitea_repository_id" not in canonical_payload
|
||||
assert "gitea_webhook_secret_file" not in canonical_payload
|
||||
assert "validation" not in canonical_payload
|
||||
assert "appops_managed_target" not in canonical_payload
|
||||
|
||||
script = command[2]
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/controller/projects.json" in script
|
||||
assert "/srv/example/appdata/gitea-deploy-runners/controller/projects.json" not in script
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script
|
||||
assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/deploy.lock" in script
|
||||
assert "os.path.samefile(registry_path, host_registry_path)" in script
|
||||
assert '"docker", "rm", stale' in script
|
||||
assert "controller.bootstrap_legacy_state(repository, project, legacy_revision)" in script
|
||||
assert "privatemuse-audio-music-1" in script
|
||||
assert "privatemuse-audio-lipsync-1" in script
|
||||
assert '"gitea_repository_id"' in script
|
||||
assert '"gitea_webhook_secret_file"' in script
|
||||
assert '"jens__privatemuse"' in script
|
||||
assert '"Managed validation / full (push)"' in script
|
||||
assert "extra_keys != allowed_metadata_keys" in script
|
||||
assert "canonical_target_match = True" in script
|
||||
assert "if not canonical_target_match:" in script
|
||||
assert create[2]["json"]["AttachStdout"] is False
|
||||
assert create[2]["json"]["AttachStderr"] is False
|
||||
|
||||
|
||||
def test_privatemuse_enrollment_uses_project_then_global_lock_order_without_child_deadlock():
|
||||
fake = FakePortainer()
|
||||
PrivateMuseDeploymentEnrollment(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
project_lock = "fcntl.flock(project_lock, fcntl.LOCK_EX)"
|
||||
cache_lock = "fcntl.flock(lock, fcntl.LOCK_EX)"
|
||||
assert script.index(project_lock) < script.index(cache_lock)
|
||||
assert "controller.bootstrap_legacy_state(repository, project, legacy_revision)" in script
|
||||
assert '"/opt/gitea-deploy/deploy.py",\n "bootstrap-legacy"' not in script
|
||||
|
||||
|
||||
def test_enrollment_pins_exact_runnerless_metadata_values():
|
||||
fake = FakePortainer()
|
||||
PrivateMuseDeploymentEnrollment(fake).run()
|
||||
script = fake.calls[0][2]["json"]["Cmd"][2]
|
||||
|
||||
assert '"gitea_repository_id",' in script
|
||||
assert '"gitea_webhook_secret_file",' in script
|
||||
assert '"validation",' in script
|
||||
assert 'current.get("gitea_webhook_secret_file") != "jens__privatemuse"' in script
|
||||
assert '"validation_gate_version": 1' in script
|
||||
assert '"required": True' in script
|
||||
assert '"contexts": ["Managed validation / full (push)"]' in script
|
||||
assert '"timeout_seconds": 1800' in script
|
||||
assert '"poll_interval_seconds": 5' in script
|
||||
assert '"max_poll_interval_seconds": 30' in script
|
||||
assert 'current.get("validation") != expected_validation' in script
|
||||
|
||||
|
||||
def test_enrollment_reports_conflicting_target_without_exec_output():
|
||||
fake = FakePortainer(exit_code=23)
|
||||
result = PrivateMuseDeploymentEnrollment(fake).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "PrivateMuse already has a different deployment target",
|
||||
}
|
||||
|
||||
|
||||
def test_enrollment_refuses_unexpected_compose_container():
|
||||
fake = FakePortainer(exit_code=45)
|
||||
result = PrivateMuseDeploymentEnrollment(fake).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "PrivateMuse Compose project contains an unexpected container",
|
||||
}
|
||||
|
||||
|
||||
def test_enrollment_refuses_running_stale_audio_provider():
|
||||
fake = FakePortainer(exit_code=48)
|
||||
result = PrivateMuseDeploymentEnrollment(fake).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "a stale PrivateMuse audio provider is still running and was not removed",
|
||||
}
|
||||
|
||||
|
||||
def test_enrollment_reports_bootstrap_rejection():
|
||||
fake = FakePortainer(exit_code=52)
|
||||
result = PrivateMuseDeploymentEnrollment(fake).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "deployment controller rejected PrivateMuse legacy-state bootstrap",
|
||||
}
|
||||
|
||||
|
||||
def test_enrollment_classifies_existing_target_field_difference_without_values():
|
||||
fake = FakePortainer(exit_code=64)
|
||||
result = PrivateMuseDeploymentEnrollment(fake).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "PrivateMuse deployment target differs in env_source",
|
||||
}
|
||||
|
||||
|
||||
def test_enrollment_classifies_controller_managed_target_collision():
|
||||
fake = FakePortainer(exit_code=55)
|
||||
result = PrivateMuseDeploymentEnrollment(fake).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "PrivateMuse deployment target is owned by the controller v2 managed-target contract",
|
||||
}
|
||||
|
||||
|
||||
def test_enrollment_classifies_unsupported_or_invalid_controller_metadata():
|
||||
fake = FakePortainer(exit_code=71)
|
||||
result = PrivateMuseDeploymentEnrollment(fake).run()
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "PrivateMuse deployment target matches canonical fields but has additional or invalid controller metadata",
|
||||
}
|
||||
@@ -0,0 +1,630 @@
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from threading import Barrier, Event, Thread
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sqlalchemy import event
|
||||
from sqlmodel import Session, SQLModel, create_engine, select
|
||||
|
||||
import backend.app.services.privatemuse_deployment_recovery as recovery
|
||||
from backend.app.db.models import JobRun
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakePortainer:
|
||||
def __init__(self, exit_code=0):
|
||||
self.exit_code = exit_code
|
||||
self.calls = []
|
||||
|
||||
def get_container_status(self, name):
|
||||
assert name == "gitea-deploy-control"
|
||||
return {
|
||||
"status": "ok",
|
||||
"container": {
|
||||
"id": "controller-id",
|
||||
"name": "gitea-deploy-control",
|
||||
"state": "running",
|
||||
},
|
||||
}
|
||||
|
||||
def _config(self):
|
||||
return SimpleNamespace(endpoint_id="3")
|
||||
|
||||
def _request(self, method, path, **kwargs):
|
||||
self.calls.append((method, path, kwargs))
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return FakeResponse({"Id": "exec-1"})
|
||||
if path.endswith("/exec/exec-1/start"):
|
||||
return FakeResponse({})
|
||||
if path.endswith("/exec/exec-1/json"):
|
||||
return FakeResponse(
|
||||
{
|
||||
"ID": "exec-1",
|
||||
"ContainerID": "controller-id",
|
||||
"Running": False,
|
||||
"ExitCode": self.exit_code,
|
||||
}
|
||||
)
|
||||
raise AssertionError(path)
|
||||
|
||||
|
||||
def recovery_job(job_id):
|
||||
return JobRun(
|
||||
id=job_id,
|
||||
type="deployment.target.register_privatemuse",
|
||||
status="queued",
|
||||
source="chatgpt-mcp",
|
||||
action_id="deployment.target.register_privatemuse",
|
||||
target="jens/privatemuse",
|
||||
risk="high",
|
||||
policy_decision="allowed",
|
||||
)
|
||||
|
||||
|
||||
def test_recovery_runs_only_the_exact_observed_privatemuse_incident():
|
||||
fake = FakePortainer()
|
||||
|
||||
result = recovery.PrivateMuseDeploymentRecovery(fake).run()
|
||||
|
||||
assert result == {
|
||||
"status": "reconciled",
|
||||
"repository": "jens/privatemuse",
|
||||
"live_revision": "2103d5795d151885edeeb964466b643b8fd4a0eb",
|
||||
"failed_revision": "6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38",
|
||||
"containers": [
|
||||
"privatemuse-backend-1",
|
||||
"privatemuse-frontend-1",
|
||||
"privatemuse-audio-enhancement-1",
|
||||
],
|
||||
"compose_project": "privatemuse",
|
||||
}
|
||||
command = fake.calls[0][2]["json"]["Cmd"]
|
||||
assert command[:3] == ["python3", "-c", recovery._REATTEST_SCRIPT]
|
||||
assert command[3:] == [
|
||||
"jens/privatemuse",
|
||||
"2103d5795d151885edeeb964466b643b8fd4a0eb",
|
||||
"6e4c959eea2e7f3bf2c6b3560a2c1658865bfc38",
|
||||
"Existing production containers require a valid healthy live deployment state",
|
||||
]
|
||||
|
||||
|
||||
def test_recovery_script_allows_only_the_two_observed_container_id_changes():
|
||||
script = recovery._REATTEST_SCRIPT
|
||||
|
||||
assert (
|
||||
'allowed_changed_ids = {"privatemuse-backend-1", "privatemuse-frontend-1"}'
|
||||
in script
|
||||
)
|
||||
assert "changed_receipt_ids != allowed_changed_ids" in script
|
||||
assert "changed_artifact_ids != allowed_changed_ids" in script
|
||||
assert 'state.get("revision") == live_revision' in script
|
||||
assert 'state.get("live_revision") == live_revision' in script
|
||||
assert 'state.get("live_status") == "unknown"' in script
|
||||
assert 'attempt.get("revision") == failed_revision' in script
|
||||
assert 'attempt.get("message") == failure_message' in script
|
||||
|
||||
|
||||
def test_recovery_reuses_controller_verifiers_and_writes_state_durably():
|
||||
script = recovery._REATTEST_SCRIPT
|
||||
|
||||
assert "controller.validate_recorded_receipt" in script
|
||||
assert "controller.verify_deployment" in script
|
||||
assert "expected_image_ids=expected_image_ids" in script
|
||||
assert "controller.deployment_artifacts" in script
|
||||
assert "controller.verified_current_receipt" in script
|
||||
assert "write_exclusive(backup, original_state, state_metadata)" in script
|
||||
assert "os.replace(temporary, state_path)" in script
|
||||
assert "os.fsync" in script
|
||||
|
||||
|
||||
def test_recovery_durably_preserves_the_original_state_before_replacement():
|
||||
script = recovery._REATTEST_SCRIPT
|
||||
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/state" in script
|
||||
assert "/srv/example/appdata/gitea-deploy-runners/deploy.lock" not in script
|
||||
assert "/mnt/cache/appdata/gitea-deploy-runners/deploy.lock" in script
|
||||
assert "original_state = state_path.read_bytes()" in script
|
||||
assert "write_exclusive(backup, original_state, state_metadata)" in script
|
||||
assert "os.fchmod(handle.fileno(), stat.S_IMODE(metadata.st_mode))" in script
|
||||
assert "os.fchown(handle.fileno(), metadata.st_uid, metadata.st_gid)" in script
|
||||
assert "fsync_directory(state_root)" in script
|
||||
|
||||
|
||||
def test_recovery_restores_original_state_when_post_write_verification_fails():
|
||||
script = recovery._REATTEST_SCRIPT
|
||||
|
||||
assert "def restore_original_state():" in script
|
||||
assert (
|
||||
"write_exclusive(restore_temporary, original_state, state_metadata)" in script
|
||||
)
|
||||
assert "os.replace(restore_temporary, state_path)" in script
|
||||
assert "return state_path.read_bytes() == original_state" in script
|
||||
assert "if not restore_original_state():" in script
|
||||
assert "raise SystemExit(45)" in script
|
||||
|
||||
|
||||
def test_recovery_restores_original_state_when_post_replace_durability_fails():
|
||||
script = recovery._REATTEST_SCRIPT
|
||||
|
||||
assert "state_replaced = False" in script
|
||||
assert "state_replaced = True" in script
|
||||
assert "if state_replaced and not restore_original_state():" in script
|
||||
|
||||
|
||||
def test_recovery_script_has_no_application_runtime_mutation_commands():
|
||||
forbidden = (
|
||||
"docker restart",
|
||||
"docker stop",
|
||||
"docker rm",
|
||||
"docker compose up",
|
||||
"docker compose down",
|
||||
'subprocess.run(["docker", "restart"',
|
||||
'subprocess.run(["docker", "rm"',
|
||||
)
|
||||
|
||||
assert all(value not in recovery._REATTEST_SCRIPT for value in forbidden)
|
||||
|
||||
|
||||
def test_recovery_reports_verified_healthy_state_without_enrollment_fallback():
|
||||
fake = FakePortainer(exit_code=35)
|
||||
|
||||
result = recovery.PrivateMuseDeploymentRecovery(fake).run()
|
||||
|
||||
assert result == {
|
||||
"status": "already-enrolled",
|
||||
"repository": "jens/privatemuse",
|
||||
}
|
||||
assert "PrivateMuseDeploymentEnrollment" not in Path(
|
||||
"backend/app/services/privatemuse_deployment_recovery.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_recovery_fails_closed_when_controller_is_unavailable():
|
||||
class UnavailablePortainer(FakePortainer):
|
||||
def get_container_status(self, name):
|
||||
assert name == "gitea-deploy-control"
|
||||
return {"status": "unavailable", "reason": "offline"}
|
||||
|
||||
def _config(self):
|
||||
raise AssertionError("configuration must not be read")
|
||||
|
||||
result = recovery.PrivateMuseDeploymentRecovery(UnavailablePortainer()).run()
|
||||
|
||||
assert result == {"status": "controller_unavailable", "reason": "offline"}
|
||||
|
||||
|
||||
def test_recovery_waits_beyond_controller_health_timeout_and_marks_timeout_uncertain(
|
||||
monkeypatch,
|
||||
):
|
||||
fake = FakePortainer()
|
||||
fake._request = lambda method, path, **kwargs: (
|
||||
FakeResponse({"Id": "exec-1"})
|
||||
if path.endswith("/containers/controller-id/exec")
|
||||
else FakeResponse({})
|
||||
if path.endswith("/exec/exec-1/start")
|
||||
else FakeResponse({"Running": True, "ExitCode": None})
|
||||
)
|
||||
monotonic_values = iter((0.0, 301.0))
|
||||
monkeypatch.setattr(recovery.time, "monotonic", lambda: next(monotonic_values))
|
||||
|
||||
result = recovery.PrivateMuseDeploymentRecovery(fake).run()
|
||||
|
||||
assert result == {
|
||||
"status": "outcome_unknown",
|
||||
"reason": (
|
||||
"PrivateMuse runtime re-attestation is still running; inspect the "
|
||||
"existing execution and do not retry"
|
||||
),
|
||||
"execution_id": "exec-1",
|
||||
}
|
||||
source = Path("backend/app/services/privatemuse_deployment_recovery.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert "deadline = time.monotonic() + 300" in source
|
||||
|
||||
|
||||
def test_recovery_marks_connector_failure_after_exec_start_as_uncertain():
|
||||
class InspectFailurePortainer(FakePortainer):
|
||||
def _request(self, method, path, **kwargs):
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return FakeResponse({"Id": "exec-uncertain"})
|
||||
if path.endswith("/exec/exec-uncertain/start"):
|
||||
return FakeResponse({})
|
||||
if path.endswith("/exec/exec-uncertain/json"):
|
||||
raise RuntimeError("connection lost")
|
||||
raise AssertionError(path)
|
||||
|
||||
result = recovery.PrivateMuseDeploymentRecovery(InspectFailurePortainer()).run()
|
||||
|
||||
assert result == {
|
||||
"status": "outcome_unknown",
|
||||
"reason": (
|
||||
"PrivateMuse runtime re-attestation outcome is unknown after execution "
|
||||
"start; inspect the existing execution and do not retry"
|
||||
),
|
||||
"execution_id": "exec-uncertain",
|
||||
}
|
||||
|
||||
|
||||
def test_recovery_reconciles_durable_unknown_exec_without_starting_another(
|
||||
tmp_path,
|
||||
):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'recovery.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
class FirstAttemptPortainer(FakePortainer):
|
||||
def _request(self, method, path, **kwargs):
|
||||
self.calls.append((method, path, kwargs))
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return FakeResponse({"Id": "exec-durable"})
|
||||
if path.endswith("/exec/exec-durable/start"):
|
||||
return FakeResponse({})
|
||||
if path.endswith("/exec/exec-durable/json"):
|
||||
raise RuntimeError("connection lost")
|
||||
raise AssertionError(path)
|
||||
|
||||
first = FirstAttemptPortainer()
|
||||
with Session(engine) as session:
|
||||
_tracked, admitted = recovery.PrivateMuseDeploymentRecovery.admit_job(
|
||||
session, recovery_job("job-original")
|
||||
)
|
||||
assert admitted is True
|
||||
uncertain = recovery.PrivateMuseDeploymentRecovery(first).run(
|
||||
session=session,
|
||||
job_id="job-original",
|
||||
)
|
||||
|
||||
assert uncertain["status"] == "outcome_unknown"
|
||||
assert uncertain["execution_id"] == "exec-durable"
|
||||
assert uncertain["original_job_id"] == "job-original"
|
||||
|
||||
class ReconcilePortainer(FakePortainer):
|
||||
def _request(self, method, path, **kwargs):
|
||||
self.calls.append((method, path, kwargs))
|
||||
assert method == "GET"
|
||||
assert path.endswith("/exec/exec-durable/json")
|
||||
return FakeResponse(
|
||||
{
|
||||
"ID": "exec-durable",
|
||||
"ContainerID": "controller-id",
|
||||
"Running": False,
|
||||
"ExitCode": 0,
|
||||
}
|
||||
)
|
||||
|
||||
reconciler = ReconcilePortainer()
|
||||
with Session(engine) as session:
|
||||
reconciled = recovery.PrivateMuseDeploymentRecovery(reconciler).run(
|
||||
session=session,
|
||||
job_id="job-fresh-key",
|
||||
)
|
||||
|
||||
assert reconciled["status"] == "reconciled"
|
||||
assert reconciled["execution_id"] == "exec-durable"
|
||||
assert reconciled["original_job_id"] == "job-original"
|
||||
assert [method for method, _path, _kwargs in reconciler.calls] == ["GET"]
|
||||
|
||||
|
||||
def test_recovery_keeps_unknown_exec_guard_when_controller_identity_changed(tmp_path):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'identity.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
session.add(
|
||||
recovery.AppSetting(
|
||||
key=recovery._RECOVERY_STATE_SETTING,
|
||||
value_json={
|
||||
"state": "outcome_unknown",
|
||||
"executionId": "exec-existing",
|
||||
"controllerId": "old-controller",
|
||||
"jobId": "job-original",
|
||||
},
|
||||
is_secret=False,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
portainer = FakePortainer()
|
||||
with Session(engine) as session:
|
||||
result = recovery.PrivateMuseDeploymentRecovery(portainer).run(
|
||||
session=session,
|
||||
job_id="job-fresh-key",
|
||||
)
|
||||
|
||||
assert result["status"] == "outcome_unknown"
|
||||
assert result["execution_id"] == "exec-existing"
|
||||
assert result["original_job_id"] == "job-original"
|
||||
assert "controller identity changed" in result["reason"]
|
||||
assert portainer.calls == []
|
||||
|
||||
|
||||
def test_recovery_admission_creates_only_the_winning_job(tmp_path):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'admission.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
with Session(engine) as first_session:
|
||||
first_state, first_admitted = recovery.PrivateMuseDeploymentRecovery.admit_job(
|
||||
first_session, recovery_job("job-winner")
|
||||
)
|
||||
with Session(engine) as second_session:
|
||||
second_state, second_admitted = (
|
||||
recovery.PrivateMuseDeploymentRecovery.admit_job(
|
||||
second_session, recovery_job("job-loser")
|
||||
)
|
||||
)
|
||||
jobs = second_session.exec(select(JobRun)).all()
|
||||
|
||||
assert first_admitted is True
|
||||
assert second_admitted is False
|
||||
assert first_state["jobId"] == "job-winner"
|
||||
assert second_state["jobId"] == "job-winner"
|
||||
assert [job.id for job in jobs] == ["job-winner"]
|
||||
|
||||
|
||||
def test_recovery_admission_cas_uses_generation_when_timestamps_collide(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
fixed = datetime(2026, 9, 20, 12, 0, 0, tzinfo=UTC).replace(tzinfo=None)
|
||||
monkeypatch.setattr(recovery, "utc_now", lambda: fixed)
|
||||
engine = create_engine(
|
||||
f"sqlite:///{tmp_path / 'admission-generation.db'}",
|
||||
connect_args={"check_same_thread": False, "timeout": 10},
|
||||
)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
session.add(
|
||||
recovery.AppSetting(
|
||||
key=recovery._RECOVERY_STATE_SETTING,
|
||||
value_json={"state": "idle", "version": 0},
|
||||
is_secret=False,
|
||||
updated_at=fixed,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
update_barrier = Barrier(2)
|
||||
|
||||
def align_cas_updates(_conn, _cursor, statement, _parameters, _context, _many):
|
||||
if statement.startswith("UPDATE app_settings SET"):
|
||||
update_barrier.wait(timeout=10)
|
||||
|
||||
event.listen(engine, "before_cursor_execute", align_cas_updates)
|
||||
admissions = []
|
||||
|
||||
def admit(job_id):
|
||||
with Session(engine) as session:
|
||||
_state, admitted = recovery.PrivateMuseDeploymentRecovery.admit_job(
|
||||
session, recovery_job(job_id)
|
||||
)
|
||||
admissions.append((job_id, admitted))
|
||||
|
||||
workers = [Thread(target=admit, args=(f"job-{index}",)) for index in range(2)]
|
||||
for worker in workers:
|
||||
worker.start()
|
||||
for worker in workers:
|
||||
worker.join(timeout=10)
|
||||
assert not worker.is_alive()
|
||||
event.remove(engine, "before_cursor_execute", align_cas_updates)
|
||||
|
||||
with Session(engine) as session:
|
||||
jobs = session.exec(select(JobRun)).all()
|
||||
state = session.get(recovery.AppSetting, recovery._RECOVERY_STATE_SETTING)
|
||||
|
||||
assert sorted(admitted for _job_id, admitted in admissions) == [False, True]
|
||||
assert len(jobs) == 1
|
||||
assert state is not None
|
||||
assert state.value_json["jobId"] == jobs[0].id
|
||||
assert state.value_json["version"] == 1
|
||||
|
||||
|
||||
def test_reserved_recovery_can_only_be_started_by_its_admitted_job(tmp_path):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'owner.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
class NoPreflightPortainer(FakePortainer):
|
||||
def get_container_status(self, name):
|
||||
raise AssertionError("a losing caller must not reach Portainer preflight")
|
||||
|
||||
with Session(engine) as session:
|
||||
recovery.PrivateMuseDeploymentRecovery.admit_job(
|
||||
session, recovery_job("job-owner")
|
||||
)
|
||||
result = recovery.PrivateMuseDeploymentRecovery(NoPreflightPortainer()).run(
|
||||
session=session,
|
||||
job_id="job-loser",
|
||||
)
|
||||
|
||||
assert result["status"] == "in_progress"
|
||||
assert result["original_job_id"] == "job-owner"
|
||||
assert "no second execution was started" in result["reason"]
|
||||
|
||||
|
||||
def test_terminal_reconciliation_cannot_regress_to_late_unknown(tmp_path):
|
||||
engine = create_engine(
|
||||
f"sqlite:///{tmp_path / 'monotonic.db'}",
|
||||
connect_args={"check_same_thread": False},
|
||||
)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
recovery.PrivateMuseDeploymentRecovery.admit_job(
|
||||
session, recovery_job("job-race")
|
||||
)
|
||||
|
||||
inspect_started = Event()
|
||||
reconciliation_finished = Event()
|
||||
worker_result = {}
|
||||
|
||||
class DelayedWorkerPortainer(FakePortainer):
|
||||
def _request(self, method, path, **kwargs):
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return FakeResponse({"Id": "exec-race"})
|
||||
if path.endswith("/exec/exec-race/start"):
|
||||
return FakeResponse({})
|
||||
if path.endswith("/exec/exec-race/json"):
|
||||
inspect_started.set()
|
||||
assert reconciliation_finished.wait(timeout=10)
|
||||
raise RuntimeError("late connection loss")
|
||||
raise AssertionError(path)
|
||||
|
||||
def run_delayed_worker():
|
||||
with Session(engine) as worker_session:
|
||||
worker_result.update(
|
||||
recovery.PrivateMuseDeploymentRecovery(
|
||||
DelayedWorkerPortainer()
|
||||
).run(session=worker_session, job_id="job-race")
|
||||
)
|
||||
|
||||
worker = Thread(target=run_delayed_worker)
|
||||
worker.start()
|
||||
assert inspect_started.wait(timeout=10)
|
||||
|
||||
class TerminalInspectPortainer(FakePortainer):
|
||||
def _request(self, method, path, **kwargs):
|
||||
assert method == "GET"
|
||||
assert path.endswith("/exec/exec-race/json")
|
||||
return FakeResponse(
|
||||
{
|
||||
"ID": "exec-race",
|
||||
"ContainerID": "controller-id",
|
||||
"Running": False,
|
||||
"ExitCode": 0,
|
||||
}
|
||||
)
|
||||
|
||||
with Session(engine) as reconcile_session:
|
||||
reconciled = recovery.PrivateMuseDeploymentRecovery(
|
||||
TerminalInspectPortainer()
|
||||
).run(session=reconcile_session, job_id="job-fresh-key")
|
||||
reconciliation_finished.set()
|
||||
worker.join(timeout=10)
|
||||
assert not worker.is_alive()
|
||||
|
||||
with Session(engine) as verification_session:
|
||||
row = verification_session.get(
|
||||
recovery.AppSetting, recovery._RECOVERY_STATE_SETTING
|
||||
)
|
||||
|
||||
assert reconciled["status"] == "reconciled"
|
||||
assert worker_result["status"] == "reconciled"
|
||||
assert row is not None
|
||||
assert row.value_json["state"] == "completed"
|
||||
assert row.value_json["result"]["status"] == "reconciled"
|
||||
|
||||
|
||||
def test_recovery_state_cas_prevents_same_timestamp_terminal_regression(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
fixed = datetime(2026, 9, 20, 12, 0, 0, tzinfo=UTC).replace(tzinfo=None)
|
||||
monkeypatch.setattr(recovery, "utc_now", lambda: fixed)
|
||||
engine = create_engine(
|
||||
f"sqlite:///{tmp_path / 'terminal-generation.db'}",
|
||||
connect_args={"check_same_thread": False, "timeout": 10},
|
||||
)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
tracked = {
|
||||
"state": "started",
|
||||
"claimId": "claim-race",
|
||||
"jobId": "job-race",
|
||||
"controllerId": "controller-id",
|
||||
"executionId": "exec-race",
|
||||
"version": 7,
|
||||
}
|
||||
with Session(engine) as session:
|
||||
session.add(
|
||||
recovery.AppSetting(
|
||||
key=recovery._RECOVERY_STATE_SETTING,
|
||||
value_json=tracked,
|
||||
is_secret=False,
|
||||
updated_at=fixed,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
completed_at_update = Event()
|
||||
unknown_at_update = Event()
|
||||
completed_finished = Event()
|
||||
|
||||
def order_cas_updates(_conn, _cursor, statement, parameters, _context, _many):
|
||||
if not statement.startswith("UPDATE app_settings SET"):
|
||||
return
|
||||
payload = str(parameters[0])
|
||||
if '"state": "completed"' in payload:
|
||||
completed_at_update.set()
|
||||
assert unknown_at_update.wait(timeout=10)
|
||||
elif '"state": "outcome_unknown"' in payload:
|
||||
unknown_at_update.set()
|
||||
assert completed_at_update.wait(timeout=10)
|
||||
assert completed_finished.wait(timeout=10)
|
||||
|
||||
event.listen(engine, "before_cursor_execute", order_cas_updates)
|
||||
results = {}
|
||||
|
||||
def persist_completed():
|
||||
with Session(engine) as session:
|
||||
results["completed"] = recovery.PrivateMuseDeploymentRecovery._persist_state(
|
||||
session,
|
||||
dict(tracked),
|
||||
state="completed",
|
||||
execution_id="exec-race",
|
||||
result={"status": "reconciled"},
|
||||
)
|
||||
completed_finished.set()
|
||||
|
||||
def persist_unknown():
|
||||
with Session(engine) as session:
|
||||
results["unknown"] = recovery.PrivateMuseDeploymentRecovery._persist_state(
|
||||
session,
|
||||
dict(tracked),
|
||||
state="outcome_unknown",
|
||||
execution_id="exec-race",
|
||||
result={"status": "outcome_unknown"},
|
||||
)
|
||||
|
||||
completed_worker = Thread(target=persist_completed)
|
||||
unknown_worker = Thread(target=persist_unknown)
|
||||
completed_worker.start()
|
||||
unknown_worker.start()
|
||||
completed_worker.join(timeout=10)
|
||||
unknown_worker.join(timeout=10)
|
||||
assert not completed_worker.is_alive()
|
||||
assert not unknown_worker.is_alive()
|
||||
event.remove(engine, "before_cursor_execute", order_cas_updates)
|
||||
|
||||
with Session(engine) as session:
|
||||
state = session.get(recovery.AppSetting, recovery._RECOVERY_STATE_SETTING)
|
||||
|
||||
assert results["completed"]["state"] == "completed"
|
||||
assert results["unknown"]["state"] == "completed"
|
||||
assert state is not None
|
||||
assert state.value_json["state"] == "completed"
|
||||
assert state.value_json["result"]["status"] == "reconciled"
|
||||
assert state.value_json["version"] == 8
|
||||
|
||||
|
||||
def test_recovery_fails_closed_when_runtime_provenance_differs():
|
||||
result = recovery.PrivateMuseDeploymentRecovery(FakePortainer(exit_code=38)).run()
|
||||
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "PrivateMuse immutable container/image provenance differs from the recorded receipt",
|
||||
}
|
||||
|
||||
|
||||
def test_privatemuse_action_execution_accepts_reconciled_status():
|
||||
source = Path("backend/app/services/action_execution.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
assert (
|
||||
"from .privatemuse_deployment_recovery import PrivateMuseDeploymentRecovery"
|
||||
in source
|
||||
)
|
||||
assert "tracked, admitted = recovery.admit_job(session, candidate_job)" in source
|
||||
assert "job_id=job.id if admitted else candidate_job.id" in source
|
||||
assert 'status in {"enrolled", "already-enrolled", "reconciled"}' in source
|
||||
@@ -0,0 +1,535 @@
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(
|
||||
0,
|
||||
str(Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker"),
|
||||
)
|
||||
|
||||
import adopt_privatemuse_f84_runtime as adoption
|
||||
import diagnose_privatemuse_f84_receipt as diagnosis
|
||||
import privatemuse_f84_receipt_diagnosis as incident_contract
|
||||
|
||||
|
||||
def _incident_evidence():
|
||||
return {
|
||||
"repository": "jens/privatemuse",
|
||||
"candidate_revision": diagnosis._FAILED_REVISION,
|
||||
"live_revision": diagnosis._LIVE_REVISION,
|
||||
"delivery_state": "failed",
|
||||
"delivery_message": diagnosis._FAILURE_MESSAGE,
|
||||
"candidate_container_mutation": False,
|
||||
"recorded_backend_container_id": adoption._RECORDED_BACKEND_ID,
|
||||
"observed_backend_container_id": adoption._OBSERVED_BACKEND_ID,
|
||||
"backend_image_id": incident_contract._EXPECTED_IMAGE_IDS["backend"],
|
||||
"frontend_image_id": incident_contract._EXPECTED_IMAGE_IDS["frontend"],
|
||||
"audio_image_id": incident_contract._EXPECTED_IMAGE_IDS["audio"],
|
||||
"all_recorded_image_ids_match": True,
|
||||
"frontend_and_audio_container_ids_match": True,
|
||||
"compose_source_sha256": incident_contract._COMPOSE_SHA256,
|
||||
"compose_source_hash_matches_receipt": True,
|
||||
"backend_health": "healthy",
|
||||
"frontend_state": "running",
|
||||
"audio_health": "healthy",
|
||||
"compose_project": "privatemuse",
|
||||
"backend_service": "backend",
|
||||
"backend_mounts": sorted(incident_contract._EXPECTED_MOUNTS),
|
||||
"backend_networks": sorted(incident_contract._EXPECTED_NETWORKS),
|
||||
"configuration_evidence": {
|
||||
"old_delivery_updated_at_epoch": 1790271038,
|
||||
"protected_env_changed_at_utc": "2026-09-24T18:13:33Z",
|
||||
"backup_filename": ".env.pre-modelforge-ready-1790273613.bak",
|
||||
"changed_key_names": [diagnosis._HEALTH_PATH_KEY],
|
||||
"live_value_matches_current_file": True,
|
||||
"live_value_matches_backup": False,
|
||||
"historical_effective_runtime_fingerprint": "not_recorded",
|
||||
"secret_values_recorded": False,
|
||||
},
|
||||
"provider_submissions": 0,
|
||||
"job_retries": 0,
|
||||
"production_mutations": 0,
|
||||
"backend_runtime_mount_network_port_and_env_match_current_compose": True,
|
||||
"backend_frontend_same_compose_project": True,
|
||||
"backend_frontend_share_network": True,
|
||||
"backend_alias_visible_to_frontend": True,
|
||||
"audio_container_identity_matches_contract": True,
|
||||
}
|
||||
|
||||
|
||||
def _post_adoption_evidence():
|
||||
evidence = _incident_evidence()
|
||||
evidence["recorded_backend_container_id"] = adoption._OBSERVED_BACKEND_ID
|
||||
return evidence
|
||||
|
||||
|
||||
def _fixture_state():
|
||||
revision = diagnosis._LIVE_REVISION
|
||||
tree = "a" * 40
|
||||
built_at = "2026-09-24T17:00:00.000Z"
|
||||
build = {
|
||||
"id": revision[:12],
|
||||
"repository": diagnosis._REPOSITORY,
|
||||
"revision": revision,
|
||||
"tree": tree,
|
||||
"treeDigest": hashlib.sha256(f"git-tree:{tree}".encode()).hexdigest(),
|
||||
"builtAt": built_at,
|
||||
}
|
||||
container_ids = {
|
||||
"backend": adoption._RECORDED_BACKEND_ID,
|
||||
"frontend": "1" * 64,
|
||||
"audio": "2" * 64,
|
||||
}
|
||||
containers = {}
|
||||
for role, name in diagnosis._CONTAINERS.items():
|
||||
containers[name] = {
|
||||
"container_id": container_ids[role],
|
||||
"image_id": incident_contract._EXPECTED_IMAGE_IDS[role],
|
||||
"image_ref": f"local/privatemuse-{role}:b79",
|
||||
"container_revision": revision,
|
||||
"image_revision": revision,
|
||||
"image_source": diagnosis._REPOSITORY,
|
||||
"image_tree": tree,
|
||||
"image_version": revision[:12],
|
||||
"image_created": built_at,
|
||||
}
|
||||
receipt = {
|
||||
"schema": "gitea-deploy-receipt.v1",
|
||||
"repository": diagnosis._REPOSITORY,
|
||||
"revision": revision,
|
||||
"tree": tree,
|
||||
"treeDigest": build["treeDigest"],
|
||||
"build": build,
|
||||
"containers": copy.deepcopy(containers),
|
||||
"live": None,
|
||||
}
|
||||
artifacts = {
|
||||
"containers": copy.deepcopy(containers),
|
||||
"compose_sha256": incident_contract._COMPOSE_SHA256,
|
||||
}
|
||||
state = {
|
||||
"schema_version": 3,
|
||||
"repository": diagnosis._REPOSITORY,
|
||||
"slug": "privatemuse",
|
||||
"compose_project": "privatemuse",
|
||||
"revision": revision,
|
||||
"live_revision": revision,
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": False,
|
||||
"live_artifacts": copy.deepcopy(artifacts),
|
||||
"live_receipt": copy.deepcopy(receipt),
|
||||
"rollback_snapshot": {},
|
||||
"status": "failed",
|
||||
"message": diagnosis._FAILURE_MESSAGE,
|
||||
"updated_at_epoch": 1790271038,
|
||||
"artifacts": copy.deepcopy(artifacts),
|
||||
"last_attempt": {
|
||||
"revision": diagnosis._FAILED_REVISION,
|
||||
"status": "failed",
|
||||
"message": diagnosis._FAILURE_MESSAGE,
|
||||
"updated_at_epoch": 1790271038,
|
||||
"metadata": {"operation": "delivery"},
|
||||
},
|
||||
}
|
||||
current_containers = copy.deepcopy(containers)
|
||||
current_containers["privatemuse-backend-1"]["container_id"] = adoption._OBSERVED_BACKEND_ID
|
||||
current_receipt = copy.deepcopy(receipt)
|
||||
current_receipt["containers"] = copy.deepcopy(current_containers)
|
||||
current_artifacts = {
|
||||
"containers": copy.deepcopy(current_containers),
|
||||
"compose_sha256": incident_contract._COMPOSE_SHA256,
|
||||
}
|
||||
return state, receipt, current_receipt, current_artifacts
|
||||
|
||||
|
||||
class FakeController:
|
||||
def __init__(self, current_receipt, current_artifacts):
|
||||
self.current_receipt = current_receipt
|
||||
self.current_artifacts = current_artifacts
|
||||
self.verified_image_maps = []
|
||||
|
||||
def repository_path(self, repository, _project):
|
||||
assert repository == diagnosis._REPOSITORY
|
||||
return Path("/unused/fixed/repository.git")
|
||||
|
||||
def validate_recorded_receipt(self, _project, receipt, repository, revision):
|
||||
assert receipt["repository"] == repository
|
||||
assert receipt["revision"] == revision
|
||||
return receipt["build"]
|
||||
|
||||
def verify_deployment(self, _project, _metadata, *, expected_image_ids):
|
||||
self.verified_image_maps.append(expected_image_ids)
|
||||
return copy.deepcopy(self.current_receipt)
|
||||
|
||||
def deployment_artifacts(self, _project):
|
||||
return copy.deepcopy(self.current_artifacts)
|
||||
|
||||
|
||||
def _install_fixture(monkeypatch, tmp_path, evidence_sequence=None):
|
||||
state, old_receipt, current_receipt, current_artifacts = _fixture_state()
|
||||
state_path = tmp_path / "privatemuse.json"
|
||||
original_bytes = adoption._json_bytes(state)
|
||||
state_path.write_bytes(original_bytes)
|
||||
state_path.chmod(0o600)
|
||||
transaction_dir = tmp_path / "privatemuse-f84-adoption"
|
||||
controller = FakeController(current_receipt, current_artifacts)
|
||||
project = {
|
||||
"slug": "privatemuse",
|
||||
"compose_project": "privatemuse",
|
||||
"containers": list(diagnosis._CONTAINERS.values()),
|
||||
}
|
||||
sequence = list(evidence_sequence or [_incident_evidence(), _incident_evidence(), _post_adoption_evidence()])
|
||||
|
||||
@contextmanager
|
||||
def no_controller_locks():
|
||||
yield
|
||||
|
||||
monkeypatch.setattr(adoption, "_STATE_PATH", state_path)
|
||||
monkeypatch.setattr(adoption, "_INCIDENT_DIR", transaction_dir)
|
||||
monkeypatch.setattr(adoption, "_BACKUP_PATH", transaction_dir / "original-state.json")
|
||||
monkeypatch.setattr(adoption, "_RECEIPT_BACKUP_PATH", transaction_dir / "original-live-receipt.json")
|
||||
monkeypatch.setattr(adoption, "_AUDIT_PATH", transaction_dir / "approval-audit.json")
|
||||
monkeypatch.setattr(adoption, "_CANDIDATE_PATH", transaction_dir / "baseline-state.json")
|
||||
monkeypatch.setattr(adoption, "_MANIFEST_PATH", transaction_dir / "transaction-manifest.json")
|
||||
monkeypatch.setattr(adoption, "_TRANSACTION_FILES", {
|
||||
"original-state.json",
|
||||
"original-live-receipt.json",
|
||||
"approval-audit.json",
|
||||
"baseline-state.json",
|
||||
"transaction-manifest.json",
|
||||
})
|
||||
monkeypatch.setattr(adoption, "exclusive_controller_locks", no_controller_locks)
|
||||
monkeypatch.setattr(adoption, "_protected_main_revision", lambda *_args: diagnosis._FAILED_REVISION)
|
||||
monkeypatch.setattr(diagnosis, "_load_controller_module", lambda: controller)
|
||||
monkeypatch.setattr(diagnosis, "_expected_project", lambda: copy.deepcopy(project))
|
||||
monkeypatch.setattr(
|
||||
diagnosis,
|
||||
"collect_live_evidence_under_lock",
|
||||
lambda: copy.deepcopy(sequence.pop(0) if sequence else _post_adoption_evidence()),
|
||||
)
|
||||
monkeypatch.setattr(adoption, "_fsync_directory", lambda _path: None)
|
||||
monkeypatch.setattr(adoption.os, "fchown", lambda *_args: None, raising=False)
|
||||
return state_path, transaction_dir, original_bytes, state, old_receipt, current_receipt, current_artifacts, controller
|
||||
|
||||
|
||||
def _run_adoption():
|
||||
return adoption.adopt_current_runtime(
|
||||
approval_reference=adoption._APPROVAL_REFERENCE,
|
||||
acknowledge_unknown_historical_configuration=True,
|
||||
apply=True,
|
||||
)
|
||||
|
||||
|
||||
def test_adoption_writes_an_audited_current_baseline_and_preserves_failure_history(monkeypatch, tmp_path):
|
||||
state_path, transaction_dir, original_bytes, original, _old_receipt, current_receipt, artifacts, controller = (
|
||||
_install_fixture(monkeypatch, tmp_path)
|
||||
)
|
||||
|
||||
result = _run_adoption()
|
||||
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
audit = json.loads((transaction_dir / "approval-audit.json").read_text(encoding="utf-8"))
|
||||
assert result["status"] == "adopted"
|
||||
assert (transaction_dir / "original-state.json").read_bytes() == original_bytes
|
||||
assert json.loads((transaction_dir / "original-live-receipt.json").read_text()) == original["live_receipt"]
|
||||
assert state["live_receipt"] == current_receipt
|
||||
assert state["live_artifacts"] == artifacts
|
||||
assert state["last_attempt"] == original["last_attempt"]
|
||||
assert state["status"] == "failed"
|
||||
assert state["live_revision"] == diagnosis._LIVE_REVISION
|
||||
assert state["f84_runtime_adoption"]["historical_effective_configuration"] == "unknown"
|
||||
assert state["f84_runtime_adoption"]["historical_equivalence_claimed"] is False
|
||||
assert audit["approval_reference"] == adoption._APPROVAL_REFERENCE
|
||||
assert audit["known_configuration_delta"] == {
|
||||
"key": diagnosis._HEALTH_PATH_KEY,
|
||||
"prior_file_value": diagnosis._OLD_HEALTH_PATH,
|
||||
"adopted_runtime_value": diagnosis._CURRENT_HEALTH_PATH,
|
||||
}
|
||||
assert "secret" not in (transaction_dir / "approval-audit.json").read_text().lower()
|
||||
assert controller.verified_image_maps[-1] == adoption._expected_image_ids_by_name()
|
||||
|
||||
|
||||
def test_adoption_is_idempotent_after_a_verified_commit(monkeypatch, tmp_path):
|
||||
state_path, _transaction_dir, *_rest = _install_fixture(monkeypatch, tmp_path)
|
||||
|
||||
assert _run_adoption()["status"] == "adopted"
|
||||
adopted_bytes = state_path.read_bytes()
|
||||
assert _run_adoption()["status"] == "already_adopted"
|
||||
assert state_path.read_bytes() == adopted_bytes
|
||||
|
||||
|
||||
def test_prepared_partial_transaction_resumes_only_after_fresh_checks(monkeypatch, tmp_path):
|
||||
bad_second = _incident_evidence()
|
||||
bad_second["audio_health"] = "unhealthy"
|
||||
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
evidence_sequence=[_incident_evidence(), bad_second, _incident_evidence(), _incident_evidence(), _post_adoption_evidence()],
|
||||
)
|
||||
|
||||
with pytest.raises(adoption.AdoptionError, match="fresh live evidence"):
|
||||
_run_adoption()
|
||||
assert state_path.read_bytes() == original_bytes
|
||||
assert (transaction_dir / "approval-audit.json").is_file()
|
||||
|
||||
resumed = _run_adoption()
|
||||
|
||||
assert resumed["status"] == "adoption_resumed"
|
||||
assert json.loads(state_path.read_text())["f84_runtime_adoption"]["approval_reference"] == adoption._APPROVAL_REFERENCE
|
||||
|
||||
|
||||
def test_incomplete_transaction_fails_closed_without_replacing_state(monkeypatch, tmp_path):
|
||||
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
|
||||
transaction_dir.mkdir()
|
||||
(transaction_dir / "unexpected-partial-file").write_bytes(b"preserve")
|
||||
|
||||
with pytest.raises(adoption.AdoptionError, match="incomplete adoption transaction"):
|
||||
_run_adoption()
|
||||
|
||||
assert state_path.read_bytes() == original_bytes
|
||||
assert (transaction_dir / "unexpected-partial-file").read_bytes() == b"preserve"
|
||||
|
||||
|
||||
def test_post_write_verification_failure_restores_exact_original_state(monkeypatch, tmp_path):
|
||||
invalid_post = _post_adoption_evidence()
|
||||
invalid_post["backend_runtime_mount_network_port_and_env_match_current_compose"] = False
|
||||
state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
evidence_sequence=[_incident_evidence(), _incident_evidence(), invalid_post],
|
||||
)
|
||||
|
||||
with pytest.raises(adoption.AdoptionError, match="post-adoption runtime"):
|
||||
_run_adoption()
|
||||
|
||||
assert state_path.read_bytes() == original_bytes
|
||||
|
||||
|
||||
def test_third_party_state_after_replace_is_preserved_without_rollback_overwrite(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
|
||||
original_collector = diagnosis.collect_live_evidence_under_lock
|
||||
calls = 0
|
||||
third_party_bytes = b"third-party-controller-update"
|
||||
|
||||
def diverge_after_state_replace():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
evidence = original_collector()
|
||||
if calls == 3:
|
||||
state_path.write_bytes(third_party_bytes)
|
||||
return evidence
|
||||
|
||||
monkeypatch.setattr(diagnosis, "collect_live_evidence_under_lock", diverge_after_state_replace)
|
||||
|
||||
with pytest.raises(adoption.AdoptionError, match="diverged after adoption failure"):
|
||||
_run_adoption()
|
||||
|
||||
assert state_path.read_bytes() == third_party_bytes
|
||||
assert (transaction_dir / "original-state.json").read_bytes() == original_bytes
|
||||
|
||||
|
||||
def test_state_rename_failure_leaves_original_state_and_transaction_evidence(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
|
||||
real_replace = adoption.os.replace
|
||||
|
||||
def fail_state_rename(source, destination):
|
||||
if Path(destination) == state_path:
|
||||
raise OSError("injected state rename failure")
|
||||
return real_replace(source, destination)
|
||||
|
||||
monkeypatch.setattr(adoption.os, "replace", fail_state_rename)
|
||||
|
||||
with pytest.raises(adoption.AdoptionError, match="state replacement did not complete"):
|
||||
_run_adoption()
|
||||
|
||||
assert state_path.read_bytes() == original_bytes
|
||||
assert (transaction_dir / "original-state.json").read_bytes() == original_bytes
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="controller directory fsync requires Linux")
|
||||
def test_directory_fsync_failure_after_rename_restores_original_state(monkeypatch, tmp_path):
|
||||
real_fsync_directory = adoption._fsync_directory
|
||||
state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
|
||||
real_replace = adoption.os.replace
|
||||
state_was_replaced = False
|
||||
injected = False
|
||||
|
||||
def track_state_replace(source, destination):
|
||||
nonlocal state_was_replaced
|
||||
real_replace(source, destination)
|
||||
if Path(destination) == state_path:
|
||||
state_was_replaced = True
|
||||
|
||||
def fail_first_post_replace_fsync(path):
|
||||
nonlocal injected
|
||||
if state_was_replaced and Path(path) == state_path.parent and not injected:
|
||||
injected = True
|
||||
raise OSError("injected directory fsync failure")
|
||||
return real_fsync_directory(path)
|
||||
|
||||
monkeypatch.setattr(adoption.os, "replace", track_state_replace)
|
||||
monkeypatch.setattr(adoption, "_fsync_directory", fail_first_post_replace_fsync)
|
||||
|
||||
with pytest.raises(adoption.AdoptionError, match="state replacement did not complete"):
|
||||
_run_adoption()
|
||||
|
||||
assert injected is True
|
||||
assert state_path.read_bytes() == original_bytes
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "replacement"),
|
||||
[
|
||||
("schema", "unexpected-schema"),
|
||||
("action", "unbounded-adoption"),
|
||||
("known_configuration_delta", {}),
|
||||
("container_ids", {"audio": "tampered"}),
|
||||
("image_ids", {"audio": "sha256:tampered"}),
|
||||
],
|
||||
)
|
||||
def test_candidate_verification_rejects_tampered_audit_fields(
|
||||
field, replacement
|
||||
):
|
||||
original_state, original_receipt, current_receipt, current_artifacts = _fixture_state()
|
||||
original_bytes = adoption._json_bytes(original_state)
|
||||
candidate, candidate_bytes, audit, audit_bytes, _manifest, _manifest_bytes = (
|
||||
adoption._prepare_transaction(
|
||||
original_bytes,
|
||||
original_state,
|
||||
original_receipt,
|
||||
receipt=current_receipt,
|
||||
artifacts=current_artifacts,
|
||||
)
|
||||
)
|
||||
tampered_audit = copy.deepcopy(audit)
|
||||
tampered_audit[field] = replacement
|
||||
|
||||
with pytest.raises(adoption.AdoptionError, match="prepared adoption baseline"):
|
||||
adoption._verify_transaction_candidate(
|
||||
original_bytes=original_bytes,
|
||||
original_state=original_state,
|
||||
original_receipt=original_receipt,
|
||||
current_receipt=current_receipt,
|
||||
current_artifacts=current_artifacts,
|
||||
candidate_bytes=candidate_bytes,
|
||||
candidate=candidate,
|
||||
audit_bytes=audit_bytes,
|
||||
audit=tampered_audit,
|
||||
)
|
||||
|
||||
|
||||
def test_state_race_immediately_before_replace_does_not_clobber_new_bytes(monkeypatch, tmp_path):
|
||||
state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
|
||||
original_main_check = adoption._protected_main_revision
|
||||
calls = 0
|
||||
|
||||
def change_state_on_final_head_check(controller, project):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 3:
|
||||
state_path.write_bytes(b"concurrent-controller-update")
|
||||
return original_main_check(controller, project)
|
||||
|
||||
monkeypatch.setattr(adoption, "_protected_main_revision", change_state_on_final_head_check)
|
||||
|
||||
with pytest.raises(adoption.AdoptionError):
|
||||
_run_adoption()
|
||||
|
||||
assert state_path.read_bytes() == b"concurrent-controller-update"
|
||||
assert (tmp_path / "privatemuse-f84-adoption" / "original-state.json").read_bytes() == original_bytes
|
||||
|
||||
|
||||
def test_mutation_requires_exact_approval_acknowledgement_and_apply(monkeypatch, tmp_path):
|
||||
state_path, _transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
|
||||
|
||||
with pytest.raises(adoption.AdoptionError, match="approval reference"):
|
||||
adoption.adopt_current_runtime(
|
||||
approval_reference="other",
|
||||
acknowledge_unknown_historical_configuration=True,
|
||||
apply=True,
|
||||
)
|
||||
with pytest.raises(adoption.AdoptionError, match="acknowledgement"):
|
||||
adoption.adopt_current_runtime(
|
||||
approval_reference=adoption._APPROVAL_REFERENCE,
|
||||
acknowledge_unknown_historical_configuration=False,
|
||||
apply=True,
|
||||
)
|
||||
with pytest.raises(adoption.AdoptionError, match="--apply"):
|
||||
adoption.adopt_current_runtime(
|
||||
approval_reference=adoption._APPROVAL_REFERENCE,
|
||||
acknowledge_unknown_historical_configuration=True,
|
||||
apply=False,
|
||||
)
|
||||
|
||||
assert state_path.read_bytes() == original_bytes
|
||||
|
||||
|
||||
def test_fresh_runtime_drift_rejects_before_creating_adoption_files(monkeypatch, tmp_path):
|
||||
bad = _incident_evidence()
|
||||
bad["backend_networks"] = ["unexpected_network"]
|
||||
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
evidence_sequence=[bad],
|
||||
)
|
||||
|
||||
with pytest.raises(adoption.AdoptionError, match="fresh live evidence"):
|
||||
_run_adoption()
|
||||
|
||||
assert state_path.read_bytes() == original_bytes
|
||||
assert not transaction_dir.exists()
|
||||
|
||||
|
||||
def test_protected_main_change_rejects_before_state_or_audit_mutation(monkeypatch, tmp_path):
|
||||
state_path, transaction_dir, original_bytes, *_rest = _install_fixture(monkeypatch, tmp_path)
|
||||
monkeypatch.setattr(adoption, "_protected_main_revision", lambda *_args: "0" * 40)
|
||||
|
||||
with pytest.raises(adoption.AdoptionError, match="protected main"):
|
||||
_run_adoption()
|
||||
|
||||
assert state_path.read_bytes() == original_bytes
|
||||
assert not transaction_dir.exists()
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="controller locks use Linux flock")
|
||||
def test_exclusive_controller_locks_are_ordered_and_refuse_contention(monkeypatch, tmp_path):
|
||||
import fcntl
|
||||
|
||||
project_lock = tmp_path / "project.lock"
|
||||
global_lock = tmp_path / "global.lock"
|
||||
project_lock.write_bytes(b"")
|
||||
global_lock.write_bytes(b"")
|
||||
monkeypatch.setattr(diagnosis, "_PROJECT_LOCK", project_lock)
|
||||
monkeypatch.setattr(diagnosis, "_GLOBAL_LOCK", global_lock)
|
||||
|
||||
real_open = os.open
|
||||
opened = []
|
||||
|
||||
def recording_open(path, *args, **kwargs):
|
||||
opened.append(Path(path))
|
||||
return real_open(path, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(adoption.os, "open", recording_open)
|
||||
with adoption.exclusive_controller_locks():
|
||||
assert opened == [project_lock, global_lock]
|
||||
|
||||
held_descriptor = real_open(project_lock, os.O_RDWR)
|
||||
try:
|
||||
fcntl.flock(held_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
opened.clear()
|
||||
with pytest.raises(adoption.AdoptionError, match="deployment is active"), adoption.exclusive_controller_locks():
|
||||
pytest.fail("contended project lock unexpectedly acquired")
|
||||
assert opened == [project_lock]
|
||||
finally:
|
||||
fcntl.flock(held_descriptor, fcntl.LOCK_UN)
|
||||
os.close(held_descriptor)
|
||||
@@ -0,0 +1,311 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(
|
||||
0,
|
||||
str(Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker"),
|
||||
)
|
||||
|
||||
from privatemuse_f84_receipt_diagnosis import (
|
||||
assess_privatemuse_f84_receipt_drift,
|
||||
)
|
||||
|
||||
_COLLECTOR_PATH = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "ops"
|
||||
/ "gitea-deploy-broker"
|
||||
/ "diagnose_privatemuse_f84_receipt.py"
|
||||
)
|
||||
_COLLECTOR_SPEC = importlib.util.spec_from_file_location(
|
||||
"privatemuse_f84_diagnostic_cli", _COLLECTOR_PATH
|
||||
)
|
||||
assert _COLLECTOR_SPEC is not None and _COLLECTOR_SPEC.loader is not None
|
||||
collector = importlib.util.module_from_spec(_COLLECTOR_SPEC)
|
||||
_COLLECTOR_SPEC.loader.exec_module(collector)
|
||||
|
||||
|
||||
def incident_evidence():
|
||||
return {
|
||||
"repository": "Jens/PrivateMuse",
|
||||
"protected_main": "f84fa7047f65346eaaee4e5a09b0973339e94a9b",
|
||||
"live_revision": "b79bdca08a138a752c414d4d6eb681b67bf6cb8f",
|
||||
"delivery_state": "failed",
|
||||
"delivery_message": "Existing deployment no longer matches its verified live receipt",
|
||||
"candidate_container_mutation": False,
|
||||
"recorded_backend_container_id": "000ea567f0f6dc6274ecaf9e7e2bf75763599f814b2b48c0cd7f806f53a5aa2a",
|
||||
"observed_backend_container_id": "4508b169cd3ee0ab398bcc3cc11fb7cb8ec49eb86642ff577fa6dbbfe8e1284b",
|
||||
"backend_image_id": "sha256:36392964088a546fc861860198c3e8f483edc89c81a3e7a6ad40c2c914fc4eda",
|
||||
"frontend_image_id": "sha256:2c64ef9571a02dfa4040ae47d2c292488173d7ef3a8d66e9b007250b704dfd79",
|
||||
"audio_image_id": "sha256:4f75a143b608dec4d19f1af800a2016710116fb917166617db0b7229ecc49afc",
|
||||
"all_recorded_image_ids_match": True,
|
||||
"frontend_and_audio_container_ids_match": True,
|
||||
"compose_source_sha256": "f0e83adea9038d9bf356ec60190e83782a7260b35251fb87f118150f09be16d2",
|
||||
"compose_source_hash_matches_receipt": True,
|
||||
"backend_health": "healthy",
|
||||
"frontend_state": "running",
|
||||
"audio_health": "healthy",
|
||||
"compose_project": "privatemuse",
|
||||
"backend_service": "backend",
|
||||
"backend_mounts": [
|
||||
"bind:/srv/example/appdata/privatemuse/data->/app/data",
|
||||
"bind:/srv/example/appdata/comfyui-nvidia/basedir/models->/comfy-models",
|
||||
],
|
||||
"backend_networks": ["privatemuse_default"],
|
||||
"configuration_evidence": {
|
||||
"old_delivery_updated_at_epoch": 1790271038,
|
||||
"protected_env_changed_at_utc": "2026-09-24T18:13:33Z",
|
||||
"backup_filename": ".env.pre-modelforge-ready-1790273613.bak",
|
||||
"changed_key_names": ["PRIVATEMUSE_MODELFORGE_HEALTH_PATH"],
|
||||
"live_value_matches_current_file": True,
|
||||
"live_value_matches_backup": False,
|
||||
"historical_effective_runtime_fingerprint": "not_recorded",
|
||||
"secret_values_recorded": False,
|
||||
},
|
||||
"provider_submissions": 0,
|
||||
"job_retries": 0,
|
||||
"production_mutations": 0,
|
||||
}
|
||||
|
||||
|
||||
def test_exact_incident_is_diagnosed_but_never_reattested():
|
||||
result = assess_privatemuse_f84_receipt_drift(incident_evidence())
|
||||
|
||||
assert result["status"] == "diagnosed"
|
||||
assert result["decision"] == "rebaseline_required"
|
||||
assert result["reattestation_permitted"] is False
|
||||
assert result["historical_effective_configuration"] == "unproven"
|
||||
assert result["configuration_delta"] == {
|
||||
"key": "PRIVATEMUSE_MODELFORGE_HEALTH_PATH",
|
||||
"recorded_before_value": "/healthz",
|
||||
"current_runtime_value": "/api/v1/health/ready",
|
||||
}
|
||||
assert result["receipt_drift"] == "backend_container_id_changed_only"
|
||||
assert result["runtime_state"] == {
|
||||
"backend": "healthy",
|
||||
"frontend": "running",
|
||||
"audio": "healthy",
|
||||
"frontend_http_probe": "not_performed",
|
||||
}
|
||||
|
||||
|
||||
def test_diagnostic_fails_closed_if_a_second_container_id_changed():
|
||||
evidence = incident_evidence()
|
||||
evidence["frontend_and_audio_container_ids_match"] = False
|
||||
|
||||
result = assess_privatemuse_f84_receipt_drift(evidence)
|
||||
|
||||
assert result["status"] == "evidence_incomplete"
|
||||
assert result["decision"] == "no_reattestation"
|
||||
assert result["reattestation_permitted"] is False
|
||||
assert "frontend_and_audio_container_ids_match" in result["failed_checks"]
|
||||
|
||||
|
||||
def test_diagnostic_fails_closed_if_config_drift_is_broader_or_unbounded():
|
||||
evidence = incident_evidence()
|
||||
evidence["configuration_evidence"]["changed_key_names"].append("DATABASE_URL")
|
||||
|
||||
result = assess_privatemuse_f84_receipt_drift(evidence)
|
||||
|
||||
assert result["status"] == "evidence_incomplete"
|
||||
assert result["decision"] == "no_reattestation"
|
||||
assert "configuration_changed_keys" in result["failed_checks"]
|
||||
|
||||
|
||||
def test_diagnostic_fails_closed_when_any_recorded_image_differs():
|
||||
evidence = incident_evidence()
|
||||
evidence["all_recorded_image_ids_match"] = False
|
||||
|
||||
result = assess_privatemuse_f84_receipt_drift(evidence)
|
||||
|
||||
assert result["status"] == "evidence_incomplete"
|
||||
assert "all_recorded_image_ids_match" in result["failed_checks"]
|
||||
|
||||
|
||||
def test_diagnostic_fails_closed_when_runtime_mounts_or_network_drift():
|
||||
evidence = incident_evidence()
|
||||
evidence["backend_mounts"] = ["bind:/unexpected->/app/data"]
|
||||
evidence["backend_networks"] = ["unexpected_default"]
|
||||
|
||||
result = assess_privatemuse_f84_receipt_drift(evidence)
|
||||
|
||||
assert result["status"] == "evidence_incomplete"
|
||||
assert "backend_mounts_match_contract" in result["failed_checks"]
|
||||
assert "backend_networks_match_contract" in result["failed_checks"]
|
||||
|
||||
|
||||
def test_diagnostic_projection_never_returns_untrusted_input_values():
|
||||
evidence = incident_evidence()
|
||||
evidence["private_key"] = "do-not-project-this"
|
||||
evidence["configuration_evidence"]["unexpected_value"] = "also-do-not-project"
|
||||
|
||||
result = assess_privatemuse_f84_receipt_drift(evidence)
|
||||
rendered = str(result)
|
||||
|
||||
assert "do-not-project-this" not in rendered
|
||||
assert "also-do-not-project" not in rendered
|
||||
|
||||
|
||||
def test_mismatched_incident_does_not_claim_a_diagnosis():
|
||||
evidence = deepcopy(incident_evidence())
|
||||
evidence["protected_main"] = "0" * 40
|
||||
|
||||
result = assess_privatemuse_f84_receipt_drift(evidence)
|
||||
|
||||
assert result["status"] == "evidence_incomplete"
|
||||
assert result["decision"] == "no_reattestation"
|
||||
assert "candidate_revision" in result["failed_checks"]
|
||||
|
||||
|
||||
def test_malformed_timestamps_and_unhashable_mount_entries_fail_closed():
|
||||
evidence = incident_evidence()
|
||||
evidence["backend_mounts"] = [{"source": "untrusted"}]
|
||||
evidence["configuration_evidence"]["old_delivery_updated_at_epoch"] = 10**100
|
||||
|
||||
result = assess_privatemuse_f84_receipt_drift(evidence)
|
||||
|
||||
assert result["status"] == "evidence_incomplete"
|
||||
assert "backend_mounts_match_contract" in result["failed_checks"]
|
||||
assert "configuration_change_after_delivery" in result["failed_checks"]
|
||||
|
||||
|
||||
def test_non_mapping_snapshot_fails_closed_without_exception():
|
||||
result = assess_privatemuse_f84_receipt_drift([]) # type: ignore[arg-type]
|
||||
|
||||
assert result["decision"] == "no_reattestation"
|
||||
assert result["failed_checks"] == ["evidence_shape"]
|
||||
|
||||
|
||||
def runtime_config_and_inspection():
|
||||
rendered = {
|
||||
"services": {
|
||||
"backend": {
|
||||
"environment": {
|
||||
"PRIVATEMUSE_MODELFORGE_HEALTH_PATH": "/api/v1/health/ready",
|
||||
"DATABASE_URL": "postgresql://redacted-fixture",
|
||||
},
|
||||
"volumes": [
|
||||
{
|
||||
"type": "bind",
|
||||
"source": "/srv/example/appdata/privatemuse/data",
|
||||
"target": "/app/data",
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"source": "/srv/example/appdata/comfyui-nvidia/basedir/models",
|
||||
"target": "/comfy-models",
|
||||
},
|
||||
],
|
||||
"networks": {"default": {}},
|
||||
"ports": [],
|
||||
}
|
||||
},
|
||||
"networks": {"default": {"name": "privatemuse_default"}},
|
||||
}
|
||||
inspection = {
|
||||
"Mounts": [
|
||||
{
|
||||
"Type": "bind",
|
||||
"Source": "/srv/example/appdata/privatemuse/data",
|
||||
"Destination": "/app/data",
|
||||
"RW": True,
|
||||
},
|
||||
{
|
||||
"Type": "bind",
|
||||
"Source": "/srv/example/appdata/comfyui-nvidia/basedir/models",
|
||||
"Destination": "/comfy-models",
|
||||
"RW": True,
|
||||
},
|
||||
],
|
||||
"Config": {
|
||||
"Env": [
|
||||
"PRIVATEMUSE_MODELFORGE_HEALTH_PATH=/api/v1/health/ready",
|
||||
"DATABASE_URL=postgresql://redacted-fixture",
|
||||
]
|
||||
},
|
||||
"NetworkSettings": {
|
||||
"Networks": {"privatemuse_default": {"Aliases": ["backend"]}}
|
||||
},
|
||||
"HostConfig": {"PortBindings": {}},
|
||||
}
|
||||
return rendered, inspection
|
||||
|
||||
|
||||
def test_live_runtime_projection_matches_declared_mounts_networks_and_ports():
|
||||
rendered, inspection = runtime_config_and_inspection()
|
||||
|
||||
assert collector._runtime_matches_compose(inspection, rendered) is True
|
||||
|
||||
|
||||
def test_live_runtime_projection_rejects_mount_network_or_port_drift():
|
||||
rendered, inspection = runtime_config_and_inspection()
|
||||
altered_mount = deepcopy(inspection)
|
||||
altered_mount["Mounts"][0]["Destination"] = "/unexpected"
|
||||
altered_network = deepcopy(inspection)
|
||||
altered_network["NetworkSettings"]["Networks"] = {"other_default": {}}
|
||||
altered_ports = deepcopy(inspection)
|
||||
altered_ports["HostConfig"]["PortBindings"] = {
|
||||
"8000/tcp": [{"HostIp": "0.0.0.0", "HostPort": "8000"}]
|
||||
}
|
||||
|
||||
assert collector._runtime_matches_compose(altered_mount, rendered) is False
|
||||
assert collector._runtime_matches_compose(altered_network, rendered) is False
|
||||
assert collector._runtime_matches_compose(altered_ports, rendered) is False
|
||||
|
||||
|
||||
def test_live_runtime_projection_checks_every_declared_backend_environment_key():
|
||||
rendered, inspection = runtime_config_and_inspection()
|
||||
wrong_env = deepcopy(inspection)
|
||||
wrong_env["Config"]["Env"][1] = "DATABASE_URL=postgresql://different-fixture"
|
||||
|
||||
assert collector._runtime_matches_compose(inspection, rendered) is True
|
||||
assert collector._runtime_matches_compose(wrong_env, rendered) is False
|
||||
|
||||
|
||||
def test_live_backend_frontend_pairing_requires_shared_compose_network_alias():
|
||||
backend = {
|
||||
"NetworkSettings": {
|
||||
"Networks": {
|
||||
"privatemuse_default": {"Aliases": ["backend"]},
|
||||
}
|
||||
}
|
||||
}
|
||||
frontend = {
|
||||
"NetworkSettings": {
|
||||
"Networks": {
|
||||
"privatemuse_default": {"Aliases": ["frontend"]},
|
||||
}
|
||||
}
|
||||
}
|
||||
disconnected_frontend = {
|
||||
"NetworkSettings": {"Networks": {"other_default": {"Aliases": ["frontend"]}}}
|
||||
}
|
||||
|
||||
assert collector._backend_alias_visible_to_frontend(backend, frontend) is True
|
||||
assert collector._backend_alias_visible_to_frontend(backend, disconnected_frontend) is False
|
||||
|
||||
|
||||
def test_protected_env_parser_normalizes_only_the_health_path_and_keeps_values_private():
|
||||
previous = collector._parse_env_file(
|
||||
b'PRIVATEMUSE_MODELFORGE_HEALTH_PATH="/healthz"\nPROVIDER_TOKEN="secret\\x"\n'
|
||||
)
|
||||
current = collector._parse_env_file(
|
||||
b'PRIVATEMUSE_MODELFORGE_HEALTH_PATH=/api/v1/health/ready\nPROVIDER_TOKEN="secret\\x"\n'
|
||||
)
|
||||
|
||||
changed_keys = sorted(key for key in previous.keys() | current.keys() if previous.get(key) != current.get(key))
|
||||
assert changed_keys == ["PRIVATEMUSE_MODELFORGE_HEALTH_PATH"]
|
||||
assert previous["PRIVATEMUSE_MODELFORGE_HEALTH_PATH"] == "/healthz"
|
||||
assert current["PRIVATEMUSE_MODELFORGE_HEALTH_PATH"] == "/api/v1/health/ready"
|
||||
|
||||
|
||||
def test_protected_env_parser_rejects_duplicate_keys():
|
||||
try:
|
||||
collector._parse_env_file(
|
||||
b"PRIVATEMUSE_MODELFORGE_HEALTH_PATH=/healthz\nPRIVATEMUSE_MODELFORGE_HEALTH_PATH=/ready\n"
|
||||
)
|
||||
except collector.DiagnosisUnavailable:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("duplicate environment keys must fail closed")
|
||||
@@ -0,0 +1,177 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
from backend.app.services import action_execution
|
||||
from backend.app.services.action_execution import approval_execution_supported
|
||||
from backend.app.services.action_registry import (
|
||||
ACTION_RISK,
|
||||
ACTIONS,
|
||||
action_default_target,
|
||||
)
|
||||
from backend.app.services.privatemuse_webhook_repair import (
|
||||
PrivateMuseWebhookRepair,
|
||||
_ENDPOINT,
|
||||
_REPAIR_SCRIPT,
|
||||
)
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakePortainer:
|
||||
def __init__(self, exit_code=0):
|
||||
self.exit_code = exit_code
|
||||
self.calls = []
|
||||
|
||||
def get_container_status(self, name):
|
||||
assert name == "gitea-deploy-control"
|
||||
return {
|
||||
"status": "ok",
|
||||
"container": {
|
||||
"id": "controller-id",
|
||||
"name": "gitea-deploy-control",
|
||||
"state": "running",
|
||||
},
|
||||
}
|
||||
|
||||
def _config(self):
|
||||
return SimpleNamespace(endpoint_id="3")
|
||||
|
||||
def _request(self, method, path, **kwargs):
|
||||
self.calls.append((method, path, kwargs))
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return FakeResponse({"Id": "exec-webhook"})
|
||||
if path.endswith("/exec/exec-webhook/start"):
|
||||
return FakeResponse({})
|
||||
if path.endswith("/exec/exec-webhook/json"):
|
||||
return FakeResponse({"Running": False, "ExitCode": self.exit_code})
|
||||
raise AssertionError(path)
|
||||
|
||||
|
||||
def test_privatemuse_webhook_repair_is_fixed_high_risk_typed_action():
|
||||
action = "deployment.webhook.repair_privatemuse"
|
||||
assert action in ACTIONS
|
||||
assert ACTION_RISK[action] == "high"
|
||||
assert action_default_target(action) == "jens/privatemuse"
|
||||
assert approval_execution_supported(action) is True
|
||||
|
||||
|
||||
def test_privatemuse_webhook_repair_executes_only_fixed_contract():
|
||||
fake = FakePortainer()
|
||||
|
||||
result = PrivateMuseWebhookRepair(fake).run()
|
||||
|
||||
assert result == {
|
||||
"status": "repaired",
|
||||
"repository": "jens/privatemuse",
|
||||
"endpoint": _ENDPOINT,
|
||||
"branch": "main",
|
||||
"event": "push",
|
||||
}
|
||||
command = fake.calls[0][2]["json"]["Cmd"]
|
||||
assert command[:3] == ["python3", "-c", _REPAIR_SCRIPT]
|
||||
assert command[3:] == ["jens/privatemuse", _ENDPOINT]
|
||||
|
||||
|
||||
def test_privatemuse_webhook_repair_script_never_returns_credentials():
|
||||
assert "runner_guard_admin_token" in _REPAIR_SCRIPT
|
||||
assert 'secret_root = Path("/opt/gitea-deploy/webhook-secrets")' in _REPAIR_SCRIPT
|
||||
assert 'secret_path = secret_root / "jens__privatemuse"' in _REPAIR_SCRIPT
|
||||
assert "secretMaterialReturned" not in _REPAIR_SCRIPT
|
||||
assert 'print(f"privatemuse_runnerless_webhook_ready hook_id={hook_id}"' in _REPAIR_SCRIPT
|
||||
assert "print(secret" not in _REPAIR_SCRIPT
|
||||
assert "print(token" not in _REPAIR_SCRIPT
|
||||
|
||||
|
||||
def test_privatemuse_webhook_repair_proves_effective_controller_reachability():
|
||||
assert "gitea_config_path" not in _REPAIR_SCRIPT
|
||||
assert '"docker", "exec", "Gitea"' in _REPAIR_SCRIPT
|
||||
assert '"http://198.19.0.1:1299/health"' in _REPAIR_SCRIPT
|
||||
assert 'health_payload.get("status") != "ok"' in _REPAIR_SCRIPT
|
||||
assert 'health_payload.get("protocolVersion") != "itworx-controller-request/v2"' in _REPAIR_SCRIPT
|
||||
|
||||
|
||||
def test_privatemuse_webhook_repair_is_fail_closed_and_non_destructive():
|
||||
assert 'repository != "jens/privatemuse"' in _REPAIR_SCRIPT
|
||||
assert 'project.get("branch") != "main"' in _REPAIR_SCRIPT
|
||||
assert '"Managed validation / full (push)"' in _REPAIR_SCRIPT
|
||||
assert 'url.endswith("/v1/gitea/push/jens/privatemuse")' in _REPAIR_SCRIPT
|
||||
assert "len(exact) > 1 or semantic_other" in _REPAIR_SCRIPT
|
||||
assert 'request("PATCH", f"/repos/Jens/PrivateMuse/hooks/{hook_id}", payload)' in _REPAIR_SCRIPT
|
||||
assert 'request(\n "POST",\n "/repos/Jens/PrivateMuse/hooks",' in _REPAIR_SCRIPT
|
||||
for forbidden in (
|
||||
"docker restart",
|
||||
"docker stop",
|
||||
"docker rm",
|
||||
"docker compose",
|
||||
'request("DELETE"',
|
||||
):
|
||||
assert forbidden not in _REPAIR_SCRIPT
|
||||
|
||||
|
||||
def test_privatemuse_webhook_repair_maps_validation_failures():
|
||||
result = PrivateMuseWebhookRepair(FakePortainer(exit_code=34)).run()
|
||||
|
||||
assert result == {
|
||||
"status": "validation_failed",
|
||||
"reason": "PrivateMuse has ambiguous or stale runnerless webhook endpoints",
|
||||
}
|
||||
|
||||
|
||||
def test_privatemuse_webhook_repair_rejects_any_other_action_target():
|
||||
result = action_execution.execute_governed_action(
|
||||
None, # type: ignore[arg-type]
|
||||
action="deployment.webhook.repair_privatemuse",
|
||||
target="jens/not-privatemuse",
|
||||
source="chatgpt-mcp",
|
||||
risk="high",
|
||||
reason="Regression test for exact-target enforcement.",
|
||||
original_policy_decision="allowed",
|
||||
)
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["result"] == "blocked"
|
||||
assert result["message"] == "PrivateMuse webhook repair accepts only jens/privatemuse."
|
||||
|
||||
|
||||
def test_privatemuse_webhook_repair_dispatches_fixed_executor(monkeypatch, tmp_path):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'webhook-action.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
class FakeRepair:
|
||||
def run(self):
|
||||
return {
|
||||
"status": "repaired",
|
||||
"repository": "jens/privatemuse",
|
||||
"endpoint": _ENDPOINT,
|
||||
"branch": "main",
|
||||
"event": "push",
|
||||
}
|
||||
|
||||
import backend.app.services.privatemuse_webhook_repair as repair_module
|
||||
|
||||
monkeypatch.setattr(repair_module, "PrivateMuseWebhookRepair", FakeRepair)
|
||||
|
||||
with Session(engine) as session:
|
||||
result = action_execution.execute_governed_action(
|
||||
session,
|
||||
action="deployment.webhook.repair_privatemuse",
|
||||
target="jens/privatemuse",
|
||||
source="chatgpt-mcp",
|
||||
risk="high",
|
||||
reason="Repair the fixed PrivateMuse runnerless hook.",
|
||||
original_policy_decision="approval_required",
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["status"] == "success"
|
||||
assert result["policyDecision"] == "allowed"
|
||||
assert result["originalPolicyDecision"] == "approval_required"
|
||||
assert result["jobId"].startswith("job-")
|
||||
assert result["auditId"].startswith("audit-")
|
||||
@@ -0,0 +1,411 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_settings_load_voice_token_from_file(tmp_path):
|
||||
voice_token_file = tmp_path / "voice-token"
|
||||
voice_token_file.write_text("persistent-voice-token-value", encoding="utf-8")
|
||||
environment = dict(os.environ)
|
||||
environment.pop("APP_OPS_VOICE_TOKEN", None)
|
||||
environment["APP_OPS_VOICE_TOKEN_FILE"] = str(voice_token_file)
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"from backend.app.core.config import Settings; "
|
||||
"loaded = Settings(); "
|
||||
"raise SystemExit(0 if loaded.voice_token == 'persistent-voice-token-value' "
|
||||
"and loaded.voice_token_file else 1)",
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=environment,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "runtime_test.db"
|
||||
data_dir = tmp_path / "data"
|
||||
data_dir.mkdir()
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APPOPS_DATA_DIR", str(data_dir))
|
||||
monkeypatch.setenv("APPOPS_ENV", "production")
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.core import security
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
from backend.app.main import app
|
||||
|
||||
previous = {
|
||||
"db_path": settings.db_path,
|
||||
"data_dir": settings.data_dir,
|
||||
"environment": settings.environment,
|
||||
"voice_token": settings.voice_token,
|
||||
"security_voice_token": security.settings.voice_token,
|
||||
"apis": dict(settings.apis),
|
||||
}
|
||||
settings.db_path = str(db_path)
|
||||
settings.data_dir = str(data_dir)
|
||||
settings.environment = "production"
|
||||
settings.voice_token = "test-voice-token"
|
||||
security.settings.voice_token = "test-voice-token"
|
||||
for key in ["PORTAINER_URL", "PORTAINER_API_KEY", "PORTAINER_ENDPOINT_ID"]:
|
||||
settings.apis[key] = None
|
||||
|
||||
engine = create_engine(
|
||||
f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False}
|
||||
)
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
settings.db_path = previous["db_path"]
|
||||
settings.data_dir = previous["data_dir"]
|
||||
settings.environment = previous["environment"]
|
||||
settings.voice_token = previous["voice_token"]
|
||||
settings.apis = previous["apis"]
|
||||
security.settings.voice_token = previous["security_voice_token"]
|
||||
|
||||
|
||||
def test_system_summary_prefers_setup_config_over_stale_connector_status(api_client):
|
||||
from sqlmodel import Session
|
||||
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.db.models import Connector
|
||||
|
||||
with Session(db_session.engine) as session:
|
||||
connector = session.get(Connector, "portainer")
|
||||
connector.status = "connected"
|
||||
session.add(connector)
|
||||
session.commit()
|
||||
|
||||
payload = api_client.get("/api/system/summary").json()
|
||||
|
||||
assert payload["portainerStatus"] == "missing_config"
|
||||
assert payload["gatewayHealth"] == "critical"
|
||||
|
||||
|
||||
def test_system_summary_reports_latest_audit_action_without_secret_leak(api_client):
|
||||
from sqlmodel import Session
|
||||
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.db.models import AuditEvent
|
||||
|
||||
secret = "SUMMARY_SECRET_TOKEN"
|
||||
with Session(db_session.engine) as session:
|
||||
session.add(
|
||||
AuditEvent(
|
||||
id="audit-summary-action",
|
||||
source="manual",
|
||||
identity="operator",
|
||||
action_id="container.restart",
|
||||
target=f"widefrog api_key={secret}",
|
||||
target_type="container",
|
||||
risk="medium",
|
||||
policy_decision="allowed",
|
||||
result="success",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
payload = api_client.get("/api/system/summary").json()
|
||||
|
||||
assert payload["lastAction"].startswith("container.restart -> widefrog")
|
||||
assert "api_key=[REDACTED]" in payload["lastAction"]
|
||||
assert secret not in payload["lastAction"]
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_runtime_info_is_secret_safe(api_client):
|
||||
secret = "PORTAINER_RUNTIME_SECRET"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={
|
||||
"url": "http://portainer.local",
|
||||
"endpointId": "1",
|
||||
"apiKey": secret,
|
||||
"verifyTls": False,
|
||||
},
|
||||
)
|
||||
|
||||
payload = api_client.get("/api/runtime/info").json()
|
||||
|
||||
assert payload["version"] == "1.8.0"
|
||||
assert payload["environment"] == "production"
|
||||
assert payload["dataDirConfigured"] is True
|
||||
assert payload["dbStatus"] == "ok"
|
||||
assert payload["connectorConfigSummary"]["portainer"]["hasSecret"] is True
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_private_request_metrics_are_bounded_and_request_ids_are_returned(api_client):
|
||||
response = api_client.get(
|
||||
"/api/runtime/info", headers={"X-Request-ID": "review-request-1234"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers["X-Request-ID"] == "review-request-1234"
|
||||
assert response.headers["Server-Timing"].startswith("app;dur=")
|
||||
|
||||
metrics = api_client.get("/api/admin/observability/requests").json()
|
||||
assert metrics["sampleLimit"] == 2_000
|
||||
assert metrics["sampleCount"] >= 1
|
||||
assert any(item["path"] == "/api/runtime/info" for item in metrics["slowEndpoints"])
|
||||
assert all("query" not in item for item in metrics["slowEndpoints"])
|
||||
openmetrics = api_client.get("/api/admin/observability/metrics")
|
||||
assert openmetrics.status_code == 200
|
||||
assert "appops_http_request_duration_p95_milliseconds" in openmetrics.text
|
||||
assert "review-request-1234" not in openmetrics.text
|
||||
|
||||
maintenance = api_client.get("/api/admin/maintenance/status").json()
|
||||
assert maintenance["status"] == "running"
|
||||
assert "notificationEvaluation" in maintenance["intervalsSeconds"]
|
||||
assert "observabilityPersistence" in maintenance["intervalsSeconds"]
|
||||
|
||||
|
||||
def test_sqlite_checkpoint_is_backup_first_and_reports_safe_sizes(api_client):
|
||||
status = api_client.get("/api/admin/database/status").json()
|
||||
result = api_client.post("/api/admin/database/checkpoint").json()
|
||||
|
||||
assert status["status"] == "ready"
|
||||
assert status["databaseBytes"] > 0
|
||||
assert status["journalMode"] in {"delete", "wal"}
|
||||
assert result["success"] is True
|
||||
assert result["status"] == "checkpointed"
|
||||
assert result["backup"]["name"].startswith("appops-db-")
|
||||
assert "auditId" in result
|
||||
|
||||
|
||||
def test_latest_backup_restore_drill_is_read_only(api_client):
|
||||
from backend.app.db.session import engine
|
||||
from backend.app.services.runtime_service import verify_latest_backup_drill
|
||||
|
||||
created = api_client.post("/api/admin/backups/sqlite").json()
|
||||
with Session(engine) as session:
|
||||
result = verify_latest_backup_drill(session)
|
||||
|
||||
assert created["success"] is True
|
||||
assert result["success"] is True
|
||||
assert result["integrityCheck"] == "ok"
|
||||
assert result["restoreCheck"] == "ok"
|
||||
assert result["backup"]["name"] == created["backup"]["name"]
|
||||
|
||||
|
||||
def test_app_connector_read_routes_load_stored_runtime_config(api_client, monkeypatch):
|
||||
from backend.app.connectors.plex import PlexConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
secret = "PLEX_RUNTIME_SECRET"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/plex",
|
||||
json={"url": "http://plex.local:32400", "apiKey": secret, "verifyTls": False},
|
||||
)
|
||||
settings.apis["PLEX_URL"] = None
|
||||
settings.apis["PLEX_TOKEN"] = None
|
||||
|
||||
monkeypatch.setattr(
|
||||
PlexConnector,
|
||||
"_request",
|
||||
lambda self, path, params=None: {
|
||||
"MediaContainer": {
|
||||
"friendlyName": "Lab Plex",
|
||||
"version": "1.2.3",
|
||||
"machineIdentifier": "machine-1",
|
||||
"size": "0",
|
||||
}
|
||||
},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
response = api_client.get("/api/apps/plex/health")
|
||||
payload = response.json()
|
||||
|
||||
assert response.status_code == 200
|
||||
assert payload["status"] == "connected"
|
||||
assert payload["friendlyName"] == "Lab Plex"
|
||||
assert secret not in response.text
|
||||
|
||||
|
||||
def test_deep_health_is_available_without_admin_token(api_client):
|
||||
payload = api_client.get("/api/health/deep").json()
|
||||
assert payload["status"] == "ok"
|
||||
assert payload["checks"]["dbReachable"] is True
|
||||
assert payload["checks"]["dataDirWritable"] is True
|
||||
assert "setup" in payload
|
||||
|
||||
|
||||
def test_operator_token_is_exchanged_for_a_revocable_browser_session(api_client):
|
||||
from backend.app.core import security
|
||||
from backend.app.core.config import settings
|
||||
|
||||
token = "strong-session-test-operator-token-value"
|
||||
previous = (settings.operator_auth_required, settings.operator_token)
|
||||
settings.operator_auth_required = True
|
||||
settings.operator_token = token
|
||||
try:
|
||||
unlocked = api_client.post(
|
||||
"/api/operator/session",
|
||||
headers={"X-AppOps-Operator-Token": token},
|
||||
)
|
||||
assert unlocked.status_code == 200
|
||||
assert unlocked.json() == {"authorized": True, "mode": "session", "identity": None}
|
||||
cookie = unlocked.headers["set-cookie"]
|
||||
assert "appops_operator_session=" in cookie
|
||||
assert "HttpOnly" in cookie
|
||||
assert "SameSite=strict" in cookie
|
||||
assert "Max-Age" not in cookie
|
||||
assert token not in cookie
|
||||
|
||||
restored = api_client.get("/api/operator/session")
|
||||
assert restored.json() == {"authorized": True, "mode": "session", "identity": None}
|
||||
|
||||
settings.operator_token = "a-new-strong-session-test-operator-token"
|
||||
assert api_client.get("/api/operator/session").json()["authorized"] is False
|
||||
settings.operator_token = token
|
||||
assert api_client.get("/api/operator/session").json()["authorized"] is False
|
||||
|
||||
wrong_token = api_client.post(
|
||||
"/api/operator/session",
|
||||
headers={"X-AppOps-Operator-Token": "wrong-operator-token"},
|
||||
)
|
||||
assert wrong_token.status_code == 403
|
||||
assert "set-cookie" not in wrong_token.headers
|
||||
|
||||
unlocked = api_client.post(
|
||||
"/api/operator/session",
|
||||
headers={"X-AppOps-Operator-Token": token},
|
||||
)
|
||||
assert unlocked.status_code == 200
|
||||
|
||||
missing_csrf = api_client.delete("/api/operator/session")
|
||||
assert missing_csrf.status_code == 403
|
||||
locked = api_client.delete(
|
||||
"/api/operator/session",
|
||||
headers={security.OPERATOR_SESSION_CSRF_HEADER: "1"},
|
||||
)
|
||||
assert locked.status_code == 200
|
||||
assert api_client.get("/api/operator/session").json()["authorized"] is False
|
||||
finally:
|
||||
settings.operator_auth_required, settings.operator_token = previous
|
||||
|
||||
|
||||
def test_operator_auth_policy_can_be_enabled_and_persists(api_client):
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db.models import AppSetting
|
||||
from backend.app.db.session import engine
|
||||
|
||||
previous = settings.operator_auth_required
|
||||
settings.operator_auth_required = False
|
||||
try:
|
||||
assert api_client.get("/api/runtime/operator-auth").json() == {"required": False}
|
||||
response = api_client.patch("/api/runtime/operator-auth", json={"required": True})
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"required": True}
|
||||
with Session(engine) as session:
|
||||
assert session.get(AppSetting, "security.operator_auth").value_json == {"required": True}
|
||||
finally:
|
||||
settings.operator_auth_required = previous
|
||||
|
||||
|
||||
def test_admin_exports_are_secret_safe(api_client):
|
||||
secret = "EXPORT_SECRET_VALUE"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={
|
||||
"url": "http://portainer.local",
|
||||
"endpointId": "1",
|
||||
"apiKey": secret,
|
||||
"verifyTls": False,
|
||||
},
|
||||
)
|
||||
|
||||
for path in [
|
||||
"/api/admin/export/config",
|
||||
"/api/admin/export/policies",
|
||||
"/api/admin/export/inventory",
|
||||
"/api/admin/export/audit-summary",
|
||||
]:
|
||||
response = api_client.get(path)
|
||||
assert response.status_code == 200
|
||||
assert secret not in response.text
|
||||
assert "API_KEY" not in response.text or "hasSecret" in response.text
|
||||
|
||||
|
||||
def test_sqlite_online_backup_creates_integrity_checked_copy(api_client):
|
||||
secret = "BACKUP_SECRET_VALUE"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={
|
||||
"url": "http://portainer.local",
|
||||
"endpointId": "1",
|
||||
"apiKey": secret,
|
||||
"verifyTls": False,
|
||||
},
|
||||
)
|
||||
|
||||
response = api_client.post("/api/admin/backups/sqlite")
|
||||
payload = response.json()
|
||||
|
||||
assert response.status_code == 200
|
||||
assert payload["success"] is True
|
||||
assert payload["integrityCheck"] == "ok"
|
||||
assert payload["restoreMode"] == "manual_stop_container"
|
||||
assert payload["backup"]["name"].startswith("appops-db-")
|
||||
assert Path(payload["backup"]["path"]).exists()
|
||||
assert payload["backup"]["manifestAvailable"] is True
|
||||
manifest_path = Path(payload["backup"]["path"]).with_suffix(".manifest.json")
|
||||
assert manifest_path.exists()
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
assert manifest["databaseFile"] == payload["backup"]["name"]
|
||||
assert manifest["sha256"] == payload["backup"]["sha256"]
|
||||
assert secret not in response.text
|
||||
|
||||
list_response = api_client.get("/api/admin/backups/sqlite")
|
||||
listed = list_response.json()
|
||||
assert list_response.status_code == 200
|
||||
assert listed["backupDir"].endswith("backups")
|
||||
assert listed["backups"][0]["name"] == payload["backup"]["name"]
|
||||
assert secret not in list_response.text
|
||||
|
||||
backup_name = payload["backup"]["name"]
|
||||
verified = api_client.post(f"/api/admin/backups/sqlite/{backup_name}/verify")
|
||||
assert verified.status_code == 200
|
||||
assert verified.json()["success"] is True
|
||||
assert verified.json()["integrityCheck"] == "ok"
|
||||
assert verified.json()["manifestCheck"] == "ok"
|
||||
assert verified.json()["restoreMode"] == "manual_stop_container"
|
||||
|
||||
downloaded = api_client.get(f"/api/admin/backups/sqlite/{backup_name}/download")
|
||||
assert downloaded.status_code == 200
|
||||
assert downloaded.headers["content-type"] == "application/vnd.sqlite3"
|
||||
assert "no-store" in downloaded.headers["cache-control"]
|
||||
assert len(downloaded.content) == payload["backup"]["sizeBytes"]
|
||||
|
||||
|
||||
def test_sqlite_backup_download_rejects_traversal(api_client):
|
||||
assert (
|
||||
api_client.get("/api/admin/backups/sqlite/not-a-backup/download").status_code
|
||||
== 404
|
||||
)
|
||||
assert (
|
||||
api_client.post("/api/admin/backups/sqlite/not-a-backup/verify").status_code
|
||||
== 404
|
||||
)
|
||||
@@ -0,0 +1,333 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "prowlarr_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.main import app
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
|
||||
previous_apis = dict(settings.apis)
|
||||
for key in ["PROWLARR_URL", "PROWLARR_API_KEY", "PROWLARR_VERIFY_TLS", "PROWLARR_TIMEOUT_SECONDS"]:
|
||||
settings.apis[key] = None
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
settings.apis = previous_apis
|
||||
|
||||
|
||||
def _set_prowlarr_config(url: str | None = "http://prowlarr:9696", api_key: str | None = "PROWLARR_SECRET_TOKEN"):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = dict(settings.apis)
|
||||
settings.apis["PROWLARR_URL"] = url
|
||||
settings.apis["PROWLARR_API_KEY"] = api_key
|
||||
settings.apis["PROWLARR_VERIFY_TLS"] = False
|
||||
settings.apis["PROWLARR_TIMEOUT_SECONDS"] = "10"
|
||||
return previous
|
||||
|
||||
|
||||
def _restore_config(previous):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous)
|
||||
|
||||
|
||||
def test_prowlarr_connector_missing_config_reports_required_env(api_client):
|
||||
response = api_client.post("/api/connectors/prowlarr/test")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "missing_config"
|
||||
assert payload["missing_config"] is True
|
||||
assert payload["required_env_vars"] == ["PROWLARR_URL", "PROWLARR_API_KEY"]
|
||||
assert payload["secretRef"] == "PROWLARR_API_KEY"
|
||||
|
||||
|
||||
def test_prowlarr_health_success_normalizes_response(monkeypatch):
|
||||
from backend.app.connectors.prowlarr import ProwlarrConnector
|
||||
|
||||
previous = _set_prowlarr_config()
|
||||
|
||||
monkeypatch.setattr(
|
||||
ProwlarrConnector,
|
||||
"_request",
|
||||
lambda self, method, path, **_kwargs: {"version": "1.14.2", "appName": "Prowlarr"},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = ProwlarrConnector().health()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "connected"
|
||||
assert payload["version"] == "1.14.2"
|
||||
assert payload["url"] == "http://prowlarr:9696"
|
||||
assert "PROWLARR_SECRET_TOKEN" not in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_prowlarr_error_redacts_api_key(monkeypatch):
|
||||
from backend.app.connectors.prowlarr import ProwlarrConnector
|
||||
|
||||
previous = _set_prowlarr_config()
|
||||
|
||||
def fail_request(self, method, path, **_kwargs):
|
||||
raise RuntimeError("api_key=PROWLARR_SECRET_TOKEN connection refused")
|
||||
|
||||
monkeypatch.setattr(ProwlarrConnector, "_request", fail_request, raising=True)
|
||||
|
||||
payload = ProwlarrConnector().health()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] in {"unreachable", "error"}
|
||||
assert "PROWLARR_SECRET_TOKEN" not in str(payload)
|
||||
assert "[REDACTED]" in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_prowlarr_indexer_status_normalizes_counts(monkeypatch):
|
||||
from backend.app.connectors.prowlarr import ProwlarrConnector
|
||||
|
||||
previous = _set_prowlarr_config()
|
||||
|
||||
monkeypatch.setattr(
|
||||
ProwlarrConnector,
|
||||
"_request",
|
||||
lambda self, method, path, **_kwargs: [
|
||||
{"id": 1, "name": "Index A", "protocol": "torrent", "enable": True, "tags": [1], "privacy": "public"},
|
||||
{"id": 2, "name": "Index B", "protocol": "usenet", "enabled": False, "status": "error", "categories": [{"name": "Movies"}], "privacy": "private"},
|
||||
{"id": 3, "name": "Index C", "protocol": "torrent", "enable": True, "status": "warning"},
|
||||
],
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = ProwlarrConnector().indexer_status()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "ok"
|
||||
assert payload["total"] == 3
|
||||
assert payload["enabled"] == 2
|
||||
assert payload["disabled"] == 1
|
||||
assert payload["failing"] == 2
|
||||
assert payload["protocols"] == {"torrent": 2, "usenet": 1, "unknown": 0}
|
||||
assert payload["items"][0]["name"] == "Index A"
|
||||
assert payload["items"][1]["categories"] == ["Movies"]
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_prowlarr_application_status_normalizes_apps(monkeypatch):
|
||||
from backend.app.connectors.prowlarr import ProwlarrConnector
|
||||
|
||||
previous = _set_prowlarr_config()
|
||||
|
||||
monkeypatch.setattr(
|
||||
ProwlarrConnector,
|
||||
"_request",
|
||||
lambda self, method, path, **_kwargs: [
|
||||
{"id": 7, "name": "Sonarr", "implementation": "Sonarr", "syncLevel": "fullSync", "enable": True},
|
||||
{"id": 8, "name": "Radarr", "implementation": "Radarr", "syncLevel": "addOnly", "enabled": False, "status": "warning"},
|
||||
],
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = ProwlarrConnector().application_status()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "ok"
|
||||
assert payload["total"] == 2
|
||||
assert payload["items"][0]["name"] == "Sonarr"
|
||||
assert payload["items"][0]["syncLevel"] == "fullSync"
|
||||
assert payload["items"][1]["enabled"] is False
|
||||
assert payload["items"][1]["status"] == "warning"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_prowlarr_indexer_health_normalizes_warnings_and_errors(monkeypatch):
|
||||
from backend.app.connectors.prowlarr import ProwlarrConnector
|
||||
|
||||
previous = _set_prowlarr_config()
|
||||
|
||||
monkeypatch.setattr(
|
||||
ProwlarrConnector,
|
||||
"_request",
|
||||
lambda self, method, path, **_kwargs: [
|
||||
{"type": "warning", "message": "Indexer A slow"},
|
||||
{"type": "error", "message": "Indexer B failing"},
|
||||
],
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = ProwlarrConnector().indexer_health()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["warnings"] == ["Indexer A slow"]
|
||||
assert payload["errors"] == ["Indexer B failing"]
|
||||
assert payload["message"] == "health_issues_detected"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_prowlarr_search_test_is_read_only_and_normalizes_results(monkeypatch):
|
||||
from backend.app.connectors.prowlarr import ProwlarrConnector
|
||||
|
||||
previous = _set_prowlarr_config()
|
||||
|
||||
def fake_request(self, method, path, **kwargs):
|
||||
assert method == "GET"
|
||||
assert path == "/api/v1/search"
|
||||
assert kwargs["params"]["query"] == "ubuntu"
|
||||
return [
|
||||
{
|
||||
"title": "Ubuntu ISO",
|
||||
"indexer": "Index A",
|
||||
"protocol": "torrent",
|
||||
"size": 123456,
|
||||
"age": 2,
|
||||
"seeders": 20,
|
||||
"leechers": 1,
|
||||
"categories": [{"name": "Linux"}],
|
||||
"downloadUrl": "https://download.example/secret",
|
||||
}
|
||||
]
|
||||
|
||||
monkeypatch.setattr(ProwlarrConnector, "_request", fake_request, raising=True)
|
||||
|
||||
payload = ProwlarrConnector().search_test("ubuntu")
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "ok"
|
||||
assert payload["total"] == 1
|
||||
assert payload["items"][0]["title"] == "Ubuntu ISO"
|
||||
assert payload["items"][0]["category"] == "Linux"
|
||||
assert payload["items"][0]["downloadAllowed"] is False
|
||||
assert "downloadUrl" not in str(payload)
|
||||
assert "PROWLARR_SECRET_TOKEN" not in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_prowlarr_sync_status_summarizes_indexers_apps_and_health(monkeypatch):
|
||||
from backend.app.connectors.prowlarr import ProwlarrConnector
|
||||
|
||||
previous = _set_prowlarr_config()
|
||||
|
||||
monkeypatch.setattr(ProwlarrConnector, "indexer_status", lambda self: {"success": True, "total": 3, "enabled": 2, "failing": 1}, raising=True)
|
||||
monkeypatch.setattr(ProwlarrConnector, "application_status", lambda self: {"success": True, "total": 2, "items": []}, raising=True)
|
||||
monkeypatch.setattr(ProwlarrConnector, "indexer_health", lambda self: {"success": True, "warnings": ["slow"], "errors": []}, raising=True)
|
||||
|
||||
payload = ProwlarrConnector().sync_status()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "ok"
|
||||
assert payload["applicationsConfigured"] == 2
|
||||
assert payload["indexersConfigured"] == 3
|
||||
assert payload["indexersEnabled"] == 2
|
||||
assert payload["healthWarnings"] == 1
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_prowlarr_search_test_route_audits_manual_read_only(api_client, monkeypatch):
|
||||
from backend.app.db.models import AuditEvent
|
||||
from backend.app.db.session import engine
|
||||
from sqlmodel import Session, select
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.prowlarr.ProwlarrConnector.search_test",
|
||||
lambda self, query: {"success": True, "status": "ok", "message": "search_ok", "total": 1, "items": [{"title": query, "downloadAllowed": False}]},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
response = api_client.get("/api/apps/prowlarr/search-test?q=ubuntu")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["success"] is True
|
||||
assert payload["items"][0]["downloadAllowed"] is False
|
||||
|
||||
with Session(engine) as session:
|
||||
events = session.exec(select(AuditEvent).where(AuditEvent.action_id == "prowlarr.search_test")).all()
|
||||
assert len(events) == 1
|
||||
assert events[0].redacted_metadata["query"] == "ubuntu"
|
||||
assert events[0].redacted_metadata["downloadAllowed"] is False
|
||||
assert "PROWLARR_SECRET_TOKEN" not in str(events[0].redacted_metadata)
|
||||
|
||||
|
||||
def test_prowlarr_voice_search_test_requires_approval_without_connector_call(api_client, monkeypatch):
|
||||
def fail_search(*_args, **_kwargs):
|
||||
raise AssertionError("connector should not run before approval")
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.prowlarr.ProwlarrConnector.search_test", fail_search, raising=True)
|
||||
|
||||
response = api_client.get("/api/apps/prowlarr/search-test?q=ubuntu&source=voice")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["success"] is False
|
||||
assert payload["decision"] == "approval_required"
|
||||
assert payload["approvalId"].startswith("appr-")
|
||||
assert payload["auditId"].startswith("audit-")
|
||||
|
||||
|
||||
def test_prowlarr_mutating_actions_are_blocked(api_client):
|
||||
for action in [
|
||||
"prowlarr.delete_indexer",
|
||||
"prowlarr.delete_application",
|
||||
"prowlarr.update_indexer",
|
||||
"prowlarr.update_application",
|
||||
"prowlarr.grab_release",
|
||||
"prowlarr.push_release",
|
||||
"prowlarr.mass_edit",
|
||||
"prowlarr.test_all_indexers",
|
||||
]:
|
||||
payload = api_client.post("/api/policies/evaluate", json={"action": action, "target": "prowlarr", "source": "manual"}).json()
|
||||
assert payload["decision"] == "blocked"
|
||||
|
||||
|
||||
def test_prowlarr_inventory_classification_and_candidate(api_client, monkeypatch):
|
||||
from backend.app.services.app_classifier import classify_container
|
||||
|
||||
result = classify_container({"name": "binhex-prowlarr", "image": "ghcr.io/hotio/prowlarr:latest", "ports": ["9696:9696"]})
|
||||
|
||||
assert result["app_type"] == "prowlarr"
|
||||
assert result["category"] == "media"
|
||||
assert result["connector"] == "prowlarr"
|
||||
assert result["policy_profile"] == "media_indexer"
|
||||
assert result["healthcheck_suggestion"] == {"port": 9696}
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.test_connection", lambda self: {"status": "connected"}, raising=True)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.portainer.PortainerConnector.list_containers",
|
||||
lambda self: {"status": "ok", "containers": [{"id": "prowlarr-container", "name": "binhex-prowlarr", "image": "linuxserver/prowlarr", "state": "running", "status": "running", "ports": ["9696:9696"]}]},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
api_client.post("/api/inventory/discover")
|
||||
candidates = api_client.get("/api/connectors/prowlarr/candidates").json()
|
||||
|
||||
assert len(candidates) == 1
|
||||
assert candidates[0]["connectorId"] == "prowlarr"
|
||||
assert candidates[0]["appType"] == "prowlarr"
|
||||
assert candidates[0]["missingSecrets"] == ["PROWLARR_URL", "PROWLARR_API_KEY"]
|
||||
@@ -0,0 +1,413 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
CLI_PATH = REPO_ROOT / "ops" / "gitea-deploy-broker" / "repopulse_adoption_cli.py"
|
||||
|
||||
|
||||
def _load_module(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
cli = _load_module("repopulse_adoption_cli_test", CLI_PATH)
|
||||
|
||||
|
||||
def _sha(value: bytes | str) -> str:
|
||||
raw = value.encode("ascii") if isinstance(value, str) else value
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def _canonical_sha(value: object) -> str:
|
||||
raw = json.dumps(
|
||||
value,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
allow_nan=False,
|
||||
).encode("ascii")
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def _projection() -> dict[str, object]:
|
||||
return {
|
||||
"repository": "jens/repopulse",
|
||||
"main_revision": "a" * 40,
|
||||
"live_revision": "b" * 40,
|
||||
"container_id_sha256": _sha("legacy-container"),
|
||||
"container_image_sha256": _sha("legacy-image"),
|
||||
"queue": {"queued": 0, "active": 0, "completed": 10, "failed": 0},
|
||||
"database": {
|
||||
"database_bytes": 12345,
|
||||
"journal_bytes": 0,
|
||||
"scan_count": 10,
|
||||
"compacted_scan_count": 4,
|
||||
},
|
||||
"container_state": "running",
|
||||
"container_health": "healthy",
|
||||
}
|
||||
|
||||
|
||||
def _payload() -> dict[str, object]:
|
||||
idempotency = _sha("itworx.repopulse-legacy-adoption/v1:jens/repopulse:cli-test")
|
||||
projection = _projection()
|
||||
return {
|
||||
"intentId": f"rpa-{idempotency[:20]}",
|
||||
"idempotencySha256": idempotency,
|
||||
"preflightSha256": _canonical_sha(projection),
|
||||
"preflightProjection": projection,
|
||||
"targetCredential": "A" * 64,
|
||||
}
|
||||
|
||||
|
||||
def _raw(payload: dict[str, object] | None = None) -> bytes:
|
||||
return json.dumps(payload or _payload(), separators=(",", ":")).encode("utf-8")
|
||||
|
||||
|
||||
def _core():
|
||||
return cli._load_core_modules()
|
||||
|
||||
|
||||
def _parsed(payload: dict[str, object] | None = None):
|
||||
core = _core()
|
||||
return cli.parse_handoff(_raw(payload), core.executor)
|
||||
|
||||
|
||||
def test_parser_accepts_exact_handoff_and_rejects_duplicate_keys_at_any_depth():
|
||||
core = _core()
|
||||
payload = _payload()
|
||||
parsed = cli.parse_handoff(_raw(payload), core.executor)
|
||||
|
||||
assert parsed.intent_id == payload["intentId"]
|
||||
assert parsed.target_credential == b"A" * 64
|
||||
assert parsed.preflight_projection == payload["preflightProjection"]
|
||||
|
||||
regular = _raw(payload)
|
||||
duplicate_top = (
|
||||
b'{"intentId":'
|
||||
+ json.dumps(payload["intentId"]).encode("ascii")
|
||||
+ bytes([44])
|
||||
+ regular[1:]
|
||||
)
|
||||
with pytest.raises(cli.ControllerCliError) as error:
|
||||
cli.parse_handoff(duplicate_top, core.executor)
|
||||
assert error.value.code == "controller_handoff_duplicate_key"
|
||||
|
||||
duplicate_nested = regular.replace(b'"queued":0', b'"queued":0,"queued":0', 1)
|
||||
with pytest.raises(cli.ControllerCliError) as error:
|
||||
cli.parse_handoff(duplicate_nested, core.executor)
|
||||
assert error.value.code == "controller_handoff_duplicate_key"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
lambda value: value.pop("targetCredential"),
|
||||
lambda value: value.update({"extra": True}),
|
||||
lambda value: value.update({"targetCredential": "short"}),
|
||||
lambda value: value.update({"targetCredential": "x" * 48 + "\n"}),
|
||||
],
|
||||
)
|
||||
def test_parser_rejects_schema_changes_and_noncontroller_credentials(change):
|
||||
core = _core()
|
||||
payload = _payload()
|
||||
change(payload)
|
||||
|
||||
with pytest.raises(cli.ControllerCliError):
|
||||
cli.parse_handoff(_raw(payload), core.executor)
|
||||
|
||||
|
||||
def test_parser_enforces_bounded_stdin_size():
|
||||
core = _core()
|
||||
with pytest.raises(cli.ControllerCliError) as error:
|
||||
cli.parse_handoff(b"x" * (cli._MAX_HANDOFF_BYTES + 1), core.executor)
|
||||
assert error.value.code == "controller_handoff_too_large"
|
||||
|
||||
|
||||
def test_reserved_binding_requires_existing_matching_intent_and_credential():
|
||||
core = _core()
|
||||
handoff = _parsed()
|
||||
binding = core.store.intent_binding_sha256(
|
||||
handoff.intent_id,
|
||||
handoff.idempotency_sha256,
|
||||
handoff.preflight_sha256,
|
||||
)
|
||||
record = {
|
||||
"intentId": handoff.intent_id,
|
||||
"idempotencySha256": handoff.idempotency_sha256,
|
||||
"preflightSha256": handoff.preflight_sha256,
|
||||
"intentBindingSha256": binding,
|
||||
"expectedTargetCredentialSha256": _sha(handoff.target_credential),
|
||||
"status": "in_progress",
|
||||
"registration": None,
|
||||
}
|
||||
|
||||
class Store:
|
||||
def read_only_status(self):
|
||||
return {
|
||||
"status": record["status"],
|
||||
"hashes": {
|
||||
"idempotencySha256": record["idempotencySha256"],
|
||||
"preflightSha256": record["preflightSha256"],
|
||||
"intentBindingSha256": record["intentBindingSha256"],
|
||||
},
|
||||
}
|
||||
|
||||
def load_intent(self):
|
||||
return record
|
||||
|
||||
assert cli._require_reserved_binding(
|
||||
store=Store(), store_module=core.store, handoff=handoff
|
||||
) is record
|
||||
|
||||
mismatched = replace(handoff, preflight_sha256=_sha("another preflight"))
|
||||
with pytest.raises(cli.ControllerCliError) as error:
|
||||
cli._require_reserved_binding(
|
||||
store=Store(), store_module=core.store, handoff=mismatched
|
||||
)
|
||||
assert error.value.code == "controller_intent_binding_mismatch"
|
||||
|
||||
wrong_credential = replace(handoff, target_credential=b"B" * 64)
|
||||
with pytest.raises(cli.ControllerCliError) as error:
|
||||
cli._require_reserved_binding(
|
||||
store=Store(), store_module=core.store, handoff=wrong_credential
|
||||
)
|
||||
assert error.value.code == "target_credential_binding_mismatch"
|
||||
|
||||
|
||||
def test_missing_reservation_is_rejected_before_loading_or_creating_journal():
|
||||
core = _core()
|
||||
handoff = _parsed()
|
||||
|
||||
class Store:
|
||||
def read_only_status(self):
|
||||
return None
|
||||
|
||||
def load_intent(self):
|
||||
raise AssertionError("missing reservation must not load mutable state")
|
||||
|
||||
with pytest.raises(cli.ControllerCliError) as error:
|
||||
cli._require_reserved_binding(
|
||||
store=Store(), store_module=core.store, handoff=handoff
|
||||
)
|
||||
assert error.value.code == "controller_intent_not_reserved"
|
||||
|
||||
|
||||
def test_fixed_storage_bridge_forwards_executor_revision_and_keeps_credential_in_memory():
|
||||
handoff = _parsed()
|
||||
|
||||
class StorageHooks:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def ensure_quiesced(self, **kwargs):
|
||||
self.calls.append(("quiesce", kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
def create_or_verify_fresh_snapshot(self, **kwargs):
|
||||
self.calls.append(("snapshot", kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
def copy_and_verify_managed_data(self, **kwargs):
|
||||
self.calls.append(("copy", kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
hooks = StorageHooks()
|
||||
bridge = cli._FixedHandoffStorageEffects(
|
||||
store=object(),
|
||||
store_module=SimpleNamespace(REPOSITORY="jens/repopulse"),
|
||||
storage_effects=hooks,
|
||||
legacy_rollback=object(),
|
||||
registry_restore=object(),
|
||||
handoff=handoff,
|
||||
)
|
||||
revision = handoff.preflight_projection["main_revision"]
|
||||
binding = "c" * 64
|
||||
|
||||
bridge.ensure_quiesced(
|
||||
intent_id=handoff.intent_id,
|
||||
intent_binding_sha256=binding,
|
||||
expected_main_revision=revision,
|
||||
expected_container_id_sha256="d" * 64,
|
||||
expected_container_image_sha256="e" * 64,
|
||||
)
|
||||
bridge.create_or_verify_fresh_snapshot(
|
||||
intent_id=handoff.intent_id,
|
||||
intent_binding_sha256=binding,
|
||||
expected_main_revision=revision,
|
||||
expected_snapshot=None,
|
||||
)
|
||||
bridge.copy_and_verify_managed_data(
|
||||
intent_id=handoff.intent_id,
|
||||
expected_main_revision=revision,
|
||||
snapshot_evidence={"intentId": handoff.intent_id},
|
||||
)
|
||||
|
||||
assert [call[0] for call in hooks.calls] == ["quiesce", "snapshot", "copy"]
|
||||
assert all(call[1]["expected_main_revision"] == revision for call in hooks.calls)
|
||||
assert bridge.registration_material(intent_id=handoff.intent_id) == (
|
||||
f"enr-{handoff.intent_id[4:]}",
|
||||
b"A" * 64,
|
||||
)
|
||||
with pytest.raises(cli.ControllerCliError):
|
||||
bridge.registration_material(intent_id="rpa-" + "f" * 20)
|
||||
|
||||
|
||||
def test_executor_store_proxy_snapshots_registry_before_registration_cas():
|
||||
events = []
|
||||
|
||||
class Store:
|
||||
def register_target(self, **kwargs):
|
||||
events.append(("register", kwargs))
|
||||
return {"status": "registered"}
|
||||
|
||||
def read_only_status(self):
|
||||
return {"status": "in_progress"}
|
||||
|
||||
class RegistryRestore:
|
||||
def preserve_before_registry_cas(self, store):
|
||||
assert isinstance(store, Store)
|
||||
events.append(("preserve", None))
|
||||
|
||||
proxy = cli._RegistrySnapshotStoreProxy(Store(), RegistryRestore())
|
||||
assert proxy.register_target(intent_id="rpa-" + "a" * 20) == {
|
||||
"status": "registered"
|
||||
}
|
||||
assert proxy.read_only_status() == {"status": "in_progress"}
|
||||
assert [name for name, _value in events] == ["preserve", "register"]
|
||||
|
||||
|
||||
def test_serialized_result_rejects_secret_leaks_and_unexpected_fields():
|
||||
credential = b"A" * 64
|
||||
intent_id = "rpa-" + "a" * 20
|
||||
result = {key: None for key in cli._RESULT_KEYS}
|
||||
result.update(
|
||||
{
|
||||
"apiVersion": "itworx.repopulse-controller-adoption-execution-result/v1",
|
||||
"repository": "jens/repopulse",
|
||||
"applicationId": "repopulse",
|
||||
"intentId": intent_id,
|
||||
"status": "deployment_verified",
|
||||
"phase": "completion_verified",
|
||||
"legacyVolume": "repopulse_data",
|
||||
"legacyVolumeRetained": True,
|
||||
}
|
||||
)
|
||||
result["deploymentReceipt"] = {"note": credential.decode("ascii")}
|
||||
|
||||
with pytest.raises(cli.ControllerCliError) as error:
|
||||
cli._serialize_result(
|
||||
result, target_credential=credential, intent_id=intent_id
|
||||
)
|
||||
assert error.value.code == "controller_result_secret_detected"
|
||||
|
||||
result["deploymentReceipt"] = {}
|
||||
result["targetCredential"] = credential.decode("ascii")
|
||||
with pytest.raises(cli.ControllerCliError) as error:
|
||||
cli._serialize_result(
|
||||
result, target_credential=credential, intent_id=intent_id
|
||||
)
|
||||
assert error.value.code == "controller_result_schema_invalid"
|
||||
|
||||
|
||||
def test_disabled_controller_does_not_read_handoff_or_load_effects():
|
||||
core = cli.CoreModules(
|
||||
store=SimpleNamespace(EXECUTION_ENABLED=False),
|
||||
executor=SimpleNamespace(EXECUTION_ENABLED=False),
|
||||
)
|
||||
effects_loaded = False
|
||||
root_checked = False
|
||||
|
||||
class UnreadableInput:
|
||||
def read(self, _size):
|
||||
raise AssertionError("disabled controller must not read stdin")
|
||||
|
||||
def load_effects(_core):
|
||||
nonlocal effects_loaded
|
||||
effects_loaded = True
|
||||
raise AssertionError("disabled controller must not construct effects")
|
||||
|
||||
def require_root():
|
||||
nonlocal root_checked
|
||||
root_checked = True
|
||||
|
||||
output = io.StringIO()
|
||||
code = cli._run_cli(
|
||||
argv=(),
|
||||
stdin=UnreadableInput(),
|
||||
stdout=output,
|
||||
load_core=lambda: core,
|
||||
load_effects=load_effects,
|
||||
require_root=require_root,
|
||||
)
|
||||
|
||||
assert code == 2
|
||||
assert json.loads(output.getvalue()) == {
|
||||
"status": "disabled",
|
||||
"reason": "controller_adoption_execution_disabled",
|
||||
}
|
||||
assert not effects_loaded
|
||||
assert root_checked
|
||||
|
||||
|
||||
def test_nonroot_controller_rejects_before_loading_modules_or_reading_handoff():
|
||||
core_loaded = False
|
||||
|
||||
class UnreadableInput:
|
||||
def read(self, _size):
|
||||
raise AssertionError("nonroot controller must not read stdin")
|
||||
|
||||
def load_core():
|
||||
nonlocal core_loaded
|
||||
core_loaded = True
|
||||
raise AssertionError("nonroot controller must not load modules")
|
||||
|
||||
def require_root():
|
||||
raise cli.ControllerCliError("controller_root_required")
|
||||
|
||||
output = io.StringIO()
|
||||
code = cli._run_cli(
|
||||
argv=(),
|
||||
stdin=UnreadableInput(),
|
||||
stdout=output,
|
||||
load_core=load_core,
|
||||
require_root=require_root,
|
||||
)
|
||||
|
||||
assert code == 2
|
||||
assert json.loads(output.getvalue()) == {
|
||||
"status": "blocked",
|
||||
"reason": "controller_root_required",
|
||||
}
|
||||
assert not core_loaded
|
||||
|
||||
|
||||
def test_real_fixed_effect_modules_compose_without_side_effects():
|
||||
core = cli._load_core_modules()
|
||||
effects = cli._load_effect_modules(core)
|
||||
handoff = _parsed()
|
||||
store = core.store.RepoPulseAdoptionStore()
|
||||
|
||||
executor = cli._compose_executor(
|
||||
core=core,
|
||||
effects=effects,
|
||||
store=store,
|
||||
handoff=handoff,
|
||||
)
|
||||
|
||||
assert isinstance(executor, core.executor.RepoPulseAdoptionExecutor)
|
||||
assert executor._effects.__class__.__name__ == "_RevisionBoundRuntimeEffects"
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,818 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
BROKER_DIR = REPO_ROOT / "ops" / "gitea-deploy-broker"
|
||||
STORE_PATH = BROKER_DIR / "repopulse_adoption_store.py"
|
||||
EXECUTOR_PATH = BROKER_DIR / "repopulse_adoption_executor.py"
|
||||
REPOSITORY = "jens/repopulse"
|
||||
APPLICATION_ID = "repopulse"
|
||||
LEGACY_VOLUME = "repopulse_data"
|
||||
PROTOCOL_VERSION = "itworx.repopulse-legacy-adoption/v1"
|
||||
|
||||
|
||||
def _load_module(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _sha(value: str) -> str:
|
||||
return hashlib.sha256(value.encode("ascii")).hexdigest()
|
||||
|
||||
|
||||
def _canonical_sha(value: object) -> str:
|
||||
payload = json.dumps(
|
||||
value,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
allow_nan=False,
|
||||
).encode("ascii")
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _preflight_projection() -> dict:
|
||||
return {
|
||||
"repository": REPOSITORY,
|
||||
"main_revision": "a" * 40,
|
||||
"live_revision": "b" * 40,
|
||||
"container_id_sha256": _sha("legacy-container"),
|
||||
"container_image_sha256": _sha("legacy-image"),
|
||||
"queue": {"queued": 0, "active": 0, "completed": 572, "failed": 0},
|
||||
"database": {
|
||||
"database_bytes": 44_576_768,
|
||||
"journal_bytes": 0,
|
||||
"scan_count": 572,
|
||||
"compacted_scan_count": 96,
|
||||
},
|
||||
"container_state": "running",
|
||||
"container_health": "healthy",
|
||||
}
|
||||
|
||||
|
||||
def _command(executor):
|
||||
idempotency_sha256 = _sha(f"{PROTOCOL_VERSION}:{REPOSITORY}:repopulse-adoption-2026")
|
||||
projection = _preflight_projection()
|
||||
return executor.AdoptionCommand(
|
||||
intent_id=f"rpa-{idempotency_sha256[:20]}",
|
||||
idempotency_sha256=idempotency_sha256,
|
||||
preflight_sha256=_canonical_sha(projection),
|
||||
preflight_projection=projection,
|
||||
)
|
||||
|
||||
|
||||
def _target_gates() -> dict:
|
||||
return {
|
||||
"profileId": "repopulse-authenticated",
|
||||
"profileVersion": "enrollment-profile/v1",
|
||||
"planSha256": _sha("reviewed fixed profile plan"),
|
||||
"profileReady": True,
|
||||
"controllerTrustReady": True,
|
||||
"controllerBootstrapSecretImported": True,
|
||||
"typedEnrollmentTrustReady": True,
|
||||
"targetSecretReady": True,
|
||||
"exactPolicyReady": True,
|
||||
"authenticatedRouteReady": True,
|
||||
"accessListReady": True,
|
||||
"mutationBoundaryReady": True,
|
||||
"managedRuntimeReady": True,
|
||||
"scanSourceMatches": True,
|
||||
"scanMountReadOnly": True,
|
||||
"scanPermissionPolicyPersistent": True,
|
||||
"scanAccessForRuntimeIdentityReady": True,
|
||||
"ownedDataTargetReady": True,
|
||||
"managedEnvironmentReady": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"maintenanceAuthReady": True,
|
||||
"controllerAdoptionCapabilityReady": True,
|
||||
"controllerRegistryWriterReady": True,
|
||||
"controllerSecretWriterReady": True,
|
||||
}
|
||||
|
||||
|
||||
def _quiescence(command, binding: str) -> dict:
|
||||
projection = command.preflight_projection
|
||||
return {
|
||||
"intentId": command.intent_id,
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"maintenanceModeActive": True,
|
||||
"queueFrozen": True,
|
||||
"queuedJobs": 0,
|
||||
"activeJobs": 0,
|
||||
"maintenanceModeAuthenticated": True,
|
||||
"containerName": "repopulse",
|
||||
"containerIdSha256": projection["container_id_sha256"],
|
||||
"containerImageSha256": projection["container_image_sha256"],
|
||||
"containerState": "exited",
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"controllerIntentDurable": True,
|
||||
"controllerIntentBindingSha256": binding,
|
||||
}
|
||||
|
||||
|
||||
def _snapshot(command, binding: str) -> dict:
|
||||
return {
|
||||
"intentId": command.intent_id,
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"legacyContainerStopped": True,
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyVolumeRetained": True,
|
||||
"sqliteBackupMethod": "sqlite_online_backup",
|
||||
"quickCheck": "ok",
|
||||
"integrityCheck": "ok",
|
||||
"sourceLogicalSha256": _sha("logical-sqlite-data"),
|
||||
"snapshotLogicalSha256": _sha("logical-sqlite-data"),
|
||||
"snapshotFileSha256": _sha("fresh-sqlite-backup"),
|
||||
"sourceScanCount": 572,
|
||||
"snapshotScanCount": 572,
|
||||
"sourceCompactedScanCount": 96,
|
||||
"snapshotCompactedScanCount": 96,
|
||||
"sourceQueueSha256": _sha("queue-state"),
|
||||
"snapshotQueueSha256": _sha("queue-state"),
|
||||
"sourceAuxiliaryTreeSha256": _sha("auxiliary-tree"),
|
||||
"sourceAuxiliaryFileCount": 42,
|
||||
"targetDataPathWasAbsent": True,
|
||||
"snapshotReceiptDurable": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
"controllerIntentBindingSha256": binding,
|
||||
}
|
||||
|
||||
|
||||
def _copy_evidence(command, snapshot: dict) -> dict:
|
||||
return {
|
||||
"intentId": command.intent_id,
|
||||
"snapshotFileSha256": snapshot["snapshotFileSha256"],
|
||||
"targetLogicalSha256": snapshot["snapshotLogicalSha256"],
|
||||
"targetScanCount": snapshot["snapshotScanCount"],
|
||||
"targetCompactedScanCount": snapshot["snapshotCompactedScanCount"],
|
||||
"targetQueueSha256": snapshot["snapshotQueueSha256"],
|
||||
"sourceAuxiliaryTreeSha256": snapshot["sourceAuxiliaryTreeSha256"],
|
||||
"targetAuxiliaryTreeSha256": snapshot["sourceAuxiliaryTreeSha256"],
|
||||
"sourceAuxiliaryFileCount": snapshot["sourceAuxiliaryFileCount"],
|
||||
"targetAuxiliaryFileCount": snapshot["sourceAuxiliaryFileCount"],
|
||||
"targetQuickCheck": "ok",
|
||||
"targetIntegrityCheck": "ok",
|
||||
"targetDataUid": 10001,
|
||||
"targetDataGid": 10001,
|
||||
"targetRecoveryStatus": "healthy",
|
||||
"managedDataMountMatches": True,
|
||||
"copySourceIsVerifiedSnapshot": True,
|
||||
"legacyVolumeRetained": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
}
|
||||
|
||||
|
||||
def _legacy_rollback_evidence(command, legacy_projects_sha256: str) -> dict:
|
||||
return {
|
||||
"intentId": command.intent_id,
|
||||
"repository": REPOSITORY,
|
||||
"legacyContainerName": "repopulse",
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyProjectsSha256": legacy_projects_sha256,
|
||||
"rollbackSnapshotSha256": _sha("durable old compose rollback snapshot"),
|
||||
"rollbackSnapshotDurable": True,
|
||||
"legacyVolumeRetained": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
}
|
||||
|
||||
|
||||
def _deployment_receipt(command, snapshot: dict, copy: dict) -> dict:
|
||||
revision = command.preflight_projection["main_revision"]
|
||||
return {
|
||||
"repository": REPOSITORY,
|
||||
"revision": revision,
|
||||
"containerRevision": revision,
|
||||
"containerIdSha256": _sha("managed-container"),
|
||||
"deploymentReceiptSchema": "gitea-deploy-receipt.v1",
|
||||
"deploymentReceiptSha256": _sha("verified-deploy-receipt"),
|
||||
"deploymentReceiptVerified": True,
|
||||
"runtimeHealthy": True,
|
||||
"publishedHostPorts": [],
|
||||
"managedDataMountMatches": copy["managedDataMountMatches"],
|
||||
"targetDataUid": copy["targetDataUid"],
|
||||
"targetDataGid": copy["targetDataGid"],
|
||||
"targetLogicalSha256": copy["targetLogicalSha256"],
|
||||
"targetScanCount": copy["targetScanCount"],
|
||||
"targetCompactedScanCount": copy["targetCompactedScanCount"],
|
||||
"targetQueueSha256": copy["targetQueueSha256"],
|
||||
"targetAuxiliaryTreeSha256": copy["targetAuxiliaryTreeSha256"],
|
||||
"targetAuxiliaryFileCount": copy["targetAuxiliaryFileCount"],
|
||||
"targetQuickCheck": copy["targetQuickCheck"],
|
||||
"targetIntegrityCheck": copy["targetIntegrityCheck"],
|
||||
"targetRecoveryStatus": copy["targetRecoveryStatus"],
|
||||
"targetQueuedJobs": 0,
|
||||
"targetActiveJobs": 0,
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyVolumeRetained": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
"scanSourceMatches": True,
|
||||
"scanMountReadOnly": True,
|
||||
"scanPermissionPolicyPersistent": True,
|
||||
"scanAccessForRuntimeIdentityReady": True,
|
||||
"containerName": "repopulse",
|
||||
"containerImageSha256": _sha("managed-image"),
|
||||
"snapshotFileSha256": snapshot["snapshotFileSha256"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def executor_env(tmp_path, monkeypatch):
|
||||
module = _load_module("repopulse_adoption_store", STORE_PATH)
|
||||
controller = tmp_path / "controller"
|
||||
controller.mkdir()
|
||||
controller.chmod(0o755)
|
||||
(controller / "projects.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
REPOSITORY: {
|
||||
"branch": "main",
|
||||
"compose_files": ["docker-compose.yml"],
|
||||
"compose_project": "repopulse",
|
||||
"containers": ["repopulse"],
|
||||
"env_source": "/srv/example/appdata/repopulse/deploy.env",
|
||||
"exact_source": True,
|
||||
"gitea_repository_id": 64,
|
||||
"gitea_webhook_secret_file": "jens__repopulse",
|
||||
"health_timeout": 180,
|
||||
"slug": "repopulse",
|
||||
"source_dir": "/mnt/cache/appdata/gitea-deploy-runners/sources/repopulse",
|
||||
"validation": {
|
||||
"contexts": ["Managed validation / full (push)"],
|
||||
"max_poll_interval_seconds": 30,
|
||||
"poll_interval_seconds": 5,
|
||||
"required": True,
|
||||
"timeout_seconds": 1800,
|
||||
"validation_gate_version": 1,
|
||||
},
|
||||
"verify_oci_revision": False,
|
||||
},
|
||||
"jens/other": {"slug": "other"},
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(controller / "projects.json").chmod(0o600)
|
||||
secrets = controller / "appops-deployment-secrets"
|
||||
secrets.mkdir()
|
||||
secrets.chmod(0o700)
|
||||
state = tmp_path / "state"
|
||||
state.mkdir()
|
||||
state.chmod(0o755)
|
||||
locks = state / "locks"
|
||||
locks.mkdir()
|
||||
locks.chmod(0o755)
|
||||
global_lock = tmp_path / "deploy.lock"
|
||||
state_store = state / "repopulse-adoption"
|
||||
|
||||
monkeypatch.setattr(module, "CONTROLLER_READONLY_ROOT", controller)
|
||||
monkeypatch.setattr(module, "CONTROLLER_RW_ROOT", controller)
|
||||
monkeypatch.setattr(module, "STATE_ROOT", state)
|
||||
monkeypatch.setattr(module, "GLOBAL_DEPLOY_LOCK", global_lock)
|
||||
monkeypatch.setattr(module, "PROJECT_LOCK", locks / "repopulse.lock")
|
||||
monkeypatch.setattr(module, "ADOPTION_STATE_ROOT", state_store)
|
||||
monkeypatch.setattr(module, "EXECUTION_ENABLED", True)
|
||||
monkeypatch.setattr(module, "_IS_POSIX_CONTROLLER", True)
|
||||
test_uid = controller.stat().st_uid
|
||||
monkeypatch.setattr(module, "_ROOT_UID", test_uid)
|
||||
monkeypatch.setattr(module, "_effective_uid", lambda: test_uid)
|
||||
if os.name != "posix":
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"_mode_bits",
|
||||
lambda path, info: (
|
||||
0o700
|
||||
if path.name in {"appops-deployment-secrets", "repopulse-adoption"}
|
||||
else 0o755
|
||||
if stat.S_ISDIR(info.st_mode)
|
||||
else 0o600
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(module, "_directory_fsync", lambda _path: None)
|
||||
|
||||
executor = _load_module("repopulse_adoption_executor_test", EXECUTOR_PATH)
|
||||
return module, module.RepoPulseAdoptionStore(), executor, controller, state_store
|
||||
|
||||
|
||||
class Effects:
|
||||
def __init__(
|
||||
self,
|
||||
store_module,
|
||||
command,
|
||||
*,
|
||||
bad_receipt_field: str | None = None,
|
||||
bad_copy=False,
|
||||
bad_validation=False,
|
||||
fail_rollback_stop=False,
|
||||
bad_live_completion=False,
|
||||
):
|
||||
self.store_module = store_module
|
||||
self.command = command
|
||||
self.bad_receipt_field = bad_receipt_field
|
||||
self.bad_copy = bad_copy
|
||||
self.bad_validation = bad_validation
|
||||
self.fail_rollback_stop = fail_rollback_stop
|
||||
self.bad_live_completion = bad_live_completion
|
||||
self.events: list[str] = []
|
||||
self.credential = b"target-secret-never-log-" + b"x" * 40
|
||||
|
||||
def _event(self, name: str) -> None:
|
||||
assert self.store_module._transaction_owner(self.store_module._lock_identity()) == threading.get_ident()
|
||||
self.events.append(name)
|
||||
|
||||
def target_gates(self):
|
||||
self._event("gates")
|
||||
return _target_gates()
|
||||
|
||||
def capture_legacy_rollback(self, *, intent_id: str, expected_legacy_projects_sha256: str):
|
||||
self._event("capture_legacy_rollback")
|
||||
assert intent_id == self.command.intent_id
|
||||
return _legacy_rollback_evidence(self.command, expected_legacy_projects_sha256)
|
||||
|
||||
def ensure_quiesced(
|
||||
self,
|
||||
*,
|
||||
intent_id: str,
|
||||
intent_binding_sha256: str,
|
||||
expected_main_revision: str,
|
||||
expected_container_id_sha256: str,
|
||||
expected_container_image_sha256: str,
|
||||
):
|
||||
self._event("quiesce")
|
||||
assert intent_id == self.command.intent_id
|
||||
assert expected_main_revision == self.command.preflight_projection["main_revision"]
|
||||
assert expected_container_id_sha256 == self.command.preflight_projection["container_id_sha256"]
|
||||
assert expected_container_image_sha256 == self.command.preflight_projection["container_image_sha256"]
|
||||
return _quiescence(self.command, intent_binding_sha256)
|
||||
|
||||
def create_or_verify_fresh_snapshot(
|
||||
self,
|
||||
*,
|
||||
intent_id: str,
|
||||
intent_binding_sha256: str,
|
||||
expected_main_revision: str,
|
||||
expected_snapshot,
|
||||
):
|
||||
self._event("snapshot")
|
||||
assert intent_id == self.command.intent_id
|
||||
assert expected_main_revision == self.command.preflight_projection["main_revision"]
|
||||
proof = _snapshot(self.command, intent_binding_sha256)
|
||||
if expected_snapshot is not None:
|
||||
assert expected_snapshot["fileSha256"] == proof["snapshotFileSha256"]
|
||||
return proof
|
||||
|
||||
def copy_and_verify_managed_data(
|
||||
self, *, intent_id: str, expected_main_revision: str, snapshot_evidence: dict
|
||||
):
|
||||
self._event("copy")
|
||||
assert intent_id == self.command.intent_id
|
||||
assert expected_main_revision == self.command.preflight_projection["main_revision"]
|
||||
proof = _copy_evidence(self.command, snapshot_evidence)
|
||||
if self.bad_copy:
|
||||
proof["copySourceIsVerifiedSnapshot"] = False
|
||||
return proof
|
||||
|
||||
def registration_material(self, *, intent_id: str):
|
||||
self._event("registration_material")
|
||||
assert intent_id == self.command.intent_id
|
||||
return "enr-0123456789abcdef0123", self.credential
|
||||
|
||||
def validate_exact_sha_ci_locked(self, *, repository: str, revision: str):
|
||||
self._event("validation_gate")
|
||||
assert repository == REPOSITORY
|
||||
assert revision == self.command.preflight_projection["main_revision"]
|
||||
_raw, projects, credential = self.store_module._check_controller_aliases()
|
||||
assert projects[REPOSITORY]["managed_profile"] == "repopulse-authenticated"
|
||||
assert credential == self.credential
|
||||
return {
|
||||
"repository": REPOSITORY,
|
||||
"revision": revision,
|
||||
"contexts": {"Managed validation / full (push)": "failure" if self.bad_validation else "success"},
|
||||
}
|
||||
|
||||
def deploy_exact_sha_locked(
|
||||
self,
|
||||
*,
|
||||
intent_id: str,
|
||||
revision: str,
|
||||
controller_registry_receipt: dict,
|
||||
snapshot_evidence: dict,
|
||||
copied_data_evidence: dict,
|
||||
):
|
||||
self._event("deploy")
|
||||
assert intent_id == self.command.intent_id
|
||||
assert revision == self.command.preflight_projection["main_revision"]
|
||||
assert controller_registry_receipt["phase"] == "registry_committed"
|
||||
proof = _deployment_receipt(self.command, snapshot_evidence, copied_data_evidence)
|
||||
if self.bad_receipt_field:
|
||||
proof[self.bad_receipt_field] = "wrong" if self.bad_receipt_field in {"repository", "revision", "containerRevision"} else False
|
||||
return proof
|
||||
|
||||
def stop_managed_writer(self, *, intent_id: str, revision: str):
|
||||
self._event("stop_managed_writer")
|
||||
assert intent_id == self.command.intent_id
|
||||
assert revision == self.command.preflight_projection["main_revision"]
|
||||
return {
|
||||
"intentId": intent_id,
|
||||
"repository": REPOSITORY,
|
||||
"containerName": "repopulse",
|
||||
"managedWriterStopped": not self.fail_rollback_stop,
|
||||
"activeWriterCount": 0 if not self.fail_rollback_stop else 1,
|
||||
"legacyVolumeRetained": True,
|
||||
}
|
||||
|
||||
def restore_legacy_registry(
|
||||
self,
|
||||
*,
|
||||
intent_id: str,
|
||||
expected_legacy_projects_sha256: str,
|
||||
controller_registry_receipt: dict | None,
|
||||
):
|
||||
self._event("restore_legacy_registry")
|
||||
assert intent_id == self.command.intent_id
|
||||
assert expected_legacy_projects_sha256 == _legacy_rollback_evidence(self.command, expected_legacy_projects_sha256)["legacyProjectsSha256"]
|
||||
projects_path = self.store_module.CONTROLLER_RW_ROOT / "projects.json"
|
||||
projects = json.loads(projects_path.read_text(encoding="utf-8"))
|
||||
projects[REPOSITORY] = self.store_module._legacy_registry_record()
|
||||
projects_path.write_text(json.dumps(projects, sort_keys=True) + "\n", encoding="utf-8")
|
||||
projects_path.chmod(0o600)
|
||||
return {
|
||||
"intentId": intent_id,
|
||||
"repository": REPOSITORY,
|
||||
"projectsSha256": expected_legacy_projects_sha256,
|
||||
"legacyRegistryRestored": True,
|
||||
"legacyVolumeRetained": True,
|
||||
}
|
||||
|
||||
def restore_legacy_runtime(
|
||||
self,
|
||||
*,
|
||||
legacy_rollback_evidence: dict,
|
||||
legacy_registry_evidence: dict,
|
||||
failed_revision: str,
|
||||
):
|
||||
self._event("restore_legacy_runtime")
|
||||
assert failed_revision == self.command.preflight_projection["main_revision"]
|
||||
assert legacy_registry_evidence["legacyRegistryRestored"] is True
|
||||
assert legacy_rollback_evidence["legacyVolume"] == LEGACY_VOLUME
|
||||
return {
|
||||
"intentId": self.command.intent_id,
|
||||
"repository": REPOSITORY,
|
||||
"containerName": "repopulse",
|
||||
"projectsSha256": legacy_registry_evidence["projectsSha256"],
|
||||
"legacyRuntimeHealthy": True,
|
||||
"activeWriterCount": 1,
|
||||
"legacyVolumeRetained": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
}
|
||||
|
||||
def verify_live_completion_locked(self, *, completion: dict):
|
||||
self._event("verify_completion")
|
||||
proof = {
|
||||
"repository": REPOSITORY,
|
||||
"revision": completion["mainRevision"],
|
||||
"containerRevision": completion["mainRevision"],
|
||||
"deploymentReceiptSha256": completion["deploymentReceiptSha256"],
|
||||
"containerIdSha256": completion["containerIdSha256"],
|
||||
"containerImageSha256": completion["containerImageSha256"],
|
||||
"runtimeHealthy": True,
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyVolumeRetained": True,
|
||||
}
|
||||
if self.bad_live_completion:
|
||||
proof["containerRevision"] = "f" * 40
|
||||
return proof
|
||||
|
||||
|
||||
def test_executor_stays_hard_disabled_without_touching_store_or_effects(executor_env, monkeypatch):
|
||||
store_module, store, executor_module, _controller, state_store = executor_env
|
||||
monkeypatch.setattr(executor_module, "EXECUTION_ENABLED", False)
|
||||
command = _command(executor_module)
|
||||
effects = object()
|
||||
executor = executor_module.RepoPulseAdoptionExecutor(store=store, effects=effects)
|
||||
|
||||
assert executor_module.EXECUTION_ENABLED is False
|
||||
assert executor_module.execution_status()["hasEffectfulRoute"] is False
|
||||
with pytest.raises(executor_module.AdoptionExecutorError) as error:
|
||||
executor.execute(command)
|
||||
|
||||
assert error.value.code == "controller_adoption_execution_disabled"
|
||||
assert not state_store.exists()
|
||||
assert store_module._transaction_owner(store_module._lock_identity()) is None
|
||||
|
||||
|
||||
def test_executor_holds_project_then_global_transaction_through_deploy_receipt(executor_env, monkeypatch):
|
||||
store_module, store, executor_module, controller, state_store = executor_env
|
||||
command = _command(executor_module)
|
||||
effects = Effects(store_module, command)
|
||||
monkeypatch.setattr(executor_module, "EXECUTION_ENABLED", True)
|
||||
executor = executor_module.RepoPulseAdoptionExecutor(store=store, effects=effects)
|
||||
|
||||
result = executor.execute(command)
|
||||
|
||||
assert effects.events == ["gates", "capture_legacy_rollback", "quiesce", "snapshot", "copy", "registration_material", "validation_gate", "deploy"]
|
||||
assert result["repository"] == REPOSITORY
|
||||
assert result["applicationId"] == APPLICATION_ID
|
||||
assert result["deploymentReceipt"]["revision"] == command.preflight_projection["main_revision"]
|
||||
assert result["deploymentReceipt"]["containerRevision"] == command.preflight_projection["main_revision"]
|
||||
assert result["legacyVolumeRetained"] is True
|
||||
intent = store.load_intent()
|
||||
assert intent is not None
|
||||
assert intent["phase"] == "snapshot_verified"
|
||||
assert intent["status"] == "in_progress"
|
||||
assert intent["registration"]["status"] == "committed"
|
||||
assert store.load_receipt()["phase"] == "registry_committed"
|
||||
persisted = (state_store / "intent.json").read_text(encoding="utf-8")
|
||||
persisted += (state_store / "receipt.json").read_text(encoding="utf-8")
|
||||
assert effects.credential.decode("ascii") not in persisted
|
||||
assert effects.credential.decode("ascii") not in json.dumps(result)
|
||||
assert result["completion"]["legacyVolumeRetained"] is True
|
||||
assert result["completion"]["mainRevision"] == command.preflight_projection["main_revision"]
|
||||
completion_path = state_store / "completion.json"
|
||||
assert completion_path.is_file()
|
||||
assert executor_module._store._mode_bits(completion_path, completion_path.stat()) == 0o600
|
||||
assert json.loads((controller / "projects.json").read_text(encoding="utf-8"))[REPOSITORY]["managed_profile"] == "repopulse-authenticated"
|
||||
|
||||
|
||||
def test_completion_replay_reverifies_registry_and_live_receipt_without_redeploy(executor_env, monkeypatch):
|
||||
store_module, store, executor_module, _controller, _state_store = executor_env
|
||||
command = _command(executor_module)
|
||||
effects = Effects(store_module, command)
|
||||
monkeypatch.setattr(executor_module, "EXECUTION_ENABLED", True)
|
||||
executor = executor_module.RepoPulseAdoptionExecutor(store=store, effects=effects)
|
||||
|
||||
executor.execute(command)
|
||||
effects.events.clear()
|
||||
replay = executor.execute(command)
|
||||
|
||||
assert effects.events == ["verify_completion"]
|
||||
assert replay["phase"] == "completion_verified"
|
||||
assert replay["completion"]["intentBindingSha256"] == store.load_intent()["intentBindingSha256"]
|
||||
|
||||
|
||||
def test_completion_replay_mismatch_fails_closed_without_starting_or_deploying(executor_env, monkeypatch):
|
||||
store_module, store, executor_module, _controller, _state_store = executor_env
|
||||
command = _command(executor_module)
|
||||
effects = Effects(store_module, command)
|
||||
monkeypatch.setattr(executor_module, "EXECUTION_ENABLED", True)
|
||||
executor = executor_module.RepoPulseAdoptionExecutor(store=store, effects=effects)
|
||||
executor.execute(command)
|
||||
effects.events.clear()
|
||||
effects.bad_live_completion = True
|
||||
|
||||
with pytest.raises(executor_module.AdoptionExecutorError) as error:
|
||||
executor.execute(command)
|
||||
|
||||
assert error.value.code == "controller_completion_live_receipt_invalid"
|
||||
assert effects.events == ["verify_completion"]
|
||||
assert store.load_intent()["status"] == "recovery_required"
|
||||
|
||||
|
||||
def test_registry_cutover_without_completion_requires_reconciliation(executor_env, monkeypatch):
|
||||
store_module, store, executor_module, _controller, state_store = executor_env
|
||||
command = _command(executor_module)
|
||||
effects = Effects(store_module, command)
|
||||
monkeypatch.setattr(executor_module, "EXECUTION_ENABLED", True)
|
||||
executor = executor_module.RepoPulseAdoptionExecutor(store=store, effects=effects)
|
||||
executor.execute(command)
|
||||
(state_store / "completion.json").unlink()
|
||||
effects.events.clear()
|
||||
|
||||
with pytest.raises(executor_module.AdoptionExecutorError) as error:
|
||||
executor.execute(command)
|
||||
|
||||
assert error.value.code == "controller_completion_reconciliation_required"
|
||||
assert effects.events == []
|
||||
assert store.load_intent()["status"] == "recovery_required"
|
||||
|
||||
|
||||
def test_private_legacy_snapshot_is_durable_and_restores_only_fixed_compose_runtime(executor_env, tmp_path, monkeypatch):
|
||||
store_module, store, executor_module, controller, state_store = executor_env
|
||||
legacy_env = tmp_path / "legacy-deploy.env"
|
||||
legacy_env.write_text("PRIVATE=not-read\n", encoding="utf-8")
|
||||
legacy_env.chmod(0o600)
|
||||
fixed_legacy = store_module._legacy_registry_record()
|
||||
fixed_legacy["env_source"] = str(legacy_env)
|
||||
monkeypatch.setattr(store_module, "_legacy_registry_record", lambda: dict(fixed_legacy))
|
||||
projects_path = controller / "projects.json"
|
||||
projects = json.loads(projects_path.read_text(encoding="utf-8"))
|
||||
projects[REPOSITORY] = fixed_legacy
|
||||
projects_path.write_text(json.dumps(projects, sort_keys=True) + "\n", encoding="utf-8")
|
||||
projects_path.chmod(0o600)
|
||||
monkeypatch.setattr(executor_module, "LEGACY_ENVIRONMENT_FILE", legacy_env)
|
||||
|
||||
class FakeDeployCore:
|
||||
def __init__(self):
|
||||
self.restored = []
|
||||
|
||||
def repository_path(self, repository, project):
|
||||
assert repository == REPOSITORY
|
||||
assert project["compose_project"] == "repopulse"
|
||||
return tmp_path / "repopulse.git"
|
||||
|
||||
def inspect_container(self, name):
|
||||
assert name == "repopulse"
|
||||
return {
|
||||
"Id": "c" * 64,
|
||||
"Image": "sha256:" + "d" * 64,
|
||||
"State": {"Running": True},
|
||||
"Mounts": [{"Type": "volume", "Name": LEGACY_VOLUME, "RW": True}],
|
||||
}
|
||||
|
||||
def output(self, command):
|
||||
assert command == [
|
||||
"docker",
|
||||
"ps",
|
||||
"--all",
|
||||
"--filter",
|
||||
f"volume={LEGACY_VOLUME}",
|
||||
"--format",
|
||||
"{{.Names}}",
|
||||
]
|
||||
return "repopulse\n"
|
||||
|
||||
def capture_compose_rollback(self, project):
|
||||
assert project["_repository_name"] == REPOSITORY
|
||||
assert project["compose_files"] == ["docker-compose.yml"]
|
||||
return {
|
||||
"revision": "b" * 40,
|
||||
"legacy_provenance": True,
|
||||
"receipt": {},
|
||||
"compose_containers": ["repopulse"],
|
||||
"containers": [{
|
||||
"name": "repopulse",
|
||||
"container_id": "c" * 64,
|
||||
"image_id": "sha256:" + "d" * 64,
|
||||
"image_ref": "repopulse:legacy",
|
||||
"rollback_ref": "gitea-deploy-rollback/repopulse-bbbbbbbbbbbb",
|
||||
"was_running": True,
|
||||
}],
|
||||
"captured_at_epoch": 1,
|
||||
}
|
||||
|
||||
def restore_compose_snapshot(self, project, snapshot, repository, failed_revision):
|
||||
assert project["compose_files"] == ["docker-compose.yml"]
|
||||
assert project["env_source"] == str(legacy_env)
|
||||
assert snapshot["legacy_provenance"] is True
|
||||
assert repository == REPOSITORY
|
||||
assert failed_revision == "a" * 40
|
||||
self.restored.append("legacy-compose")
|
||||
|
||||
def verify_containers(self, project, *, expected_revision, expected_image_ids):
|
||||
assert project["compose_project"] == "repopulse"
|
||||
assert expected_revision is None
|
||||
assert expected_image_ids == {"repopulse": "sha256:" + "d" * 64}
|
||||
return {"repopulse": {"image_id": "sha256:" + "d" * 64}}
|
||||
|
||||
def container_state(self, name):
|
||||
assert name == "repopulse"
|
||||
return True, "healthy", "repopulse"
|
||||
|
||||
core = FakeDeployCore()
|
||||
adapter = executor_module.FixedLegacyRollbackAdapter(store=store, deployment_core=core)
|
||||
intent_id = "rpa-0123456789abcdef0123"
|
||||
projects_sha = hashlib.sha256(projects_path.read_bytes()).hexdigest()
|
||||
with store.transaction():
|
||||
evidence = adapter.capture_legacy_rollback(
|
||||
intent_id=intent_id,
|
||||
expected_legacy_projects_sha256=projects_sha,
|
||||
)
|
||||
snapshot_file = state_store / f"legacy-runtime-{intent_id}.json"
|
||||
assert snapshot_file.is_file()
|
||||
assert executor_module._store._mode_bits(snapshot_file, snapshot_file.stat()) == 0o600
|
||||
raw_record = json.loads(snapshot_file.read_text(encoding="utf-8"))
|
||||
assert raw_record["snapshot"]["containers"][0]["name"] == "repopulse"
|
||||
proof = adapter.restore_legacy_runtime(
|
||||
legacy_rollback_evidence=evidence,
|
||||
legacy_registry_evidence={
|
||||
"intentId": intent_id,
|
||||
"repository": REPOSITORY,
|
||||
"projectsSha256": projects_sha,
|
||||
"legacyRegistryRestored": True,
|
||||
"legacyVolumeRetained": True,
|
||||
},
|
||||
failed_revision="a" * 40,
|
||||
)
|
||||
|
||||
assert core.restored == ["legacy-compose"]
|
||||
assert proof["legacyRuntimeHealthy"] is True
|
||||
assert proof["activeWriterCount"] == 1
|
||||
assert "legacy-deploy.env" not in json.dumps(proof)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ["repository", "revision", "containerRevision"])
|
||||
def test_unverified_exact_sha_controller_receipt_enters_durable_recovery(executor_env, monkeypatch, field):
|
||||
store_module, store, executor_module, _controller, state_store = executor_env
|
||||
command = _command(executor_module)
|
||||
effects = Effects(store_module, command, bad_receipt_field=field)
|
||||
monkeypatch.setattr(executor_module, "EXECUTION_ENABLED", True)
|
||||
executor = executor_module.RepoPulseAdoptionExecutor(store=store, effects=effects)
|
||||
|
||||
with pytest.raises(executor_module.AdoptionExecutorError) as error:
|
||||
executor.execute(command)
|
||||
|
||||
assert error.value.code == "controller_deployment_receipt_mismatch"
|
||||
intent = store.load_intent()
|
||||
assert intent is not None
|
||||
assert intent["phase"] == "snapshot_verified"
|
||||
assert intent["status"] == "recovery_required"
|
||||
assert intent["recovery"]["reason"] == "target_health_failed"
|
||||
assert LEGACY_VOLUME in str(intent)
|
||||
assert "target-secret-never-log" not in str(error.value)
|
||||
assert effects.events[-4:] == ["deploy", "stop_managed_writer", "restore_legacy_registry", "restore_legacy_runtime"]
|
||||
assert effects.events.index("stop_managed_writer") < effects.events.index("restore_legacy_registry") < effects.events.index("restore_legacy_runtime")
|
||||
assert (state_store / "receipt.json").is_file()
|
||||
|
||||
|
||||
def test_unverified_managed_copy_blocks_registry_cas_and_records_recovery(executor_env, monkeypatch):
|
||||
store_module, store, executor_module, controller, _state_store = executor_env
|
||||
command = _command(executor_module)
|
||||
effects = Effects(store_module, command, bad_copy=True)
|
||||
monkeypatch.setattr(executor_module, "EXECUTION_ENABLED", True)
|
||||
executor = executor_module.RepoPulseAdoptionExecutor(store=store, effects=effects)
|
||||
|
||||
with pytest.raises(executor_module.AdoptionExecutorError) as error:
|
||||
executor.execute(command)
|
||||
|
||||
assert error.value.code == "managed_data_copy_not_verified"
|
||||
assert effects.events == ["gates", "capture_legacy_rollback", "quiesce", "snapshot", "copy", "stop_managed_writer", "restore_legacy_registry", "restore_legacy_runtime"]
|
||||
assert json.loads((controller / "projects.json").read_text(encoding="utf-8"))[REPOSITORY]["env_source"] == "/srv/example/appdata/repopulse/deploy.env"
|
||||
assert not (controller / "appops-deployment-secrets" / "jens__repopulse").exists()
|
||||
intent = store.load_intent()
|
||||
assert intent is not None
|
||||
assert intent["phase"] == "quiesced"
|
||||
assert intent["status"] == "recovery_required"
|
||||
assert intent["recovery"]["reason"] == "database_backup_failed"
|
||||
|
||||
|
||||
def test_exact_sha_ci_gate_is_required_before_deploy_and_failed_gate_restores_legacy(executor_env, monkeypatch):
|
||||
_store_module, store, executor_module, controller, _state_store = executor_env
|
||||
command = _command(executor_module)
|
||||
effects = Effects(_store_module, command, bad_validation=True)
|
||||
monkeypatch.setattr(executor_module, "EXECUTION_ENABLED", True)
|
||||
|
||||
with pytest.raises(executor_module.AdoptionExecutorError) as error:
|
||||
executor_module.RepoPulseAdoptionExecutor(store=store, effects=effects).execute(command)
|
||||
|
||||
assert error.value.code == "exact_sha_validation_not_passed"
|
||||
assert effects.events[-3:] == ["stop_managed_writer", "restore_legacy_registry", "restore_legacy_runtime"]
|
||||
assert "deploy" not in effects.events
|
||||
assert json.loads((controller / "projects.json").read_text(encoding="utf-8"))[REPOSITORY]["env_source"] == "/srv/example/appdata/repopulse/deploy.env"
|
||||
assert store.load_intent()["status"] == "recovery_required"
|
||||
|
||||
|
||||
def test_legacy_runtime_is_never_restarted_when_managed_writer_stop_is_unproven(executor_env, monkeypatch):
|
||||
_store_module, store, executor_module, _controller, _state_store = executor_env
|
||||
command = _command(executor_module)
|
||||
effects = Effects(_store_module, command, bad_receipt_field="revision", fail_rollback_stop=True)
|
||||
monkeypatch.setattr(executor_module, "EXECUTION_ENABLED", True)
|
||||
|
||||
with pytest.raises(executor_module.AdoptionExecutorError) as error:
|
||||
executor_module.RepoPulseAdoptionExecutor(store=store, effects=effects).execute(command)
|
||||
|
||||
assert error.value.code == "legacy_rollback_incomplete"
|
||||
assert effects.events[-1] == "stop_managed_writer"
|
||||
assert "restore_legacy_registry" not in effects.events
|
||||
assert "restore_legacy_runtime" not in effects.events
|
||||
assert store.load_intent()["status"] == "recovery_required"
|
||||
|
||||
|
||||
def test_invalid_or_changed_fixed_preflight_is_rejected_before_external_effects(executor_env, monkeypatch):
|
||||
_store_module, store, executor_module, _controller, state_store = executor_env
|
||||
command = _command(executor_module)
|
||||
changed_projection = dict(command.preflight_projection)
|
||||
changed_projection["repository"] = "jens/other"
|
||||
changed = executor_module.AdoptionCommand(
|
||||
intent_id=command.intent_id,
|
||||
idempotency_sha256=command.idempotency_sha256,
|
||||
preflight_sha256=command.preflight_sha256,
|
||||
preflight_projection=changed_projection,
|
||||
)
|
||||
effects = object()
|
||||
monkeypatch.setattr(executor_module, "EXECUTION_ENABLED", True)
|
||||
|
||||
with pytest.raises(executor_module.AdoptionExecutorError) as error:
|
||||
executor_module.RepoPulseAdoptionExecutor(store=store, effects=effects).execute(changed)
|
||||
|
||||
assert error.value.code == "preflight_projection_invalid"
|
||||
assert not state_store.exists()
|
||||
@@ -0,0 +1,497 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
from backend.app.db.models import RepoPulseAdoptionJournal
|
||||
from backend.app.services import repopulse_adoption_journal as journal
|
||||
from backend.app.services.enrollment_profiles import PROFILE_VERSION
|
||||
from backend.app.services.repopulse_legacy_adoption_protocol import (
|
||||
APPLICATION_ID,
|
||||
CONTROLLER_RECEIPT_VERSION,
|
||||
LEGACY_VOLUME,
|
||||
PROTOCOL_VERSION,
|
||||
REPOSITORY,
|
||||
advance_intent,
|
||||
create_intent,
|
||||
intent_binding_sha256,
|
||||
)
|
||||
|
||||
|
||||
def _sha(char: str) -> str:
|
||||
return char * 64
|
||||
|
||||
|
||||
def _passing_preflight() -> dict:
|
||||
check_codes = (
|
||||
"legacy_registry",
|
||||
"controller_state",
|
||||
"gitea_ancestry",
|
||||
"runtime_identity_and_health",
|
||||
"legacy_storage_topology",
|
||||
"legacy_appdata",
|
||||
"product_api",
|
||||
"database_health",
|
||||
"queue_quiescence",
|
||||
"database_baseline",
|
||||
)
|
||||
return {
|
||||
"apiVersion": "itworx.repopulse-legacy-adoption-preflight/v1",
|
||||
"repository": REPOSITORY,
|
||||
"status": "preflight_passed",
|
||||
"legacyCompatibility": {
|
||||
"status": "passed",
|
||||
"checks": [
|
||||
{"code": code, "status": "passed", "gate": "legacy_compatibility"}
|
||||
for code in check_codes
|
||||
],
|
||||
},
|
||||
"targetAcceptance": {"status": "blocked", "checks": []},
|
||||
"readOnly": True,
|
||||
"adoptionAuthorized": False,
|
||||
"evidence": {
|
||||
"main_revision": "a" * 40,
|
||||
"live_revision": "b" * 40,
|
||||
"container_id_sha256": _sha("d"),
|
||||
"container_image_sha256": _sha("1"),
|
||||
"container_state": "running",
|
||||
"container_health": "healthy",
|
||||
"queue": {"queued": 0, "active": 0, "completed": 569, "failed": 0},
|
||||
"database": {
|
||||
"database_bytes": 44_576_768,
|
||||
"journal_bytes": 0,
|
||||
"scan_count": 572,
|
||||
"compacted_scan_count": 96,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _target_gates() -> dict:
|
||||
return {
|
||||
"profileId": "repopulse-authenticated",
|
||||
"profileVersion": PROFILE_VERSION,
|
||||
"planSha256": _sha("c"),
|
||||
**{
|
||||
key: True
|
||||
for key in (
|
||||
"profileReady",
|
||||
"controllerTrustReady",
|
||||
"controllerBootstrapSecretImported",
|
||||
"typedEnrollmentTrustReady",
|
||||
"targetSecretReady",
|
||||
"exactPolicyReady",
|
||||
"authenticatedRouteReady",
|
||||
"accessListReady",
|
||||
"mutationBoundaryReady",
|
||||
"managedRuntimeReady",
|
||||
"scanSourceMatches",
|
||||
"scanMountReadOnly",
|
||||
"scanPermissionPolicyPersistent",
|
||||
"scanAccessForRuntimeIdentityReady",
|
||||
"ownedDataTargetReady",
|
||||
"managedEnvironmentReady",
|
||||
"legacyEnvironmentRetained",
|
||||
"maintenanceAuthReady",
|
||||
"controllerAdoptionCapabilityReady",
|
||||
"controllerRegistryWriterReady",
|
||||
"controllerSecretWriterReady",
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _quiescence(intent: dict) -> dict:
|
||||
return {
|
||||
"intentId": intent["intentId"],
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"maintenanceModeActive": True,
|
||||
"queueFrozen": True,
|
||||
"queuedJobs": 0,
|
||||
"activeJobs": 0,
|
||||
"maintenanceModeAuthenticated": True,
|
||||
"containerName": "repopulse",
|
||||
"containerIdSha256": _sha("d"),
|
||||
"containerImageSha256": _sha("1"),
|
||||
"containerState": "exited",
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"controllerIntentDurable": True,
|
||||
"controllerIntentBindingSha256": intent_binding_sha256(intent),
|
||||
}
|
||||
|
||||
|
||||
def _snapshot(intent: dict) -> dict:
|
||||
return {
|
||||
"intentId": intent["intentId"],
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"legacyContainerStopped": True,
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyVolumeRetained": True,
|
||||
"sqliteBackupMethod": "sqlite_online_backup",
|
||||
"quickCheck": "ok",
|
||||
"integrityCheck": "ok",
|
||||
"sourceLogicalSha256": _sha("e"),
|
||||
"snapshotLogicalSha256": _sha("e"),
|
||||
"snapshotFileSha256": _sha("f"),
|
||||
"sourceScanCount": 572,
|
||||
"snapshotScanCount": 572,
|
||||
"sourceCompactedScanCount": 96,
|
||||
"snapshotCompactedScanCount": 96,
|
||||
"sourceQueueSha256": _sha("1"),
|
||||
"snapshotQueueSha256": _sha("1"),
|
||||
"sourceAuxiliaryTreeSha256": _sha("3"),
|
||||
"sourceAuxiliaryFileCount": 42,
|
||||
"targetDataPathWasAbsent": True,
|
||||
"snapshotReceiptDurable": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
"controllerIntentBindingSha256": intent_binding_sha256(intent),
|
||||
}
|
||||
|
||||
|
||||
def _registration(intent: dict) -> dict:
|
||||
gates = intent["evidence"]["targetGates"]
|
||||
snapshot = intent["evidence"]["snapshot"]
|
||||
return {
|
||||
"receiptVersion": "itworx.repopulse-legacy-adoption-receipt/v1",
|
||||
"intentId": intent["intentId"],
|
||||
"receiptSha256": _sha("2"),
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"enrollmentId": "enr-0123456789abcdefabcd",
|
||||
"profileId": gates["profileId"],
|
||||
"planSha256": gates["planSha256"],
|
||||
"snapshotFileSha256": snapshot["snapshotFileSha256"],
|
||||
"targetLogicalSha256": snapshot["snapshotLogicalSha256"],
|
||||
"targetScanCount": snapshot["snapshotScanCount"],
|
||||
"targetCompactedScanCount": snapshot["snapshotCompactedScanCount"],
|
||||
"targetQueueSha256": snapshot["snapshotQueueSha256"],
|
||||
"targetAuxiliaryTreeSha256": snapshot["sourceAuxiliaryTreeSha256"],
|
||||
"targetAuxiliaryFileCount": snapshot["sourceAuxiliaryFileCount"],
|
||||
"targetQueuedJobs": 0,
|
||||
"targetActiveJobs": 0,
|
||||
"targetQuickCheck": "ok",
|
||||
"targetIntegrityCheck": "ok",
|
||||
"targetRecoveryStatus": "healthy",
|
||||
"targetDataUid": 10001,
|
||||
"targetDataGid": 10001,
|
||||
"managedDataMountMatches": True,
|
||||
"copySourceIsVerifiedSnapshot": True,
|
||||
"registryTargetMatches": True,
|
||||
"controllerStateReceiptVerified": True,
|
||||
"deploymentReceiptSchema": "gitea-deploy-receipt.v1",
|
||||
"deploymentReceiptSha256": _sha("4"),
|
||||
"deploymentReceiptVerified": True,
|
||||
"runtimeHealthy": True,
|
||||
"publishedHostPorts": [],
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyVolumeRetained": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
"scanSourceMatches": True,
|
||||
"scanMountReadOnly": True,
|
||||
"scanPermissionPolicyPersistent": True,
|
||||
"scanAccessForRuntimeIdentityReady": True,
|
||||
"controllerIntentDurable": True,
|
||||
"controllerIntentBindingSha256": intent_binding_sha256(intent),
|
||||
}
|
||||
|
||||
|
||||
def _ownership(intent: dict, registration: dict) -> dict:
|
||||
return {
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"enrollmentId": registration["enrollmentId"],
|
||||
"profileId": registration["profileId"],
|
||||
"planSha256": intent["evidence"]["targetGates"]["planSha256"],
|
||||
"controllerReceiptSha256": registration["receiptSha256"],
|
||||
"enrollmentPersisted": True,
|
||||
"exactApplyPolicyPersisted": True,
|
||||
"exactRollbackPolicyPersisted": True,
|
||||
"secretReferenceReady": True,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine(tmp_path):
|
||||
database_engine = create_engine(f"sqlite:///{tmp_path / 'appops.db'}")
|
||||
SQLModel.metadata.create_all(database_engine)
|
||||
yield database_engine
|
||||
database_engine.dispose()
|
||||
|
||||
|
||||
def _reserve(engine, key: str = "repopulse-adoption-2026") -> dict:
|
||||
with Session(engine) as session:
|
||||
return journal.reserve_repopulse_adoption(session, _passing_preflight(), key)
|
||||
|
||||
|
||||
def test_create_all_registers_singleton_table_and_reservation_survives_restart(engine) -> None:
|
||||
assert inspect(engine).has_table("repopulse_adoption_journal")
|
||||
|
||||
first = _reserve(engine)
|
||||
assert first["apiVersion"] == PROTOCOL_VERSION
|
||||
assert first["status"] == "in_progress"
|
||||
assert first["phase"] == "reserved"
|
||||
assert first["repository"] == REPOSITORY
|
||||
assert first["applicationId"] == APPLICATION_ID
|
||||
assert first["revision"] == 1
|
||||
|
||||
# A new Session models a process restart: the slot and idempotent replay
|
||||
# are read back from SQLite, not held in adapter memory.
|
||||
replay = _reserve(engine)
|
||||
assert replay == first
|
||||
|
||||
with Session(engine) as session:
|
||||
status = journal.get_repopulse_adoption_status(session)
|
||||
assert status == first
|
||||
|
||||
|
||||
def test_changed_preflight_and_different_key_conflict_without_replacing_slot(engine) -> None:
|
||||
first = _reserve(engine)
|
||||
changed = _passing_preflight()
|
||||
changed["evidence"]["live_revision"] = "c" * 40
|
||||
|
||||
with Session(engine) as session, pytest.raises(journal.RepoPulseAdoptionJournalError) as error:
|
||||
journal.reserve_repopulse_adoption(session, changed, "repopulse-adoption-2026")
|
||||
assert error.value.code == "idempotency_conflict"
|
||||
|
||||
with Session(engine) as session, pytest.raises(journal.RepoPulseAdoptionJournalError) as error:
|
||||
journal.reserve_repopulse_adoption(
|
||||
session, _passing_preflight(), "repopulse-adoption-2027"
|
||||
)
|
||||
assert error.value.code == "one_time_adoption_already_reserved"
|
||||
assert _reserve(engine) == first
|
||||
|
||||
|
||||
def test_reservation_rejects_another_repository_identity_without_creating_a_row(engine) -> None:
|
||||
other_repository = _passing_preflight()
|
||||
other_repository["repository"] = "someone-else/repopulse"
|
||||
|
||||
with Session(engine) as session, pytest.raises(journal.RepoPulseAdoptionJournalError) as error:
|
||||
journal.reserve_repopulse_adoption(
|
||||
session, other_repository, "repopulse-adoption-2026"
|
||||
)
|
||||
assert error.value.code == "legacy_preflight_not_compatible"
|
||||
|
||||
with Session(engine) as session:
|
||||
assert journal.get_repopulse_adoption_status(session)["status"] == "unreserved"
|
||||
|
||||
intent = create_intent(_passing_preflight(), "repopulse-adoption-2026")
|
||||
with Session(engine) as session, pytest.raises(IntegrityError):
|
||||
session.add(
|
||||
RepoPulseAdoptionJournal(
|
||||
slot="other",
|
||||
repository="someone-else/repopulse",
|
||||
application_id="other-app",
|
||||
intent_id=intent["intentId"],
|
||||
idempotency_sha256=intent["idempotencySha256"],
|
||||
preflight_sha256=intent["preflightSha256"],
|
||||
phase=intent["phase"],
|
||||
status=intent["status"],
|
||||
revision=1,
|
||||
intent_json=intent,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
|
||||
def test_controller_intent_confirmation_requires_exact_binding_and_replays(engine) -> None:
|
||||
_reserve(engine)
|
||||
with Session(engine) as session:
|
||||
after_gates = journal.advance_repopulse_adoption(
|
||||
session, "target_gates", _target_gates(), expected_revision=1
|
||||
)
|
||||
assert after_gates["phase"] == "target_gates_ready"
|
||||
assert after_gates["revision"] == 2
|
||||
|
||||
current = create_intent(_passing_preflight(), "repopulse-adoption-2026")
|
||||
valid_confirmation = _quiescence(current)
|
||||
invalid_confirmation = {**valid_confirmation, "controllerIntentBindingSha256": _sha("0")}
|
||||
|
||||
with Session(engine) as session, pytest.raises(journal.RepoPulseAdoptionJournalError) as error:
|
||||
journal.confirm_repopulse_controller_intent(
|
||||
session, invalid_confirmation, expected_revision=2
|
||||
)
|
||||
assert error.value.code == "controller_intent_binding_mismatch"
|
||||
|
||||
with Session(engine) as session:
|
||||
confirmed = journal.confirm_repopulse_controller_intent(
|
||||
session, valid_confirmation, expected_revision=2
|
||||
)
|
||||
assert confirmed["phase"] == "quiesced"
|
||||
assert confirmed["revision"] == 3
|
||||
|
||||
with Session(engine) as session:
|
||||
replay = journal.confirm_repopulse_controller_intent(
|
||||
session, valid_confirmation, expected_revision=2
|
||||
)
|
||||
assert replay == confirmed
|
||||
|
||||
|
||||
def test_compare_and_swap_rejects_stale_checkpoint_without_overwriting_state(engine) -> None:
|
||||
first = _reserve(engine)
|
||||
with Session(engine) as session:
|
||||
updated = journal.advance_repopulse_adoption(
|
||||
session,
|
||||
"target_gates",
|
||||
_target_gates(),
|
||||
expected_revision=1,
|
||||
)
|
||||
assert updated["status"] == "in_progress"
|
||||
assert updated["revision"] == 2
|
||||
|
||||
candidate = create_intent(_passing_preflight(), "repopulse-adoption-2026")
|
||||
valid_confirmation = _quiescence(candidate)
|
||||
|
||||
with Session(engine) as session, pytest.raises(journal.RepoPulseAdoptionJournalError) as error:
|
||||
journal.confirm_repopulse_controller_intent(
|
||||
session,
|
||||
valid_confirmation,
|
||||
expected_revision=1,
|
||||
)
|
||||
assert error.value.code == "journal_revision_conflict"
|
||||
|
||||
with Session(engine) as session:
|
||||
assert journal.get_repopulse_adoption_status(session) == updated
|
||||
assert updated["intentId"] == first["intentId"]
|
||||
|
||||
|
||||
def test_recovery_resume_checks_controller_binding_and_persists_across_sessions(engine) -> None:
|
||||
_reserve(engine)
|
||||
with Session(engine) as session:
|
||||
recovery = journal.mark_repopulse_adoption_recovery(
|
||||
session,
|
||||
"operator_interruption",
|
||||
{"operatorNote": "resume review", "path": "private/path"},
|
||||
expected_revision=1,
|
||||
)
|
||||
assert "private/path" not in str(recovery)
|
||||
|
||||
receipt = {
|
||||
"apiVersion": CONTROLLER_RECEIPT_VERSION,
|
||||
"intentId": recovery["intentId"],
|
||||
"phase": "reserved",
|
||||
"receiptSha256": hashlib.sha256(b"durable controller journal row").hexdigest(),
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"legacyVolumeRetained": True,
|
||||
"intentDurable": True,
|
||||
"intentBindingSha256": recovery["intentBindingSha256"],
|
||||
"registrationEvidence": None,
|
||||
}
|
||||
invalid_receipt = {**receipt, "intentBindingSha256": _sha("0")}
|
||||
with Session(engine) as session, pytest.raises(journal.RepoPulseAdoptionJournalError) as error:
|
||||
journal.resume_repopulse_adoption(session, invalid_receipt, expected_revision=2)
|
||||
assert error.value.code == "controller_recovery_receipt_unbound"
|
||||
|
||||
with Session(engine) as session:
|
||||
resumed = journal.resume_repopulse_adoption(
|
||||
session, receipt, expected_revision=2
|
||||
)
|
||||
assert resumed["status"] == "in_progress"
|
||||
assert resumed["phase"] == "reserved"
|
||||
assert resumed["revision"] == 3
|
||||
|
||||
with Session(engine) as session:
|
||||
assert journal.get_repopulse_adoption_status(session) == resumed
|
||||
|
||||
|
||||
def test_controller_registration_reconciles_before_appops_ownership_can_advance(engine) -> None:
|
||||
_reserve(engine)
|
||||
candidate = create_intent(_passing_preflight(), "repopulse-adoption-2026")
|
||||
with Session(engine) as session:
|
||||
current = journal.advance_repopulse_adoption(
|
||||
session, "target_gates", _target_gates(), expected_revision=1
|
||||
)
|
||||
candidate = advance_intent(candidate, "target_gates", _target_gates())
|
||||
with Session(engine) as session:
|
||||
current = journal.confirm_repopulse_controller_intent(
|
||||
session, _quiescence(candidate), expected_revision=2
|
||||
)
|
||||
candidate = advance_intent(candidate, "quiescence", _quiescence(candidate))
|
||||
with Session(engine) as session:
|
||||
current = journal.advance_repopulse_adoption(
|
||||
session, "snapshot", _snapshot(candidate), expected_revision=3
|
||||
)
|
||||
candidate = advance_intent(candidate, "snapshot", _snapshot(candidate))
|
||||
|
||||
early_ownership = {
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"enrollmentId": "enr-0123456789abcdefabcd",
|
||||
"profileId": "repopulse-authenticated",
|
||||
"planSha256": _sha("c"),
|
||||
"controllerReceiptSha256": _sha("2"),
|
||||
"enrollmentPersisted": True,
|
||||
"exactApplyPolicyPersisted": True,
|
||||
"exactRollbackPolicyPersisted": True,
|
||||
"secretReferenceReady": True,
|
||||
}
|
||||
with Session(engine) as session, pytest.raises(journal.RepoPulseAdoptionJournalError) as error:
|
||||
journal.advance_repopulse_adoption(
|
||||
session, "appops_ownership", early_ownership, expected_revision=4
|
||||
)
|
||||
assert error.value.code == "controller_registration_missing"
|
||||
assert current["phase"] == "snapshot_verified"
|
||||
|
||||
registration = _registration(candidate)
|
||||
with Session(engine) as session:
|
||||
registered = journal.advance_repopulse_adoption(
|
||||
session,
|
||||
"controller_registration",
|
||||
registration,
|
||||
expected_revision=4,
|
||||
)
|
||||
candidate = advance_intent(candidate, "controller_registration", registration)
|
||||
assert registered["phase"] == "target_registered"
|
||||
assert registered["revision"] == 5
|
||||
|
||||
with Session(engine) as session:
|
||||
recovery = journal.mark_repopulse_adoption_recovery(
|
||||
session,
|
||||
"appops_commit_uncertain",
|
||||
{"uncertain": True, "path": "private/path"},
|
||||
expected_revision=5,
|
||||
)
|
||||
assert recovery["status"] == "recovery_required"
|
||||
assert recovery["phase"] == "target_registered"
|
||||
assert recovery["legacyVolumePreserved"] is True
|
||||
assert recovery["controllerReconciliationRequired"] is True
|
||||
assert "private/path" not in str(recovery)
|
||||
|
||||
controller_receipt = {
|
||||
"apiVersion": CONTROLLER_RECEIPT_VERSION,
|
||||
"intentId": candidate["intentId"],
|
||||
"phase": "target_registered",
|
||||
"receiptSha256": registration["receiptSha256"],
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"legacyVolumeRetained": True,
|
||||
"intentDurable": True,
|
||||
"intentBindingSha256": intent_binding_sha256(candidate),
|
||||
"registrationEvidence": registration,
|
||||
}
|
||||
with Session(engine) as session:
|
||||
resumed = journal.resume_repopulse_adoption(
|
||||
session, controller_receipt, expected_revision=6
|
||||
)
|
||||
assert resumed["status"] == "in_progress"
|
||||
assert resumed["phase"] == "target_registered"
|
||||
assert resumed["revision"] == 7
|
||||
|
||||
ownership = _ownership(candidate, registration)
|
||||
with Session(engine) as session:
|
||||
owned = journal.advance_repopulse_adoption(
|
||||
session, "appops_ownership", ownership, expected_revision=7
|
||||
)
|
||||
assert owned["phase"] == "appops_owned"
|
||||
assert owned["revision"] == 8
|
||||
assert owned["status"] != "completed"
|
||||
@@ -0,0 +1,606 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from sqlmodel import Session, SQLModel, create_engine, select
|
||||
|
||||
from backend.app.db.models import (
|
||||
AppSetting,
|
||||
Enrollment,
|
||||
Policy,
|
||||
RepoPulseAdoptionJournal,
|
||||
)
|
||||
from backend.app.schemas.enrollment import EnrollmentPlanV1
|
||||
from backend.app.services import repopulse_adoption_journal as journal
|
||||
from backend.app.services import repopulse_adoption_ownership as ownership
|
||||
from backend.app.services.repopulse_legacy_adoption_protocol import (
|
||||
APPLICATION_ID,
|
||||
CONTROLLER_RECEIPT_VERSION,
|
||||
LEGACY_VOLUME,
|
||||
REPOSITORY,
|
||||
intent_binding_sha256,
|
||||
)
|
||||
from backend.app.services.secret_store import set_secret
|
||||
|
||||
|
||||
def _sha(char: str) -> str:
|
||||
return char * 64
|
||||
|
||||
|
||||
def _plan_payload(revision: str) -> dict:
|
||||
return EnrollmentPlanV1.model_validate(
|
||||
{
|
||||
"application_id": "repopulse",
|
||||
"display_name": "RepoPulse",
|
||||
"environment": "production",
|
||||
"repository": "Jens/RepoPulse",
|
||||
"revision": revision,
|
||||
"profile": "repopulse-authenticated",
|
||||
"target": {
|
||||
"target_identity": "unraid-controller",
|
||||
"container_name": "repopulse",
|
||||
},
|
||||
"storage": {"runtime_uid": 10001, "runtime_gid": 10001},
|
||||
"network": {"docker_network": "itworx-mcp-hub-npm", "internal_port": 8000},
|
||||
"artifact": {
|
||||
"release_version": "1.5.0",
|
||||
"image_repository": "repopulse-managed",
|
||||
"image_tag": "production",
|
||||
"health_path": "/health",
|
||||
},
|
||||
"exposure": {
|
||||
"hostname": "repopulse.example.com",
|
||||
"upstream_service": "repopulse",
|
||||
},
|
||||
"secrets": {"additional_secret_refs": ["REPOPULSE_GITEA_TOKEN"]},
|
||||
}
|
||||
).model_dump(mode="json")
|
||||
|
||||
|
||||
def _plan_sha(revision: str) -> str:
|
||||
payload = _plan_payload(revision)
|
||||
canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
|
||||
return hashlib.sha256(canonical).hexdigest()
|
||||
|
||||
|
||||
def _preflight() -> dict:
|
||||
check_codes = (
|
||||
"legacy_registry",
|
||||
"controller_state",
|
||||
"gitea_ancestry",
|
||||
"runtime_identity_and_health",
|
||||
"legacy_storage_topology",
|
||||
"legacy_appdata",
|
||||
"product_api",
|
||||
"database_health",
|
||||
"queue_quiescence",
|
||||
"database_baseline",
|
||||
)
|
||||
return {
|
||||
"apiVersion": "itworx.repopulse-legacy-adoption-preflight/v1",
|
||||
"repository": REPOSITORY,
|
||||
"status": "preflight_passed",
|
||||
"legacyCompatibility": {
|
||||
"status": "passed",
|
||||
"checks": [
|
||||
{"code": code, "status": "passed", "gate": "legacy_compatibility"}
|
||||
for code in check_codes
|
||||
],
|
||||
},
|
||||
"targetAcceptance": {"status": "blocked", "checks": []},
|
||||
"readOnly": True,
|
||||
"adoptionAuthorized": False,
|
||||
"evidence": {
|
||||
"main_revision": "a" * 40,
|
||||
"live_revision": "b" * 40,
|
||||
"container_id_sha256": _sha("d"),
|
||||
"container_image_sha256": _sha("1"),
|
||||
"container_state": "running",
|
||||
"container_health": "healthy",
|
||||
"queue": {"queued": 0, "active": 0, "completed": 569, "failed": 0},
|
||||
"database": {
|
||||
"database_bytes": 44_576_768,
|
||||
"journal_bytes": 0,
|
||||
"scan_count": 572,
|
||||
"compacted_scan_count": 96,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _target_gates() -> dict:
|
||||
return {
|
||||
"profileId": "repopulse-authenticated",
|
||||
"profileVersion": "enrollment-profile/v1",
|
||||
"planSha256": _plan_sha("a" * 40),
|
||||
**{
|
||||
key: True
|
||||
for key in (
|
||||
"profileReady",
|
||||
"controllerTrustReady",
|
||||
"controllerBootstrapSecretImported",
|
||||
"typedEnrollmentTrustReady",
|
||||
"targetSecretReady",
|
||||
"exactPolicyReady",
|
||||
"authenticatedRouteReady",
|
||||
"accessListReady",
|
||||
"mutationBoundaryReady",
|
||||
"managedRuntimeReady",
|
||||
"scanSourceMatches",
|
||||
"scanMountReadOnly",
|
||||
"scanPermissionPolicyPersistent",
|
||||
"scanAccessForRuntimeIdentityReady",
|
||||
"ownedDataTargetReady",
|
||||
"managedEnvironmentReady",
|
||||
"legacyEnvironmentRetained",
|
||||
"maintenanceAuthReady",
|
||||
"controllerAdoptionCapabilityReady",
|
||||
"controllerRegistryWriterReady",
|
||||
"controllerSecretWriterReady",
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _quiescence(intent: dict) -> dict:
|
||||
return {
|
||||
"intentId": intent["intentId"],
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"maintenanceModeActive": True,
|
||||
"queueFrozen": True,
|
||||
"queuedJobs": 0,
|
||||
"activeJobs": 0,
|
||||
"maintenanceModeAuthenticated": True,
|
||||
"containerName": "repopulse",
|
||||
"containerIdSha256": _sha("d"),
|
||||
"containerImageSha256": _sha("1"),
|
||||
"containerState": "exited",
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"controllerIntentDurable": True,
|
||||
"controllerIntentBindingSha256": intent_binding_sha256(intent),
|
||||
}
|
||||
|
||||
|
||||
def _snapshot(intent: dict) -> dict:
|
||||
return {
|
||||
"intentId": intent["intentId"],
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"legacyContainerStopped": True,
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyVolumeRetained": True,
|
||||
"sqliteBackupMethod": "sqlite_online_backup",
|
||||
"quickCheck": "ok",
|
||||
"integrityCheck": "ok",
|
||||
"sourceLogicalSha256": _sha("e"),
|
||||
"snapshotLogicalSha256": _sha("e"),
|
||||
"snapshotFileSha256": _sha("f"),
|
||||
"sourceScanCount": 572,
|
||||
"snapshotScanCount": 572,
|
||||
"sourceCompactedScanCount": 96,
|
||||
"snapshotCompactedScanCount": 96,
|
||||
"sourceQueueSha256": _sha("1"),
|
||||
"snapshotQueueSha256": _sha("1"),
|
||||
"sourceAuxiliaryTreeSha256": _sha("3"),
|
||||
"sourceAuxiliaryFileCount": 42,
|
||||
"targetDataPathWasAbsent": True,
|
||||
"snapshotReceiptDurable": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
"controllerIntentBindingSha256": intent_binding_sha256(intent),
|
||||
}
|
||||
|
||||
|
||||
def _registration(intent: dict, *, enrollment_id: str | None = None) -> dict:
|
||||
gates = intent["evidence"]["targetGates"]
|
||||
snapshot = intent["evidence"]["snapshot"]
|
||||
return {
|
||||
"receiptVersion": CONTROLLER_RECEIPT_VERSION,
|
||||
"intentId": intent["intentId"],
|
||||
"receiptSha256": _sha("2"),
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"enrollmentId": enrollment_id or f"enr-{intent['intentId'][4:]}",
|
||||
"profileId": gates["profileId"],
|
||||
"planSha256": gates["planSha256"],
|
||||
"snapshotFileSha256": snapshot["snapshotFileSha256"],
|
||||
"targetLogicalSha256": snapshot["snapshotLogicalSha256"],
|
||||
"targetScanCount": snapshot["snapshotScanCount"],
|
||||
"targetCompactedScanCount": snapshot["snapshotCompactedScanCount"],
|
||||
"targetQueueSha256": snapshot["snapshotQueueSha256"],
|
||||
"targetAuxiliaryTreeSha256": snapshot["sourceAuxiliaryTreeSha256"],
|
||||
"targetAuxiliaryFileCount": snapshot["sourceAuxiliaryFileCount"],
|
||||
"targetQueuedJobs": 0,
|
||||
"targetActiveJobs": 0,
|
||||
"targetQuickCheck": "ok",
|
||||
"targetIntegrityCheck": "ok",
|
||||
"targetRecoveryStatus": "healthy",
|
||||
"targetDataUid": 10001,
|
||||
"targetDataGid": 10001,
|
||||
"managedDataMountMatches": True,
|
||||
"copySourceIsVerifiedSnapshot": True,
|
||||
"registryTargetMatches": True,
|
||||
"controllerStateReceiptVerified": True,
|
||||
"deploymentReceiptSchema": "gitea-deploy-receipt.v1",
|
||||
"deploymentReceiptSha256": _sha("4"),
|
||||
"deploymentReceiptVerified": True,
|
||||
"runtimeHealthy": True,
|
||||
"publishedHostPorts": [],
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyVolumeRetained": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
"scanSourceMatches": True,
|
||||
"scanMountReadOnly": True,
|
||||
"scanPermissionPolicyPersistent": True,
|
||||
"scanAccessForRuntimeIdentityReady": True,
|
||||
"controllerIntentDurable": True,
|
||||
"controllerIntentBindingSha256": intent_binding_sha256(intent),
|
||||
}
|
||||
|
||||
|
||||
def _ready_engine():
|
||||
engine = create_engine("sqlite://")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
session.add(
|
||||
AppSetting(
|
||||
key="deployment_enrollment:trust",
|
||||
is_secret=False,
|
||||
value_json={
|
||||
"controller_identity": "unraid-controller",
|
||||
"bootstrap_secret_ref": "ENROLLMENT_BOOTSTRAP",
|
||||
"approved_networks": ["itworx-mcp-hub-npm"],
|
||||
"allowed_domain_suffixes": ["example.com"],
|
||||
"autonomous_profiles": ["repopulse-authenticated"],
|
||||
"repopulse_proxy_access_list_id": 1,
|
||||
},
|
||||
)
|
||||
)
|
||||
set_secret(
|
||||
session,
|
||||
"ENROLL_REPOPULSE_TARGET_HMAC",
|
||||
"test-target-credential-only-for-fixture",
|
||||
commit=False,
|
||||
)
|
||||
session.commit()
|
||||
return engine
|
||||
|
||||
|
||||
def _register_target(
|
||||
engine, *, enrollment_id: str | None = None
|
||||
) -> tuple[dict, dict, int]:
|
||||
preflight = _preflight()
|
||||
with Session(engine) as session:
|
||||
reserved = journal.reserve_repopulse_adoption(
|
||||
session, preflight, "repopulse-adoption-2026"
|
||||
)
|
||||
intent = journal.protocol.create_intent(preflight, "repopulse-adoption-2026")
|
||||
gates = _target_gates()
|
||||
with Session(engine) as session:
|
||||
journal.advance_repopulse_adoption(
|
||||
session, "target_gates", gates, expected_revision=reserved["revision"]
|
||||
)
|
||||
intent = journal.protocol.advance_intent(intent, "target_gates", gates)
|
||||
quiescence = _quiescence(intent)
|
||||
with Session(engine) as session:
|
||||
journal.confirm_repopulse_controller_intent(
|
||||
session, quiescence, expected_revision=2
|
||||
)
|
||||
intent = journal.protocol.advance_intent(intent, "quiescence", quiescence)
|
||||
snapshot = _snapshot(intent)
|
||||
with Session(engine) as session:
|
||||
journal.advance_repopulse_adoption(
|
||||
session, "snapshot", snapshot, expected_revision=3
|
||||
)
|
||||
intent = journal.protocol.advance_intent(intent, "snapshot", snapshot)
|
||||
registration = _registration(intent, enrollment_id=enrollment_id)
|
||||
with Session(engine) as session:
|
||||
registered = journal.advance_repopulse_adoption(
|
||||
session,
|
||||
"controller_registration",
|
||||
registration,
|
||||
expected_revision=4,
|
||||
)
|
||||
return intent, registration, registered["revision"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine():
|
||||
db = _ready_engine()
|
||||
yield db
|
||||
db.dispose()
|
||||
|
||||
|
||||
def test_ownership_commit_is_atomic_fixed_and_replayable(engine) -> None:
|
||||
intent, registration, revision = _register_target(engine)
|
||||
|
||||
with Session(engine) as session:
|
||||
result = ownership.commit_repopulse_adoption_ownership(
|
||||
session,
|
||||
registration,
|
||||
expected_revision=revision,
|
||||
)
|
||||
|
||||
assert result["phase"] == "appops_owned"
|
||||
assert result["revision"] == revision + 1
|
||||
with Session(engine) as session:
|
||||
row = session.get(Enrollment, registration["enrollmentId"])
|
||||
assert row is not None
|
||||
assert row.repository == "jens/repopulse"
|
||||
assert row.application_id == "repopulse"
|
||||
assert row.revision == "a" * 40
|
||||
assert row.profile == "repopulse-authenticated"
|
||||
assert row.controller_identity == "unraid-controller"
|
||||
assert row.plan_sha256 == registration["planSha256"]
|
||||
assert row.target_secret_ref == "ENROLL_REPOPULSE_TARGET_HMAC"
|
||||
assert row.oidc_secret_ref is None
|
||||
assert row.plan_json == _plan_payload("a" * 40)
|
||||
assert row.provenance_json["createdBy"] == "repopulse-legacy-adoption/v1"
|
||||
assert (
|
||||
row.provenance_json["controllerReceiptSha256"]
|
||||
== registration["receiptSha256"]
|
||||
)
|
||||
policies = session.exec(select(Policy).order_by(Policy.id)).all()
|
||||
assert [policy.id for policy in policies] == [
|
||||
"enrollment-repopulse-deployment-apply",
|
||||
"enrollment-repopulse-deployment-rollback",
|
||||
]
|
||||
assert {
|
||||
(item.action, item.target, item.voice, item.manual, item.risk)
|
||||
for item in policies
|
||||
} == {
|
||||
("deployment.apply", "repopulse", "blocked", "allowed", "high"),
|
||||
("deployment.rollback", "repopulse", "blocked", "allowed", "high"),
|
||||
}
|
||||
assert all(
|
||||
item.provenance_json["adoptionIntentId"] == intent["intentId"]
|
||||
for item in policies
|
||||
)
|
||||
stored_journal = session.get(RepoPulseAdoptionJournal, "repopulse")
|
||||
assert stored_journal.phase == "appops_owned"
|
||||
assert (
|
||||
stored_journal.intent_json["evidence"]["appopsOwnership"][
|
||||
"secretReferenceReady"
|
||||
]
|
||||
is True
|
||||
)
|
||||
|
||||
with Session(engine) as session:
|
||||
replay = ownership.commit_repopulse_adoption_ownership(
|
||||
session,
|
||||
registration,
|
||||
expected_revision=revision + 1,
|
||||
)
|
||||
assert replay["phase"] == "appops_owned"
|
||||
assert replay["revision"] == revision + 1
|
||||
|
||||
|
||||
def test_ownership_commit_rejects_live_controller_receipt_drift(engine) -> None:
|
||||
_, registration, revision = _register_target(engine)
|
||||
changed = {**registration, "deploymentReceiptSha256": _sha("5")}
|
||||
|
||||
with (
|
||||
Session(engine) as session,
|
||||
pytest.raises(ownership.RepoPulseOwnershipError) as error,
|
||||
):
|
||||
ownership.commit_repopulse_adoption_ownership(
|
||||
session, changed, expected_revision=revision
|
||||
)
|
||||
|
||||
assert error.value.code == "idempotency_conflict"
|
||||
with Session(engine) as session:
|
||||
assert session.exec(select(Enrollment)).all() == []
|
||||
assert session.exec(select(Policy)).all() == []
|
||||
|
||||
|
||||
def test_enrollment_identity_is_derived_from_persisted_intent(engine) -> None:
|
||||
_, registration, revision = _register_target(
|
||||
engine, enrollment_id="enr-0123456789abcdefabcd"
|
||||
)
|
||||
|
||||
with (
|
||||
Session(engine) as session,
|
||||
pytest.raises(ownership.RepoPulseOwnershipError) as error,
|
||||
):
|
||||
ownership.commit_repopulse_adoption_ownership(
|
||||
session, registration, expected_revision=revision
|
||||
)
|
||||
|
||||
assert error.value.code == "fixed_enrollment_id_mismatch"
|
||||
with Session(engine) as session:
|
||||
assert session.exec(select(Enrollment)).all() == []
|
||||
assert session.exec(select(Policy)).all() == []
|
||||
|
||||
|
||||
def test_preexisting_exact_policy_id_is_not_adopted(engine) -> None:
|
||||
_, registration, revision = _register_target(engine)
|
||||
with Session(engine) as session:
|
||||
session.add(
|
||||
Policy(
|
||||
id="enrollment-repopulse-deployment-apply",
|
||||
action="deployment.apply",
|
||||
target="repopulse",
|
||||
voice="blocked",
|
||||
manual="denied",
|
||||
risk="high",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
with (
|
||||
Session(engine) as session,
|
||||
pytest.raises(ownership.RepoPulseOwnershipError) as error,
|
||||
):
|
||||
ownership.commit_repopulse_adoption_ownership(
|
||||
session, registration, expected_revision=revision
|
||||
)
|
||||
assert error.value.code == "ownership_resource_conflict"
|
||||
with Session(engine) as session:
|
||||
assert session.exec(select(Enrollment)).all() == []
|
||||
assert (
|
||||
session.get(Policy, "enrollment-repopulse-deployment-apply").manual
|
||||
== "denied"
|
||||
)
|
||||
assert session.get(Policy, "enrollment-repopulse-deployment-rollback") is None
|
||||
assert (
|
||||
session.get(RepoPulseAdoptionJournal, "repopulse").phase
|
||||
== "target_registered"
|
||||
)
|
||||
|
||||
|
||||
def test_ownership_commit_rejects_missing_fixed_secret_ref(engine) -> None:
|
||||
_, registration, revision = _register_target(engine)
|
||||
with Session(engine) as session:
|
||||
from backend.app.db.models import Secret
|
||||
|
||||
session.delete(session.get(Secret, "ENROLL_REPOPULSE_TARGET_HMAC"))
|
||||
session.commit()
|
||||
|
||||
with (
|
||||
Session(engine) as session,
|
||||
pytest.raises(ownership.RepoPulseOwnershipError) as error,
|
||||
):
|
||||
ownership.commit_repopulse_adoption_ownership(
|
||||
session, registration, expected_revision=revision
|
||||
)
|
||||
assert error.value.code == "target_secret_reference_unavailable"
|
||||
with Session(engine) as session:
|
||||
assert session.exec(select(Enrollment)).all() == []
|
||||
assert session.exec(select(Policy)).all() == []
|
||||
|
||||
|
||||
def test_completion_is_a_separate_idempotent_step_bound_to_live_receipt(engine) -> None:
|
||||
_, registration, revision = _register_target(engine)
|
||||
with Session(engine) as session:
|
||||
owned = ownership.commit_repopulse_adoption_ownership(
|
||||
session, registration, expected_revision=revision
|
||||
)
|
||||
|
||||
with Session(engine) as session:
|
||||
completed = ownership.complete_repopulse_adoption(
|
||||
session,
|
||||
registration,
|
||||
expected_revision=owned["revision"],
|
||||
)
|
||||
assert completed["phase"] == "completed"
|
||||
assert completed["status"] == "completed"
|
||||
assert completed["revision"] == owned["revision"] + 1
|
||||
|
||||
with Session(engine) as session:
|
||||
replay = ownership.complete_repopulse_adoption(
|
||||
session,
|
||||
registration,
|
||||
expected_revision=completed["revision"],
|
||||
)
|
||||
assert replay["revision"] == completed["revision"]
|
||||
assert replay["phase"] == "completed"
|
||||
|
||||
with Session(engine) as session:
|
||||
ownership_replay = ownership.commit_repopulse_adoption_ownership(
|
||||
session, registration, expected_revision=completed["revision"]
|
||||
)
|
||||
assert ownership_replay["revision"] == completed["revision"]
|
||||
assert ownership_replay["phase"] == "completed"
|
||||
|
||||
changed = {**registration, "receiptSha256": _sha("9")}
|
||||
with (
|
||||
Session(engine) as session,
|
||||
pytest.raises(ownership.RepoPulseOwnershipError) as error,
|
||||
):
|
||||
ownership.complete_repopulse_adoption(
|
||||
session, changed, expected_revision=completed["revision"]
|
||||
)
|
||||
assert error.value.code == "idempotency_conflict"
|
||||
|
||||
|
||||
def test_completion_replay_rechecks_owned_policy_rows(engine) -> None:
|
||||
_, registration, revision = _register_target(engine)
|
||||
with Session(engine) as session:
|
||||
owned = ownership.commit_repopulse_adoption_ownership(
|
||||
session, registration, expected_revision=revision
|
||||
)
|
||||
with Session(engine) as session:
|
||||
completed = ownership.complete_repopulse_adoption(
|
||||
session, registration, expected_revision=owned["revision"]
|
||||
)
|
||||
with Session(engine) as session:
|
||||
policy = session.get(Policy, "enrollment-repopulse-deployment-apply")
|
||||
policy.manual = "denied"
|
||||
session.add(policy)
|
||||
session.commit()
|
||||
|
||||
with (
|
||||
Session(engine) as session,
|
||||
pytest.raises(ownership.RepoPulseOwnershipError) as error,
|
||||
):
|
||||
ownership.complete_repopulse_adoption(
|
||||
session, registration, expected_revision=completed["revision"]
|
||||
)
|
||||
assert error.value.code == "ownership_policy_readback_mismatch"
|
||||
|
||||
|
||||
def test_half_write_failure_rolls_back_all_ownership_rows_and_marks_recovery(
|
||||
engine, monkeypatch
|
||||
) -> None:
|
||||
_, registration, revision = _register_target(engine)
|
||||
original_commit = Session.commit
|
||||
calls = 0
|
||||
|
||||
def fail_first_commit(session):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise RuntimeError("simulated commit interruption")
|
||||
return original_commit(session)
|
||||
|
||||
monkeypatch.setattr(Session, "commit", fail_first_commit)
|
||||
with (
|
||||
Session(engine) as session,
|
||||
pytest.raises(ownership.RepoPulseOwnershipError) as error,
|
||||
):
|
||||
ownership.commit_repopulse_adoption_ownership(
|
||||
session, registration, expected_revision=revision
|
||||
)
|
||||
assert error.value.code == "appops_commit_uncertain"
|
||||
with Session(engine) as session:
|
||||
assert session.exec(select(Enrollment)).all() == []
|
||||
assert session.exec(select(Policy)).all() == []
|
||||
stored = session.get(RepoPulseAdoptionJournal, "repopulse")
|
||||
assert stored.phase == "target_registered"
|
||||
assert stored.status == "recovery_required"
|
||||
assert stored.intent_json["recovery"]["reason"] == "appops_commit_uncertain"
|
||||
|
||||
|
||||
def test_commit_exception_after_durable_commit_is_resolved_by_readback(
|
||||
engine, monkeypatch
|
||||
) -> None:
|
||||
_, registration, revision = _register_target(engine)
|
||||
original_commit = Session.commit
|
||||
calls = 0
|
||||
|
||||
def commit_then_lose_response(session):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
original_commit(session)
|
||||
raise RuntimeError("simulated lost commit response")
|
||||
return original_commit(session)
|
||||
|
||||
monkeypatch.setattr(Session, "commit", commit_then_lose_response)
|
||||
with Session(engine) as session:
|
||||
result = ownership.commit_repopulse_adoption_ownership(
|
||||
session, registration, expected_revision=revision
|
||||
)
|
||||
|
||||
assert result["phase"] == "appops_owned"
|
||||
with Session(engine) as session:
|
||||
assert session.get(Enrollment, registration["enrollmentId"]) is not None
|
||||
stored = session.get(RepoPulseAdoptionJournal, "repopulse")
|
||||
assert stored.phase == "appops_owned"
|
||||
assert stored.status == "in_progress"
|
||||
@@ -0,0 +1,368 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
|
||||
from backend.app.services.repopulse_adoption_preflight import (
|
||||
_EXPECTED_REGISTRY_KEYS,
|
||||
_EXPECTED_TARGET,
|
||||
_EXPECTED_VALIDATION,
|
||||
_PREFLIGHT_SCRIPT,
|
||||
RepoPulseAdoptionPreflight,
|
||||
evaluate_facts,
|
||||
)
|
||||
|
||||
|
||||
def _healthy_facts() -> dict:
|
||||
return {
|
||||
"registry": {
|
||||
"present": True,
|
||||
"alias_matches": True,
|
||||
"entry_type": "object",
|
||||
"keys": sorted(_EXPECTED_REGISTRY_KEYS),
|
||||
"target": {
|
||||
key: value
|
||||
for key, value in _EXPECTED_TARGET.items()
|
||||
if key != "environment_keys"
|
||||
},
|
||||
"environment_keys": [],
|
||||
"validation": deepcopy(_EXPECTED_VALIDATION),
|
||||
"repository_id": 64,
|
||||
"webhook_secret_file_matches": True,
|
||||
},
|
||||
"state": {
|
||||
"schema_version": 3,
|
||||
"repository": "jens/repopulse",
|
||||
"revision": "12431667760d2bce8c7533b57f462aac8d2e289e",
|
||||
"live_revision": "12431667760d2bce8c7533b57f462aac8d2e289e",
|
||||
"live_status": "healthy",
|
||||
"live_legacy_provenance": False,
|
||||
"status": "success",
|
||||
},
|
||||
"git": {
|
||||
"bare_repo_present": True,
|
||||
"main_revision": "12431667760d2bce8c7533b57f462aac8d2e289e",
|
||||
"live_revision": "12431667760d2bce8c7533b57f462aac8d2e289e",
|
||||
"live_revision_resolves": True,
|
||||
"live_is_ancestor": True,
|
||||
"compose_present_on_main": True,
|
||||
},
|
||||
"runtime": {
|
||||
"name_matches": True,
|
||||
"compose_project_matches": True,
|
||||
"container_id_sha256": "a" * 64,
|
||||
"image_sha256": "b" * 64,
|
||||
"state": "running",
|
||||
"health": "healthy",
|
||||
"image": "repopulse-repopulse:latest",
|
||||
"network_mode": "repopulse_default",
|
||||
"mounts": [
|
||||
{
|
||||
"destination": "/data",
|
||||
"type": "volume",
|
||||
"name": "repopulse_data",
|
||||
"read_write": True,
|
||||
},
|
||||
{
|
||||
"destination": "/scan",
|
||||
"type": "bind",
|
||||
"source": "/mnt/cache/appdata/gitea-deploy-runners/sources",
|
||||
"read_write": False,
|
||||
},
|
||||
],
|
||||
"read_only_root": False,
|
||||
"cap_drop_all": False,
|
||||
"no_new_privileges": True,
|
||||
"host_ports": [{"ip": "0.0.0.0", "port": 1254}],
|
||||
},
|
||||
"volume": {"name_matches": True, "mountpoint_matches": True},
|
||||
"scan_access": {
|
||||
"runtime_uid": 10001,
|
||||
"runtime_gid": 10001,
|
||||
"root_readable": True,
|
||||
"repopulse_source_readable": True,
|
||||
"privatemuse_source_readable": True,
|
||||
},
|
||||
"appdata": {
|
||||
"root_is_safe_directory": True,
|
||||
"deploy_env_is_regular_file": True,
|
||||
"managed_env_exists": False,
|
||||
"managed_env_is_regular_file": False,
|
||||
"managed_env_is_private": False,
|
||||
"target_data_exists": False,
|
||||
"target_data_is_symlink": False,
|
||||
"sources_exists": False,
|
||||
"sources_is_safe_directory": True,
|
||||
"unexpected_entry_count": 0,
|
||||
},
|
||||
"health": {"http_ok": True, "status": "ok"},
|
||||
"meta": {
|
||||
"http_ok": True,
|
||||
"product_matches": True,
|
||||
"repository_count": 71,
|
||||
"queue": {"queued": 0, "active": 0, "completed": 568, "failed": 0},
|
||||
"storage": {
|
||||
"database_bytes": 44_408_832,
|
||||
"journal_bytes": 0,
|
||||
"scan_count": 567,
|
||||
"compacted_scan_count": 94,
|
||||
"mirror_count": 69,
|
||||
"mirror_cache_bytes": 3_000_000_000,
|
||||
"worktree_count": 0,
|
||||
},
|
||||
"database_recovery_status": "healthy",
|
||||
"mutation_authentication": "perimeter_required",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _blocked_codes(report: dict) -> set[str]:
|
||||
return {check["code"] for check in report["checks"] if check["status"] != "passed"}
|
||||
|
||||
|
||||
def test_healthy_legacy_facts_pass_compatibility_but_not_target_acceptance() -> None:
|
||||
report = evaluate_facts(_healthy_facts())
|
||||
|
||||
assert report["status"] == "preflight_passed"
|
||||
assert report["legacyCompatibility"]["status"] == "passed"
|
||||
assert report["targetAcceptance"]["status"] == "blocked"
|
||||
assert report["repository"] == "jens/repopulse"
|
||||
assert report["readOnly"] is True
|
||||
assert report["adoptionAuthorized"] is False
|
||||
assert report["evidence"]["host_ports"] == [{"ip": "0.0.0.0", "port": 1254}]
|
||||
assert report["evidence"]["operator_token_configured"] is False
|
||||
assert report["evidence"]["queue"] == {
|
||||
"queued": 0,
|
||||
"active": 0,
|
||||
"completed": 568,
|
||||
"failed": 0,
|
||||
}
|
||||
|
||||
|
||||
def test_registry_extra_keys_and_webhook_reference_mismatch_block() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["registry"]["keys"].append("unreviewed_target_option")
|
||||
facts["registry"]["webhook_secret_file_matches"] = False
|
||||
|
||||
assert "legacy_registry" in _blocked_codes(evaluate_facts(facts))
|
||||
|
||||
|
||||
def test_malformed_registry_key_list_fails_closed() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["registry"]["keys"] = [[], "branch"]
|
||||
|
||||
assert "legacy_registry" in _blocked_codes(evaluate_facts(facts))
|
||||
|
||||
|
||||
def test_registry_json_types_must_match_the_fixed_contract() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["registry"]["target"]["exact_source"] = 1
|
||||
facts["registry"]["validation"]["validation_gate_version"] = True
|
||||
|
||||
assert "legacy_registry" in _blocked_codes(evaluate_facts(facts))
|
||||
|
||||
|
||||
def test_missing_registry_state_and_runtime_facts_fail_closed() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["registry"]["present"] = False
|
||||
facts["state"] = None
|
||||
facts["runtime"] = None
|
||||
|
||||
blocked = _blocked_codes(evaluate_facts(facts))
|
||||
assert {"legacy_registry", "controller_state", "runtime_identity_and_health"} <= blocked
|
||||
|
||||
|
||||
def test_wrong_live_sha_or_non_ancestor_blocks() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["git"]["live_is_ancestor"] = False
|
||||
|
||||
assert "gitea_ancestry" in _blocked_codes(evaluate_facts(facts))
|
||||
|
||||
|
||||
def test_changed_data_volume_or_writable_scan_mount_blocks() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["runtime"]["mounts"][0]["name"] = "other_volume"
|
||||
facts["runtime"]["mounts"][1]["read_write"] = True
|
||||
|
||||
assert "legacy_storage_topology" in _blocked_codes(evaluate_facts(facts))
|
||||
|
||||
|
||||
def test_unhealthy_or_exposed_container_blocks() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["runtime"]["health"] = "unhealthy"
|
||||
facts["runtime"]["host_ports"] = [{"ip": "0.0.0.0", "port": 1254}]
|
||||
|
||||
report = evaluate_facts(facts)
|
||||
assert report["status"] == "blocked"
|
||||
assert "runtime_identity_and_health" in _blocked_codes(report)
|
||||
assert "host_exposure" in _blocked_codes(report)
|
||||
|
||||
|
||||
def test_container_identity_requires_redacted_id_and_image_fingerprints() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["runtime"]["container_id_sha256"] = None
|
||||
facts["runtime"]["image_sha256"] = "not-a-sha256"
|
||||
|
||||
report = evaluate_facts(facts)
|
||||
|
||||
assert report["legacyCompatibility"]["status"] == "blocked"
|
||||
assert "runtime_identity_and_health" in _blocked_codes(report)
|
||||
assert report["evidence"]["container_id_sha256"] is None
|
||||
assert report["evidence"]["container_image_sha256"] is None
|
||||
|
||||
|
||||
def test_host_network_container_blocks_even_without_port_bindings() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["runtime"]["network_mode"] = "host"
|
||||
|
||||
report = evaluate_facts(facts)
|
||||
assert report["legacyCompatibility"]["status"] == "passed"
|
||||
assert "managed_network" in _blocked_codes(report)
|
||||
|
||||
|
||||
def test_data_target_collision_and_unsafe_legacy_runtime_block_target_acceptance_only() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["appdata"]["target_data_exists"] = True
|
||||
facts["runtime"]["read_only_root"] = False
|
||||
facts["runtime"]["cap_drop_all"] = False
|
||||
facts["runtime"]["no_new_privileges"] = False
|
||||
|
||||
report = evaluate_facts(facts)
|
||||
assert report["status"] == "preflight_passed"
|
||||
assert report["legacyCompatibility"]["status"] == "passed"
|
||||
blocked = _blocked_codes(report)
|
||||
assert {"managed_data_target", "runtime_hardening"} <= blocked
|
||||
|
||||
|
||||
def test_optional_sources_staging_directory_must_be_safe() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["appdata"]["sources_exists"] = True
|
||||
facts["appdata"]["sources_is_safe_directory"] = True
|
||||
assert evaluate_facts(facts)["legacyCompatibility"]["status"] == "passed"
|
||||
|
||||
facts["appdata"]["sources_is_safe_directory"] = False
|
||||
assert "legacy_appdata" in _blocked_codes(evaluate_facts(facts))
|
||||
|
||||
|
||||
def test_managed_env_may_be_staged_only_as_private_regular_file() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["appdata"].update(
|
||||
managed_env_exists=True,
|
||||
managed_env_is_regular_file=True,
|
||||
managed_env_is_private=True,
|
||||
managed_env_contents="SECRET_VALUE",
|
||||
)
|
||||
report = evaluate_facts(facts)
|
||||
assert report["legacyCompatibility"]["status"] == "passed"
|
||||
assert "managed_environment_file" not in _blocked_codes(report)
|
||||
assert "SECRET_VALUE" not in str(report)
|
||||
|
||||
facts["appdata"]["managed_env_is_private"] = False
|
||||
report = evaluate_facts(facts)
|
||||
assert "legacy_appdata" in _blocked_codes(report)
|
||||
assert "managed_environment_file" in _blocked_codes(report)
|
||||
assert report["evidence"]["managed_environment_file_ready"] is False
|
||||
|
||||
|
||||
def test_scan_access_is_a_managed_acceptance_gate_for_runtime_uid() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["scan_access"]["repopulse_source_readable"] = False
|
||||
|
||||
report = evaluate_facts(facts)
|
||||
|
||||
assert report["legacyCompatibility"]["status"] == "passed"
|
||||
assert "scan_access" in _blocked_codes(report)
|
||||
assert report["evidence"]["scan_access_uid_10001"] is False
|
||||
|
||||
|
||||
def test_missing_or_busy_queue_and_bad_database_recovery_block() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["meta"]["queue"]["active"] = 1
|
||||
facts["meta"]["database_recovery_status"] = "unknown"
|
||||
|
||||
blocked = _blocked_codes(evaluate_facts(facts))
|
||||
assert {"queue_quiescence", "database_health"} <= blocked
|
||||
|
||||
facts["meta"]["queue"].pop("failed")
|
||||
assert "queue_quiescence" in _blocked_codes(evaluate_facts(facts))
|
||||
|
||||
|
||||
def test_report_never_echoes_untrusted_or_secret_values() -> None:
|
||||
facts = _healthy_facts()
|
||||
facts["state"]["last_attempt"] = {"message": "TOP_SECRET_VALUE"}
|
||||
facts["meta"]["gitea"] = {"token": "TOP_SECRET_VALUE"}
|
||||
facts["runtime"]["operator_token"] = "TOP_SECRET_VALUE"
|
||||
|
||||
report_text = json.dumps(evaluate_facts(facts))
|
||||
assert "TOP_SECRET_VALUE" not in report_text
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, *, payload=None, text=""):
|
||||
self.payload = payload
|
||||
self.text = text
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class _FakePortainer:
|
||||
def __init__(self, facts):
|
||||
self.facts = facts
|
||||
self.calls = []
|
||||
|
||||
def get_container_status(self, name):
|
||||
assert name == "gitea-deploy-control"
|
||||
return {
|
||||
"status": "ok",
|
||||
"container": {"id": "controller-id", "name": name, "state": "running"},
|
||||
}
|
||||
|
||||
def _config(self):
|
||||
return SimpleNamespace(endpoint_id="3")
|
||||
|
||||
def _request(self, method, path, **kwargs):
|
||||
self.calls.append((method, path, kwargs))
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return _FakeResponse(payload={"Id": "exec-1"})
|
||||
if path.endswith("/exec/exec-1/start"):
|
||||
return _FakeResponse(text=json.dumps(self.facts))
|
||||
raise AssertionError(path)
|
||||
|
||||
|
||||
def test_live_service_uses_only_fixed_read_only_collector() -> None:
|
||||
fake = _FakePortainer(_healthy_facts())
|
||||
report = RepoPulseAdoptionPreflight(fake).run()
|
||||
|
||||
assert report["status"] == "preflight_passed"
|
||||
assert len(fake.calls) == 2
|
||||
create = fake.calls[0]
|
||||
assert create[0] == "POST"
|
||||
assert create[2]["json"]["Cmd"] == ["python3", "-c", _PREFLIGHT_SCRIPT]
|
||||
assert create[2]["json"]["AttachStdout"] is True
|
||||
assert create[2]["json"]["AttachStderr"] is False
|
||||
assert create[2]["json"]["Tty"] is True
|
||||
start = fake.calls[1]
|
||||
assert start[2]["json"] == {"Detach": False, "Tty": True}
|
||||
assert "docker rm" not in _PREFLIGHT_SCRIPT
|
||||
assert "docker start" not in _PREFLIGHT_SCRIPT
|
||||
assert "write_text(" not in _PREFLIGHT_SCRIPT
|
||||
|
||||
|
||||
def test_fixed_collector_source_is_valid_python() -> None:
|
||||
ast.parse(_PREFLIGHT_SCRIPT)
|
||||
|
||||
|
||||
def test_collector_failure_is_sanitized() -> None:
|
||||
class BrokenPortainer(_FakePortainer):
|
||||
def _request(self, method, path, **kwargs):
|
||||
if path.endswith("/containers/controller-id/exec"):
|
||||
return _FakeResponse(payload={"Id": "exec-1"})
|
||||
return _FakeResponse(text="not json with secret=TOP_SECRET_VALUE")
|
||||
|
||||
report = RepoPulseAdoptionPreflight(BrokenPortainer(_healthy_facts())).run()
|
||||
report_text = json.dumps(report)
|
||||
assert report["status"] == "unavailable"
|
||||
assert "TOP_SECRET_VALUE" not in report_text
|
||||
@@ -0,0 +1,631 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db.models import AppSetting
|
||||
from backend.app.schemas.enrollment import EnrollmentPlanV1
|
||||
from backend.app.services import repopulse_adoption_journal as journal
|
||||
from backend.app.services import repopulse_legacy_adoption_protocol as protocol
|
||||
from backend.app.services.secret_store import set_secret
|
||||
|
||||
|
||||
def _sha(char: str) -> str:
|
||||
return char * 64
|
||||
|
||||
|
||||
def _passing_preflight() -> dict:
|
||||
check_codes = (
|
||||
"legacy_registry",
|
||||
"controller_state",
|
||||
"gitea_ancestry",
|
||||
"runtime_identity_and_health",
|
||||
"legacy_storage_topology",
|
||||
"legacy_appdata",
|
||||
"product_api",
|
||||
"database_health",
|
||||
"queue_quiescence",
|
||||
"database_baseline",
|
||||
)
|
||||
return {
|
||||
"apiVersion": "itworx.repopulse-legacy-adoption-preflight/v1",
|
||||
"repository": "jens/repopulse",
|
||||
"status": "preflight_passed",
|
||||
"legacyCompatibility": {
|
||||
"status": "passed",
|
||||
"checks": [
|
||||
{"code": code, "status": "passed", "gate": "legacy_compatibility"}
|
||||
for code in check_codes
|
||||
],
|
||||
},
|
||||
"targetAcceptance": {"status": "blocked", "checks": []},
|
||||
"readOnly": True,
|
||||
"adoptionAuthorized": False,
|
||||
"evidence": {
|
||||
"main_revision": "a" * 40,
|
||||
"live_revision": "b" * 40,
|
||||
"container_id_sha256": _sha("d"),
|
||||
"container_image_sha256": _sha("1"),
|
||||
"container_state": "running",
|
||||
"container_health": "healthy",
|
||||
"queue": {"queued": 0, "active": 0, "completed": 569, "failed": 0},
|
||||
"database": {
|
||||
"database_bytes": 44_576_768,
|
||||
"journal_bytes": 0,
|
||||
"scan_count": 572,
|
||||
"compacted_scan_count": 96,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _plan_sha(revision: str) -> str:
|
||||
plan = EnrollmentPlanV1.model_validate(
|
||||
{
|
||||
"application_id": "repopulse",
|
||||
"display_name": "RepoPulse",
|
||||
"environment": "production",
|
||||
"repository": "Jens/RepoPulse",
|
||||
"revision": revision,
|
||||
"profile": "repopulse-authenticated",
|
||||
"target": {
|
||||
"target_identity": "unraid-controller",
|
||||
"container_name": "repopulse",
|
||||
},
|
||||
"storage": {"runtime_uid": 10001, "runtime_gid": 10001},
|
||||
"network": {"docker_network": "itworx-mcp-hub-npm", "internal_port": 8000},
|
||||
"artifact": {
|
||||
"release_version": "1.5.0",
|
||||
"image_repository": "repopulse-managed",
|
||||
"image_tag": "production",
|
||||
"health_path": "/health",
|
||||
},
|
||||
"exposure": {
|
||||
"hostname": "repopulse.example.com",
|
||||
"upstream_service": "repopulse",
|
||||
},
|
||||
"secrets": {"additional_secret_refs": ["REPOPULSE_GITEA_TOKEN"]},
|
||||
}
|
||||
).model_dump(mode="json")
|
||||
encoded = json.dumps(plan, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _document_sha(value: dict) -> str:
|
||||
encoded = (json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + "\n").encode("ascii")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _completion_sha(value: dict) -> str:
|
||||
encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("ascii")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def _controller_result(intent_id: str, binding: str) -> dict:
|
||||
plan_sha = _plan_sha("a" * 40)
|
||||
gates = {
|
||||
"profileId": "repopulse-authenticated",
|
||||
"profileVersion": "enrollment-profile/v1",
|
||||
"planSha256": plan_sha,
|
||||
**{
|
||||
key: True
|
||||
for key in (
|
||||
"profileReady",
|
||||
"controllerTrustReady",
|
||||
"controllerBootstrapSecretImported",
|
||||
"typedEnrollmentTrustReady",
|
||||
"targetSecretReady",
|
||||
"exactPolicyReady",
|
||||
"authenticatedRouteReady",
|
||||
"accessListReady",
|
||||
"mutationBoundaryReady",
|
||||
"managedRuntimeReady",
|
||||
"scanSourceMatches",
|
||||
"scanMountReadOnly",
|
||||
"scanPermissionPolicyPersistent",
|
||||
"scanAccessForRuntimeIdentityReady",
|
||||
"ownedDataTargetReady",
|
||||
"managedEnvironmentReady",
|
||||
"legacyEnvironmentRetained",
|
||||
"maintenanceAuthReady",
|
||||
"controllerAdoptionCapabilityReady",
|
||||
"controllerRegistryWriterReady",
|
||||
"controllerSecretWriterReady",
|
||||
)
|
||||
},
|
||||
}
|
||||
quiescence = {
|
||||
"intentId": intent_id,
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"maintenanceModeActive": True,
|
||||
"queueFrozen": True,
|
||||
"queuedJobs": 0,
|
||||
"activeJobs": 0,
|
||||
"maintenanceModeAuthenticated": True,
|
||||
"containerName": "repopulse",
|
||||
"containerIdSha256": _sha("d"),
|
||||
"containerImageSha256": _sha("1"),
|
||||
"containerState": "exited",
|
||||
"legacyVolume": "repopulse_data",
|
||||
"controllerIntentDurable": True,
|
||||
"controllerIntentBindingSha256": binding,
|
||||
}
|
||||
snapshot = {
|
||||
"intentId": intent_id,
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"legacyContainerStopped": True,
|
||||
"legacyVolume": "repopulse_data",
|
||||
"legacyVolumeRetained": True,
|
||||
"sqliteBackupMethod": "sqlite_online_backup",
|
||||
"quickCheck": "ok",
|
||||
"integrityCheck": "ok",
|
||||
"sourceLogicalSha256": _sha("e"),
|
||||
"snapshotLogicalSha256": _sha("e"),
|
||||
"snapshotFileSha256": _sha("f"),
|
||||
"sourceScanCount": 572,
|
||||
"snapshotScanCount": 572,
|
||||
"sourceCompactedScanCount": 96,
|
||||
"snapshotCompactedScanCount": 96,
|
||||
"sourceQueueSha256": _sha("1"),
|
||||
"snapshotQueueSha256": _sha("1"),
|
||||
"sourceAuxiliaryTreeSha256": _sha("3"),
|
||||
"sourceAuxiliaryFileCount": 42,
|
||||
"targetDataPathWasAbsent": True,
|
||||
"snapshotReceiptDurable": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
"controllerIntentBindingSha256": binding,
|
||||
}
|
||||
copied = {
|
||||
"intentId": intent_id,
|
||||
"snapshotFileSha256": _sha("f"),
|
||||
"targetLogicalSha256": _sha("e"),
|
||||
"targetScanCount": 572,
|
||||
"targetCompactedScanCount": 96,
|
||||
"targetQueueSha256": _sha("1"),
|
||||
"sourceAuxiliaryTreeSha256": _sha("3"),
|
||||
"targetAuxiliaryTreeSha256": _sha("3"),
|
||||
"sourceAuxiliaryFileCount": 42,
|
||||
"targetAuxiliaryFileCount": 42,
|
||||
"targetQuickCheck": "ok",
|
||||
"targetIntegrityCheck": "ok",
|
||||
"targetDataUid": 10001,
|
||||
"targetDataGid": 10001,
|
||||
"targetRecoveryStatus": "healthy",
|
||||
"managedDataMountMatches": True,
|
||||
"copySourceIsVerifiedSnapshot": True,
|
||||
"legacyVolumeRetained": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
}
|
||||
enrollment_id = f"enr-{intent_id[4:]}"
|
||||
registry_receipt = {
|
||||
"apiVersion": "itworx.repopulse-controller-adoption-store-receipt/v1",
|
||||
"intentId": intent_id,
|
||||
"repository": "jens/repopulse",
|
||||
"applicationId": "repopulse",
|
||||
"phase": "registry_committed",
|
||||
"intentBindingSha256": binding,
|
||||
"legacyVolumeRetained": True,
|
||||
"intentDurable": True,
|
||||
"managedEnrollmentId": enrollment_id,
|
||||
"projectsSha256": _sha("2"),
|
||||
"snapshotFileSha256": _sha("f"),
|
||||
}
|
||||
registry_receipt["receiptSha256"] = _document_sha(registry_receipt)
|
||||
deployment_receipt = {
|
||||
"repository": "jens/repopulse",
|
||||
"revision": "a" * 40,
|
||||
"containerRevision": "a" * 40,
|
||||
"containerIdSha256": _sha("4"),
|
||||
"deploymentReceiptSchema": "gitea-deploy-receipt.v1",
|
||||
"deploymentReceiptSha256": _sha("5"),
|
||||
"deploymentReceiptVerified": True,
|
||||
"runtimeHealthy": True,
|
||||
"publishedHostPorts": [],
|
||||
"managedDataMountMatches": True,
|
||||
"targetDataUid": 10001,
|
||||
"targetDataGid": 10001,
|
||||
"targetLogicalSha256": _sha("e"),
|
||||
"targetScanCount": 572,
|
||||
"targetCompactedScanCount": 96,
|
||||
"targetQueueSha256": _sha("1"),
|
||||
"targetAuxiliaryTreeSha256": _sha("3"),
|
||||
"targetAuxiliaryFileCount": 42,
|
||||
"targetQuickCheck": "ok",
|
||||
"targetIntegrityCheck": "ok",
|
||||
"targetRecoveryStatus": "healthy",
|
||||
"targetQueuedJobs": 0,
|
||||
"targetActiveJobs": 0,
|
||||
"legacyVolume": "repopulse_data",
|
||||
"legacyVolumeRetained": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
"scanSourceMatches": True,
|
||||
"scanMountReadOnly": True,
|
||||
"scanPermissionPolicyPersistent": True,
|
||||
"scanAccessForRuntimeIdentityReady": True,
|
||||
"containerName": "repopulse",
|
||||
"containerImageSha256": _sha("6"),
|
||||
"snapshotFileSha256": _sha("f"),
|
||||
}
|
||||
completion = {
|
||||
"apiVersion": "itworx.repopulse-controller-adoption-completion/v1",
|
||||
"intentId": intent_id,
|
||||
"intentBindingSha256": binding,
|
||||
"controllerRegistrationReceiptSha256": registry_receipt["receiptSha256"],
|
||||
"mainRevision": "a" * 40,
|
||||
"deploymentReceiptSha256": deployment_receipt["deploymentReceiptSha256"],
|
||||
"snapshotFileSha256": snapshot["snapshotFileSha256"],
|
||||
"snapshotLogicalSha256": snapshot["snapshotLogicalSha256"],
|
||||
"snapshotQueueSha256": snapshot["snapshotQueueSha256"],
|
||||
"auxiliaryTreeSha256": copied["targetAuxiliaryTreeSha256"],
|
||||
"containerIdSha256": deployment_receipt["containerIdSha256"],
|
||||
"containerImageSha256": deployment_receipt["containerImageSha256"],
|
||||
"legacyVolume": "repopulse_data",
|
||||
"legacyVolumeRetained": True,
|
||||
}
|
||||
completion["completionSha256"] = _completion_sha(completion)
|
||||
return {
|
||||
"apiVersion": "itworx.repopulse-controller-adoption-execution-result/v1",
|
||||
"repository": "jens/repopulse",
|
||||
"applicationId": "repopulse",
|
||||
"intentId": intent_id,
|
||||
"status": "deployment_verified",
|
||||
"phase": "completion_verified",
|
||||
"legacyVolume": "repopulse_data",
|
||||
"legacyVolumeRetained": True,
|
||||
"targetGates": gates,
|
||||
"exactShaValidation": {
|
||||
"repository": "jens/repopulse",
|
||||
"revision": "a" * 40,
|
||||
"contexts": {"Managed validation / full (push)": "success"},
|
||||
},
|
||||
"quiescence": quiescence,
|
||||
"snapshot": snapshot,
|
||||
"copy": copied,
|
||||
"registryReceipt": registry_receipt,
|
||||
"deploymentReceipt": deployment_receipt,
|
||||
"completion": completion,
|
||||
}
|
||||
|
||||
|
||||
class FakePreflight:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
def run(self) -> dict:
|
||||
self.calls += 1
|
||||
return _passing_preflight()
|
||||
|
||||
|
||||
class FakeBroker:
|
||||
base_url = "https://controller.test"
|
||||
verify_tls = True
|
||||
ca_file = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
reserve_result: dict | None = None,
|
||||
status_result: dict | None = None,
|
||||
) -> None:
|
||||
self.reserve_result = reserve_result
|
||||
self.status_result = status_result
|
||||
self.execute_calls: list[dict] = []
|
||||
self.status_calls = 0
|
||||
self.reserve_calls: list[dict] = []
|
||||
|
||||
def configuration_error(self):
|
||||
return None
|
||||
|
||||
def repopulse_adoption_capabilities(self, *, secret: str) -> dict:
|
||||
return {
|
||||
"apiVersion": "itworx.repopulse-adoption-capabilities/v1",
|
||||
"status": "ready",
|
||||
"repository": "jens/repopulse",
|
||||
"adoptionStore": {"executionEnabled": True},
|
||||
"privateTls": {"status": "ready"},
|
||||
"controller": {
|
||||
"aliases": "ready",
|
||||
"registry": "ready",
|
||||
"secretRoot": "ready",
|
||||
},
|
||||
}
|
||||
|
||||
def repopulse_adoption_reserve(self, **kwargs) -> dict:
|
||||
self.reserve_calls.append(kwargs)
|
||||
if self.reserve_result is not None:
|
||||
return self.reserve_result
|
||||
binding = hashlib.sha256(
|
||||
json.dumps(
|
||||
{
|
||||
"intentId": kwargs["intent_id"],
|
||||
"repository": "jens/repopulse",
|
||||
"applicationId": "repopulse",
|
||||
"preflightSha256": kwargs["preflight_sha256"],
|
||||
"idempotencySha256": kwargs["idempotency_sha256"],
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
).encode("ascii")
|
||||
).hexdigest()
|
||||
return {
|
||||
"apiVersion": "itworx.repopulse-adoption-reserve/v1",
|
||||
"status": "reserved",
|
||||
"intentId": kwargs["intent_id"],
|
||||
"phase": "reserved",
|
||||
"intentSha256": _sha("7"),
|
||||
"intentBindingSha256": binding,
|
||||
}
|
||||
|
||||
def repopulse_adoption_execute(self, **kwargs) -> dict:
|
||||
self.execute_calls.append(kwargs)
|
||||
raise AssertionError("AppOps must not invoke the controller cutover")
|
||||
|
||||
def repopulse_adoption_status(self, *, secret: str) -> dict:
|
||||
self.status_calls += 1
|
||||
return self.status_result or {
|
||||
"apiVersion": "itworx.repopulse-adoption-status/v1",
|
||||
"status": "not_reserved",
|
||||
"phase": None,
|
||||
"hashes": {},
|
||||
}
|
||||
|
||||
|
||||
def _runner_type():
|
||||
try:
|
||||
module = importlib.import_module("backend.app.services.repopulse_adoption_runner")
|
||||
except ModuleNotFoundError:
|
||||
pytest.fail("fixed RepoPulse AppOps runner is missing")
|
||||
return module.RepoPulseAdoptionRunner
|
||||
|
||||
|
||||
def _controller_status(intent: dict, controller_result: dict) -> dict:
|
||||
snapshot = controller_result["snapshot"]
|
||||
gates = controller_result["targetGates"]
|
||||
registry = controller_result["registryReceipt"]
|
||||
return {
|
||||
"apiVersion": "itworx.repopulse-adoption-status/v1",
|
||||
"status": "in_progress",
|
||||
"phase": "snapshot_verified",
|
||||
"hashes": {
|
||||
"intentSha256": _sha("7"),
|
||||
"intentBindingSha256": protocol.intent_binding_sha256(intent),
|
||||
"idempotencySha256": intent["idempotencySha256"],
|
||||
"preflightSha256": intent["preflightSha256"],
|
||||
"expectedProjectsSha256": registry["projectsSha256"],
|
||||
"quiescenceSha256": _sha("d"),
|
||||
"planSha256": gates["planSha256"],
|
||||
"snapshotFileSha256": snapshot["snapshotFileSha256"],
|
||||
"snapshotLogicalSha256": snapshot["snapshotLogicalSha256"],
|
||||
"snapshotQueueSha256": snapshot["snapshotQueueSha256"],
|
||||
"targetProjectsSha256": registry["projectsSha256"],
|
||||
"receiptSha256": registry["receiptSha256"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _session(monkeypatch) -> Session:
|
||||
monkeypatch.setattr(settings, "environment", "production")
|
||||
monkeypatch.setattr(settings, "secret_key", "appops-test-key-with-at-least-32-bytes")
|
||||
monkeypatch.setattr(settings, "secret_key_source", "env")
|
||||
engine = create_engine("sqlite://")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
session = Session(engine)
|
||||
session.add(
|
||||
AppSetting(
|
||||
key="deployment_enrollment:trust",
|
||||
is_secret=False,
|
||||
value_json={
|
||||
"controller_identity": "unraid-controller",
|
||||
"bootstrap_secret_ref": "APPOPS_DEPLOYMENT_BOOTSTRAP",
|
||||
"approved_networks": ["itworx-mcp-hub-npm"],
|
||||
"allowed_domain_suffixes": ["example.com"],
|
||||
"autonomous_profiles": ["repopulse-authenticated"],
|
||||
"repopulse_proxy_access_list_id": 1,
|
||||
},
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
set_secret(session, "APPOPS_DEPLOYMENT_BOOTSTRAP", "B" * 64)
|
||||
return session
|
||||
|
||||
|
||||
def test_synthetic_prepare_and_finalize_commits_protocol_ownership_and_completion(monkeypatch):
|
||||
runner_type = _runner_type()
|
||||
monkeypatch.setattr(protocol, "EXECUTION_ENABLED", True)
|
||||
session = _session(monkeypatch)
|
||||
preflight = FakePreflight()
|
||||
report = _passing_preflight()
|
||||
candidate = protocol.create_intent(report, "repopulse-adoption-2026")
|
||||
binding = protocol.intent_binding_sha256(candidate)
|
||||
controller_result = _controller_result(candidate["intentId"], binding)
|
||||
broker = FakeBroker()
|
||||
runner = runner_type(session=session, preflight=preflight, broker=broker)
|
||||
|
||||
handoff = runner.prepare_controller_cli_handoff("repopulse-adoption-2026")
|
||||
prepared = journal.get_repopulse_adoption_status(session)
|
||||
broker.status_result = _controller_status(candidate, controller_result)
|
||||
result = runner.finalize(controller_result)
|
||||
|
||||
assert prepared["status"] == "in_progress"
|
||||
assert prepared["phase"] == "reserved"
|
||||
assert handoff["intentId"] == candidate["intentId"]
|
||||
assert result["status"] == "completed"
|
||||
assert result["phase"] == "completed"
|
||||
assert result["revision"] == 7
|
||||
assert preflight.calls == 1
|
||||
assert len(broker.reserve_calls) == 1
|
||||
assert broker.reserve_calls[0]["intent_id"] == candidate["intentId"]
|
||||
assert broker.reserve_calls[0]["preflight_sha256"] == candidate["preflightSha256"]
|
||||
assert broker.execute_calls == []
|
||||
|
||||
|
||||
def test_prepared_and_completed_replays_never_repeat_external_mutations(monkeypatch):
|
||||
runner_type = _runner_type()
|
||||
monkeypatch.setattr(protocol, "EXECUTION_ENABLED", True)
|
||||
session = _session(monkeypatch)
|
||||
report = _passing_preflight()
|
||||
candidate = protocol.create_intent(report, "repopulse-adoption-2026")
|
||||
binding = protocol.intent_binding_sha256(candidate)
|
||||
controller_result = _controller_result(candidate["intentId"], binding)
|
||||
broker = FakeBroker()
|
||||
runner = runner_type(session=session, preflight=FakePreflight(), broker=broker)
|
||||
|
||||
broker.status_result = _controller_status(candidate, controller_result)
|
||||
handoff = runner.prepare_controller_cli_handoff("repopulse-adoption-2026")
|
||||
replay = runner.prepare("repopulse-adoption-2026")
|
||||
completed = runner.finalize(controller_result)
|
||||
completed_replay = runner.prepare("repopulse-adoption-2026")
|
||||
|
||||
assert handoff["intentId"] == candidate["intentId"]
|
||||
assert replay["status"] == "prepared"
|
||||
assert completed["status"] == completed_replay["status"] == "completed"
|
||||
assert completed_replay["revision"] == completed["revision"]
|
||||
assert len(broker.reserve_calls) == 1
|
||||
assert broker.execute_calls == []
|
||||
|
||||
|
||||
def test_controller_cli_handoff_is_exact_in_memory_contract_and_not_printed(monkeypatch, capsys):
|
||||
runner_type = _runner_type()
|
||||
monkeypatch.setattr(protocol, "EXECUTION_ENABLED", True)
|
||||
session = _session(monkeypatch)
|
||||
report = _passing_preflight()
|
||||
candidate = protocol.create_intent(report, "repopulse-adoption-2026")
|
||||
broker = FakeBroker()
|
||||
runner = runner_type(session=session, preflight=FakePreflight(), broker=broker)
|
||||
|
||||
handoff = runner.prepare_controller_cli_handoff("repopulse-adoption-2026")
|
||||
|
||||
assert set(handoff) == {
|
||||
"intentId",
|
||||
"idempotencySha256",
|
||||
"preflightSha256",
|
||||
"preflightProjection",
|
||||
"targetCredential",
|
||||
}
|
||||
assert handoff["intentId"] == candidate["intentId"]
|
||||
assert handoff["idempotencySha256"] == candidate["idempotencySha256"]
|
||||
assert handoff["preflightSha256"] == candidate["preflightSha256"]
|
||||
assert handoff["preflightProjection"] == candidate["evidence"]["preflight"]
|
||||
assert isinstance(handoff["targetCredential"], str)
|
||||
assert 32 <= len(handoff["targetCredential"]) <= 256
|
||||
assert capsys.readouterr().out == ""
|
||||
assert len(broker.reserve_calls) == 1
|
||||
assert broker.execute_calls == []
|
||||
|
||||
|
||||
def test_controller_cli_handoff_cannot_be_reissued(monkeypatch):
|
||||
runner_type = _runner_type()
|
||||
monkeypatch.setattr(protocol, "EXECUTION_ENABLED", True)
|
||||
session = _session(monkeypatch)
|
||||
candidate = protocol.create_intent(_passing_preflight(), "repopulse-adoption-2026")
|
||||
controller_result = _controller_result(
|
||||
candidate["intentId"], protocol.intent_binding_sha256(candidate)
|
||||
)
|
||||
broker = FakeBroker(status_result=_controller_status(candidate, controller_result))
|
||||
runner = runner_type(session=session, preflight=FakePreflight(), broker=broker)
|
||||
|
||||
first = runner.prepare_controller_cli_handoff("repopulse-adoption-2026")
|
||||
with pytest.raises(ValueError, match="controller_cli_handoff_replay_requires_recovery"):
|
||||
runner.prepare_controller_cli_handoff("repopulse-adoption-2026")
|
||||
|
||||
status = journal.get_repopulse_adoption_status(session)
|
||||
assert first["intentId"] == candidate["intentId"]
|
||||
assert status["status"] == "recovery_required"
|
||||
assert status["phase"] == "reserved"
|
||||
assert len(broker.reserve_calls) == 1
|
||||
assert broker.execute_calls == []
|
||||
|
||||
|
||||
def test_operator_runner_hydrates_portainer_settings_without_fastapi_lifespan(monkeypatch):
|
||||
runner_type = _runner_type()
|
||||
monkeypatch.setattr(protocol, "EXECUTION_ENABLED", True)
|
||||
session = _session(monkeypatch)
|
||||
calls = []
|
||||
runner_module = importlib.import_module(
|
||||
"backend.app.services.repopulse_adoption_runner"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
runner_module,
|
||||
"apply_stored_connector_settings",
|
||||
lambda active_session, connector_ids: calls.append(
|
||||
(active_session, connector_ids)
|
||||
),
|
||||
)
|
||||
|
||||
runner = runner_type(session=session, preflight=FakePreflight(), broker=FakeBroker())
|
||||
assert calls == []
|
||||
runner.prepare("repopulse-adoption-2026")
|
||||
|
||||
assert calls == [(session, {"portainer"})]
|
||||
|
||||
|
||||
def test_execution_flag_remains_fail_closed_in_source(monkeypatch):
|
||||
runner_type = _runner_type()
|
||||
monkeypatch.setattr(protocol, "EXECUTION_ENABLED", False)
|
||||
session = _session(monkeypatch)
|
||||
preflight = FakePreflight()
|
||||
broker = FakeBroker()
|
||||
runner = runner_type(session=session, preflight=preflight, broker=broker)
|
||||
|
||||
assert runner.prepare("repopulse-adoption-2026")["status"] == "disabled"
|
||||
assert preflight.calls == 0
|
||||
assert broker.reserve_calls == []
|
||||
assert broker.execute_calls == []
|
||||
|
||||
|
||||
def test_tampered_completion_is_rejected_and_marked_for_recovery(monkeypatch):
|
||||
runner_type = _runner_type()
|
||||
monkeypatch.setattr(protocol, "EXECUTION_ENABLED", True)
|
||||
session = _session(monkeypatch)
|
||||
candidate = protocol.create_intent(_passing_preflight(), "repopulse-adoption-2026")
|
||||
binding = protocol.intent_binding_sha256(candidate)
|
||||
controller_result = _controller_result(candidate["intentId"], binding)
|
||||
broker = FakeBroker()
|
||||
runner = runner_type(session=session, preflight=FakePreflight(), broker=broker)
|
||||
runner.prepare_controller_cli_handoff("repopulse-adoption-2026")
|
||||
broker.status_result = _controller_status(candidate, controller_result)
|
||||
controller_result["completion"]["completionSha256"] = _sha("0")
|
||||
|
||||
result = runner.finalize(controller_result)
|
||||
|
||||
assert result["status"] == "recovery_required"
|
||||
assert result["phase"] == "reserved"
|
||||
assert broker.status_calls == 1
|
||||
assert broker.execute_calls == []
|
||||
|
||||
|
||||
def test_ambiguous_controller_reserve_queries_status_and_requires_recovery(monkeypatch):
|
||||
runner_type = _runner_type()
|
||||
monkeypatch.setattr(protocol, "EXECUTION_ENABLED", True)
|
||||
session = _session(monkeypatch)
|
||||
broker = FakeBroker(
|
||||
reserve_result={
|
||||
"status": "unreachable",
|
||||
"reason": "secret-bearing raw transport error",
|
||||
}
|
||||
)
|
||||
runner = runner_type(session=session, preflight=FakePreflight(), broker=broker)
|
||||
|
||||
result = runner.prepare("repopulse-adoption-2026")
|
||||
|
||||
assert result["status"] == "recovery_required"
|
||||
assert result["reason"] == "unknown_external_effect"
|
||||
assert broker.status_calls == 1
|
||||
assert len(broker.reserve_calls) == 1
|
||||
assert broker.execute_calls == []
|
||||
assert "secret-bearing raw transport error" not in json.dumps(result)
|
||||
|
||||
replay = runner.prepare("repopulse-adoption-2026")
|
||||
assert replay["status"] == "recovery_required"
|
||||
assert len(broker.reserve_calls) == 1
|
||||
@@ -0,0 +1,345 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import http.client
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
BROKER_DIR = REPO_ROOT / "ops" / "gitea-deploy-broker"
|
||||
|
||||
|
||||
def _load_module(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage_effects():
|
||||
_load_module("repopulse_adoption_store", BROKER_DIR / "repopulse_adoption_store.py")
|
||||
return _load_module(
|
||||
"repopulse_adoption_storage_effects_test",
|
||||
BROKER_DIR / "repopulse_adoption_storage_effects.py",
|
||||
)
|
||||
|
||||
|
||||
def _run_git(*args: str, cwd: Path | None = None) -> str:
|
||||
result = subprocess.run(
|
||||
["git", *args],
|
||||
cwd=cwd,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def _bare_repo_with_helper(tmp_path: Path, helper: bytes) -> tuple[Path, str]:
|
||||
worktree = tmp_path / "worktree"
|
||||
bare_repo = tmp_path / "repositories" / "jens" / "repopulse.git"
|
||||
worktree.mkdir()
|
||||
bare_repo.parent.mkdir(parents=True)
|
||||
_run_git("init", "--bare", str(bare_repo))
|
||||
_run_git("init", "--initial-branch=main", str(worktree))
|
||||
_run_git("config", "user.name", "Storage effects test", cwd=worktree)
|
||||
_run_git("config", "user.email", "storage-effects@example.invalid", cwd=worktree)
|
||||
helper_path = worktree / "scripts" / "repopulse_db_migration.py"
|
||||
helper_path.parent.mkdir()
|
||||
helper_path.write_bytes(helper)
|
||||
_run_git("add", "scripts/repopulse_db_migration.py", cwd=worktree)
|
||||
_run_git("commit", "-m", "Add reviewed migration helper", cwd=worktree)
|
||||
_run_git("remote", "add", "origin", str(bare_repo), cwd=worktree)
|
||||
_run_git("push", "origin", "main", cwd=worktree)
|
||||
return bare_repo, _run_git("rev-parse", "HEAD", cwd=worktree)
|
||||
|
||||
|
||||
def test_maintenance_http_error_is_sanitized(storage_effects):
|
||||
with pytest.raises(storage_effects.StorageEffectsError) as error:
|
||||
storage_effects._maintenance_is_drained(
|
||||
{
|
||||
"before": {
|
||||
"status": 200,
|
||||
"body": {
|
||||
"enabled": False,
|
||||
"queue_depth": 0,
|
||||
"active_jobs": 0,
|
||||
},
|
||||
},
|
||||
"enter": {"status": [], "body": None},
|
||||
"status": {"status": 200, "body": {"enabled": True}},
|
||||
}
|
||||
)
|
||||
|
||||
assert error.value.code == "maintenance_response_invalid"
|
||||
|
||||
|
||||
def test_maintenance_queue_precheck_avoids_freezing_queued_work(
|
||||
storage_effects, monkeypatch
|
||||
):
|
||||
token = "test-maintenance-token-value-0123456789"
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
class Response:
|
||||
status = 200
|
||||
|
||||
def __init__(self, payload):
|
||||
self.payload = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def read(self, _limit):
|
||||
return self.payload
|
||||
|
||||
class Connection:
|
||||
def __init__(self, _host, *args, **kwargs):
|
||||
assert args == (8000,)
|
||||
assert kwargs["timeout"] == 5
|
||||
|
||||
def request(self, method, path, **_kwargs):
|
||||
calls.append((method, path))
|
||||
|
||||
def getresponse(self):
|
||||
if len(calls) == 1:
|
||||
return Response(
|
||||
{
|
||||
"enabled": False,
|
||||
"queue_depth": 2,
|
||||
"active_jobs": 0,
|
||||
"active_drained": True,
|
||||
"drained": False,
|
||||
}
|
||||
)
|
||||
return Response(
|
||||
{
|
||||
"enabled": True,
|
||||
"queue_depth": 2,
|
||||
"active_jobs": 0,
|
||||
"active_drained": True,
|
||||
"drained": False,
|
||||
}
|
||||
)
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(http.client, "HTTPConnection", Connection)
|
||||
monkeypatch.setattr(sys, "stdin", io.StringIO(token))
|
||||
output = io.StringIO()
|
||||
with redirect_stdout(output):
|
||||
exec( # noqa: S102 - exercise the exact fixed in-container exchange script
|
||||
storage_effects._REMOTE_MAINTENANCE_EXCHANGE, {"__name__": "__main__"}
|
||||
)
|
||||
|
||||
response = json.loads(output.getvalue())
|
||||
assert calls == [("GET", storage_effects.MAINTENANCE_STATUS_PATH)]
|
||||
assert response["enter"] is None
|
||||
assert token not in output.getvalue()
|
||||
|
||||
|
||||
def test_legacy_main_revision_mismatch_stops_before_loading_helper(
|
||||
storage_effects, tmp_path, monkeypatch
|
||||
):
|
||||
helper_bytes = b"# reviewed test helper\n"
|
||||
bare_repo, actual_revision = _bare_repo_with_helper(tmp_path, helper_bytes)
|
||||
source_root = tmp_path / "sources" / "repopulse"
|
||||
helper_source = source_root / "scripts" / "repopulse_db_migration.py"
|
||||
helper_source.parent.mkdir(parents=True)
|
||||
helper_source.write_bytes(helper_bytes)
|
||||
expected_revision = "0" * 40
|
||||
assert actual_revision != expected_revision
|
||||
|
||||
core = type("GitCore", (), {})()
|
||||
core.repository_path = lambda _repository, _project: bare_repo
|
||||
core.output = lambda command: subprocess.run(
|
||||
command,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
core.validate_commit = lambda repo, branch, revision: storage_effects._fixed_core(
|
||||
None
|
||||
).validate_commit(repo, branch, revision)
|
||||
|
||||
monkeypatch.setattr(storage_effects, "EXACT_SOURCE_ROOT", source_root.parent)
|
||||
monkeypatch.setattr(storage_effects._store, "_ROOT_UID", source_root.stat().st_uid)
|
||||
|
||||
def controller_directory(path: Path, *, root_owned: bool = False):
|
||||
storage_effects._store._assert_no_symlink_components(path)
|
||||
info = path.lstat()
|
||||
assert info.st_mode
|
||||
return info
|
||||
|
||||
monkeypatch.setattr(storage_effects, "_validate_directory", controller_directory)
|
||||
monkeypatch.setattr(
|
||||
storage_effects,
|
||||
"_validate_git_repository_directory",
|
||||
lambda path: path.lstat(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
storage_effects._store,
|
||||
"_safe_directory",
|
||||
lambda path, **_kwargs: (
|
||||
storage_effects._store._assert_no_symlink_components(path) or path.lstat()
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(storage_effects.StorageEffectsError) as error:
|
||||
storage_effects._load_reviewed_migration_helper(
|
||||
core,
|
||||
{"source_dir": str(source_root)},
|
||||
expected_revision,
|
||||
)
|
||||
|
||||
assert error.value.code == "main_revision_mismatch"
|
||||
|
||||
|
||||
def test_oversized_helper_is_rejected_before_git_returns_blob(
|
||||
storage_effects, tmp_path, monkeypatch
|
||||
):
|
||||
helper_bytes = b"#" * (storage_effects._HELPER_MAX_BYTES + 1)
|
||||
bare_repo, revision = _bare_repo_with_helper(tmp_path, helper_bytes)
|
||||
real_run = subprocess.run
|
||||
commands: list[list[str]] = []
|
||||
|
||||
def recording_run(args, *positional, **keyword):
|
||||
commands.append(list(args))
|
||||
return real_run(args, *positional, **keyword)
|
||||
|
||||
monkeypatch.setattr(storage_effects.subprocess, "run", recording_run)
|
||||
|
||||
with pytest.raises(storage_effects.StorageEffectsError) as error:
|
||||
storage_effects._read_git_blob(bare_repo, revision)
|
||||
|
||||
assert error.value.code == "reviewed_migration_helper_invalid"
|
||||
assert len(commands) == 1
|
||||
assert commands[0][2:4] == ["cat-file", "-s"]
|
||||
|
||||
|
||||
def test_legacy_maintenance_404_blocks_stop(storage_effects, tmp_path, monkeypatch):
|
||||
revision = "a" * 40
|
||||
container_id = "b" * 64
|
||||
image_id = "sha256:" + "c" * 64
|
||||
image_ref = "ghcr.io/jens/repopulse:legacy"
|
||||
token = "fixed-maintenance-token-value-0123456789"
|
||||
intent_id = "rpa-0123456789abcdef0123"
|
||||
binding = "d" * 64
|
||||
container_id_sha256 = hashlib.sha256(container_id.encode("ascii")).hexdigest()
|
||||
image_sha256 = hashlib.sha256(image_ref.encode("utf-8")).hexdigest()
|
||||
|
||||
class Store:
|
||||
def _lock_identity(self):
|
||||
return ("project-lock", "global-lock")
|
||||
|
||||
def _transaction_owner(self, _identity):
|
||||
return threading.get_ident()
|
||||
|
||||
def load_intent(self):
|
||||
return {
|
||||
"intentId": intent_id,
|
||||
"intentBindingSha256": binding,
|
||||
"status": "in_progress",
|
||||
"phase": "target_gates_ready",
|
||||
}
|
||||
|
||||
class Core:
|
||||
def __init__(self):
|
||||
self.commands: list[list[str]] = []
|
||||
|
||||
def inspect_container(self, _name):
|
||||
return {
|
||||
"Id": container_id,
|
||||
"Name": "/repopulse",
|
||||
"Image": image_id,
|
||||
"Config": {
|
||||
"Image": image_ref,
|
||||
"Env": [f"REPOPULSE_MAINTENANCE_TOKEN={token}"],
|
||||
"Labels": {
|
||||
"com.docker.compose.project": "repopulse",
|
||||
"org.opencontainers.image.revision": revision,
|
||||
},
|
||||
},
|
||||
"State": {
|
||||
"Status": "running",
|
||||
"Running": True,
|
||||
"Health": {"Status": "healthy"},
|
||||
},
|
||||
"Mounts": [
|
||||
{
|
||||
"Type": "volume",
|
||||
"Name": "repopulse_data",
|
||||
"Destination": "/data",
|
||||
"Source": str(storage_effects.LEGACY_VOLUME_MOUNTPOINT),
|
||||
"RW": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
def inspect_image(self, _image_id):
|
||||
return {
|
||||
"Id": image_id,
|
||||
"Config": {
|
||||
"Labels": {
|
||||
"org.opencontainers.image.revision": revision,
|
||||
"org.opencontainers.image.source": "jens/repopulse",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def run(self, command):
|
||||
self.commands.append(list(command))
|
||||
|
||||
core = Core()
|
||||
effects = storage_effects.FixedRepoPulseStorageEffects(
|
||||
store=Store(), deployment_core=core
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
effects,
|
||||
"_project_and_helper",
|
||||
lambda _intent, _revision: ({}, ModuleType("unused"), "e" * 64),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
storage_effects, "_private_environment_facts", lambda: (token, True, True)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
storage_effects,
|
||||
"_private_record_path",
|
||||
lambda _name, _intent: tmp_path / "unused-receipt.json",
|
||||
)
|
||||
monkeypatch.setattr(storage_effects, "_read_private_record", lambda *_args: None)
|
||||
monkeypatch.setattr(
|
||||
storage_effects,
|
||||
"_assert_legacy_volume_retained",
|
||||
lambda _core: storage_effects.LEGACY_VOLUME_MOUNTPOINT,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
storage_effects,
|
||||
"_maintenance_exchange",
|
||||
lambda *_args: {
|
||||
"before": {"status": 404, "body": None},
|
||||
"enter": None,
|
||||
"status": None,
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(storage_effects.StorageEffectsError) as error:
|
||||
effects.ensure_quiesced(
|
||||
intent_id=intent_id,
|
||||
intent_binding_sha256=binding,
|
||||
expected_main_revision=revision,
|
||||
expected_container_id_sha256=container_id_sha256,
|
||||
expected_container_image_sha256=image_sha256,
|
||||
)
|
||||
|
||||
assert error.value.code == "maintenance_endpoint_unavailable"
|
||||
assert core.commands == []
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,512 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services.enrollment_profiles import PROFILE_VERSION
|
||||
from backend.app.services.repopulse_legacy_adoption_protocol import (
|
||||
APPLICATION_ID,
|
||||
CONTROLLER_RECEIPT_VERSION,
|
||||
EXECUTION_ENABLED,
|
||||
LEGACY_VOLUME,
|
||||
REPOSITORY,
|
||||
AdoptionProtocolError,
|
||||
advance_intent,
|
||||
complete_intent,
|
||||
create_intent,
|
||||
execution_status,
|
||||
intent_binding_sha256,
|
||||
mark_recovery_required,
|
||||
reserve_or_replay,
|
||||
resume_intent,
|
||||
)
|
||||
|
||||
|
||||
def _sha(value: str) -> str:
|
||||
return value * 64
|
||||
|
||||
|
||||
def _passing_preflight() -> dict:
|
||||
checks = [
|
||||
{
|
||||
"code": code,
|
||||
"status": "passed",
|
||||
"gate": "legacy_compatibility",
|
||||
"detail": "sanitized fixed check",
|
||||
}
|
||||
for code in (
|
||||
"legacy_registry",
|
||||
"controller_state",
|
||||
"gitea_ancestry",
|
||||
"runtime_identity_and_health",
|
||||
"legacy_storage_topology",
|
||||
"legacy_appdata",
|
||||
"product_api",
|
||||
"database_health",
|
||||
"queue_quiescence",
|
||||
"database_baseline",
|
||||
)
|
||||
]
|
||||
return {
|
||||
"apiVersion": "itworx.repopulse-legacy-adoption-preflight/v1",
|
||||
"repository": REPOSITORY,
|
||||
"status": "preflight_passed",
|
||||
"legacyCompatibility": {"status": "passed", "checks": checks},
|
||||
"targetAcceptance": {"status": "blocked", "checks": []},
|
||||
"readOnly": True,
|
||||
"adoptionAuthorized": False,
|
||||
"evidence": {
|
||||
"main_revision": "a" * 40,
|
||||
"live_revision": "b" * 40,
|
||||
"container_id_sha256": _sha("d"),
|
||||
"container_image_sha256": _sha("1"),
|
||||
"container_state": "running",
|
||||
"container_health": "healthy",
|
||||
"queue": {"queued": 0, "active": 0, "completed": 569, "failed": 0},
|
||||
"database": {
|
||||
"database_bytes": 44_576_768,
|
||||
"journal_bytes": 0,
|
||||
"scan_count": 572,
|
||||
"compacted_scan_count": 96,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _intent() -> dict:
|
||||
return create_intent(_passing_preflight(), "repopulse-adoption-2026")
|
||||
|
||||
|
||||
def _target_gates() -> dict:
|
||||
return {
|
||||
"profileId": "repopulse-authenticated",
|
||||
"profileVersion": PROFILE_VERSION,
|
||||
"planSha256": _sha("c"),
|
||||
"profileReady": True,
|
||||
"controllerTrustReady": True,
|
||||
"controllerBootstrapSecretImported": True,
|
||||
"typedEnrollmentTrustReady": True,
|
||||
"targetSecretReady": True,
|
||||
"exactPolicyReady": True,
|
||||
"authenticatedRouteReady": True,
|
||||
"accessListReady": True,
|
||||
"mutationBoundaryReady": True,
|
||||
"managedRuntimeReady": True,
|
||||
"scanSourceMatches": True,
|
||||
"scanMountReadOnly": True,
|
||||
"scanPermissionPolicyPersistent": True,
|
||||
"scanAccessForRuntimeIdentityReady": True,
|
||||
"ownedDataTargetReady": True,
|
||||
"managedEnvironmentReady": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"maintenanceAuthReady": True,
|
||||
"controllerAdoptionCapabilityReady": True,
|
||||
"controllerRegistryWriterReady": True,
|
||||
"controllerSecretWriterReady": True,
|
||||
}
|
||||
|
||||
|
||||
def _quiescence(intent: dict) -> dict:
|
||||
return {
|
||||
"intentId": intent["intentId"],
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"maintenanceModeActive": True,
|
||||
"queueFrozen": True,
|
||||
"queuedJobs": 0,
|
||||
"activeJobs": 0,
|
||||
"maintenanceModeAuthenticated": True,
|
||||
"containerName": "repopulse",
|
||||
"containerIdSha256": _sha("d"),
|
||||
"containerImageSha256": _sha("1"),
|
||||
"containerState": "exited",
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"controllerIntentDurable": True,
|
||||
"controllerIntentBindingSha256": intent_binding_sha256(intent),
|
||||
}
|
||||
|
||||
|
||||
def _snapshot(intent: dict) -> dict:
|
||||
return {
|
||||
"intentId": intent["intentId"],
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"legacyContainerStopped": True,
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyVolumeRetained": True,
|
||||
"sqliteBackupMethod": "sqlite_online_backup",
|
||||
"quickCheck": "ok",
|
||||
"integrityCheck": "ok",
|
||||
"sourceLogicalSha256": _sha("e"),
|
||||
"snapshotLogicalSha256": _sha("e"),
|
||||
"snapshotFileSha256": _sha("f"),
|
||||
"sourceScanCount": 572,
|
||||
"snapshotScanCount": 572,
|
||||
"sourceCompactedScanCount": 96,
|
||||
"snapshotCompactedScanCount": 96,
|
||||
"sourceQueueSha256": _sha("1"),
|
||||
"snapshotQueueSha256": _sha("1"),
|
||||
"sourceAuxiliaryTreeSha256": _sha("3"),
|
||||
"sourceAuxiliaryFileCount": 42,
|
||||
"targetDataPathWasAbsent": True,
|
||||
"snapshotReceiptDurable": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
"controllerIntentBindingSha256": intent_binding_sha256(intent),
|
||||
}
|
||||
|
||||
|
||||
def _registration(intent: dict) -> dict:
|
||||
gates = intent["evidence"]["targetGates"]
|
||||
snapshot = intent["evidence"]["snapshot"]
|
||||
return {
|
||||
"receiptVersion": CONTROLLER_RECEIPT_VERSION,
|
||||
"intentId": intent["intentId"],
|
||||
"receiptSha256": _sha("2"),
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"enrollmentId": "enr-0123456789abcdefabcd",
|
||||
"profileId": gates["profileId"],
|
||||
"planSha256": gates["planSha256"],
|
||||
"snapshotFileSha256": snapshot["snapshotFileSha256"],
|
||||
"targetLogicalSha256": snapshot["snapshotLogicalSha256"],
|
||||
"targetScanCount": snapshot["snapshotScanCount"],
|
||||
"targetCompactedScanCount": snapshot["snapshotCompactedScanCount"],
|
||||
"targetQueueSha256": snapshot["snapshotQueueSha256"],
|
||||
"targetAuxiliaryTreeSha256": snapshot["sourceAuxiliaryTreeSha256"],
|
||||
"targetAuxiliaryFileCount": snapshot["sourceAuxiliaryFileCount"],
|
||||
"targetQueuedJobs": 0,
|
||||
"targetActiveJobs": 0,
|
||||
"targetQuickCheck": "ok",
|
||||
"targetIntegrityCheck": "ok",
|
||||
"targetRecoveryStatus": "healthy",
|
||||
"targetDataUid": 10001,
|
||||
"targetDataGid": 10001,
|
||||
"managedDataMountMatches": True,
|
||||
"copySourceIsVerifiedSnapshot": True,
|
||||
"registryTargetMatches": True,
|
||||
"controllerStateReceiptVerified": True,
|
||||
"deploymentReceiptSchema": "gitea-deploy-receipt.v1",
|
||||
"deploymentReceiptSha256": _sha("4"),
|
||||
"deploymentReceiptVerified": True,
|
||||
"runtimeHealthy": True,
|
||||
"publishedHostPorts": [],
|
||||
"globalDeployLockHeld": True,
|
||||
"projectLockHeld": True,
|
||||
"legacyVolume": LEGACY_VOLUME,
|
||||
"legacyVolumeRetained": True,
|
||||
"legacyEnvironmentRetained": True,
|
||||
"managedEnvironmentFilePrivate": True,
|
||||
"scanSourceMatches": True,
|
||||
"scanMountReadOnly": True,
|
||||
"scanPermissionPolicyPersistent": True,
|
||||
"scanAccessForRuntimeIdentityReady": True,
|
||||
"controllerIntentDurable": True,
|
||||
"controllerIntentBindingSha256": intent_binding_sha256(intent),
|
||||
}
|
||||
|
||||
|
||||
def _to_snapshot_verified() -> dict:
|
||||
intent = advance_intent(_intent(), "target_gates", _target_gates())
|
||||
intent = advance_intent(intent, "quiescence", _quiescence(intent))
|
||||
return advance_intent(intent, "snapshot", _snapshot(intent))
|
||||
|
||||
|
||||
def _to_target_registered() -> dict:
|
||||
intent = _to_snapshot_verified()
|
||||
return advance_intent(intent, "controller_registration", _registration(intent))
|
||||
|
||||
|
||||
def test_passing_legacy_preflight_can_reserve_but_never_authorizes_adoption() -> None:
|
||||
intent = _intent()
|
||||
|
||||
assert intent["repository"] == REPOSITORY
|
||||
assert intent["applicationId"] == APPLICATION_ID
|
||||
assert intent["legacyVolume"] == LEGACY_VOLUME
|
||||
assert intent["phase"] == "reserved"
|
||||
assert intent["status"] == "in_progress"
|
||||
assert "adoptionAuthorized" not in intent
|
||||
assert intent["evidence"]["preflight"]["queue"]["queued"] == 0
|
||||
|
||||
|
||||
def test_reservation_is_idempotent_and_binds_the_preflight_facts() -> None:
|
||||
first = create_intent(_passing_preflight(), "repopulse-adoption-2026")
|
||||
replay = reserve_or_replay(first, _passing_preflight(), "repopulse-adoption-2026")
|
||||
changed = _passing_preflight()
|
||||
changed["evidence"]["main_revision"] = "c" * 40
|
||||
|
||||
assert replay == first
|
||||
with pytest.raises(AdoptionProtocolError, match="idempotency_conflict"):
|
||||
reserve_or_replay(first, changed, "repopulse-adoption-2026")
|
||||
with pytest.raises(AdoptionProtocolError, match="one_time_adoption_already_reserved"):
|
||||
reserve_or_replay(first, _passing_preflight(), "repopulse-adoption-2027")
|
||||
assert intent_binding_sha256(first) == intent_binding_sha256(replay)
|
||||
|
||||
|
||||
def test_modified_persisted_intent_evidence_fails_integrity_check() -> None:
|
||||
intent = _intent()
|
||||
intent["evidence"]["preflight"]["repository"] = "jens/other"
|
||||
|
||||
with pytest.raises(AdoptionProtocolError, match="intent_evidence_invalid"):
|
||||
advance_intent(intent, "target_gates", _target_gates())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutate,code",
|
||||
[
|
||||
(lambda report: report.update(repository="jens/other"), "legacy_preflight_not_compatible"),
|
||||
(lambda report: report.update(adoptionAuthorized=True), "legacy_preflight_not_compatible"),
|
||||
(lambda report: report.update(status="blocked"), "legacy_preflight_not_compatible"),
|
||||
(lambda report: report["evidence"]["queue"].update(active=1), "legacy_preflight_queue_not_quiescent"),
|
||||
(lambda report: report["legacyCompatibility"]["checks"].pop(), "legacy_preflight_check_incomplete"),
|
||||
],
|
||||
)
|
||||
def test_invalid_or_tampered_preflight_cannot_reserve(mutate, code: str) -> None:
|
||||
report = _passing_preflight()
|
||||
mutate(report)
|
||||
|
||||
with pytest.raises(AdoptionProtocolError) as error:
|
||||
create_intent(report, "repopulse-adoption-2026")
|
||||
assert error.value.code == code
|
||||
|
||||
|
||||
def test_changed_or_unready_profile_trust_secret_acl_or_runtime_gate_blocks() -> None:
|
||||
intent = _intent()
|
||||
gates = _target_gates()
|
||||
gates["accessListReady"] = False
|
||||
|
||||
with pytest.raises(AdoptionProtocolError, match="target_gates_not_ready"):
|
||||
advance_intent(intent, "target_gates", gates)
|
||||
with pytest.raises(AdoptionProtocolError, match="adoption_step_out_of_order"):
|
||||
advance_intent(intent, "quiescence", _quiescence(intent))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value"),
|
||||
[("profileId", "internal-webapp"), ("profileVersion", "unreviewed/v9")],
|
||||
)
|
||||
def test_target_gates_require_the_exact_managed_profile(field: str, value: str) -> None:
|
||||
gates = _target_gates()
|
||||
gates[field] = value
|
||||
|
||||
with pytest.raises(AdoptionProtocolError, match="target_gates_not_ready"):
|
||||
advance_intent(_intent(), "target_gates", gates)
|
||||
|
||||
|
||||
def test_quiescence_requires_locks_maintenance_drain_stop_and_the_named_volume() -> None:
|
||||
intent = advance_intent(_intent(), "target_gates", _target_gates())
|
||||
evidence = _quiescence(intent)
|
||||
evidence["globalDeployLockHeld"] = False
|
||||
|
||||
with pytest.raises(AdoptionProtocolError, match="quiescence_not_proven"):
|
||||
advance_intent(intent, "quiescence", evidence)
|
||||
|
||||
evidence = _quiescence(intent)
|
||||
evidence["activeJobs"] = 1
|
||||
with pytest.raises(AdoptionProtocolError, match="quiescence_not_proven"):
|
||||
advance_intent(intent, "quiescence", evidence)
|
||||
|
||||
evidence = _quiescence(intent)
|
||||
evidence["controllerIntentBindingSha256"] = _sha("9")
|
||||
with pytest.raises(AdoptionProtocolError, match="controller_intent_binding_mismatch"):
|
||||
advance_intent(intent, "quiescence", evidence)
|
||||
|
||||
evidence = _quiescence(intent)
|
||||
evidence["containerIdSha256"] = _sha("9")
|
||||
with pytest.raises(AdoptionProtocolError, match="quiescence_not_proven"):
|
||||
advance_intent(intent, "quiescence", evidence)
|
||||
|
||||
|
||||
def test_quiesced_snapshot_requires_sqlite_online_backup_and_matching_logical_data() -> None:
|
||||
intent = advance_intent(_intent(), "target_gates", _target_gates())
|
||||
intent = advance_intent(intent, "quiescence", _quiescence(intent))
|
||||
evidence = _snapshot(intent)
|
||||
evidence["sqliteBackupMethod"] = "raw_file_copy"
|
||||
|
||||
with pytest.raises(AdoptionProtocolError, match="sqlite_snapshot_not_verified"):
|
||||
advance_intent(intent, "snapshot", evidence)
|
||||
|
||||
evidence = _snapshot(intent)
|
||||
evidence["sourceLogicalSha256"] = _sha("a")
|
||||
with pytest.raises(AdoptionProtocolError, match="sqlite_snapshot_not_verified"):
|
||||
advance_intent(intent, "snapshot", evidence)
|
||||
|
||||
|
||||
def test_controller_receipt_must_bind_exact_target_keep_legacy_volume_and_have_no_ports() -> None:
|
||||
intent = _to_snapshot_verified()
|
||||
evidence = _registration(intent)
|
||||
evidence["publishedHostPorts"] = [{"ip": "0.0.0.0", "port": 1254}]
|
||||
|
||||
with pytest.raises(AdoptionProtocolError, match="controller_adoption_receipt_not_verified"):
|
||||
advance_intent(intent, "controller_registration", evidence)
|
||||
|
||||
evidence = _registration(intent)
|
||||
evidence["legacyVolumeRetained"] = False
|
||||
with pytest.raises(AdoptionProtocolError, match="controller_adoption_receipt_not_verified"):
|
||||
advance_intent(intent, "controller_registration", evidence)
|
||||
|
||||
evidence = _registration(intent)
|
||||
evidence["targetDataUid"] = 0
|
||||
with pytest.raises(AdoptionProtocolError, match="controller_adoption_receipt_not_verified"):
|
||||
advance_intent(intent, "controller_registration", evidence)
|
||||
|
||||
evidence = _registration(intent)
|
||||
evidence["targetActiveJobs"] = 1
|
||||
with pytest.raises(AdoptionProtocolError, match="controller_adoption_receipt_not_verified"):
|
||||
advance_intent(intent, "controller_registration", evidence)
|
||||
|
||||
for field, invalid in (
|
||||
("targetAuxiliaryTreeSha256", _sha("5")),
|
||||
("targetAuxiliaryFileCount", 41),
|
||||
("deploymentReceiptSchema", "unknown"),
|
||||
("deploymentReceiptSha256", "missing"),
|
||||
("deploymentReceiptVerified", False),
|
||||
):
|
||||
evidence = _registration(intent)
|
||||
evidence[field] = invalid
|
||||
with pytest.raises(AdoptionProtocolError, match="controller_adoption_receipt_not_verified"):
|
||||
advance_intent(intent, "controller_registration", evidence)
|
||||
|
||||
|
||||
def test_exact_controller_and_appops_ownership_are_separate_checkpoints() -> None:
|
||||
intent = _to_target_registered()
|
||||
registration = intent["evidence"]["controllerRegistration"]
|
||||
gates = intent["evidence"]["targetGates"]
|
||||
evidence = {
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"enrollmentId": registration["enrollmentId"],
|
||||
"profileId": registration["profileId"],
|
||||
"planSha256": gates["planSha256"],
|
||||
"controllerReceiptSha256": registration["receiptSha256"],
|
||||
"enrollmentPersisted": True,
|
||||
"exactApplyPolicyPersisted": True,
|
||||
"exactRollbackPolicyPersisted": True,
|
||||
"secretReferenceReady": True,
|
||||
}
|
||||
appops_owned = advance_intent(intent, "appops_ownership", evidence)
|
||||
complete = complete_intent(
|
||||
appops_owned,
|
||||
{
|
||||
"intentId": intent["intentId"],
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"enrollmentId": registration["enrollmentId"],
|
||||
"controllerReceiptSha256": registration["receiptSha256"],
|
||||
"enrollmentStillPersisted": True,
|
||||
"exactApplyPolicyStillPersisted": True,
|
||||
"exactRollbackPolicyStillPersisted": True,
|
||||
"controllerReceiptStillMatches": True,
|
||||
"legacyVolumeRetained": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert complete["phase"] == "completed"
|
||||
assert complete["status"] == "completed"
|
||||
assert "appopsOwnership" in complete["evidence"]
|
||||
assert "completion" in complete["evidence"]
|
||||
|
||||
|
||||
def test_step_replay_is_idempotent_but_changed_evidence_is_a_conflict() -> None:
|
||||
intent = advance_intent(_intent(), "target_gates", _target_gates())
|
||||
replay = advance_intent(intent, "target_gates", _target_gates())
|
||||
changed = _target_gates()
|
||||
changed["planSha256"] = _sha("9")
|
||||
|
||||
assert replay == intent
|
||||
with pytest.raises(AdoptionProtocolError, match="idempotency_conflict"):
|
||||
advance_intent(intent, "target_gates", changed)
|
||||
|
||||
|
||||
def test_recovery_marker_keeps_last_good_phase_and_never_deletes_legacy_data() -> None:
|
||||
intent = _to_snapshot_verified()
|
||||
failed = mark_recovery_required(
|
||||
intent,
|
||||
"registry_receipt_uncertain",
|
||||
{"rawError": "SECRET_OR_PATH_MUST_NOT_BE_STORED"},
|
||||
)
|
||||
|
||||
assert failed["status"] == "recovery_required"
|
||||
assert failed["phase"] == "snapshot_verified"
|
||||
assert failed["legacyVolumePreserved"] is True
|
||||
assert "legacyVolumePreserved" not in failed["recovery"] or failed["recovery"]["legacyVolumePreserved"] is True
|
||||
assert "SECRET_OR_PATH_MUST_NOT_BE_STORED" not in str(failed)
|
||||
with pytest.raises(AdoptionProtocolError, match="intent_requires_recovery"):
|
||||
advance_intent(failed, "controller_registration", _registration(intent))
|
||||
|
||||
|
||||
def test_recovery_reconciles_only_exact_controller_committed_registration() -> None:
|
||||
intent = _to_snapshot_verified()
|
||||
failed = mark_recovery_required(intent, "appops_commit_uncertain", {"timeout": True})
|
||||
registration = _registration(intent)
|
||||
receipt = {
|
||||
"apiVersion": CONTROLLER_RECEIPT_VERSION,
|
||||
"intentId": intent["intentId"],
|
||||
"phase": "target_registered",
|
||||
"receiptSha256": registration["receiptSha256"],
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"legacyVolumeRetained": True,
|
||||
"intentDurable": True,
|
||||
"intentBindingSha256": intent_binding_sha256(intent),
|
||||
"registrationEvidence": registration,
|
||||
}
|
||||
|
||||
resumed = resume_intent(failed, receipt)
|
||||
|
||||
assert resumed["status"] == "in_progress"
|
||||
assert resumed["phase"] == "target_registered"
|
||||
assert resumed["evidence"]["controllerRegistration"]["receiptSha256"] == registration["receiptSha256"]
|
||||
|
||||
|
||||
def test_recovery_replays_registered_target_without_repeating_cutover() -> None:
|
||||
intent = _to_target_registered()
|
||||
failed = mark_recovery_required(intent, "appops_commit_uncertain", {"commit": "timeout"})
|
||||
registration = intent["evidence"]["controllerRegistration"]
|
||||
receipt = {
|
||||
"apiVersion": CONTROLLER_RECEIPT_VERSION,
|
||||
"intentId": intent["intentId"],
|
||||
"phase": "target_registered",
|
||||
"receiptSha256": registration["receiptSha256"],
|
||||
"repository": REPOSITORY,
|
||||
"applicationId": APPLICATION_ID,
|
||||
"legacyVolumeRetained": True,
|
||||
"intentDurable": True,
|
||||
"intentBindingSha256": intent_binding_sha256(intent),
|
||||
"registrationEvidence": registration,
|
||||
}
|
||||
|
||||
resumed = resume_intent(failed, receipt)
|
||||
|
||||
assert resumed["phase"] == "target_registered"
|
||||
assert resumed["status"] == "in_progress"
|
||||
assert resumed["receipts"]["controller_registration"] == intent["receipts"]["controller_registration"]
|
||||
|
||||
|
||||
def test_recovery_receipt_for_another_repository_or_intent_is_rejected() -> None:
|
||||
intent = mark_recovery_required(_to_snapshot_verified(), "registry_receipt_uncertain", {})
|
||||
receipt = {
|
||||
"apiVersion": CONTROLLER_RECEIPT_VERSION,
|
||||
"intentId": "rpa-00000000000000000000",
|
||||
"phase": "target_registered",
|
||||
"receiptSha256": _sha("2"),
|
||||
"repository": "jens/other",
|
||||
"applicationId": APPLICATION_ID,
|
||||
"legacyVolumeRetained": True,
|
||||
"intentDurable": True,
|
||||
"intentBindingSha256": intent_binding_sha256(intent),
|
||||
"registrationEvidence": {},
|
||||
}
|
||||
|
||||
with pytest.raises(AdoptionProtocolError, match="controller_recovery_receipt_unbound"):
|
||||
resume_intent(intent, receipt)
|
||||
|
||||
|
||||
def test_protocol_activation_reports_enabled_without_claiming_readiness() -> None:
|
||||
assert EXECUTION_ENABLED is True
|
||||
assert execution_status() == {
|
||||
"apiVersion": "itworx.repopulse-legacy-adoption/v1",
|
||||
"repository": REPOSITORY,
|
||||
"status": "enabled",
|
||||
"reason": None,
|
||||
"executionEnabled": True,
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Verify that the legacy maintenance stager preserves existing environment bytes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "ops"
|
||||
/ "gitea-deploy-broker"
|
||||
/ "stage_repopulse_maintenance_environment.py"
|
||||
)
|
||||
|
||||
|
||||
def _load_stage():
|
||||
sys.path.insert(0, str(SCRIPT.parent))
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location("repopulse_maintenance_stage_test", SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
finally:
|
||||
sys.path.pop(0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "posix" or os.geteuid() != 0, reason="root-owned Linux files required")
|
||||
def test_mixed_legacy_line_endings_are_backed_up_byte_for_byte_and_staged_idempotently(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
stage = _load_stage()
|
||||
legacy_env = tmp_path / "deploy.env"
|
||||
managed_env = tmp_path / "deploy.managed.env"
|
||||
backup_root = tmp_path / "environment-backup"
|
||||
original = (
|
||||
b"REPOPULSE_GITEA_URL=http://198.18.10.150:3000\n"
|
||||
b"REPOPULSE_GITEA_TOKEN=validation-only-token-1234567890\n"
|
||||
b"REPOPULSE_MAX_CONCURRENT_SCANS=1\r\n"
|
||||
)
|
||||
legacy_env.write_bytes(original)
|
||||
legacy_env.chmod(0o600)
|
||||
monkeypatch.setattr(stage, "LEGACY_ENV", legacy_env)
|
||||
monkeypatch.setattr(stage, "MANAGED_ENV", managed_env)
|
||||
monkeypatch.setattr(stage, "BACKUP_ROOT", backup_root)
|
||||
monkeypatch.setattr(stage, "BACKUP_FILE", backup_root / "legacy-deploy.env")
|
||||
monkeypatch.setattr(stage, "BACKUP_SHA_FILE", backup_root / "legacy-deploy.env.sha256")
|
||||
|
||||
first = stage.prepare()
|
||||
assert first == {"status": "prepared", "legacyBackupSha256": hashlib.sha256(original).hexdigest()}
|
||||
assert stage.BACKUP_FILE.read_bytes() == original
|
||||
assert stage.BACKUP_SHA_FILE.read_text() == hashlib.sha256(original).hexdigest() + "\n"
|
||||
staged_legacy = legacy_env.read_bytes()
|
||||
assert staged_legacy.startswith(original)
|
||||
assert staged_legacy[len(original) :].endswith(b"\r\n")
|
||||
legacy_values = stage._parse_environment(staged_legacy)
|
||||
managed_values = stage._parse_environment(managed_env.read_bytes())
|
||||
assert legacy_values["REPOPULSE_GITEA_URL"] == "http://198.18.10.150:3000"
|
||||
assert managed_values["REPOPULSE_GITEA_URL"] == "https://gitea.example.com"
|
||||
assert managed_values["REPOPULSE_MAINTENANCE_TOKEN"] == legacy_values["REPOPULSE_MAINTENANCE_TOKEN"]
|
||||
|
||||
assert stage.prepare() == first
|
||||
assert legacy_env.read_bytes() == staged_legacy
|
||||
assert stage.BACKUP_FILE.read_bytes() == original
|
||||
|
||||
|
||||
def test_bare_carriage_return_is_rejected() -> None:
|
||||
stage = _load_stage()
|
||||
with pytest.raises(ValueError):
|
||||
stage._parse_environment(b"REPOPULSE_GITEA_URL=https://gitea.example.com\rREPOPULSE_GITEA_TOKEN=x\n")
|
||||
@@ -0,0 +1,95 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.services import repopulse_proxy_stage as stage
|
||||
|
||||
|
||||
class _Npm:
|
||||
def __init__(self, *, valid_acl: bool = True) -> None:
|
||||
self.valid_acl = valid_acl
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return True
|
||||
|
||||
def validate_managed_access_list(self, access_list_id: int) -> dict:
|
||||
assert access_list_id == 1
|
||||
return {
|
||||
"success": self.valid_acl,
|
||||
"status": "verified" if self.valid_acl else "blocked",
|
||||
"accessListId": access_list_id,
|
||||
}
|
||||
|
||||
def ensure_managed_proxy_host(self, **kwargs: object) -> dict:
|
||||
self.calls.append(kwargs)
|
||||
return {
|
||||
"success": True,
|
||||
"status": "created",
|
||||
"hostname": "repopulse.example.com",
|
||||
"hostId": 7,
|
||||
}
|
||||
|
||||
|
||||
def _ready(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
intent_id = "rpa-" + "a" * 20
|
||||
monkeypatch.setattr(stage.journal, "_load", lambda _session: (object(), {"intentId": intent_id, "status": "in_progress"}))
|
||||
monkeypatch.setattr(
|
||||
stage.EnrollmentPlane,
|
||||
"get_trust_config",
|
||||
lambda _session: SimpleNamespace(
|
||||
controller_identity="unraid-controller",
|
||||
repopulse_proxy_access_list_id=1,
|
||||
autonomous_profiles=("repopulse-authenticated",),
|
||||
approved_networks=("itworx-mcp-hub-npm",),
|
||||
allowed_domain_suffixes=("example.com",),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(stage, "apply_stored_connector_settings", lambda *_args: None)
|
||||
monkeypatch.setitem(settings.apis, "NPM_IDENTITY", "operator@example.org")
|
||||
|
||||
|
||||
def test_stages_only_reserved_fixed_authenticated_host(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ready(monkeypatch)
|
||||
npm = _Npm()
|
||||
result = stage.stage_repopulse_proxy_host(object(), connector=npm)
|
||||
assert result == {
|
||||
"status": "ready",
|
||||
"intentId": "rpa-" + "a" * 20,
|
||||
"hostname": "repopulse.example.com",
|
||||
"accessListId": 1,
|
||||
"hostId": 7,
|
||||
}
|
||||
assert npm.calls == [
|
||||
{
|
||||
"hostname": "repopulse.example.com",
|
||||
"forward_host": "repopulse",
|
||||
"forward_port": 8000,
|
||||
"certificate_mode": "managed",
|
||||
"force_ssl": True,
|
||||
"http2": True,
|
||||
"upload_limit_mb": 128,
|
||||
"read_timeout_seconds": 120,
|
||||
"ownership_id": "enr-" + "a" * 20,
|
||||
"access_list_id": 1,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_rejects_unverified_access_list_before_host_mutation(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_ready(monkeypatch)
|
||||
npm = _Npm(valid_acl=False)
|
||||
with pytest.raises(stage.RepoPulseProxyStageError, match="repopulse_access_list_unavailable"):
|
||||
stage.stage_repopulse_proxy_host(object(), connector=npm)
|
||||
assert npm.calls == []
|
||||
|
||||
|
||||
def test_rejects_unreserved_intent_before_connector_mutation(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(stage.journal, "_load", lambda _session: (None, None))
|
||||
npm = _Npm()
|
||||
with pytest.raises(stage.RepoPulseProxyStageError, match="adoption_intent_unavailable"):
|
||||
stage.stage_repopulse_proxy_host(object(), connector=npm)
|
||||
assert npm.calls == []
|
||||
@@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
from backend.app.db.models import Secret
|
||||
from backend.app.services import repopulse_secret_import as importer
|
||||
from backend.app.services.secret_store import get_secret
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def session(tmp_path, monkeypatch):
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'secrets.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
monkeypatch.setattr(importer, "secret_storage_status", lambda: {"productionReady": True})
|
||||
with Session(engine) as value:
|
||||
yield value
|
||||
|
||||
|
||||
def test_import_is_write_only_and_idempotent(session):
|
||||
assert importer.import_repopulse_gitea_token(session, "test-token") == "imported"
|
||||
assert importer.import_repopulse_gitea_token(session, "test-token") == "already_present"
|
||||
assert get_secret(session, importer.SECRET_REF, touch=False) == "test-token"
|
||||
|
||||
|
||||
def test_import_rejects_replacement_and_malformed_input(session):
|
||||
importer.import_repopulse_gitea_token(session, "first")
|
||||
with pytest.raises(ValueError, match="different value"):
|
||||
importer.import_repopulse_gitea_token(session, "second")
|
||||
for value in ("", " leading", "trailing ", "line\nbreak", "x" * 8193):
|
||||
with pytest.raises(ValueError):
|
||||
importer.import_repopulse_gitea_token(session, value)
|
||||
assert get_secret(session, importer.SECRET_REF, touch=False) == "first"
|
||||
|
||||
|
||||
def test_import_requires_production_secret_storage(session, monkeypatch):
|
||||
monkeypatch.setattr(importer, "secret_storage_status", lambda: {"productionReady": False})
|
||||
with pytest.raises(ValueError, match="not ready"):
|
||||
importer.import_repopulse_gitea_token(session, "test-token")
|
||||
assert get_secret(session, importer.SECRET_REF, touch=False) is None
|
||||
|
||||
|
||||
def test_import_does_not_overwrite_an_undecryptable_secret(session):
|
||||
importer.import_repopulse_gitea_token(session, "original")
|
||||
row = session.get(Secret, importer.SECRET_REF)
|
||||
assert row is not None
|
||||
row.encrypted_value = "corrupted-ciphertext"
|
||||
session.add(row)
|
||||
session.commit()
|
||||
with pytest.raises(ValueError, match="cannot be decrypted"):
|
||||
importer.import_repopulse_gitea_token(session, "replacement")
|
||||
assert session.get(Secret, importer.SECRET_REF).encrypted_value == "corrupted-ciphertext"
|
||||
|
||||
|
||||
def test_controller_bootstrap_import_uses_fixed_distinct_reference(session):
|
||||
assert importer.import_controller_bootstrap(session, "bootstrap-test-token") == "imported"
|
||||
assert importer.import_controller_bootstrap(session, "bootstrap-test-token") == "already_present"
|
||||
assert get_secret(session, importer.BOOTSTRAP_SECRET_REF, touch=False) == "bootstrap-test-token"
|
||||
assert get_secret(session, importer.SECRET_REF, touch=False) is None
|
||||
with pytest.raises(ValueError, match="different value"):
|
||||
importer.import_controller_bootstrap(session, "other-token")
|
||||
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "import_repopulse_gitea_token.py"
|
||||
SPEC = importlib.util.spec_from_file_location("repopulse_import_host", SCRIPT)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
host_import = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(host_import)
|
||||
|
||||
|
||||
def test_reads_only_fixed_token_key(tmp_path, monkeypatch):
|
||||
path = tmp_path / "deploy.env"
|
||||
path.write_text("REPOPULSE_GITEA_URL=https://example.invalid\nREPOPULSE_GITEA_TOKEN=test-value\n")
|
||||
path.chmod(0o600)
|
||||
monkeypatch.setattr(host_import.os, "geteuid", lambda: 0, raising=False)
|
||||
assert host_import.read_legacy_token(path) == "test-value"
|
||||
path.write_text("REPOPULSE_GITEA_TOKEN=one\nREPOPULSE_GITEA_TOKEN=two\n")
|
||||
with pytest.raises(ValueError, match="duplicated"):
|
||||
host_import.read_legacy_token(path)
|
||||
|
||||
|
||||
def test_import_passes_token_only_on_stdin(monkeypatch):
|
||||
observed = {}
|
||||
|
||||
def fake_run(argv, *, input, capture_output, check, timeout):
|
||||
observed.update({"argv": argv, "input": input, "capture_output": capture_output})
|
||||
|
||||
class Result:
|
||||
returncode = 0
|
||||
stdout = json.dumps({"status": "imported", "secretRef": "REPOPULSE_GITEA_TOKEN"}).encode()
|
||||
|
||||
return Result()
|
||||
|
||||
monkeypatch.setattr(host_import.subprocess, "run", fake_run)
|
||||
assert host_import.import_token("sensitive-test-value") == "imported"
|
||||
assert observed["input"] == b"sensitive-test-value"
|
||||
assert "sensitive-test-value" not in " ".join(observed["argv"])
|
||||
assert observed["capture_output"] is True
|
||||
@@ -0,0 +1,308 @@
|
||||
"""Cryptographic and intent-binding checks for the fixed target attestation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import importlib
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _module(monkeypatch):
|
||||
root = Path(__file__).resolve().parents[2] / "ops" / "gitea-deploy-broker"
|
||||
monkeypatch.syspath_prepend(str(root))
|
||||
return importlib.import_module("repopulse_target_attestation")
|
||||
|
||||
|
||||
def _sample(now: datetime) -> dict:
|
||||
return {
|
||||
"schema": "itworx.repopulse-target-attestation/v1",
|
||||
"attestation_id": "rpa-target-0123456789abcdef0123",
|
||||
"intent_id": "rpa-0123456789abcdef0123",
|
||||
"nonce": "1" * 32,
|
||||
"issued_at": now.isoformat(),
|
||||
"expires_at": (now + timedelta(minutes=2)).isoformat(),
|
||||
"controller_identity": "unraid-controller",
|
||||
"controller_trust_profile_id": "repopulse-authenticated",
|
||||
"controller_trust_profile_version": "enrollment-profile/v1",
|
||||
"controller_bootstrap_key_ref": "ENROLLMENT_BOOTSTRAP",
|
||||
"controller_tls_certificate_sha256": "a" * 64,
|
||||
"repository": "jens/repopulse",
|
||||
"application_id": "repopulse",
|
||||
"profile_id": "repopulse-authenticated",
|
||||
"profile_version": "enrollment-profile/v1",
|
||||
"plan_sha256": "b" * 64,
|
||||
"target_secret_ref": "jens__repopulse",
|
||||
"npm_hostname": "repopulse.example.com",
|
||||
"npm_access_list_route_hostname": "repopulse.example.com",
|
||||
"npm_upstream_host": "repopulse",
|
||||
"npm_upstream_network": "itworx-mcp-hub-npm",
|
||||
"npm_upstream_port": 8000,
|
||||
"npm_certificate_sha256": "c" * 64,
|
||||
"npm_certificate_sans": ["repopulse.example.com"],
|
||||
"npm_certificate_not_before": (now - timedelta(days=1)).isoformat(),
|
||||
"npm_certificate_not_after": (now + timedelta(days=1)).isoformat(),
|
||||
"npm_access_list_id": 1,
|
||||
"npm_access_list_satisfy_any": 0,
|
||||
"npm_access_list_is_deleted": 0,
|
||||
"npm_access_list_pass_auth": 1,
|
||||
"npm_access_list_client_count": 0,
|
||||
"npm_access_list_rule_count": 1,
|
||||
"npm_access_list_rules_sha256": "d" * 64,
|
||||
"npm_access_list_rules": [
|
||||
{
|
||||
"action": "authenticate",
|
||||
"subject_type": "basic_user_sha256",
|
||||
"subject": "e" * 64,
|
||||
}
|
||||
],
|
||||
"maintenance_method": "GET",
|
||||
"maintenance_path": "/api/v1/operator/maintenance",
|
||||
"maintenance_status_code": 200,
|
||||
"maintenance_auth_mechanism": "operator_token",
|
||||
"maintenance_authenticated_principal": "root_operator",
|
||||
"maintenance_response_sha256": "f" * 64,
|
||||
"maintenance_observed_at": now.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def test_signed_target_attestation_is_bound_to_one_intent_and_nonce(
|
||||
monkeypatch, tmp_path
|
||||
):
|
||||
module = _module(monkeypatch)
|
||||
secret = b"fixed-test-bootstrap-secret-value-0123456789"
|
||||
claim_path = tmp_path / "target-attestation.json"
|
||||
bootstrap_path = tmp_path / "bootstrap"
|
||||
bootstrap_path.write_bytes(secret + b"\n")
|
||||
monkeypatch.setattr(module, "ATTESTATION_PATH", claim_path)
|
||||
monkeypatch.setattr(module, "BOOTSTRAP_PATH", bootstrap_path)
|
||||
monkeypatch.setattr(
|
||||
module, "_private_file", lambda path, maximum: path.read_bytes()
|
||||
)
|
||||
consumed: dict[str, tuple[str, str]] = {}
|
||||
|
||||
def consume(nonce, digest, intent):
|
||||
value = (digest, intent)
|
||||
if nonce in consumed and consumed[nonce] != value:
|
||||
raise ValueError("nonce_replayed")
|
||||
consumed[nonce] = value
|
||||
|
||||
monkeypatch.setattr(module, "_record_nonce", consume)
|
||||
now = datetime.now(timezone.utc)
|
||||
claims = _sample(now)
|
||||
|
||||
def stage(value):
|
||||
raw = module._canonical(value)
|
||||
claim_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema": module.ENVELOPE_SCHEMA,
|
||||
"claims": value,
|
||||
"signatureSha256": hmac.new(
|
||||
secret, module._SIGNING_DOMAIN + raw, hashlib.sha256
|
||||
).hexdigest(),
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
expected = {
|
||||
"repository": "jens/repopulse",
|
||||
"applicationId": "repopulse",
|
||||
"profileId": "repopulse-authenticated",
|
||||
"profileVersion": "enrollment-profile/v1",
|
||||
"planSha256": "b" * 64,
|
||||
"intentId": claims["intent_id"],
|
||||
"targetSecretRef": "jens__repopulse",
|
||||
"controllerTlsCertificateSha256": "a" * 64,
|
||||
}
|
||||
verifier = module.FixedTargetAttestationVerifier()
|
||||
stage(claims)
|
||||
result = verifier.verify_target_attestation(expected=expected)
|
||||
assert result is not None and result.intent_id == claims["intent_id"]
|
||||
assert verifier.verify_target_attestation(expected=expected) == result
|
||||
|
||||
tampered = dict(claims, plan_sha256="0" * 64)
|
||||
claim_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema": module.ENVELOPE_SCHEMA,
|
||||
"claims": tampered,
|
||||
"signatureSha256": hmac.new(
|
||||
secret,
|
||||
module._SIGNING_DOMAIN + module._canonical(claims),
|
||||
hashlib.sha256,
|
||||
).hexdigest(),
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert verifier.verify_target_attestation(expected=expected) is None
|
||||
|
||||
expired = dict(
|
||||
claims,
|
||||
issued_at=(now - timedelta(minutes=9)).isoformat(),
|
||||
expires_at=(now - timedelta(minutes=8)).isoformat(),
|
||||
)
|
||||
stage(expired)
|
||||
assert verifier.verify_target_attestation(expected=expected) is None
|
||||
|
||||
changed_intent = dict(claims, intent_id="rpa-ffffffffffffffffffff")
|
||||
stage(changed_intent)
|
||||
assert verifier.verify_target_attestation(expected=expected) is None
|
||||
|
||||
reused_nonce = dict(claims, attestation_id="rpa-target-elsewhere")
|
||||
stage(reused_nonce)
|
||||
assert verifier.verify_target_attestation(expected=expected) is None
|
||||
|
||||
|
||||
def test_npm_producer_requires_the_fixed_authenticated_route(monkeypatch, tmp_path):
|
||||
_module(monkeypatch)
|
||||
producer = importlib.import_module("stage_repopulse_target_attestation")
|
||||
database_path = tmp_path / "npm.sqlite"
|
||||
with sqlite3.connect(database_path) as database:
|
||||
database.executescript("""
|
||||
CREATE TABLE proxy_host (id INTEGER, domain_names TEXT, forward_host TEXT,
|
||||
forward_port INTEGER, access_list_id INTEGER, certificate_id INTEGER,
|
||||
ssl_forced INTEGER, enabled INTEGER, forward_scheme TEXT, is_deleted INTEGER);
|
||||
CREATE TABLE certificate (id INTEGER, domain_names TEXT, is_deleted INTEGER);
|
||||
CREATE TABLE access_list (id INTEGER, is_deleted INTEGER, satisfy_any INTEGER,
|
||||
pass_auth INTEGER);
|
||||
CREATE TABLE access_list_auth (id INTEGER, access_list_id INTEGER, username TEXT);
|
||||
CREATE TABLE access_list_client (id INTEGER, access_list_id INTEGER,
|
||||
address TEXT, directive TEXT);
|
||||
INSERT INTO proxy_host VALUES (7, '["repopulse.example.com"]', 'repopulse',
|
||||
8000, 1, 9, 1, 1, 'http', 0);
|
||||
INSERT INTO certificate VALUES (9, '["repopulse.example.com"]', 0);
|
||||
INSERT INTO access_list VALUES (1, 0, 0, 1);
|
||||
INSERT INTO access_list_auth VALUES (3, 1, 'private-user');
|
||||
""")
|
||||
monkeypatch.setattr(producer, "NPM_DATABASE", database_path)
|
||||
rules, cert_id = producer._npm_host_and_rules(1)
|
||||
assert cert_id == 9
|
||||
assert rules == [
|
||||
{
|
||||
"action": "authenticate",
|
||||
"subjectType": "basic_user_sha256",
|
||||
"subject": hashlib.sha256(b"private-user").hexdigest(),
|
||||
}
|
||||
]
|
||||
with sqlite3.connect(database_path) as database:
|
||||
database.execute("UPDATE access_list SET satisfy_any=1 WHERE id=1")
|
||||
try:
|
||||
producer._npm_host_and_rules(1)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("permissive access list must fail closed")
|
||||
|
||||
|
||||
def test_maintenance_proof_timestamp_is_captured_with_the_probe(monkeypatch):
|
||||
_module(monkeypatch)
|
||||
producer = importlib.import_module("stage_repopulse_target_attestation")
|
||||
monkeypatch.setattr(
|
||||
producer,
|
||||
"_private_bytes",
|
||||
lambda *_args, **_kwargs: (
|
||||
b"REPOPULSE_GITEA_URL=https://gitea.example.com\n"
|
||||
b"REPOPULSE_GITEA_TOKEN=token-value\n"
|
||||
b"REPOPULSE_MAINTENANCE_TOKEN=maintenance-token-value-123456789\n"
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
producer,
|
||||
"_docker_python",
|
||||
lambda *_args, **_kwargs: {"status": 200, "responseSha256": "a" * 64},
|
||||
)
|
||||
before = datetime.now(timezone.utc)
|
||||
mechanism, digest, observed = producer._maintenance()
|
||||
after = datetime.now(timezone.utc)
|
||||
assert mechanism == "operator_token"
|
||||
assert digest == "a" * 64
|
||||
assert before <= datetime.fromisoformat(observed) <= after
|
||||
|
||||
|
||||
def test_producer_main_builds_verifier_shape_with_proof_before_issue(
|
||||
monkeypatch, capsys
|
||||
):
|
||||
verifier = _module(monkeypatch)
|
||||
producer = importlib.import_module("stage_repopulse_target_attestation")
|
||||
intent_id = "rpa-" + "a" * 20
|
||||
now = datetime.now(timezone.utc)
|
||||
observed = now.isoformat()
|
||||
captured: dict = {}
|
||||
|
||||
monkeypatch.setattr(producer.os, "geteuid", lambda: 0, raising=False)
|
||||
monkeypatch.setattr(producer.sys, "argv", ["stage_repopulse_target_attestation.py"])
|
||||
monkeypatch.setattr(
|
||||
producer,
|
||||
"_private_bytes",
|
||||
lambda path, _maximum: (
|
||||
b"x" * 64
|
||||
if path == verifier.BOOTSTRAP_PATH
|
||||
else b'{"certificate_file":"/fixed/server.crt"}'
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
producer.store,
|
||||
"RepoPulseAdoptionStore",
|
||||
lambda: type(
|
||||
"Store",
|
||||
(),
|
||||
{
|
||||
"load_intent": lambda self: {
|
||||
"status": "in_progress",
|
||||
"intentId": intent_id,
|
||||
}
|
||||
},
|
||||
)(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
producer, "_appops_trust", lambda _secret: {"repopulseProxyAccessListId": 1}
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
producer,
|
||||
"_npm_host_and_rules",
|
||||
lambda _acl: (
|
||||
[
|
||||
{
|
||||
"action": "authenticate",
|
||||
"subjectType": "basic_user_sha256",
|
||||
"subject": "c" * 64,
|
||||
}
|
||||
],
|
||||
9,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(producer, "_npm_network", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
producer,
|
||||
"_npm_tls",
|
||||
lambda: (
|
||||
"d" * 64,
|
||||
("repopulse.example.com",),
|
||||
(now - timedelta(days=1)).isoformat(),
|
||||
(now + timedelta(days=1)).isoformat(),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
producer, "_maintenance", lambda: ("operator_token", "e" * 64, observed)
|
||||
)
|
||||
monkeypatch.setattr(producer, "_certificate_bytes", lambda _path: b"pem")
|
||||
monkeypatch.setattr(producer.ssl, "PEM_cert_to_DER_cert", lambda _pem: b"der")
|
||||
monkeypatch.setattr(producer, "_plan", lambda _intent: "b" * 64)
|
||||
|
||||
def capture(claims, _secret):
|
||||
captured.update(claims)
|
||||
return "f" * 64
|
||||
|
||||
monkeypatch.setattr(producer, "_stage", capture)
|
||||
assert producer.main() == 0
|
||||
assert '"status":"staged"' in capsys.readouterr().out
|
||||
assert verifier._claims(captured).intent_id == intent_id
|
||||
assert datetime.fromisoformat(
|
||||
captured["maintenance_observed_at"]
|
||||
) <= datetime.fromisoformat(captured["issued_at"])
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
from backend.app.db.models import Secret
|
||||
from backend.app.services import repopulse_target_credential as target
|
||||
|
||||
|
||||
def test_fixed_target_credential_is_durable_and_collision_safe(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
target.secret_store,
|
||||
"secret_storage_status",
|
||||
lambda: {"productionReady": True},
|
||||
)
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'credentials.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
first_intent = "rpa-" + "a" * 20
|
||||
|
||||
with Session(engine) as session:
|
||||
first_id, first_secret = target.get_or_create_repopulse_target_credential(
|
||||
session, intent_id=first_intent
|
||||
)
|
||||
row = session.get(Secret, "ENROLL_REPOPULSE_TARGET_HMAC")
|
||||
assert row is not None
|
||||
assert first_secret.decode("ascii") not in row.encrypted_value
|
||||
|
||||
with Session(engine) as session:
|
||||
replay_id, replay_secret = target.get_or_create_repopulse_target_credential(
|
||||
session, intent_id=first_intent
|
||||
)
|
||||
assert (replay_id, replay_secret) == (first_id, first_secret)
|
||||
with pytest.raises(
|
||||
target.RepoPulseTargetCredentialError, match="target_credential_collision"
|
||||
):
|
||||
target.get_or_create_repopulse_target_credential(
|
||||
session, intent_id="rpa-" + "b" * 20
|
||||
)
|
||||
|
||||
with Session(engine) as session:
|
||||
row = session.get(Secret, "ENROLL_REPOPULSE_TARGET_HMAC")
|
||||
assert row is not None
|
||||
row.metadata_json = {}
|
||||
session.add(row)
|
||||
session.commit()
|
||||
with pytest.raises(
|
||||
target.RepoPulseTargetCredentialError, match="target_credential_collision"
|
||||
):
|
||||
target.get_or_create_repopulse_target_credential(
|
||||
session, intent_id=first_intent
|
||||
)
|
||||
|
||||
|
||||
def test_fixed_target_credential_requires_production_secret_store(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
target.secret_store,
|
||||
"secret_storage_status",
|
||||
lambda: {"productionReady": False},
|
||||
)
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'credentials.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
with pytest.raises(
|
||||
target.RepoPulseTargetCredentialError, match="secret_store_not_ready"
|
||||
):
|
||||
target.get_or_create_repopulse_target_credential(
|
||||
session, intent_id="rpa-" + "a" * 20
|
||||
)
|
||||
assert session.get(Secret, "ENROLL_REPOPULSE_TARGET_HMAC") is None
|
||||
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.app.services import repopulse_trust_stage as stage
|
||||
|
||||
|
||||
class _Response:
|
||||
status = 200
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self, _limit: int) -> bytes:
|
||||
return b'{"status":"ok"}'
|
||||
|
||||
|
||||
class _Opener:
|
||||
def open(self, url: str, *, timeout: int):
|
||||
assert url == "https://203.0.113.1:1300/health"
|
||||
assert timeout == 8
|
||||
return _Response()
|
||||
|
||||
|
||||
def _ready(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
stage, "secret_storage_status", lambda: {"productionReady": True}
|
||||
)
|
||||
monkeypatch.setattr(stage, "get_secret", lambda *_args, **_kwargs: "x" * 64)
|
||||
monkeypatch.setattr(stage, "has_secret", lambda *_args: True)
|
||||
monkeypatch.setattr(stage.ssl, "create_default_context", lambda *, cafile: object())
|
||||
monkeypatch.setattr(stage.urllib.request, "build_opener", lambda *_args: _Opener())
|
||||
|
||||
|
||||
def test_stages_exact_trust_once_after_tls_and_secret_checks(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_ready(monkeypatch)
|
||||
state: dict = {}
|
||||
monkeypatch.setattr(
|
||||
stage.EnrollmentPlane, "get_trust_config", lambda _session: state.get("trust")
|
||||
)
|
||||
|
||||
def save(_session, trust):
|
||||
state["trust"] = trust
|
||||
return {"status": "ready", "bootstrapSecretConfigured": True}
|
||||
|
||||
monkeypatch.setattr(stage.EnrollmentPlane, "set_trust_config", save)
|
||||
broker = SimpleNamespace(
|
||||
base_url="https://203.0.113.1:1300",
|
||||
verify_tls=True,
|
||||
ca_file="/data/controller-ca.pem",
|
||||
configuration_error=lambda: None,
|
||||
)
|
||||
assert stage.stage(object(), broker=broker) == {
|
||||
"status": "configured",
|
||||
"controllerIdentity": "unraid-controller",
|
||||
}
|
||||
assert state["trust"].repopulse_proxy_access_list_id == 1
|
||||
assert stage.stage(object(), broker=broker)["status"] == "already_configured"
|
||||
|
||||
|
||||
def test_rejects_wrong_broker_before_trust_write(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_ready(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
stage.EnrollmentPlane,
|
||||
"set_trust_config",
|
||||
lambda *_args: pytest.fail("trust must not be written"),
|
||||
)
|
||||
broker = SimpleNamespace(
|
||||
base_url="http://127.0.0.1:1299",
|
||||
verify_tls=False,
|
||||
ca_file=None,
|
||||
configuration_error=lambda: None,
|
||||
)
|
||||
with pytest.raises(ValueError, match="repopulse_trust_not_ready"):
|
||||
stage.stage(object(), broker=broker)
|
||||
@@ -0,0 +1,13 @@
|
||||
from backend.app.services.request_metrics import _percentile
|
||||
|
||||
|
||||
def test_nearest_rank_p95_uses_the_maximum_for_small_samples():
|
||||
assert _percentile([10.0], 0.95) == 10.0
|
||||
assert _percentile([10.0, 40.0], 0.95) == 40.0
|
||||
assert _percentile([10.0, 20.0, 40.0], 0.95) == 40.0
|
||||
|
||||
|
||||
def test_nearest_rank_percentile_is_bounded():
|
||||
assert _percentile([1.0, 2.0, 3.0], 0.0) == 1.0
|
||||
assert _percentile([1.0, 2.0, 3.0], 2.0) == 3.0
|
||||
assert _percentile([], 0.95) == 0.0
|
||||
@@ -0,0 +1,123 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "run_repopulse_adoption.py"
|
||||
_SPEC = importlib.util.spec_from_file_location("run_repopulse_adoption", _SCRIPT)
|
||||
assert _SPEC is not None and _SPEC.loader is not None
|
||||
handoff = importlib.util.module_from_spec(_SPEC)
|
||||
_SPEC.loader.exec_module(handoff)
|
||||
|
||||
|
||||
def test_fixed_operator_sequence_keeps_credential_out_of_result(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(handoff, "_require_root", lambda: None)
|
||||
intent_id = "rpa-" + "a" * 20
|
||||
calls: list[str] = []
|
||||
secret = "operator-secret-value" * 4
|
||||
|
||||
def appops(
|
||||
*arguments: str, payload: bytes | None = None, **_kwargs: object
|
||||
) -> dict:
|
||||
if "prepare" in arguments:
|
||||
calls.append("prepare")
|
||||
return {"status": "prepared", "intentId": intent_id}
|
||||
if "backend.app.services.repopulse_proxy_stage" in arguments:
|
||||
calls.append("proxy")
|
||||
return {"status": "ready", "intentId": intent_id}
|
||||
if "-c" in arguments:
|
||||
calls.append("handoff")
|
||||
return {
|
||||
"intentId": intent_id,
|
||||
"idempotencySha256": "b" * 64,
|
||||
"preflightSha256": "c" * 64,
|
||||
"preflightProjection": {},
|
||||
"targetCredential": secret,
|
||||
}
|
||||
assert "finalize" in arguments
|
||||
assert payload is not None and secret.encode() not in payload
|
||||
calls.append("finalize")
|
||||
return {"status": "completed", "intentId": intent_id}
|
||||
|
||||
def controller(
|
||||
command: list[str], *, payload: bytes | None = None, **_kwargs: object
|
||||
) -> dict:
|
||||
if command[-1].endswith("stage_repopulse_target_attestation.py"):
|
||||
calls.append("attestation")
|
||||
return {"status": "staged", "intentId": intent_id}
|
||||
assert command[-1].endswith("repopulse_adoption_cli.py")
|
||||
assert json.loads(payload or b"{}")["targetCredential"] == secret
|
||||
calls.append("controller")
|
||||
return {"status": "deployment_verified", "intentId": intent_id}
|
||||
|
||||
monkeypatch.setattr(handoff, "_appops", appops)
|
||||
monkeypatch.setattr(handoff, "_run", controller)
|
||||
result = handoff.run()
|
||||
assert result == {"status": "completed", "intentId": intent_id}
|
||||
assert secret not in json.dumps(result)
|
||||
assert calls == [
|
||||
"prepare",
|
||||
"proxy",
|
||||
"attestation",
|
||||
"handoff",
|
||||
"controller",
|
||||
"finalize",
|
||||
]
|
||||
|
||||
|
||||
def test_proxy_failure_prevents_secret_handoff(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(handoff, "_require_root", lambda: None)
|
||||
calls: list[str] = []
|
||||
|
||||
def appops(*arguments: str, **_kwargs: object) -> dict:
|
||||
calls.append(arguments[-1])
|
||||
if "prepare" in arguments:
|
||||
return {"status": "prepared", "intentId": "rpa-" + "a" * 20}
|
||||
return {"status": "blocked"}
|
||||
|
||||
monkeypatch.setattr(handoff, "_appops", appops)
|
||||
monkeypatch.setattr(
|
||||
handoff,
|
||||
"_run",
|
||||
lambda *_args, **_kwargs: pytest.fail("controller must not execute"),
|
||||
)
|
||||
with pytest.raises(handoff.AdoptionHandoffError, match="operator_step_blocked"):
|
||||
handoff.run()
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
def test_operator_subprocess_uses_a_fixed_environment(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("PYTHONPATH", "/untrusted/modules")
|
||||
observed: dict = {}
|
||||
|
||||
def invoke(command, **kwargs):
|
||||
observed.update(kwargs)
|
||||
assert command == ["python3", "fixed.py"]
|
||||
return SimpleNamespace(returncode=0, stdout=b'{"status":"ready"}', stderr=b"")
|
||||
|
||||
monkeypatch.setattr(handoff.subprocess, "run", invoke)
|
||||
assert handoff._run(["python3", "fixed.py"]) == {"status": "ready"}
|
||||
assert "PYTHONPATH" not in observed["env"]
|
||||
assert observed["env"]["PATH"].startswith("/usr/local/sbin:")
|
||||
|
||||
|
||||
def test_operator_rejects_ambiguous_child_json(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
handoff.subprocess,
|
||||
"run",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=b'{"status":"ready","status":"blocked"}',
|
||||
stderr=b"",
|
||||
),
|
||||
)
|
||||
with pytest.raises(handoff.AdoptionHandoffError, match="operator_result_invalid"):
|
||||
handoff._run(["python3", "fixed.py"])
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
IN_PLACE = ROOT / "scripts" / "upgrade_gitea_runner_3_5.sh"
|
||||
FLEET_BOOTSTRAP = ROOT / "scripts" / "upgrade_ci_runner_fleet.sh"
|
||||
|
||||
OLD_IMAGE = (
|
||||
"gitea/runner@sha256:"
|
||||
"3ebc3ad9654552377896e1edbc77ac420025bb01ff1736bc43232d0bf2866f5a"
|
||||
)
|
||||
NEW_IMAGE = (
|
||||
"gitea/runner:3.5.0@sha256:"
|
||||
"66b7da94dc7dcadb2e076bec6928221336a9a637196399281c4b766fe1288242"
|
||||
)
|
||||
|
||||
|
||||
def test_runner_3_5_upgrade_script_is_syntactically_valid() -> None:
|
||||
for script in (IN_PLACE, FLEET_BOOTSTRAP):
|
||||
subprocess.run(
|
||||
["bash", "-n"],
|
||||
input=script.read_bytes().replace(b"\r\n", b"\n"),
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def test_future_fleet_bootstrap_uses_reviewed_runner_3_5_pin() -> None:
|
||||
text = FLEET_BOOTSTRAP.read_text(encoding="utf-8")
|
||||
assert NEW_IMAGE in text
|
||||
assert OLD_IMAGE not in text
|
||||
assert "gitea/runner:latest" not in text
|
||||
assert "# v3.5.0" in text
|
||||
|
||||
|
||||
def test_in_place_upgrade_is_fixed_target_and_fail_closed() -> None:
|
||||
text = IN_PLACE.read_text(encoding="utf-8")
|
||||
|
||||
assert 'readonly PRIMARY="itworx-security-runner"' in text
|
||||
assert 'readonly SECONDARY="itworx-validation-runner-2"' in text
|
||||
assert 'readonly DIND="gitea-runner-dind"' in text
|
||||
assert OLD_IMAGE in text
|
||||
assert NEW_IMAGE in text
|
||||
assert 'readonly CONFIRMATION="UPGRADE_GITEA_RUNNER_3_5"' in text
|
||||
|
||||
assert 'active DinD workload prevents runner upgrade' in text
|
||||
assert 'grep -F -c "$OLD_IMAGE"' in text
|
||||
assert 'runner image baseline is not the reviewed 2.0.0 pin' in text
|
||||
assert 'docker pull "$NEW_IMAGE"' in text
|
||||
assert 'gitea-runner "$NEW_IMAGE" --version' in text
|
||||
|
||||
assert 'set_runner_disabled "$PRIMARY" true' in text
|
||||
assert 'set_runner_disabled "$SECONDARY" true' in text
|
||||
assert 'set_runner_disabled "$PRIMARY" false' in text
|
||||
assert 'set_runner_disabled "$SECONDARY" false' in text
|
||||
|
||||
assert (
|
||||
'up -d --no-deps --force-recreate "$PRIMARY_SERVICE" "$SECONDARY_SERVICE"'
|
||||
in text
|
||||
)
|
||||
assert 'docker compose --project-directory "$ROOT"' in text
|
||||
assert 'cp -a -- "$backup/docker-compose.yml" "$COMPOSE_FILE"' in text
|
||||
|
||||
# The maintenance path may inspect DinD for active CI work, but must never
|
||||
# recreate or prune the daemon, Gitea, unrelated networks, or host storage.
|
||||
forbidden = (
|
||||
'force-recreate "$DIND"',
|
||||
'force-recreate "$GITEA_CONTAINER"',
|
||||
"docker system prune",
|
||||
"docker network prune",
|
||||
"docker volume prune",
|
||||
'docker rm --force "$DIND"',
|
||||
'docker rm --force "$GITEA_CONTAINER"',
|
||||
)
|
||||
for token in forbidden:
|
||||
assert token not in text
|
||||
|
||||
|
||||
def test_runner_3_5_upgrade_preserves_separate_runner_credentials() -> None:
|
||||
text = IN_PLACE.read_text(encoding="utf-8")
|
||||
assert (
|
||||
'PRIMARY_RUNNER_FILE="/mnt/cache/appdata/itworx-security-runner/.runner"'
|
||||
in text
|
||||
)
|
||||
assert (
|
||||
'SECONDARY_RUNNER_FILE="/mnt/cache/appdata/itworx-validation-runner-2/.runner"'
|
||||
in text
|
||||
)
|
||||
assert (
|
||||
'readlink -f "$PRIMARY_RUNNER_FILE"'
|
||||
in text
|
||||
and 'readlink -f "$SECONDARY_RUNNER_FILE"' in text
|
||||
)
|
||||
@@ -0,0 +1,220 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CONTROLLER_ROOT = ROOT / "ops" / "gitea-deploy-broker"
|
||||
CONTROLLER_PATH = CONTROLLER_ROOT / "runner_capacity_controller.py"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def capacity_module(monkeypatch):
|
||||
monkeypatch.syspath_prepend(str(CONTROLLER_ROOT))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"appops_runner_capacity_controller_test", CONTROLLER_PATH
|
||||
)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _wire_common(service, monkeypatch, *, running=False, busy=False, disabled=True):
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_burst_container",
|
||||
lambda: {
|
||||
"name": "itworx-validation-runner-burst-1",
|
||||
"running": running,
|
||||
"health": "healthy" if running else "none",
|
||||
"startedAt": "",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_runner_registration",
|
||||
lambda: {
|
||||
"id": 44,
|
||||
"status": "online" if running else "offline",
|
||||
"busy": busy,
|
||||
"disabled": disabled,
|
||||
"labels": ["linux-validation", "ubuntu-latest"],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(service, "_read_state", lambda: {})
|
||||
monkeypatch.setattr(service, "_write_state", lambda _state: None)
|
||||
monkeypatch.setattr(service, "_record", lambda _event: None)
|
||||
|
||||
|
||||
def test_cycle_starts_and_enables_burst_for_queue_pressure(capacity_module, monkeypatch):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_common(service, monkeypatch, running=False, disabled=True)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_queue",
|
||||
lambda: {"depth": 2, "oldestSeconds": 20},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_host",
|
||||
lambda _previous, now: {
|
||||
"cpuCount": 20,
|
||||
"loadOne": 5.0,
|
||||
"loadRatio": 0.25,
|
||||
"availableMemoryBytes": 20 * 1024**3,
|
||||
"cacheFreeBytes": 100 * 1024**3,
|
||||
"swapInKbps": 0.0,
|
||||
"swapOutKbps": 0.0,
|
||||
"swapPressureSamples": 0,
|
||||
"swapCritical": False,
|
||||
"swapSampleAt": now,
|
||||
"swapInPages": 0,
|
||||
"swapOutPages": 0,
|
||||
},
|
||||
)
|
||||
commands = []
|
||||
disabled = []
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_run",
|
||||
lambda args, timeout=20, check=True: commands.append(args) or "",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_set_disabled",
|
||||
lambda runner_id, value: disabled.append((runner_id, value)),
|
||||
)
|
||||
|
||||
result = service.cycle()
|
||||
|
||||
assert result["decision"]["action"] == "start"
|
||||
assert commands == [["docker", "start", "itworx-validation-runner-burst-1"]]
|
||||
assert disabled == [(44, False)]
|
||||
|
||||
|
||||
def test_cycle_drains_busy_burst_under_host_pressure(capacity_module, monkeypatch):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_common(service, monkeypatch, running=True, busy=True, disabled=False)
|
||||
monkeypatch.setattr(service, "_queue", lambda: {"depth": 3, "oldestSeconds": 90})
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_host",
|
||||
lambda _previous, now: {
|
||||
"cpuCount": 20,
|
||||
"loadOne": 19.0,
|
||||
"loadRatio": 0.95,
|
||||
"availableMemoryBytes": 20 * 1024**3,
|
||||
"cacheFreeBytes": 100 * 1024**3,
|
||||
"swapInKbps": 0.0,
|
||||
"swapOutKbps": 0.0,
|
||||
"swapPressureSamples": 0,
|
||||
"swapCritical": False,
|
||||
"swapSampleAt": now,
|
||||
"swapInPages": 0,
|
||||
"swapOutPages": 0,
|
||||
},
|
||||
)
|
||||
disabled = []
|
||||
commands = []
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_set_disabled",
|
||||
lambda runner_id, value: disabled.append((runner_id, value)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_run",
|
||||
lambda args, timeout=20, check=True: commands.append(args) or "",
|
||||
)
|
||||
|
||||
result = service.cycle()
|
||||
|
||||
assert result["decision"]["action"] == "drain"
|
||||
assert disabled == [(44, True)]
|
||||
assert commands == []
|
||||
|
||||
|
||||
def test_cycle_stops_only_disabled_idle_runner_without_active_assignment(
|
||||
capacity_module, monkeypatch
|
||||
):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_common(service, monkeypatch, running=True, busy=False, disabled=True)
|
||||
monkeypatch.setattr(service, "_read_state", lambda: {"lastDemandAt": 1, "lastActionAt": 1})
|
||||
monkeypatch.setattr(service, "_queue", lambda: {"depth": 0, "oldestSeconds": None})
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_host",
|
||||
lambda _previous, now: {
|
||||
"cpuCount": 20,
|
||||
"loadOne": 5.0,
|
||||
"loadRatio": 0.25,
|
||||
"availableMemoryBytes": 20 * 1024**3,
|
||||
"cacheFreeBytes": 100 * 1024**3,
|
||||
"swapInKbps": 0.0,
|
||||
"swapOutKbps": 0.0,
|
||||
"swapPressureSamples": 0,
|
||||
"swapCritical": False,
|
||||
"swapSampleAt": now,
|
||||
"swapInPages": 0,
|
||||
"swapOutPages": 0,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(service, "_active_on_runner", lambda _runner_id: False)
|
||||
commands = []
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_run",
|
||||
lambda args, timeout=20, check=True: commands.append(args) or "",
|
||||
)
|
||||
|
||||
result = service.cycle()
|
||||
|
||||
assert result["decision"]["action"] == "stop"
|
||||
assert commands == [
|
||||
["docker", "stop", "--time", "30", "itworx-validation-runner-burst-1"]
|
||||
]
|
||||
|
||||
|
||||
def test_cycle_never_stops_when_gitea_still_has_active_assignment(
|
||||
capacity_module, monkeypatch
|
||||
):
|
||||
service = capacity_module.RunnerCapacityController()
|
||||
_wire_common(service, monkeypatch, running=True, busy=False, disabled=True)
|
||||
monkeypatch.setattr(service, "_read_state", lambda: {"lastDemandAt": 1, "lastActionAt": 1})
|
||||
monkeypatch.setattr(service, "_queue", lambda: {"depth": 0, "oldestSeconds": None})
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_host",
|
||||
lambda _previous, now: {
|
||||
"cpuCount": 20,
|
||||
"loadOne": 5.0,
|
||||
"loadRatio": 0.25,
|
||||
"availableMemoryBytes": 20 * 1024**3,
|
||||
"cacheFreeBytes": 100 * 1024**3,
|
||||
"swapInKbps": 0.0,
|
||||
"swapOutKbps": 0.0,
|
||||
"swapPressureSamples": 0,
|
||||
"swapCritical": False,
|
||||
"swapSampleAt": now,
|
||||
"swapInPages": 0,
|
||||
"swapOutPages": 0,
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(service, "_active_on_runner", lambda _runner_id: True)
|
||||
commands = []
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_run",
|
||||
lambda args, timeout=20, check=True: commands.append(args) or "",
|
||||
)
|
||||
|
||||
result = service.cycle()
|
||||
|
||||
assert result["decision"]["result"] == "stop_deferred_not_proven_idle"
|
||||
assert commands == []
|
||||
@@ -0,0 +1,189 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
POLICY_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_capacity_policy.py"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def policy_module():
|
||||
spec = importlib.util.spec_from_file_location("appops_runner_capacity_policy_test", POLICY_PATH)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _state(policy_module, **overrides):
|
||||
values = {
|
||||
"queue_depth": 0,
|
||||
"oldest_queued_seconds": None,
|
||||
"load_ratio": 0.30,
|
||||
"available_memory_bytes": 20 * policy_module.GIB,
|
||||
"burst_running": False,
|
||||
"burst_busy": False,
|
||||
"burst_disabled": True,
|
||||
"seconds_since_demand": 600,
|
||||
"seconds_since_action": 600,
|
||||
"swap_critical": False,
|
||||
"cache_free_bytes": 100 * policy_module.GIB,
|
||||
}
|
||||
values.update(overrides)
|
||||
return policy_module.BurstCapacityInput(**values)
|
||||
|
||||
|
||||
def test_burst_starts_only_for_queue_demand_with_headroom(policy_module):
|
||||
decision = policy_module.decide_burst_capacity(
|
||||
_state(policy_module, queue_depth=2)
|
||||
)
|
||||
|
||||
assert decision.action == "start"
|
||||
assert decision.reason == "queue_pressure_with_host_headroom"
|
||||
assert decision.pressure == "headroom"
|
||||
|
||||
|
||||
def test_old_job_can_trigger_burst_even_with_single_queued_job(policy_module):
|
||||
decision = policy_module.decide_burst_capacity(
|
||||
_state(policy_module, queue_depth=1, oldest_queued_seconds=60)
|
||||
)
|
||||
|
||||
assert decision.action == "start"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("load_ratio", "memory_gib", "swap_critical"),
|
||||
[
|
||||
(0.75, 20, False),
|
||||
(0.30, 10, False),
|
||||
(0.30, 20, True),
|
||||
],
|
||||
)
|
||||
def test_burst_does_not_start_without_headroom(
|
||||
policy_module, load_ratio, memory_gib, swap_critical
|
||||
):
|
||||
decision = policy_module.decide_burst_capacity(
|
||||
_state(
|
||||
policy_module,
|
||||
queue_depth=3,
|
||||
load_ratio=load_ratio,
|
||||
available_memory_bytes=memory_gib * policy_module.GIB,
|
||||
swap_critical=swap_critical,
|
||||
)
|
||||
)
|
||||
|
||||
assert decision.action == "hold"
|
||||
assert decision.reason == "queue_pressure_but_host_headroom_insufficient"
|
||||
|
||||
|
||||
def test_high_pressure_drains_busy_burst_instead_of_stopping_it(policy_module):
|
||||
decision = policy_module.decide_burst_capacity(
|
||||
_state(
|
||||
policy_module,
|
||||
queue_depth=3,
|
||||
load_ratio=0.95,
|
||||
burst_running=True,
|
||||
burst_busy=True,
|
||||
burst_disabled=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert decision.action == "drain"
|
||||
assert decision.reason == "host_pressure_drain_after_current_job"
|
||||
|
||||
|
||||
def test_high_pressure_stops_only_already_drained_idle_burst(policy_module):
|
||||
decision = policy_module.decide_burst_capacity(
|
||||
_state(
|
||||
policy_module,
|
||||
load_ratio=0.95,
|
||||
burst_running=True,
|
||||
burst_busy=False,
|
||||
burst_disabled=True,
|
||||
seconds_since_action=600,
|
||||
)
|
||||
)
|
||||
|
||||
assert decision.action == "stop"
|
||||
assert decision.reason == "host_pressure_idle_burst_can_stop"
|
||||
|
||||
|
||||
def test_idle_burst_is_drained_before_stop(policy_module):
|
||||
first = policy_module.decide_burst_capacity(
|
||||
_state(
|
||||
policy_module,
|
||||
burst_running=True,
|
||||
burst_disabled=False,
|
||||
seconds_since_demand=600,
|
||||
seconds_since_action=600,
|
||||
)
|
||||
)
|
||||
second = policy_module.decide_burst_capacity(
|
||||
_state(
|
||||
policy_module,
|
||||
burst_running=True,
|
||||
burst_disabled=True,
|
||||
seconds_since_demand=600,
|
||||
seconds_since_action=600,
|
||||
)
|
||||
)
|
||||
|
||||
assert first.action == "drain"
|
||||
assert second.action == "stop"
|
||||
|
||||
|
||||
def test_disabled_running_burst_reenables_when_demand_returns(policy_module):
|
||||
decision = policy_module.decide_burst_capacity(
|
||||
_state(
|
||||
policy_module,
|
||||
queue_depth=2,
|
||||
burst_running=True,
|
||||
burst_disabled=True,
|
||||
seconds_since_action=600,
|
||||
)
|
||||
)
|
||||
|
||||
assert decision.action == "enable"
|
||||
|
||||
|
||||
def test_start_cooldown_prevents_flapping(policy_module):
|
||||
decision = policy_module.decide_burst_capacity(
|
||||
_state(
|
||||
policy_module,
|
||||
queue_depth=2,
|
||||
seconds_since_action=30,
|
||||
)
|
||||
)
|
||||
|
||||
assert decision.action == "hold"
|
||||
|
||||
|
||||
def test_burst_does_not_start_when_cache_headroom_is_low(policy_module):
|
||||
decision = policy_module.decide_burst_capacity(
|
||||
_state(
|
||||
policy_module,
|
||||
queue_depth=3,
|
||||
cache_free_bytes=20 * policy_module.GIB,
|
||||
)
|
||||
)
|
||||
assert decision.action == "hold"
|
||||
assert decision.reason == "queue_pressure_but_host_headroom_insufficient"
|
||||
|
||||
|
||||
def test_low_cache_headroom_drains_busy_burst(policy_module):
|
||||
decision = policy_module.decide_burst_capacity(
|
||||
_state(
|
||||
policy_module,
|
||||
queue_depth=3,
|
||||
burst_running=True,
|
||||
burst_busy=True,
|
||||
burst_disabled=False,
|
||||
cache_free_bytes=10 * policy_module.GIB,
|
||||
)
|
||||
)
|
||||
assert decision.action == "drain"
|
||||
assert decision.pressure == "critical"
|
||||
@@ -0,0 +1,180 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
FLEET_PATH = ROOT / "ops" / "gitea-deploy-broker" / "runner_fleet.py"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fleet_module():
|
||||
spec = importlib.util.spec_from_file_location("appops_runner_fleet_test", FLEET_PATH)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_cleanup_candidates_exclude_unknown_attached_and_labeled_networks(
|
||||
fleet_module, monkeypatch
|
||||
):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
now = 1_800_000_000
|
||||
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
||||
old = "2026-01-01T00:00:00Z"
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_network_inventory",
|
||||
lambda: (
|
||||
[
|
||||
{
|
||||
"id": "a" * 64,
|
||||
"name": "GITEA-ACTIONS-TASK-12-WORKFLOW-ci-JOB-test-network",
|
||||
"created": old,
|
||||
"attached": 0,
|
||||
"labels": {},
|
||||
},
|
||||
{
|
||||
"id": "b" * 64,
|
||||
"name": "production_default",
|
||||
"created": old,
|
||||
"attached": 0,
|
||||
"labels": {},
|
||||
},
|
||||
{
|
||||
"id": "c" * 64,
|
||||
"name": "GITEA-ACTIONS-TASK-13-WORKFLOW-ci-JOB-test-network",
|
||||
"created": old,
|
||||
"attached": 1,
|
||||
"labels": {},
|
||||
},
|
||||
{
|
||||
"id": "d" * 64,
|
||||
"name": "GITEA-ACTIONS-TASK-14-WORKFLOW-ci-JOB-test-network",
|
||||
"created": old,
|
||||
"attached": 0,
|
||||
"labels": {"com.example.keep": "true"},
|
||||
},
|
||||
],
|
||||
2048,
|
||||
),
|
||||
)
|
||||
|
||||
candidates = service.cleanup_candidates()
|
||||
|
||||
assert [candidate["id"] for candidate in candidates] == ["a" * 64]
|
||||
assert candidates[0]["reason"] == "empty_gitea_actions_task_network_in_dedicated_dind"
|
||||
|
||||
|
||||
def test_automatic_cycle_only_deletes_exact_preview_candidates(fleet_module, monkeypatch):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
candidate = {
|
||||
"id": "a" * 64,
|
||||
"name": "GITEA-ACTIONS-TASK-12-WORKFLOW-ci-JOB-test-network",
|
||||
"ageSeconds": 3600,
|
||||
"reason": "empty_gitea_actions_task_network_in_dedicated_dind",
|
||||
}
|
||||
snapshot = {
|
||||
"status": "healthy",
|
||||
"networks": {"cleanupCandidates": [candidate]},
|
||||
"storage": {"status": "healthy"},
|
||||
"topology": {
|
||||
"dind": {
|
||||
"name": "gitea-runner-dind",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
"startedAt": "2026-09-16T18:00:00Z",
|
||||
"restartPolicy": "unless-stopped",
|
||||
},
|
||||
"runners": [],
|
||||
},
|
||||
}
|
||||
calls: list[list[str]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(service, "_dind", lambda args, timeout=20: calls.append(args) or "")
|
||||
monkeypatch.setattr(service, "_record", lambda **_kwargs: None)
|
||||
|
||||
result = service.automatic_cycle()
|
||||
|
||||
assert result["removedNetworks"] == 1
|
||||
assert calls == [["network", "rm", "a" * 64]]
|
||||
|
||||
|
||||
def test_recovery_budget_stops_restart_loops(fleet_module, monkeypatch):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
now = 1_800_000_000
|
||||
monkeypatch.setattr(fleet_module.time, "time", lambda: now)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"_events",
|
||||
lambda: [
|
||||
{
|
||||
"action": "restart",
|
||||
"target": "itworx-security-runner",
|
||||
"at": "2027-01-15T08:00:00Z",
|
||||
}
|
||||
for _ in range(fleet_module.RECOVERY_BUDGET)
|
||||
],
|
||||
)
|
||||
|
||||
assert service._recovery_available("itworx-security-runner") is False
|
||||
|
||||
|
||||
def test_fresh_starting_runner_is_not_restarted_before_health_grace_expires(
|
||||
fleet_module,
|
||||
):
|
||||
fresh = {
|
||||
"name": "itworx-validation-runner-2",
|
||||
"running": True,
|
||||
"health": "starting",
|
||||
"startedAt": "1970-01-01T00:16:00Z",
|
||||
}
|
||||
stale = {**fresh, "startedAt": "1970-01-01T00:13:00Z"}
|
||||
|
||||
assert fleet_module._container_is_healthy(fresh, now=1_000)
|
||||
assert not fleet_module._container_is_healthy(stale, now=1_000)
|
||||
|
||||
|
||||
def test_automatic_cycle_starts_only_an_exited_known_runner(fleet_module, monkeypatch):
|
||||
service = fleet_module.RunnerFleetService()
|
||||
snapshot = {
|
||||
"status": "healthy",
|
||||
"networks": {"cleanupCandidates": []},
|
||||
"storage": {"status": "healthy"},
|
||||
"topology": {
|
||||
"dind": {
|
||||
"name": "gitea-runner-dind",
|
||||
"running": True,
|
||||
"health": "healthy",
|
||||
"startedAt": "2026-09-16T18:00:00Z",
|
||||
"restartPolicy": "unless-stopped",
|
||||
},
|
||||
"runners": [
|
||||
{
|
||||
"name": "itworx-validation-runner-2",
|
||||
"running": False,
|
||||
"health": "unhealthy",
|
||||
},
|
||||
{
|
||||
"name": "itworx-security-runner",
|
||||
"running": True,
|
||||
"health": "unhealthy",
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
commands: list[list[str]] = []
|
||||
events: list[dict[str, object]] = []
|
||||
monkeypatch.setattr(service, "snapshot", lambda: snapshot)
|
||||
monkeypatch.setattr(service, "_recovery_available", lambda _target: True)
|
||||
monkeypatch.setattr(service, "_run", lambda args, timeout=20: commands.append(args) or "")
|
||||
monkeypatch.setattr(service, "_record", lambda **event: events.append(event))
|
||||
|
||||
service.automatic_cycle()
|
||||
|
||||
assert commands == [["docker", "start", "itworx-validation-runner-2"]]
|
||||
assert {event["action"] for event in events} == {"start", "restart.deferred"}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def test_runner_fleet_upgrade_action_is_fixed_and_registered():
|
||||
registry = (ROOT / "backend/app/services/action_registry.py").read_text(encoding="utf-8")
|
||||
execution = (ROOT / "backend/app/services/action_execution.py").read_text(encoding="utf-8")
|
||||
assert "'runner.fleet.upgrade_v2'" in registry
|
||||
assert "return 'tower/runner-fleet-v2'" in registry
|
||||
assert 'action == "runner.fleet.upgrade_v2"' in execution
|
||||
assert 'target.strip().lower() != "tower/runner-fleet-v2"' in execution
|
||||
|
||||
|
||||
def test_runner_fleet_upgrade_uses_signed_fixed_broker_path():
|
||||
broker = (ROOT / "backend/app/services/deployment_broker.py").read_text(encoding="utf-8")
|
||||
assert '"/v1/appops/runner-fleet/upgrade-v2"' in broker
|
||||
assert "runner_fleet_upgrade_v2" in broker
|
||||
|
||||
|
||||
def test_root_controller_upgrade_route_is_exact_revision_bound():
|
||||
status = (ROOT / "ops/gitea-deploy-broker/status_server.py").read_text(encoding="utf-8")
|
||||
assert 'RUNNER_FLEET_UPGRADE_PATH = "/v1/appops/runner-fleet/upgrade-v2"' in status
|
||||
assert "_self_upgrade_signature_is_valid" in status
|
||||
assert "_self_upgrade_preflight(revision)" in status
|
||||
assert 'f"{revision}:scripts/upgrade_ci_runner_fleet_v2.sh"' in status
|
||||
assert "execute_runner_fleet_v2(script_result.stdout)" in status
|
||||
|
||||
|
||||
def test_controller_artifact_carries_fixed_fleet_executor():
|
||||
status = (ROOT / "ops/gitea-deploy-broker/status_server.py").read_text(encoding="utf-8")
|
||||
prepare = (ROOT / "scripts/prepare_controller_artifact.py").read_text(encoding="utf-8")
|
||||
upgrade = (ROOT / "scripts/upgrade_deployment_controller.sh").read_text(encoding="utf-8")
|
||||
for text in (status, prepare, upgrade):
|
||||
assert "runner_capacity_controller.py" in text
|
||||
assert "runner_capacity_policy.py" in text
|
||||
assert "runner_fleet_upgrade_v2.py" in text
|
||||
|
||||
|
||||
def test_root_controller_sources_compile():
|
||||
for relative in (
|
||||
"ops/gitea-deploy-broker/status_server.py",
|
||||
"ops/gitea-deploy-broker/runner_fleet_upgrade_v2.py",
|
||||
"ops/gitea-deploy-broker/runner_capacity_controller.py",
|
||||
"ops/gitea-deploy-broker/runner_capacity_policy.py",
|
||||
):
|
||||
path = ROOT / relative
|
||||
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
||||
@@ -0,0 +1,101 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from backend.app.connectors import gitea_actions
|
||||
from backend.app.connectors.gitea_actions import GiteaActionsConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
|
||||
def _configure(monkeypatch) -> GiteaActionsConnector:
|
||||
monkeypatch.setitem(settings.apis, "GITEA_URL", "https://gitea.test")
|
||||
monkeypatch.setitem(settings.apis, "GITEA_ADMIN_TOKEN", "fixture-secret")
|
||||
monkeypatch.setitem(
|
||||
settings.apis,
|
||||
"GITEA_RUNNER_TARGETS_JSON",
|
||||
'{"linux-validation":"itworx-validation-runner-2",'
|
||||
'"mcp-validation":"itworx-security-runner"}',
|
||||
)
|
||||
monkeypatch.setattr(gitea_actions, "_CACHE", None)
|
||||
return GiteaActionsConnector()
|
||||
|
||||
|
||||
def _requests(created_at: str, *, busy: bool = False):
|
||||
def fake_request(path, params=None):
|
||||
params = params or {}
|
||||
if path.endswith("/runners"):
|
||||
return {
|
||||
"total_count": 1,
|
||||
"runners": [
|
||||
{
|
||||
"id": 17,
|
||||
"name": "validation-secondary",
|
||||
"status": "online",
|
||||
"busy": busy,
|
||||
"disabled": False,
|
||||
"labels": ["linux-validation", "ubuntu-latest"],
|
||||
}
|
||||
],
|
||||
}
|
||||
if params.get("status") == "queued":
|
||||
return {
|
||||
"total_count": 1,
|
||||
"jobs": [
|
||||
{
|
||||
"id": 91,
|
||||
"name": "full",
|
||||
"status": "queued",
|
||||
"created_at": created_at,
|
||||
"labels": ["linux-validation"],
|
||||
}
|
||||
],
|
||||
}
|
||||
return {"total_count": 0, "jobs": []}
|
||||
|
||||
return fake_request
|
||||
|
||||
|
||||
def test_old_unassigned_job_on_idle_matching_runner_is_recovery_candidate(monkeypatch):
|
||||
connector = _configure(monkeypatch)
|
||||
created_at = (datetime.now(UTC) - timedelta(minutes=12)).isoformat()
|
||||
monkeypatch.setattr(connector, "_request", _requests(created_at))
|
||||
|
||||
result = connector.snapshot(refresh=True)
|
||||
|
||||
assert result["severity"] == "critical"
|
||||
assert result["queue"]["items"][0]["reason"] == "awaiting_scheduler_assignment"
|
||||
assert result["queue"]["staleAssignmentRunnerLabels"] == ["linux-validation"]
|
||||
assert result["recommendedAction"] == {
|
||||
"intent": "runner.recover",
|
||||
"target": "itworx-validation-runner-2",
|
||||
"requiresApproval": False,
|
||||
"autoExecute": True,
|
||||
"reason": "Gitea Actions scheduler left a compatible idle runner unassigned for at least ten minutes.",
|
||||
"evidence": {
|
||||
"jobId": 91,
|
||||
"label": "linux-validation",
|
||||
"minimumAgeSeconds": 600,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_recent_unassigned_job_does_not_trigger_recovery(monkeypatch):
|
||||
connector = _configure(monkeypatch)
|
||||
created_at = (datetime.now(UTC) - timedelta(minutes=2)).isoformat()
|
||||
monkeypatch.setattr(connector, "_request", _requests(created_at))
|
||||
|
||||
result = connector.snapshot(refresh=True)
|
||||
|
||||
assert result["queue"]["items"][0]["reason"] == "awaiting_scheduler_assignment"
|
||||
assert result["queue"]["staleAssignmentRunnerLabels"] == []
|
||||
assert result["recommendedAction"] is None
|
||||
|
||||
|
||||
def test_busy_matching_runner_is_never_restarted_for_waiting_queue(monkeypatch):
|
||||
connector = _configure(monkeypatch)
|
||||
created_at = (datetime.now(UTC) - timedelta(minutes=12)).isoformat()
|
||||
monkeypatch.setattr(connector, "_request", _requests(created_at, busy=True))
|
||||
|
||||
result = connector.snapshot(refresh=True)
|
||||
|
||||
assert result["queue"]["items"][0]["reason"] == "all_matching_runners_are_busy"
|
||||
assert result["queue"]["staleAssignmentRunnerLabels"] == []
|
||||
assert result["recommendedAction"] is None
|
||||
@@ -0,0 +1,271 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import Session, SQLModel, create_engine, select
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "secret_storage.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APPOPS_DATA_DIR", str(tmp_path / "data"))
|
||||
monkeypatch.setenv("APPOPS_ENV", "development")
|
||||
monkeypatch.setenv("APPOPS_SECRET_KEY", "test-secret-storage-master-key")
|
||||
monkeypatch.delenv("APPOPS_SECRET_KEY_FILE", raising=False)
|
||||
for name in [
|
||||
"PORTAINER_URL",
|
||||
"PORTAINER_API_KEY",
|
||||
"PORTAINER_ENDPOINT_ID",
|
||||
"UNRAID_URL",
|
||||
"UNRAID_API_KEY",
|
||||
"SONARR_URL",
|
||||
"SONARR_API_KEY",
|
||||
"WIDEFROG_URL",
|
||||
"WIDEFROG_API_KEY",
|
||||
]:
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
from backend.app.main import app
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
|
||||
previous = {
|
||||
"db_path": settings.db_path,
|
||||
"data_dir": settings.data_dir,
|
||||
"environment": settings.environment,
|
||||
"secret_key": getattr(settings, "secret_key", None),
|
||||
"secret_key_source": getattr(settings, "secret_key_source", None),
|
||||
"apis": dict(settings.apis),
|
||||
}
|
||||
settings.db_path = str(db_path)
|
||||
settings.data_dir = str(tmp_path / "data")
|
||||
settings.environment = "development"
|
||||
settings.secret_key = "test-secret-storage-master-key"
|
||||
settings.secret_key_source = "env"
|
||||
settings.secret_key_file = None
|
||||
for key in previous["apis"]:
|
||||
if key.endswith("_API_KEY") or key in {"PORTAINER_URL", "PORTAINER_ENDPOINT_ID", "UNRAID_URL", "SONARR_URL", "WIDEFROG_URL"}:
|
||||
settings.apis[key] = None
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
settings.db_path = previous["db_path"]
|
||||
settings.data_dir = previous["data_dir"]
|
||||
settings.environment = previous["environment"]
|
||||
settings.secret_key = previous["secret_key"]
|
||||
settings.secret_key_source = previous["secret_key_source"]
|
||||
settings.apis = previous["apis"]
|
||||
|
||||
|
||||
def test_secret_store_encrypts_at_rest_and_decrypts_internally(api_client):
|
||||
from backend.app.db.models import Secret
|
||||
from backend.app.db.session import engine
|
||||
from backend.app.services.secret_store import get_secret, has_secret, list_secret_status, set_secret
|
||||
|
||||
secret_value = "PORTAINER_SECRET_AT_REST_VALUE"
|
||||
with Session(engine) as session:
|
||||
set_secret(session, "PORTAINER_API_KEY", secret_value, used_by=["portainer"])
|
||||
row = session.get(Secret, "PORTAINER_API_KEY")
|
||||
|
||||
assert row is not None
|
||||
assert row.encrypted_value
|
||||
assert row.encrypted_value != secret_value
|
||||
assert secret_value not in row.encrypted_value
|
||||
assert has_secret(session, "PORTAINER_API_KEY") is True
|
||||
assert get_secret(session, "PORTAINER_API_KEY") == secret_value
|
||||
|
||||
statuses = list_secret_status(session)
|
||||
portainer_status = next(item for item in statuses if item["secretRef"] == "PORTAINER_API_KEY")
|
||||
assert portainer_status["hasSecret"] is True
|
||||
assert portainer_status["status"] == "configured"
|
||||
assert "value" not in portainer_status
|
||||
assert secret_value not in str(statuses)
|
||||
|
||||
|
||||
def test_connector_settings_store_secret_encrypted_and_keep_response_safe(api_client):
|
||||
from backend.app.db.models import AppSetting, Secret
|
||||
from backend.app.db.session import engine
|
||||
|
||||
secret_value = "PORTAINER_PATCH_SECRET"
|
||||
response = api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={
|
||||
"url": "http://198.18.10.150:9200",
|
||||
"endpointId": "1",
|
||||
"verifyTls": False,
|
||||
"apiKey": secret_value,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["connectorId"] == "portainer"
|
||||
assert payload["configured"] is True
|
||||
assert payload["hasSecret"] is True
|
||||
assert payload["secretRef"] == "PORTAINER_API_KEY"
|
||||
assert payload["secretStatus"]["status"] == "configured"
|
||||
assert payload["safeConfig"] == {
|
||||
"url": "http://198.18.10.150:9200",
|
||||
"endpointId": "1",
|
||||
"verifyTls": False,
|
||||
}
|
||||
assert secret_value not in str(payload)
|
||||
|
||||
with Session(engine) as session:
|
||||
row = session.get(Secret, "PORTAINER_API_KEY")
|
||||
assert row is not None
|
||||
assert secret_value not in row.encrypted_value
|
||||
|
||||
settings_rows = session.exec(select(AppSetting)).all()
|
||||
assert secret_value not in str([item.value_json for item in settings_rows])
|
||||
|
||||
listed = api_client.get("/api/settings/connectors").json()
|
||||
assert secret_value not in str(listed)
|
||||
assert next(item for item in listed if item["connectorId"] == "portainer")["hasSecret"] is True
|
||||
|
||||
|
||||
def test_runtime_exports_and_secrets_endpoint_never_show_secret_values(api_client):
|
||||
secret_values = {
|
||||
"PORTAINER_API_KEY": "PORTAINER_EXPORT_SECRET",
|
||||
"UNRAID_API_KEY": "UNRAID_EXPORT_SECRET",
|
||||
"SONARR_API_KEY": "SONARR_EXPORT_SECRET",
|
||||
"WIDEFROG_API_KEY": "WIDEFROG_EXPORT_SECRET",
|
||||
}
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={"url": "http://portainer.local", "endpointId": "1", "verifyTls": False, "apiKey": secret_values["PORTAINER_API_KEY"]},
|
||||
)
|
||||
api_client.patch("/api/settings/connectors/unraid", json={"url": "http://unraid.local", "apiKey": secret_values["UNRAID_API_KEY"]})
|
||||
api_client.patch("/api/settings/connectors/sonarr", json={"url": "http://sonarr.local", "apiKey": secret_values["SONARR_API_KEY"]})
|
||||
api_client.patch("/api/settings/connectors/widefrog", json={"url": "http://widefrog.local", "apiKey": secret_values["WIDEFROG_API_KEY"]})
|
||||
|
||||
for path in [
|
||||
"/api/settings/connectors",
|
||||
"/api/settings/secrets",
|
||||
"/api/setup/status",
|
||||
"/api/runtime/info",
|
||||
"/api/admin/export/config",
|
||||
"/api/admin/export/policies",
|
||||
"/api/admin/export/inventory",
|
||||
"/api/admin/export/audit-summary",
|
||||
]:
|
||||
response = api_client.get(path)
|
||||
assert response.status_code == 200
|
||||
body = response.text
|
||||
for secret in secret_values.values():
|
||||
assert secret not in body
|
||||
assert "test-secret-storage-master-key" not in body
|
||||
|
||||
secrets_payload = api_client.get("/api/settings/secrets").json()
|
||||
refs = {item["secretRef"] for item in secrets_payload["secrets"]}
|
||||
assert {"PORTAINER_API_KEY", "UNRAID_API_KEY", "SONARR_API_KEY", "WIDEFROG_API_KEY"}.issubset(refs)
|
||||
assert secrets_payload["storage"]["encryptedAtRest"] is True
|
||||
assert secrets_payload["storage"]["keySource"] == "env"
|
||||
|
||||
|
||||
def test_setup_status_does_not_touch_secret_last_used(api_client):
|
||||
from backend.app.db.models import Secret
|
||||
from backend.app.db.session import engine
|
||||
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={
|
||||
"url": "http://portainer.local",
|
||||
"endpointId": "1",
|
||||
"verifyTls": False,
|
||||
"apiKey": "PORTAINER_READONLY_STATUS_SECRET",
|
||||
},
|
||||
)
|
||||
|
||||
with Session(engine) as session:
|
||||
row = session.get(Secret, "PORTAINER_API_KEY")
|
||||
assert row is not None
|
||||
row.last_used_at = None
|
||||
session.add(row)
|
||||
session.commit()
|
||||
|
||||
response = api_client.get("/api/setup/status")
|
||||
|
||||
assert response.status_code == 200
|
||||
with Session(engine) as session:
|
||||
row = session.get(Secret, "PORTAINER_API_KEY")
|
||||
assert row is not None
|
||||
assert row.last_used_at is None
|
||||
|
||||
|
||||
def test_legacy_plaintext_connector_secret_is_migrated(api_client):
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db.models import AppSetting, Secret
|
||||
from backend.app.db.session import engine
|
||||
from backend.app.services.setup_service import apply_stored_connector_settings, serialize_connector_settings
|
||||
|
||||
legacy_secret = "LEGACY_PORTAINER_PLAINTEXT"
|
||||
with Session(engine) as session:
|
||||
session.add(AppSetting(key="connector:portainer:config", value_json={"url": "http://legacy.local", "endpointId": "1"}, is_secret=False))
|
||||
session.add(AppSetting(key="connector:portainer:secret", value_json={"value": legacy_secret}, is_secret=True))
|
||||
session.commit()
|
||||
|
||||
apply_stored_connector_settings(session)
|
||||
payload = serialize_connector_settings(session, "portainer")
|
||||
legacy_row = session.get(AppSetting, "connector:portainer:secret")
|
||||
secret_row = session.get(Secret, "PORTAINER_API_KEY")
|
||||
|
||||
assert payload["hasSecret"] is True
|
||||
assert payload["secretRef"] == "PORTAINER_API_KEY"
|
||||
assert settings.apis["PORTAINER_API_KEY"] == legacy_secret
|
||||
assert secret_row is not None
|
||||
assert legacy_secret not in secret_row.encrypted_value
|
||||
assert legacy_secret not in str(legacy_row.value_json)
|
||||
assert legacy_row.value_json["secretRef"] == "PORTAINER_API_KEY"
|
||||
|
||||
response = api_client.get("/api/settings/connectors")
|
||||
assert response.status_code == 200
|
||||
assert legacy_secret not in response.text
|
||||
|
||||
|
||||
def test_secret_redaction_masks_common_tokens_and_known_values(api_client):
|
||||
from backend.app.db.session import engine
|
||||
from backend.app.services.secret_store import redact_text, set_secret
|
||||
|
||||
with Session(engine) as session:
|
||||
set_secret(session, "SONARR_API_KEY", "KNOWN_SONARR_SECRET", used_by=["sonarr"])
|
||||
message = (
|
||||
"Bearer abcdefghijklmnop x-api-key: XAPISECRET "
|
||||
"apiKey=INLINESECRET password=hunter2 known=KNOWN_SONARR_SECRET"
|
||||
)
|
||||
redacted = redact_text(message, session=session)
|
||||
|
||||
assert "abcdefghijklmnop" not in redacted
|
||||
assert "XAPISECRET" not in redacted
|
||||
assert "INLINESECRET" not in redacted
|
||||
assert "hunter2" not in redacted
|
||||
assert "KNOWN_SONARR_SECRET" not in redacted
|
||||
assert "[REDACTED]" in redacted
|
||||
|
||||
|
||||
def test_production_missing_secret_key_reports_warning(api_client):
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.services.secret_store import secret_storage_status
|
||||
|
||||
previous = (settings.environment, settings.secret_key, settings.secret_key_source)
|
||||
settings.environment = "production"
|
||||
settings.secret_key = None
|
||||
settings.secret_key_source = "missing"
|
||||
|
||||
status = secret_storage_status()
|
||||
|
||||
assert status["productionReady"] is False
|
||||
assert status["keySource"] == "missing"
|
||||
assert status["warnings"]
|
||||
assert "APPOPS_SECRET_KEY" in status["warnings"][0]
|
||||
|
||||
settings.environment, settings.secret_key, settings.secret_key_source = previous
|
||||
@@ -0,0 +1,122 @@
|
||||
import socket
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from starlette.requests import Request
|
||||
|
||||
from backend.app.core.network_security import UnsafeConnectorUrl, validate_connector_url
|
||||
|
||||
|
||||
def test_connector_url_allows_private_lan_hosts(monkeypatch):
|
||||
monkeypatch.setattr(socket, "getaddrinfo", lambda *_args, **_kwargs: (_ for _ in ()).throw(socket.gaierror()))
|
||||
assert validate_connector_url("http://198.18.1.10:9000/") == "http://198.18.1.10:9000"
|
||||
assert validate_connector_url("https://sonarr.internal/api") == "https://sonarr.internal/api"
|
||||
|
||||
|
||||
def test_connector_url_blocks_unsafe_targets():
|
||||
blocked = [
|
||||
"file:///etc/passwd",
|
||||
"http://localhost:8080",
|
||||
"http://127.0.0.1:8080",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://user:pass@host.local",
|
||||
"http://0.0.0.0:80",
|
||||
]
|
||||
for value in blocked:
|
||||
try:
|
||||
validate_connector_url(value)
|
||||
except UnsafeConnectorUrl:
|
||||
continue
|
||||
raise AssertionError(f"unsafe connector URL accepted: {value}")
|
||||
|
||||
|
||||
def test_connector_url_blocks_dns_names_resolving_to_loopback(monkeypatch):
|
||||
monkeypatch.setattr(socket, "getaddrinfo", lambda *_args, **_kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 80))])
|
||||
with pytest.raises(UnsafeConnectorUrl):
|
||||
validate_connector_url("http://connector.internal")
|
||||
|
||||
|
||||
def _request(headers: list[tuple[bytes, bytes]] | None = None, client_host: str = "198.18.10.20") -> Request:
|
||||
return Request({"type": "http", "method": "POST", "path": "/api/actions/plan", "headers": headers or [], "client": (client_host, 12345)})
|
||||
|
||||
|
||||
def test_operator_boundary_protects_mutations_and_accepts_valid_token():
|
||||
from backend.app.core import security
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = (settings.operator_auth_required, settings.operator_token, security.settings.operator_auth_required, security.settings.operator_token)
|
||||
settings.operator_auth_required = True
|
||||
settings.operator_token = "test-operator-boundary-token"
|
||||
security.settings.operator_auth_required = True
|
||||
security.settings.operator_token = "test-operator-boundary-token"
|
||||
try:
|
||||
with pytest.raises(HTTPException) as missing:
|
||||
security.require_operator_request(_request())
|
||||
assert missing.value.status_code == 401
|
||||
request = _request([(b"x-appops-operator-token", b"test-operator-boundary-token")])
|
||||
assert security.require_operator_request(request) == "operator"
|
||||
finally:
|
||||
settings.operator_auth_required, settings.operator_token, security.settings.operator_auth_required, security.settings.operator_token = previous
|
||||
|
||||
|
||||
def test_disabled_operator_boundary_ignores_stale_configured_token():
|
||||
from backend.app.core import security
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = (settings.operator_auth_required, settings.operator_token)
|
||||
settings.operator_auth_required = False
|
||||
settings.operator_token = "stale-operator-token-that-no-longer-unlocks"
|
||||
try:
|
||||
assert security.require_operator_request(_request()) == "private_unlocked"
|
||||
assert security.operator_session_status(_request()) == {
|
||||
"authorized": True,
|
||||
"mode": "private",
|
||||
"identity": None,
|
||||
}
|
||||
finally:
|
||||
settings.operator_auth_required, settings.operator_token = previous
|
||||
|
||||
|
||||
def test_operator_boundary_accepts_allowed_authentik_account_only_from_trusted_proxy():
|
||||
from backend.app.core import security
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = (
|
||||
settings.operator_auth_required,
|
||||
settings.operator_token,
|
||||
settings.operator_proxy_auth_enabled,
|
||||
settings.operator_proxy_trusted_networks,
|
||||
settings.operator_proxy_users,
|
||||
settings.operator_proxy_groups,
|
||||
)
|
||||
settings.operator_auth_required = True
|
||||
settings.operator_token = "fallback-operator-token-value"
|
||||
settings.operator_proxy_auth_enabled = True
|
||||
settings.operator_proxy_trusted_networks = ["203.0.113.0/16"]
|
||||
settings.operator_proxy_users = ["example-host"]
|
||||
settings.operator_proxy_groups = ["appops-operators"]
|
||||
headers = [(b"x-authentik-username", b"example-host"), (b"x-authentik-groups", b"users|appops-operators")]
|
||||
try:
|
||||
assert security.require_operator_request(_request(headers, "203.0.113.8")) == "authentik:example-host"
|
||||
assert security.operator_session_status(_request(headers, "203.0.113.8")) == {
|
||||
"authorized": True,
|
||||
"mode": "account",
|
||||
"identity": "example-host",
|
||||
}
|
||||
assert security.require_operator_request(_request(headers + [(b"x-appops-operator-token", b"stale-token")], "203.0.113.8")) == "authentik:example-host"
|
||||
with pytest.raises(HTTPException) as spoofed:
|
||||
security.require_operator_request(_request(headers, "198.18.10.20"))
|
||||
assert spoofed.value.status_code == 401
|
||||
denied_headers = [(b"x-authentik-username", b"someone-else"), (b"x-authentik-groups", b"users")]
|
||||
with pytest.raises(HTTPException) as denied:
|
||||
security.require_operator_request(_request(denied_headers, "203.0.113.8"))
|
||||
assert denied.value.status_code == 401
|
||||
finally:
|
||||
(
|
||||
settings.operator_auth_required,
|
||||
settings.operator_token,
|
||||
settings.operator_proxy_auth_enabled,
|
||||
settings.operator_proxy_trusted_networks,
|
||||
settings.operator_proxy_users,
|
||||
settings.operator_proxy_groups,
|
||||
) = previous
|
||||
@@ -0,0 +1,343 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "setup_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
for name in [
|
||||
"PORTAINER_URL",
|
||||
"PORTAINER_API_KEY",
|
||||
"PORTAINER_ENDPOINT_ID",
|
||||
"UNRAID_URL",
|
||||
"UNRAID_API_KEY",
|
||||
"AUTHENTIK_URL",
|
||||
"AUTHENTIK_TOKEN",
|
||||
"AUTHENTIK_OUTPOST_URL",
|
||||
"RADARR_URL",
|
||||
"RADARR_API_KEY",
|
||||
"PROWLARR_URL",
|
||||
"PROWLARR_API_KEY",
|
||||
]:
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
from backend.app.main import app
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
|
||||
for name in [
|
||||
"PORTAINER_URL",
|
||||
"PORTAINER_API_KEY",
|
||||
"PORTAINER_ENDPOINT_ID",
|
||||
"UNRAID_URL",
|
||||
"UNRAID_API_KEY",
|
||||
"AUTHENTIK_URL",
|
||||
"AUTHENTIK_TOKEN",
|
||||
"AUTHENTIK_OUTPOST_URL",
|
||||
"RADARR_URL",
|
||||
"RADARR_API_KEY",
|
||||
"PROWLARR_URL",
|
||||
"PROWLARR_API_KEY",
|
||||
]:
|
||||
settings.apis[name] = None
|
||||
settings.verify_tls = False
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
|
||||
def test_setup_status_initial_is_pending_and_secret_safe(api_client):
|
||||
payload = api_client.get("/api/setup/status").json()
|
||||
|
||||
assert payload["firstRunCompleted"] is False
|
||||
assert payload["summary"]["containersDiscovered"] == 0
|
||||
assert payload["summary"]["appsManaged"] >= 0
|
||||
assert payload["summary"]["connectorsMissingConfig"] >= 1
|
||||
assert {step["id"]: step["status"] for step in payload["steps"]}["portainer"] == "pending"
|
||||
assert {step["id"]: step["status"] for step in payload["steps"]}["discovery"] == "pending"
|
||||
assert "test-voice-token" not in str(payload)
|
||||
|
||||
|
||||
def test_portainer_config_save_never_returns_secret(api_client):
|
||||
secret = "PORTAINER_SUPER_SECRET"
|
||||
|
||||
payload = api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={
|
||||
"url": "http://198.18.10.150:9443",
|
||||
"endpointId": "1",
|
||||
"verifyTls": False,
|
||||
"apiKey": secret,
|
||||
},
|
||||
).json()
|
||||
|
||||
assert payload["connectorId"] == "portainer"
|
||||
assert payload["configured"] is True
|
||||
assert payload["hasSecret"] is True
|
||||
assert payload["missing"] == []
|
||||
assert payload["safeConfig"] == {
|
||||
"url": "http://198.18.10.150:9443",
|
||||
"endpointId": "1",
|
||||
"verifyTls": False,
|
||||
}
|
||||
assert secret not in str(payload)
|
||||
|
||||
all_settings = api_client.get("/api/settings/connectors").json()
|
||||
assert secret not in str(all_settings)
|
||||
assert next(item for item in all_settings if item["connectorId"] == "portainer")["hasSecret"] is True
|
||||
|
||||
|
||||
def test_portainer_config_save_trims_safe_fields(api_client):
|
||||
payload = api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={
|
||||
"url": " http://198.18.10.150:9200/ ",
|
||||
"endpointId": " 1 ",
|
||||
"verifyTls": False,
|
||||
"apiKey": "PORTAINER_SUPER_SECRET",
|
||||
},
|
||||
).json()
|
||||
|
||||
assert payload["safeConfig"]["url"] == "http://198.18.10.150:9200"
|
||||
assert payload["safeConfig"]["endpointId"] == "1"
|
||||
|
||||
|
||||
def test_portainer_config_save_defaults_endpoint_id(api_client):
|
||||
payload = api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={
|
||||
"url": "http://198.18.10.150:9200",
|
||||
"endpointId": "",
|
||||
"verifyTls": False,
|
||||
"apiKey": "PORTAINER_SUPER_SECRET",
|
||||
},
|
||||
).json()
|
||||
|
||||
assert payload["safeConfig"]["endpointId"] == "1"
|
||||
assert "PORTAINER_ENDPOINT_ID" not in payload["missing"]
|
||||
|
||||
|
||||
def test_setup_test_portainer_missing_config_and_redaction(api_client):
|
||||
response = api_client.post("/api/setup/test/portainer")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "missing_config"
|
||||
assert "PORTAINER_API_KEY" in payload["requiredFields"]
|
||||
|
||||
|
||||
def test_setup_test_portainer_success_summary(api_client, monkeypatch):
|
||||
secret = "PORTAINER_SUPER_SECRET"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={"url": "http://portainer.local", "endpointId": "1", "verifyTls": False, "apiKey": secret},
|
||||
)
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.test_connection", lambda self: {"status": "connected"}, raising=True)
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.list_containers", lambda self: {"status": "ok", "containers": [{"id": "one"}]}, raising=True)
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.list_stacks", lambda self: {"status": "ok", "stacks": [{"id": "stack"}]}, raising=True)
|
||||
|
||||
payload = api_client.post("/api/setup/test/portainer").json()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "connected"
|
||||
assert payload["summary"]["containers"] == 1
|
||||
assert payload["summary"]["stacks"] == 1
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_setup_test_portainer_endpoint_error_is_actionable(api_client, monkeypatch):
|
||||
secret = "PORTAINER_SUPER_SECRET"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={"url": "http://portainer.local", "endpointId": "99", "verifyTls": False, "apiKey": secret},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.portainer.PortainerConnector.test_connection",
|
||||
lambda self: {"status": "endpoint_error", "reason": "Portainer API error: HTTP 404 endpoint not found"},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = api_client.post("/api/setup/test/portainer").json()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "endpoint_error"
|
||||
assert payload["requiredFields"] == ["PORTAINER_ENDPOINT_ID"]
|
||||
assert payload["safeConfig"]["endpointId"] == "99"
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_setup_test_portainer_container_list_error_marks_endpoint(api_client, monkeypatch):
|
||||
secret = "PORTAINER_SUPER_SECRET"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={"url": "http://portainer.local", "endpointId": "1", "verifyTls": False, "apiKey": secret},
|
||||
)
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.test_connection", lambda self: {"status": "connected"}, raising=True)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.portainer.PortainerConnector.list_containers",
|
||||
lambda self: {"status": "error", "reason": "Portainer API error: HTTP 404 endpoint not found", "containers": []},
|
||||
raising=True,
|
||||
)
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.list_stacks", lambda self: {"status": "ok", "stacks": []}, raising=True)
|
||||
|
||||
payload = api_client.post("/api/setup/test/portainer").json()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "endpoint_error"
|
||||
assert payload["summary"]["endpointReachable"] is False
|
||||
assert payload["requiredFields"] == ["PORTAINER_ENDPOINT_ID"]
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_unraid_config_save_never_returns_secret(api_client):
|
||||
secret = "UNRAID_SUPER_SECRET"
|
||||
|
||||
payload = api_client.patch(
|
||||
"/api/settings/connectors/unraid",
|
||||
json={"url": "http://unraid.local", "verifyTls": False, "apiKey": secret},
|
||||
).json()
|
||||
|
||||
assert payload["connectorId"] == "unraid"
|
||||
assert payload["configured"] is True
|
||||
assert payload["hasSecret"] is True
|
||||
assert payload["safeConfig"] == {"url": "http://unraid.local", "verifyTls": False}
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_setup_test_authentik_404_marks_url_error(api_client, monkeypatch):
|
||||
secret = "AUTHENTIK_SUPER_SECRET"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/authentik",
|
||||
json={"url": "https://auth.example.local/if/flow/default-authentication-flow", "verifyTls": False, "apiKey": secret},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.authentik.AuthentikConnector.test_connection",
|
||||
lambda self: {"status": "unreachable", "reason": "Authentik API error: HTTP 404 <!DOCTYPE html>"},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = api_client.post("/api/setup/test/connector/authentik").json()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "url_error"
|
||||
assert payload["requiredFields"] == ["AUTHENTIK_URL"]
|
||||
assert "root/base URL" in payload["message"]
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_setup_test_authentik_auth_failed_marks_token(api_client, monkeypatch):
|
||||
secret = "AUTHENTIK_SUPER_SECRET"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/authentik",
|
||||
json={"url": "https://auth.example.local", "verifyTls": False, "apiKey": secret},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.authentik.AuthentikConnector.test_connection",
|
||||
lambda self: {"status": "auth_failed", "reason": "auth_failed"},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = api_client.post("/api/setup/test/connector/authentik").json()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "auth_failed"
|
||||
assert payload["requiredFields"] == ["AUTHENTIK_TOKEN"]
|
||||
assert "token ongeldig of verlopen" in payload["message"]
|
||||
assert "API Token" in payload["message"]
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_unraid_partial_config_update_preserves_url_and_secret(api_client):
|
||||
secret = "UNRAID_SUPER_SECRET"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/unraid",
|
||||
json={"url": "https://unraid.local", "verifyTls": True, "apiKey": secret},
|
||||
)
|
||||
|
||||
payload = api_client.patch(
|
||||
"/api/settings/connectors/unraid",
|
||||
json={"verifyTls": False},
|
||||
).json()
|
||||
|
||||
assert payload["configured"] is True
|
||||
assert payload["hasSecret"] is True
|
||||
assert payload["safeConfig"] == {"url": "https://unraid.local", "verifyTls": False}
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_setup_test_prowlarr_uses_connector_test(api_client, monkeypatch):
|
||||
secret = "PROWLARR_SUPER_SECRET"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/prowlarr",
|
||||
json={"url": "http://prowlarr.local:9696", "verifyTls": False, "apiKey": secret},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.prowlarr.ProwlarrConnector.test_connection",
|
||||
lambda self: {"status": "connected", "version": "1.2.3"},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = api_client.post("/api/setup/test/connector/prowlarr").json()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "connected"
|
||||
assert payload["connected"] is True
|
||||
assert payload["settings"]["safeConfig"]["url"] == "http://prowlarr.local:9696"
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_setup_test_prowlarr_duplicate_radarr_url_is_actionable(api_client, monkeypatch):
|
||||
prowlarr_secret = "PROWLARR_SUPER_SECRET"
|
||||
radarr_secret = "RADARR_SUPER_SECRET"
|
||||
shared_url = "http://media.local:7878"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/radarr",
|
||||
json={"url": shared_url, "verifyTls": False, "apiKey": radarr_secret},
|
||||
)
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/prowlarr",
|
||||
json={"url": shared_url, "verifyTls": False, "apiKey": prowlarr_secret},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.prowlarr.ProwlarrConnector.test_connection",
|
||||
lambda self: {"status": "error", "reason": "unauthorized"},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = api_client.post("/api/setup/test/connector/prowlarr").json()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "url_mismatch"
|
||||
assert payload["requiredFields"] == ["PROWLARR_URL", "PROWLARR_API_KEY"]
|
||||
assert "Radarr" in payload["message"]
|
||||
assert "Prowlarr" in payload["message"]
|
||||
assert prowlarr_secret not in str(payload)
|
||||
assert radarr_secret not in str(payload)
|
||||
|
||||
|
||||
def test_complete_setup_sets_first_run_completed(api_client):
|
||||
payload = api_client.post("/api/setup/complete").json()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"]["firstRunCompleted"] is True
|
||||
assert payload["status"]["setupCompletedAt"]
|
||||
@@ -0,0 +1,535 @@
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, Session, create_engine, select
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "setup_reconcile.db"
|
||||
data_dir = tmp_path / "data"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APPOPS_DATA_DIR", str(data_dir))
|
||||
monkeypatch.setenv("APPOPS_ENV", "production")
|
||||
monkeypatch.setenv("APPOPS_PORT", "1216")
|
||||
monkeypatch.setenv("APPOPS_SECRET_KEY", "test-secret-key-with-enough-length")
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
for name in [
|
||||
"PORTAINER_URL",
|
||||
"PORTAINER_API_KEY",
|
||||
"PORTAINER_ENDPOINT_ID",
|
||||
"UNRAID_URL",
|
||||
"UNRAID_API_KEY",
|
||||
"NPM_URL",
|
||||
"NPM_IDENTITY",
|
||||
"NPM_SECRET",
|
||||
"AUTHENTIK_URL",
|
||||
"AUTHENTIK_TOKEN",
|
||||
"HOME_ASSISTANT_URL",
|
||||
"HOME_ASSISTANT_TOKEN",
|
||||
]:
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
from backend.app.main import app
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
|
||||
settings.db_path = str(db_path)
|
||||
settings.data_dir = str(data_dir)
|
||||
settings.environment = "production"
|
||||
settings.port = 1216
|
||||
settings.secret_key = "test-secret-key-with-enough-length"
|
||||
settings.secret_key_source = "env"
|
||||
settings.voice_token = "test-voice-token"
|
||||
for name in [
|
||||
"PORTAINER_URL",
|
||||
"PORTAINER_API_KEY",
|
||||
"PORTAINER_ENDPOINT_ID",
|
||||
"UNRAID_URL",
|
||||
"UNRAID_API_KEY",
|
||||
"NPM_URL",
|
||||
"NPM_IDENTITY",
|
||||
"NPM_SECRET",
|
||||
"AUTHENTIK_URL",
|
||||
"AUTHENTIK_TOKEN",
|
||||
"HOME_ASSISTANT_URL",
|
||||
"HOME_ASSISTANT_TOKEN",
|
||||
]:
|
||||
settings.apis[name] = None
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
try:
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
finally:
|
||||
settings.environment = "development"
|
||||
settings.port = 1216
|
||||
settings.secret_key = None
|
||||
settings.secret_key_source = "missing"
|
||||
settings.voice_token = None
|
||||
|
||||
|
||||
def _make_required_complete() -> None:
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.db.models import App, AppContainerMapping, Connector, DiscoveryRun
|
||||
|
||||
with Session(db_session.engine) as session:
|
||||
portainer = session.get(Connector, "portainer")
|
||||
portainer.status = "connected"
|
||||
session.add(portainer)
|
||||
run = DiscoveryRun(
|
||||
id="run-required-complete",
|
||||
source="portainer",
|
||||
finished_at=datetime.now(UTC).replace(tzinfo=None),
|
||||
status="success",
|
||||
containers_found=1,
|
||||
apps_matched=1,
|
||||
)
|
||||
session.add(run)
|
||||
app = session.get(App, "widefrog")
|
||||
app.needs_review = False
|
||||
app.connector_status = "connected"
|
||||
app.container_status = "running"
|
||||
session.add(app)
|
||||
session.add(
|
||||
AppContainerMapping(
|
||||
id="mapping-widefrog",
|
||||
app_id="widefrog",
|
||||
container_id="container-widefrog",
|
||||
container_name="widefrog",
|
||||
last_seen_run_id=run.id,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
|
||||
def test_required_complete_optional_missing_is_live_ready_after_reconcile(api_client):
|
||||
_make_required_complete()
|
||||
|
||||
result = api_client.post("/api/setup/reconcile").json()
|
||||
status = result["after"]
|
||||
|
||||
assert result["success"] is True
|
||||
assert status["firstRunCompleted"] is True
|
||||
assert status["setupComplete"] is True
|
||||
assert status["liveReady"] is True
|
||||
assert status["coverageComplete"] is False
|
||||
assert status["securityCoveragePartial"] is True
|
||||
assert status["requiredStepsComplete"] is True
|
||||
assert status["reviewBlockers"] == 0
|
||||
assert status["managedApps"] == len(api_client.get("/api/apps").json())
|
||||
assert status["summary"]["appsManaged"] == status["managedApps"]
|
||||
assert "nginx_proxy_manager" in status["missingOptionalSources"]
|
||||
assert "AUTHENTIK_TOKEN" not in str(status)
|
||||
|
||||
|
||||
def test_setup_counts_current_container_names_not_historical_docker_ids(api_client):
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.db.models import AppContainerMapping, DiscoveredContainer, DiscoveryRun
|
||||
from backend.app.services.inventory_discovery import invalidate_discovery_cache
|
||||
|
||||
with Session(db_session.engine) as session:
|
||||
old_run = DiscoveryRun(
|
||||
id="run-old-container-id",
|
||||
source="portainer",
|
||||
started_at=datetime(2026, 1, 1, 10, 0, 0),
|
||||
finished_at=datetime(2026, 1, 1, 10, 1, 0),
|
||||
status="success",
|
||||
)
|
||||
current_run = DiscoveryRun(
|
||||
id="run-current-container-id",
|
||||
source="portainer",
|
||||
started_at=datetime(2026, 1, 2, 10, 0, 0),
|
||||
finished_at=datetime(2026, 1, 2, 10, 1, 0),
|
||||
status="success",
|
||||
)
|
||||
session.add(old_run)
|
||||
session.add(current_run)
|
||||
session.add(DiscoveredContainer(
|
||||
id="discovered-old-id",
|
||||
discovery_run_id=old_run.id,
|
||||
container_id="docker-old-id",
|
||||
name="recreated-app",
|
||||
image="local/recreated:old",
|
||||
state="exited",
|
||||
status="exited",
|
||||
import_status="unmapped",
|
||||
))
|
||||
session.add(DiscoveredContainer(
|
||||
id="discovered-current-id",
|
||||
discovery_run_id=current_run.id,
|
||||
container_id="docker-current-id",
|
||||
name="recreated-app",
|
||||
image="local/recreated:current",
|
||||
state="running",
|
||||
status="running",
|
||||
import_status="managed",
|
||||
matched_app_id="widefrog",
|
||||
))
|
||||
session.add(AppContainerMapping(
|
||||
id="mapping-recreated-current",
|
||||
app_id="widefrog",
|
||||
container_id="docker-current-id",
|
||||
container_name="recreated-app",
|
||||
last_seen_run_id=current_run.id,
|
||||
))
|
||||
session.commit()
|
||||
|
||||
invalidate_discovery_cache()
|
||||
status = api_client.get("/api/setup/status").json()
|
||||
|
||||
assert status["summary"]["containersDiscovered"] == 1
|
||||
assert status["summary"]["appsNeedReview"] == 0
|
||||
assert status["needsReview"] == status["summary"]["appsNeedReview"]
|
||||
|
||||
|
||||
def test_apps_without_registered_connector_use_none_status(api_client):
|
||||
apps = {app["id"]: app for app in api_client.get("/api/apps").json()}
|
||||
|
||||
assert apps["gitea"]["connectorStatus"] == "none"
|
||||
assert apps["jellyfin"]["connectorStatus"] == "none"
|
||||
|
||||
|
||||
def test_missing_optional_unraid_npm_auth_does_not_block_required_setup(api_client):
|
||||
_make_required_complete()
|
||||
status = api_client.post("/api/setup/reconcile").json()["after"]
|
||||
|
||||
assert status["setupComplete"] is True
|
||||
assert status["liveReady"] is True
|
||||
assert status["coverageComplete"] is False
|
||||
assert {step["id"]: step["status"] for step in status["steps"]}["unraid"] == "warning"
|
||||
assert {step["id"]: step["status"] for step in status["steps"]}["connectors"] == "warning"
|
||||
|
||||
|
||||
def test_connector_status_sync_applies_stored_settings_once(api_client, monkeypatch):
|
||||
_make_required_complete()
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.services import inventory_discovery, setup_service
|
||||
|
||||
calls = 0
|
||||
|
||||
def apply_once(session):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
|
||||
monkeypatch.setattr(setup_service, "apply_stored_connector_settings", apply_once)
|
||||
with Session(db_session.engine) as session:
|
||||
result = inventory_discovery.sync_app_connector_statuses(session)
|
||||
|
||||
assert result["success"] is True
|
||||
assert calls == 1
|
||||
|
||||
|
||||
def test_status_sync_refreshes_running_and_ignored_mapped_apps(api_client):
|
||||
_make_required_complete()
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.db.models import App, AppContainerMapping, DiscoveredContainer
|
||||
from backend.app.services import inventory_discovery
|
||||
|
||||
with Session(db_session.engine) as session:
|
||||
widefrog = session.get(App, "widefrog")
|
||||
widefrog.status = "warning"
|
||||
widefrog.container_status = "exited"
|
||||
session.add(widefrog)
|
||||
gitea = session.get(App, "gitea")
|
||||
gitea.status = "warning"
|
||||
gitea.container_status = "exited"
|
||||
session.add(gitea)
|
||||
session.add(
|
||||
DiscoveredContainer(
|
||||
id="discovered-widefrog-running",
|
||||
discovery_run_id="run-required-complete",
|
||||
container_id="container-widefrog",
|
||||
name="widefrog",
|
||||
image="widefrog:test",
|
||||
state="running",
|
||||
status="healthy",
|
||||
import_status="managed",
|
||||
matched_app_id="widefrog",
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
DiscoveredContainer(
|
||||
id="discovered-widefrog-retired-helper",
|
||||
discovery_run_id="run-required-complete",
|
||||
container_id="container-widefrog-retired-helper",
|
||||
name="widefrog-retired-helper",
|
||||
image="widefrog-helper:test",
|
||||
state="missing",
|
||||
status="missing",
|
||||
import_status="missing",
|
||||
matched_app_id="widefrog",
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
AppContainerMapping(
|
||||
id="mapping-widefrog-retired-helper",
|
||||
app_id="widefrog",
|
||||
container_id="container-widefrog-retired-helper",
|
||||
container_name="widefrog-retired-helper",
|
||||
last_seen_run_id="run-required-complete",
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
DiscoveredContainer(
|
||||
id="discovered-gitea-migrate",
|
||||
discovery_run_id="run-required-complete",
|
||||
container_id="container-gitea-migrate",
|
||||
name="gitea-migrate",
|
||||
image="gitea-migrate:test",
|
||||
state="exited",
|
||||
status="unhealthy",
|
||||
import_status="ignored",
|
||||
matched_app_id="gitea",
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
AppContainerMapping(
|
||||
id="mapping-gitea-migrate",
|
||||
app_id="gitea",
|
||||
container_id="container-gitea-migrate",
|
||||
container_name="gitea-migrate",
|
||||
last_seen_run_id="run-required-complete",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
inventory_discovery.invalidate_discovery_cache()
|
||||
with Session(db_session.engine) as session:
|
||||
result = inventory_discovery.sync_app_connector_statuses(session)
|
||||
widefrog = session.get(App, "widefrog")
|
||||
gitea = session.get(App, "gitea")
|
||||
|
||||
assert result["success"] is True
|
||||
assert widefrog.status == "healthy"
|
||||
assert widefrog.container_status == "running"
|
||||
assert gitea.status == "healthy"
|
||||
assert gitea.container_status == "ignored"
|
||||
|
||||
|
||||
def test_latest_discovery_cache_reuses_short_lived_snapshot(api_client, monkeypatch):
|
||||
_make_required_complete()
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.services import inventory_discovery
|
||||
|
||||
calls = 0
|
||||
original_exec = Session.exec
|
||||
|
||||
def counting_exec(self, statement, *args, **kwargs):
|
||||
nonlocal calls
|
||||
if "discovered_containers" in str(statement):
|
||||
calls += 1
|
||||
return original_exec(self, statement, *args, **kwargs)
|
||||
|
||||
inventory_discovery.invalidate_discovery_cache()
|
||||
monkeypatch.setattr(Session, "exec", counting_exec)
|
||||
with Session(db_session.engine) as session:
|
||||
first = inventory_discovery._latest_by_container(session)
|
||||
second = inventory_discovery._latest_by_container(session)
|
||||
inventory_discovery.invalidate_discovery_cache()
|
||||
third = inventory_discovery._latest_by_container(session)
|
||||
|
||||
assert len(first) == len(second) == len(third)
|
||||
assert calls == 2
|
||||
|
||||
|
||||
def test_connectors_list_is_snapshot_by_default(api_client, monkeypatch):
|
||||
from backend.app.api import routes_connectors
|
||||
|
||||
def fail_if_live_tested(self):
|
||||
raise AssertionError("plain connector list should not run live connector tests")
|
||||
|
||||
monkeypatch.setattr(routes_connectors.PortainerConnector, "test_connection", fail_if_live_tested)
|
||||
|
||||
response = api_client.get("/api/connectors")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert any(item["id"] == "portainer" for item in response.json())
|
||||
|
||||
|
||||
def test_setup_status_is_fast_snapshot_without_connector_sync(api_client, monkeypatch):
|
||||
_make_required_complete()
|
||||
from backend.app.services import inventory_discovery
|
||||
|
||||
def fail_if_synced(session):
|
||||
raise AssertionError("setup status should not sync connector statuses")
|
||||
|
||||
monkeypatch.setattr(inventory_discovery, "sync_app_connector_statuses", fail_if_synced)
|
||||
|
||||
payload = api_client.get("/api/setup/status").json()
|
||||
|
||||
assert payload["requiredStepsComplete"] is True
|
||||
assert payload["managedApps"] >= 1
|
||||
|
||||
|
||||
def test_voice_step_warns_when_home_assistant_connector_is_missing(api_client):
|
||||
_make_required_complete()
|
||||
status = api_client.post("/api/setup/reconcile").json()["after"]
|
||||
steps = {step["id"]: step["status"] for step in status["steps"]}
|
||||
|
||||
assert status["liveReady"] is True
|
||||
assert "home_assistant" in status["missingOptionalSources"]
|
||||
assert steps["voice"] == "warning"
|
||||
|
||||
|
||||
def test_home_assistant_connector_settings_use_token_secret_contract(api_client):
|
||||
payload = api_client.get("/api/settings/connectors").json()
|
||||
home_assistant = next(item for item in payload if item["connectorId"] == "home_assistant")
|
||||
|
||||
assert home_assistant["secretRef"] == "HOME_ASSISTANT_TOKEN"
|
||||
assert home_assistant["missing"] == ["HOME_ASSISTANT_URL", "HOME_ASSISTANT_TOKEN"]
|
||||
assert "HOME_ASSISTANT_API_KEY" not in str(home_assistant)
|
||||
|
||||
|
||||
def test_reconcile_sets_first_run_completed_and_audit(api_client):
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.db.models import AuditEvent
|
||||
|
||||
_make_required_complete()
|
||||
payload = api_client.post("/api/setup/reconcile").json()
|
||||
|
||||
assert payload["before"]["firstRunCompleted"] is False
|
||||
assert payload["after"]["firstRunCompleted"] is True
|
||||
assert payload["after"]["setupCompletedAt"]
|
||||
with Session(db_session.engine) as session:
|
||||
audit = session.exec(select(AuditEvent).where(AuditEvent.action_id == "setup.reconcile")).first()
|
||||
assert audit is not None
|
||||
assert audit.result == "success"
|
||||
|
||||
|
||||
def test_reconcile_with_blockers_does_not_set_first_run_completed(api_client):
|
||||
payload = api_client.post("/api/setup/reconcile").json()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["after"]["firstRunCompleted"] is False
|
||||
assert payload["after"]["setupComplete"] is False
|
||||
assert payload["blockers"]
|
||||
|
||||
|
||||
def test_mark_complete_requires_confirm(api_client):
|
||||
response = api_client.post("/api/setup/mark-complete", json={"confirm": "wrong"})
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json()["detail"] == "confirm_required"
|
||||
|
||||
|
||||
def test_mark_complete_writes_audit(api_client):
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.db.models import AuditEvent
|
||||
|
||||
payload = api_client.post("/api/setup/mark-complete", json={"confirm": "MARK_SETUP_COMPLETE"}).json()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"]["firstRunCompleted"] is True
|
||||
with Session(db_session.engine) as session:
|
||||
audit = session.exec(select(AuditEvent).where(AuditEvent.action_id == "setup.mark_complete")).first()
|
||||
assert audit is not None
|
||||
assert audit.result == "success"
|
||||
|
||||
|
||||
def test_runtime_info_uses_setup_status(api_client):
|
||||
_make_required_complete()
|
||||
api_client.post("/api/setup/reconcile")
|
||||
|
||||
runtime = api_client.get("/api/runtime/info").json()
|
||||
|
||||
assert runtime["setupComplete"] is True
|
||||
assert runtime["liveReady"] is True
|
||||
assert runtime["firstRunCompleted"] is True
|
||||
assert runtime["coverageComplete"] is False
|
||||
assert runtime["canonicalPort"] == 1216
|
||||
assert runtime["setupStatus"]["setupComplete"] is True
|
||||
assert "nginx_proxy_manager" in runtime["missingOptionalSources"]
|
||||
assert "test-secret-key" not in str(runtime)
|
||||
|
||||
|
||||
def test_setup_status_does_not_return_secret_values(api_client):
|
||||
secret = "PORTAINER_RECONCILE_SECRET"
|
||||
api_client.patch(
|
||||
"/api/settings/connectors/portainer",
|
||||
json={"url": "http://portainer.local", "endpointId": "2", "verifyTls": False, "apiKey": secret},
|
||||
)
|
||||
|
||||
payload = api_client.get("/api/setup/status").json()
|
||||
|
||||
assert secret not in str(payload)
|
||||
|
||||
|
||||
def test_existing_first_run_completed_preserved_after_init_db(api_client):
|
||||
from backend.app.db.init_db import init_db
|
||||
|
||||
_make_required_complete()
|
||||
api_client.post("/api/setup/reconcile")
|
||||
init_db()
|
||||
|
||||
payload = api_client.get("/api/setup/status").json()
|
||||
assert payload["firstRunCompleted"] is True
|
||||
|
||||
|
||||
def test_completed_first_run_keeps_live_ready_with_later_review_items(api_client):
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.db.models import DiscoveredContainer
|
||||
|
||||
_make_required_complete()
|
||||
api_client.post("/api/setup/reconcile")
|
||||
with Session(db_session.engine) as session:
|
||||
session.add(
|
||||
DiscoveredContainer(
|
||||
id="dc-new-after-setup",
|
||||
discovery_run_id="run-required-complete",
|
||||
container_id="container-new-after-setup",
|
||||
name="new-lab-app",
|
||||
image="local/new-lab-app:latest",
|
||||
state="running",
|
||||
status="running",
|
||||
import_status="unmapped",
|
||||
suggested_app_type="unknown",
|
||||
suggested_category="unknown",
|
||||
suggested_criticality="low",
|
||||
suggested_connector="none",
|
||||
suggested_policy_profile="generic_unknown",
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
from backend.app.services.inventory_discovery import invalidate_discovery_cache
|
||||
|
||||
invalidate_discovery_cache()
|
||||
|
||||
status = api_client.get("/api/setup/status").json()
|
||||
|
||||
assert status["firstRunCompleted"] is True
|
||||
assert status["setupComplete"] is True
|
||||
assert status["liveReady"] is True
|
||||
assert status["requiredStepsComplete"] is True
|
||||
assert status["reviewBlockers"] == 0
|
||||
assert status["summary"]["appsNeedReview"] == 1
|
||||
assert status["needsReview"] == 1
|
||||
review_step = {step["id"]: step for step in status["steps"]}["review"]
|
||||
assert review_step["status"] == "complete"
|
||||
assert review_step["label"] == "Reviewbaseline vastgelegd"
|
||||
|
||||
|
||||
def test_notification_rules_do_not_reset_first_run_completed(api_client):
|
||||
from backend.app.db import session as db_session
|
||||
from backend.app.db.init_db import init_db
|
||||
from backend.app.db.models import NotificationRule
|
||||
|
||||
_make_required_complete()
|
||||
api_client.post("/api/setup/reconcile")
|
||||
with Session(db_session.engine) as session:
|
||||
rule = session.get(NotificationRule, "notify-coverage-incomplete")
|
||||
rule.channels_json = ["dashboard", "home_assistant"]
|
||||
session.add(rule)
|
||||
session.commit()
|
||||
init_db()
|
||||
|
||||
payload = api_client.get("/api/setup/status").json()
|
||||
assert payload["firstRunCompleted"] is True
|
||||
@@ -0,0 +1,27 @@
|
||||
from pathlib import Path
|
||||
|
||||
from sqlmodel import SQLModel
|
||||
|
||||
from backend.app.db.session import create_app_engine
|
||||
|
||||
|
||||
def test_sqlite_engine_waits_for_busy_database_and_uses_wal(tmp_path):
|
||||
db_path = tmp_path / "runtime.db"
|
||||
engine = create_app_engine(Path(db_path))
|
||||
SQLModel.metadata.create_all(engine)
|
||||
|
||||
with engine.connect() as connection:
|
||||
busy_timeout = connection.exec_driver_sql("PRAGMA busy_timeout").scalar_one()
|
||||
journal_mode = str(connection.exec_driver_sql("PRAGMA journal_mode").scalar_one()).lower()
|
||||
|
||||
assert int(busy_timeout) >= 30000
|
||||
assert journal_mode == "wal"
|
||||
|
||||
|
||||
def test_sqlite_engine_allows_ui_request_bursts(tmp_path):
|
||||
db_path = tmp_path / "runtime.db"
|
||||
engine = create_app_engine(Path(db_path))
|
||||
|
||||
assert engine.pool.size() >= 20
|
||||
assert engine.pool._max_overflow >= 20
|
||||
assert engine.pool._timeout <= 10
|
||||
@@ -0,0 +1,325 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "tautulli_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.core.config import settings
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
from backend.app.main import app
|
||||
|
||||
previous_apis = dict(settings.apis)
|
||||
for key in ["TAUTULLI_URL", "TAUTULLI_API_KEY", "TAUTULLI_VERIFY_TLS", "TAUTULLI_TIMEOUT_SECONDS"]:
|
||||
settings.apis[key] = None
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
settings.apis = previous_apis
|
||||
|
||||
|
||||
def _set_tautulli_config(url: str | None = "http://tautulli:8181", api_key: str | None = "TAUTULLI_SECRET_KEY"):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = dict(settings.apis)
|
||||
settings.apis["TAUTULLI_URL"] = url
|
||||
settings.apis["TAUTULLI_API_KEY"] = api_key
|
||||
settings.apis["TAUTULLI_VERIFY_TLS"] = False
|
||||
settings.apis["TAUTULLI_TIMEOUT_SECONDS"] = "10"
|
||||
return previous
|
||||
|
||||
|
||||
def _restore_config(previous):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous)
|
||||
|
||||
|
||||
def _api_payload(data, result="success", message="ok"):
|
||||
return {"response": {"result": result, "message": message, "data": data}}
|
||||
|
||||
|
||||
def _server_info():
|
||||
return _api_payload({
|
||||
"tautulli_version": "2.14.0",
|
||||
"pms_name": "Widefrog Plex",
|
||||
"pms_version": "1.41.0",
|
||||
"pms_platform": "Linux",
|
||||
"pms_identifier": "machine-abcdef123456",
|
||||
"pms_is_plexpass": True,
|
||||
})
|
||||
|
||||
|
||||
def _activity():
|
||||
return _api_payload({
|
||||
"stream_count": 2,
|
||||
"lan_bandwidth": 12000,
|
||||
"wan_bandwidth": 4500,
|
||||
"total_bandwidth": 16500,
|
||||
"sessions": [
|
||||
{
|
||||
"session_key": "1",
|
||||
"friendly_name": "Jens",
|
||||
"full_title": "Direct Movie",
|
||||
"media_type": "movie",
|
||||
"player": "Apple TV",
|
||||
"platform": "tvOS",
|
||||
"state": "playing",
|
||||
"progress_percent": 55,
|
||||
"quality_profile": "1080p",
|
||||
"transcode_decision": "direct play",
|
||||
"video_decision": "direct play",
|
||||
"audio_decision": "direct play",
|
||||
"bandwidth": 12000,
|
||||
"ip_address": "198.18.10.55",
|
||||
"location": "lan",
|
||||
},
|
||||
{
|
||||
"session_key": "2",
|
||||
"username": "Remote",
|
||||
"full_title": "Transcoded Episode",
|
||||
"media_type": "episode",
|
||||
"player": "Chrome",
|
||||
"platform": "Windows",
|
||||
"state": "playing",
|
||||
"progress_percent": 10,
|
||||
"quality_profile": "720p",
|
||||
"transcode_decision": "transcode",
|
||||
"video_decision": "transcode",
|
||||
"audio_decision": "copy",
|
||||
"bandwidth": 4500,
|
||||
"ip_address": "203.0.113.9",
|
||||
"location": "wan",
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
|
||||
def _history():
|
||||
return _api_payload({"data": [
|
||||
{"row_id": 10, "friendly_name": "Jens", "full_title": "Arrival", "media_type": "movie", "watched_status": 1, "date": 1700000000, "duration": 7200, "platform": "tvOS", "transcode_decision": "direct play"},
|
||||
{"row_id": 11, "username": "Remote", "full_title": "Example Show - Pilot", "media_type": "episode", "watched_status": 0, "date": 1700000500, "duration": 1800, "platform": "Web", "transcode_decision": "transcode"},
|
||||
]})
|
||||
|
||||
|
||||
def _users():
|
||||
return _api_payload([
|
||||
{"user_id": 1, "username": "jens", "friendly_name": "Jens", "email": "secret@example.test", "thumb": "http://avatar", "last_seen": 1700000000, "plays": 42, "duration": 15000},
|
||||
{"user_id": 2, "username": "remote", "friendly_name": "Remote", "last_seen": 1700000500, "plays": 7, "duration": 2400},
|
||||
])
|
||||
|
||||
|
||||
def _libraries():
|
||||
return _api_payload([
|
||||
{"section_id": 1, "section_name": "Movies", "section_type": "movie", "count": 1200, "parent_count": 0, "child_count": 0, "last_accessed": 1700000000},
|
||||
{"section_id": 2, "section_name": "Shows", "section_type": "show", "count": 120, "parent_count": 120, "child_count": 1200},
|
||||
])
|
||||
|
||||
|
||||
def _recently_added():
|
||||
return _api_payload({"recently_added": [
|
||||
{"title": "Dune", "media_type": "movie", "section_name": "Movies", "added_at": 1700000000, "year": 2024, "thumb": "/library/metadata/1/thumb"},
|
||||
{"title": "Pilot", "grandparent_title": "Example Show", "media_type": "episode", "section_name": "Shows", "added_at": 1700000500, "year": 2025},
|
||||
]})
|
||||
|
||||
|
||||
def _plays_by_date():
|
||||
return _api_payload({"data": [{"date": "2026-06-24", "plays": 3, "duration": 3000}, {"date": "2026-06-25", "plays": 5, "duration": 6000}]})
|
||||
|
||||
|
||||
def _top_users():
|
||||
return _api_payload({"data": [{"friendly_name": "Jens", "plays": 8, "duration": 9000}]})
|
||||
|
||||
|
||||
def _top_platforms():
|
||||
return _api_payload({"data": [{"platform": "tvOS", "plays": 6}, {"platform": "Web", "plays": 2}]})
|
||||
|
||||
|
||||
def test_tautulli_connector_missing_config_reports_required_env(api_client):
|
||||
response = api_client.post("/api/connectors/tautulli/test")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "missing_config"
|
||||
assert payload["missing_config"] is True
|
||||
assert payload["required_env_vars"] == ["TAUTULLI_URL", "TAUTULLI_API_KEY"]
|
||||
assert payload["secretRef"] == "TAUTULLI_API_KEY"
|
||||
|
||||
|
||||
def test_tautulli_health_success_normalizes_response(monkeypatch):
|
||||
from backend.app.connectors.tautulli import TautulliConnector
|
||||
|
||||
previous = _set_tautulli_config()
|
||||
monkeypatch.setattr(TautulliConnector, "_command", lambda self, cmd, params=None: _server_info(), raising=True)
|
||||
|
||||
payload = TautulliConnector().health()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["status"] == "connected"
|
||||
assert payload["tautulliVersion"] == "2.14.0"
|
||||
assert payload["plexServerName"] == "Widefrog Plex"
|
||||
assert payload["url"] == "http://tautulli:8181"
|
||||
assert "TAUTULLI_SECRET_KEY" not in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_tautulli_auth_failure_redacts_api_key(monkeypatch):
|
||||
from backend.app.connectors.tautulli import TautulliConnector
|
||||
|
||||
previous = _set_tautulli_config()
|
||||
|
||||
def fail_command(self, cmd, params=None):
|
||||
raise RuntimeError("auth_failed apikey=TAUTULLI_SECRET_KEY")
|
||||
|
||||
monkeypatch.setattr(TautulliConnector, "_command", fail_command, raising=True)
|
||||
|
||||
payload = TautulliConnector().health()
|
||||
|
||||
assert payload["success"] is False
|
||||
assert payload["status"] == "auth_failed"
|
||||
assert "TAUTULLI_SECRET_KEY" not in str(payload)
|
||||
assert "[REDACTED]" in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_tautulli_activity_and_stream_summary(monkeypatch):
|
||||
from backend.app.connectors.tautulli import TautulliConnector
|
||||
|
||||
previous = _set_tautulli_config()
|
||||
monkeypatch.setattr(TautulliConnector, "_command", lambda self, cmd, params=None: _activity(), raising=True)
|
||||
|
||||
activity = TautulliConnector().activity()
|
||||
summary = TautulliConnector().stream_summary()
|
||||
|
||||
assert activity["success"] is True
|
||||
assert activity["streamCount"] == 2
|
||||
assert activity["sessions"][0]["ipAddress"] == "192.168.10.x"
|
||||
assert activity["sessions"][1]["transcodeDecision"] == "transcode"
|
||||
assert summary["activeStreams"] == 2
|
||||
assert summary["directPlay"] == 1
|
||||
assert summary["transcoding"] == 1
|
||||
assert summary["lanStreams"] == 1
|
||||
assert summary["wanStreams"] == 1
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_tautulli_history_users_libraries_recent_and_stats(monkeypatch):
|
||||
from backend.app.connectors.tautulli import TautulliConnector
|
||||
|
||||
previous = _set_tautulli_config()
|
||||
|
||||
def fake_command(self, cmd, params=None):
|
||||
return {
|
||||
"get_history": _history(),
|
||||
"get_users": _users(),
|
||||
"get_libraries": _libraries(),
|
||||
"get_recently_added": _recently_added(),
|
||||
"get_plays_by_date": _plays_by_date(),
|
||||
"get_plays_by_top_10_users": _top_users(),
|
||||
"get_plays_by_top_10_platforms": _top_platforms(),
|
||||
}[cmd]
|
||||
|
||||
monkeypatch.setattr(TautulliConnector, "_command", fake_command, raising=True)
|
||||
connector = TautulliConnector()
|
||||
|
||||
history = connector.history()
|
||||
users = connector.users()
|
||||
libraries = connector.libraries()
|
||||
recent = connector.recently_added()
|
||||
stats = connector.watch_stats()
|
||||
|
||||
assert history["items"][0]["rowId"] == 10
|
||||
assert history["items"][1]["transcodeDecision"] == "transcode"
|
||||
assert users["items"][0]["friendlyName"] == "Jens"
|
||||
assert "email" not in users["items"][0]
|
||||
assert "secret@example.test" not in str(users)
|
||||
assert libraries["items"][0]["sectionId"] == "1"
|
||||
assert recent["items"][1]["title"] == "Example Show - Pilot"
|
||||
assert stats["totalPlays"] == 8
|
||||
assert stats["topUsers"][0]["name"] == "Jens"
|
||||
assert stats["topPlatforms"][0]["platform"] == "tvOS"
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_tautulli_plex_info_normalizes_server_info(monkeypatch):
|
||||
from backend.app.connectors.tautulli import TautulliConnector
|
||||
|
||||
previous = _set_tautulli_config()
|
||||
monkeypatch.setattr(TautulliConnector, "_command", lambda self, cmd, params=None: _server_info(), raising=True)
|
||||
|
||||
payload = TautulliConnector().plex_info()
|
||||
|
||||
assert payload["success"] is True
|
||||
assert payload["name"] == "Widefrog Plex"
|
||||
assert payload["version"] == "1.41.0"
|
||||
assert payload["machineIdentifier"] == "machine-..."
|
||||
assert payload["plexPass"] is True
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_tautulli_privacy_policies_and_destructive_actions_are_blocked(api_client):
|
||||
for action in ["tautulli.history", "tautulli.users", "tautulli.watch_stats"]:
|
||||
voice = api_client.post("/api/policies/evaluate", json={"action": action, "target": "tautulli", "source": "voice"}).json()
|
||||
assert voice["decision"] in {"approval_required", "blocked"}
|
||||
|
||||
for action in [
|
||||
"tautulli.terminate_session",
|
||||
"tautulli.download_database",
|
||||
"tautulli.download_config",
|
||||
"tautulli.delete_notifier",
|
||||
"tautulli.notify",
|
||||
"tautulli.user_management",
|
||||
"tautulli.mass_edit",
|
||||
"tautulli.unknown_mutation",
|
||||
]:
|
||||
payload = api_client.post("/api/policies/evaluate", json={"action": action, "target": "tautulli", "source": "manual"}).json()
|
||||
assert payload["decision"] == "blocked"
|
||||
|
||||
|
||||
def test_tautulli_inventory_classification_and_candidate(api_client, monkeypatch):
|
||||
from backend.app.services.app_classifier import classify_container
|
||||
|
||||
result = classify_container({"name": "binhex-tautulli", "image": "linuxserver/tautulli:latest", "ports": ["8181:8181"]})
|
||||
|
||||
assert result["app_type"] == "tautulli"
|
||||
assert result["category"] == "media"
|
||||
assert result["subcategory"] == "plex_analytics"
|
||||
assert result["connector"] == "tautulli"
|
||||
assert result["policy_profile"] == "media_analytics"
|
||||
assert result["privacy_sensitive"] is True
|
||||
|
||||
monkeypatch.setattr("backend.app.connectors.portainer.PortainerConnector.test_connection", lambda self: {"status": "connected"}, raising=True)
|
||||
monkeypatch.setattr(
|
||||
"backend.app.connectors.portainer.PortainerConnector.list_containers",
|
||||
lambda self: {"status": "ok", "containers": [{"id": "tautulli-container", "name": "binhex-tautulli", "image": "linuxserver/tautulli", "state": "running", "status": "running", "ports": ["8181:8181"]}]},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
api_client.post("/api/inventory/discover")
|
||||
candidates = api_client.get("/api/connectors/tautulli/candidates").json()
|
||||
|
||||
assert len(candidates) == 1
|
||||
assert candidates[0]["connectorId"] == "tautulli"
|
||||
assert candidates[0]["appType"] == "tautulli"
|
||||
assert candidates[0]["missingSecrets"] == ["TAUTULLI_URL", "TAUTULLI_API_KEY"]
|
||||
@@ -0,0 +1,478 @@
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from backend.app.connectors.unraid import UnraidConnector
|
||||
from backend.app.core.config import settings
|
||||
|
||||
|
||||
def test_unraid_request_preserves_api_key_for_same_host_https_redirect(monkeypatch):
|
||||
calls = []
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def post(self, url, *, headers, json):
|
||||
calls.append((url, headers.copy(), json.copy()))
|
||||
request = httpx.Request('POST', url)
|
||||
if len(calls) == 1:
|
||||
return httpx.Response(
|
||||
302,
|
||||
headers={'location': 'https://example-host:5001/graphql'},
|
||||
request=request,
|
||||
)
|
||||
return httpx.Response(200, json={'data': {'__typename': 'Query'}}, request=request)
|
||||
|
||||
monkeypatch.setitem(settings.apis, 'UNRAID_URL', 'http://example-host:5000')
|
||||
monkeypatch.setitem(settings.apis, 'UNRAID_API_KEY', 'test-api-key')
|
||||
monkeypatch.setattr('backend.app.connectors.unraid.httpx.Client', FakeClient)
|
||||
|
||||
result = UnraidConnector()._request('query { __typename }')
|
||||
|
||||
assert result == {'__typename': 'Query'}
|
||||
assert [call[0] for call in calls] == [
|
||||
'http://example-host:5000/graphql',
|
||||
'https://example-host:5001/graphql',
|
||||
]
|
||||
assert calls[1][1]['x-api-key'] == 'test-api-key'
|
||||
|
||||
|
||||
def test_unraid_request_refuses_cross_host_redirect(monkeypatch):
|
||||
class FakeClient:
|
||||
def __init__(self, **_kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def post(self, url, **_kwargs):
|
||||
return httpx.Response(
|
||||
302,
|
||||
headers={'location': 'https://attacker.example/graphql'},
|
||||
request=httpx.Request('POST', url),
|
||||
)
|
||||
|
||||
monkeypatch.setitem(settings.apis, 'UNRAID_URL', 'http://example-host:5000')
|
||||
monkeypatch.setitem(settings.apis, 'UNRAID_API_KEY', 'test-api-key')
|
||||
monkeypatch.setattr('backend.app.connectors.unraid.httpx.Client', FakeClient)
|
||||
|
||||
with pytest.raises(RuntimeError, match='untrusted redirect'):
|
||||
UnraidConnector()._request('query { __typename }')
|
||||
|
||||
|
||||
def test_unraid_disk_warnings_maps_live_array_disk_schema(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
|
||||
def fake_query(_queries):
|
||||
return (
|
||||
{
|
||||
"array": {
|
||||
"disks": [
|
||||
{
|
||||
"name": "disk1",
|
||||
"status": "DISK_OK",
|
||||
"warning": None,
|
||||
"critical": None,
|
||||
"fsUsed": 2_097_152,
|
||||
"fsFree": 1_048_576,
|
||||
"fsSize": 3_145_728,
|
||||
},
|
||||
{
|
||||
"name": "disk2",
|
||||
"status": "DISK_DSBL",
|
||||
"warning": "SMART warning",
|
||||
"critical": None,
|
||||
},
|
||||
]
|
||||
}
|
||||
},
|
||||
"ok",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(connector, "_query", fake_query)
|
||||
|
||||
result = connector.get_disk_warnings()
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["diskWarnings"] == 1
|
||||
assert result["disks"][0] | {"type": None, "temperature": None} == {
|
||||
"name": "disk1",
|
||||
"status": "DISK_OK",
|
||||
"healthy": True,
|
||||
"type": None,
|
||||
"usedGiB": 2,
|
||||
"sizeGiB": 3,
|
||||
"freeGiB": 1,
|
||||
"temperature": None,
|
||||
}
|
||||
assert result["disks"][1]["name"] == "disk2"
|
||||
assert result["disks"][1]["healthy"] is False
|
||||
|
||||
|
||||
def test_unraid_share_summary_maps_live_share_schema(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
|
||||
def fake_query(_queries):
|
||||
return (
|
||||
{
|
||||
"shares": [
|
||||
{
|
||||
"name": "media",
|
||||
"used": 2_097_152,
|
||||
"free": 1_048_576,
|
||||
"size": 0,
|
||||
},
|
||||
{
|
||||
"name": "backups",
|
||||
"used": 10,
|
||||
"free": 5,
|
||||
"size": 20,
|
||||
},
|
||||
]
|
||||
},
|
||||
"ok",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(connector, "_query", fake_query)
|
||||
|
||||
result = connector.get_share_summary()
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["shares"] == [
|
||||
{"name": "media", "usedGiB": 2, "sizeGiB": 3, "critical": False},
|
||||
{"name": "backups", "usedGiB": 10, "sizeGiB": 20, "critical": False},
|
||||
]
|
||||
|
||||
|
||||
def test_unraid_docker_status_maps_live_docker_schema(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
|
||||
def fake_query(_queries):
|
||||
return (
|
||||
{
|
||||
"docker": {
|
||||
"containers": [
|
||||
{"id": "1", "names": ["plex"], "state": "running"},
|
||||
{"id": "2", "names": ["radarr"], "state": "exited"},
|
||||
],
|
||||
"networks": [{"id": "n1", "name": "bridge"}],
|
||||
}
|
||||
},
|
||||
"ok",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(connector, "_query", fake_query)
|
||||
|
||||
result = connector.get_docker_service_status()
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["running"] is True
|
||||
assert result["serviceStatus"] == "running"
|
||||
assert result["containers"] == 2
|
||||
|
||||
|
||||
def test_unraid_vm_summary_maps_live_vms_domains_schema(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
|
||||
def fake_query(queries):
|
||||
assert "vms { domains" in queries[0]
|
||||
return ({"vms": {"domains": [{"id": "vm-1", "uuid": "uuid-1", "name": "Windows", "state": "RUNNING"}]}}, "ok")
|
||||
|
||||
monkeypatch.setattr(connector, "_query", fake_query)
|
||||
|
||||
result = connector.get_vm_summary()
|
||||
|
||||
assert result == {"status": "ok", "vms": [{"id": "vm-1", "name": "Windows", "state": "RUNNING"}]}
|
||||
|
||||
|
||||
def test_unraid_vm_summary_does_not_claim_vm_manager_is_disabled(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
monkeypatch.setattr(connector, "_query", lambda _queries: ({"error": "GraphQL error: Failed to retrieve VM domains: VMs are not available"}, "error"))
|
||||
|
||||
result = connector.get_vm_summary()
|
||||
|
||||
assert result["status"] == "api_unavailable"
|
||||
assert "vm:read" in result["reason"]
|
||||
assert "kan nog steeds actief zijn" in result["reason"]
|
||||
assert result["vms"] == []
|
||||
|
||||
|
||||
def test_unraid_vm_mutation_uses_escaped_graphql_id(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
monkeypatch.setattr(connector, "is_configured", lambda: True)
|
||||
captured = {}
|
||||
|
||||
def fake_request(query):
|
||||
captured["query"] = query
|
||||
return {"vm": {"start": True}}
|
||||
|
||||
monkeypatch.setattr(connector, "_request", fake_request)
|
||||
|
||||
result = connector.mutate_vm('server:uuid-"quoted"', "start")
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["status"] == "ok"
|
||||
assert 'start(id: "server:uuid-\\\"quoted\\\"")' in captured["query"]
|
||||
|
||||
|
||||
def test_unraid_vm_mutation_blocks_unsafe_actions_without_request(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
monkeypatch.setattr(connector, "_request", lambda _query: (_ for _ in ()).throw(AssertionError("must not call Unraid")))
|
||||
|
||||
result = connector.mutate_vm("vm-1", "forceStop")
|
||||
|
||||
assert result == {
|
||||
"success": False,
|
||||
"status": "blocked",
|
||||
"message": "unsupported_or_blocked_vm_action",
|
||||
"action": "forcestop",
|
||||
}
|
||||
|
||||
|
||||
def test_unraid_query_stops_after_vm_api_reports_unavailable(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
calls = []
|
||||
|
||||
def fake_request(query):
|
||||
calls.append(query)
|
||||
raise RuntimeError("GraphQL error: Failed to retrieve VM domains: VMs are not available")
|
||||
|
||||
monkeypatch.setattr(connector, "_request", fake_request)
|
||||
payload, status = connector._query(["vm-query", "legacy-fallback"])
|
||||
|
||||
assert status == "unavailable"
|
||||
assert "VMs are not available" in payload["error"]
|
||||
assert calls == ["vm-query"]
|
||||
|
||||
|
||||
def test_unraid_cache_status_maps_live_array_cache_schema(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
|
||||
def fake_query(_queries):
|
||||
return (
|
||||
{
|
||||
"array": {
|
||||
"caches": [
|
||||
{"name": "cache", "fsUsed": 25, "fsSize": 100},
|
||||
{"name": "cache2", "fsUsed": 10, "fsSize": 100},
|
||||
]
|
||||
}
|
||||
},
|
||||
"ok",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(connector, "_query", fake_query)
|
||||
|
||||
result = connector.get_cache_status()
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["cacheUsage"] == 17
|
||||
assert [pool["name"] for pool in result["pools"]] == ["cache", "cache2"]
|
||||
|
||||
|
||||
def test_unraid_cache_status_returns_cache_pool_details(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
|
||||
def fake_query(_queries):
|
||||
return (
|
||||
{
|
||||
"array": {
|
||||
"caches": [
|
||||
{
|
||||
"name": "cache",
|
||||
"status": "DISK_OK",
|
||||
"fsUsed": 25,
|
||||
"fsFree": 75,
|
||||
"fsSize": 100,
|
||||
"type": "CACHE",
|
||||
},
|
||||
{
|
||||
"name": "fastpool",
|
||||
"status": "DISK_DSBL",
|
||||
"fsUsed": 90,
|
||||
"fsFree": 10,
|
||||
"fsSize": 100,
|
||||
"warning": "pool warning",
|
||||
},
|
||||
]
|
||||
}
|
||||
},
|
||||
"ok",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(connector, "_query", fake_query)
|
||||
|
||||
result = connector.get_cache_status()
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["cacheUsage"] == 57
|
||||
assert result["pools"] == [
|
||||
{
|
||||
"name": "cache",
|
||||
"status": "DISK_OK",
|
||||
"healthy": True,
|
||||
"type": "CACHE",
|
||||
"usedGiB": 25,
|
||||
"sizeGiB": 100,
|
||||
"freeGiB": 75,
|
||||
"usagePercent": 25,
|
||||
},
|
||||
{
|
||||
"name": "fastpool",
|
||||
"status": "DISK_DSBL",
|
||||
"healthy": False,
|
||||
"type": None,
|
||||
"usedGiB": 90,
|
||||
"sizeGiB": 100,
|
||||
"freeGiB": 10,
|
||||
"usagePercent": 90,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_unraid_parity_status_uses_live_parity_schema(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
|
||||
def fake_query(queries):
|
||||
assert "parityCheckStatus" in queries[0]
|
||||
assert "parities" in queries[0]
|
||||
return (
|
||||
{
|
||||
"array": {
|
||||
"parityCheckStatus": {
|
||||
"status": "IDLE",
|
||||
"running": False,
|
||||
"paused": False,
|
||||
"correcting": False,
|
||||
"progress": 100,
|
||||
"speed": 0,
|
||||
"errors": 3,
|
||||
},
|
||||
"parities": [{"name": "parity", "status": "DISK_OK", "warning": None, "critical": None}],
|
||||
}
|
||||
},
|
||||
"ok",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(connector, "_query", fake_query)
|
||||
|
||||
result = connector.get_parity_status()
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["parityStatus"] == "valid"
|
||||
assert result["checkStatus"] == "idle"
|
||||
assert result["errors"] == 3
|
||||
|
||||
|
||||
def test_unraid_parity_status_reports_running_check(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
monkeypatch.setattr(
|
||||
connector,
|
||||
"_query",
|
||||
lambda _queries: (
|
||||
{
|
||||
"array": {
|
||||
"parityCheckStatus": {
|
||||
"status": "RUNNING",
|
||||
"running": None,
|
||||
"paused": False,
|
||||
"correcting": False,
|
||||
"progress": 42,
|
||||
"speed": 1234,
|
||||
"errors": 0,
|
||||
},
|
||||
"parities": [{"name": "parity", "status": "DISK_OK"}],
|
||||
}
|
||||
},
|
||||
"ok",
|
||||
),
|
||||
)
|
||||
|
||||
result = connector.get_parity_status()
|
||||
|
||||
assert result["parityStatus"] == "checking"
|
||||
assert result["running"] is True
|
||||
assert result["progress"] == 42
|
||||
|
||||
|
||||
def test_unraid_parity_status_is_invalid_for_unhealthy_or_missing_parity(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
|
||||
for parities in ([{"name": "parity", "status": "DISK_DSBL"}], []):
|
||||
monkeypatch.setattr(
|
||||
connector,
|
||||
"_query",
|
||||
lambda _queries, value=parities: ({"array": {"parities": value}}, "ok"),
|
||||
)
|
||||
result = connector.get_parity_status()
|
||||
assert result["parityStatus"] == "invalid"
|
||||
|
||||
|
||||
def test_unraid_system_metrics_maps_live_metrics_schema(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
|
||||
def fake_query(_queries):
|
||||
return (
|
||||
{
|
||||
"metrics": {
|
||||
"cpu": {
|
||||
"cpus": [
|
||||
{"percentIdle": 80.0, "percentUser": 10.0, "percentSystem": 10.0},
|
||||
{"percentIdle": 60.0, "percentUser": 25.0, "percentSystem": 15.0},
|
||||
]
|
||||
},
|
||||
"memory": {
|
||||
"total": 1000,
|
||||
"used": 640,
|
||||
"free": 360,
|
||||
"available": 300,
|
||||
},
|
||||
}
|
||||
},
|
||||
"ok",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(connector, "_query", fake_query)
|
||||
|
||||
result = connector.get_system_metrics()
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert result["cpuUsage"] == 30
|
||||
assert result["ramUsage"] == 70
|
||||
assert result["sources"] == {
|
||||
"cpu": "metrics.cpu.cpus.percentIdle",
|
||||
"ram": "metrics.memory.available_total",
|
||||
}
|
||||
|
||||
|
||||
def test_unraid_system_metrics_reports_partial_instead_of_inventing_zero(monkeypatch):
|
||||
connector = UnraidConnector()
|
||||
|
||||
def fake_query(_queries):
|
||||
return (
|
||||
{
|
||||
"metrics": {
|
||||
"cpu": {"cpus": []},
|
||||
"memory": {"total": 1000, "used": 920},
|
||||
}
|
||||
},
|
||||
"ok",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(connector, "_query", fake_query)
|
||||
|
||||
result = connector.get_system_metrics()
|
||||
|
||||
assert result["status"] == "partial"
|
||||
assert result["cpuUsage"] is None
|
||||
assert result["ramUsage"] == 92
|
||||
assert "cpu_unavailable" in result["reason"]
|
||||
@@ -0,0 +1,154 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "unraid_vm_actions.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
from backend.app.main import app
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
|
||||
def test_vm_capabilities_expose_safe_execution_boundary(api_client):
|
||||
response = api_client.get("/api/unraid/vms/capabilities")
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
actions = {item["id"]: item for item in payload["actions"]}
|
||||
assert actions["start"]["policyDecision"] == "allowed"
|
||||
assert actions["start"]["executionSupported"] is True
|
||||
assert actions["pause"]["policyDecision"] == "approval_required"
|
||||
assert actions["pause"]["executionSupported"] is True
|
||||
assert actions["stop"]["policyDecision"] == "blocked"
|
||||
assert actions["stop"]["executionSupported"] is False
|
||||
assert actions["force_stop"]["policyDecision"] == "blocked"
|
||||
|
||||
|
||||
def test_manual_vm_start_runs_job_postcheck_and_audit(api_client, monkeypatch):
|
||||
from backend.app.services import unraid_vm_service
|
||||
|
||||
inventory_calls = {"count": 0}
|
||||
|
||||
def fake_inventory(_self):
|
||||
inventory_calls["count"] += 1
|
||||
state = "SHUTOFF" if inventory_calls["count"] == 1 else "RUNNING"
|
||||
return {"status": "ok", "vms": [{"id": "server:vm-1", "name": "Test VM", "state": state}]}
|
||||
|
||||
monkeypatch.setattr(unraid_vm_service.UnraidConnector, "get_vm_summary", fake_inventory)
|
||||
monkeypatch.setattr(
|
||||
unraid_vm_service.UnraidConnector,
|
||||
"mutate_vm",
|
||||
lambda _self, vm_id, action: {
|
||||
"success": vm_id == "server:vm-1" and action == "start",
|
||||
"status": "ok",
|
||||
"message": "vm_start_accepted",
|
||||
},
|
||||
)
|
||||
|
||||
response = api_client.post(
|
||||
"/api/unraid/vms/server%3Avm-1/actions/start",
|
||||
json={"source": "manual", "confirm": True},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["success"] is True
|
||||
assert payload["decision"] == "allowed"
|
||||
assert payload["before_state"]["state"] == "SHUTOFF"
|
||||
assert payload["after_state"] == {"state": "RUNNING", "verified": True}
|
||||
assert payload["jobId"].startswith("job-")
|
||||
assert payload["auditId"].startswith("audit-")
|
||||
|
||||
jobs = api_client.get("/api/jobs").json()
|
||||
assert any(job["id"] == payload["jobId"] and job["status"] == "succeeded" for job in jobs)
|
||||
|
||||
audits = api_client.get("/api/audit").json()
|
||||
audit = next(item for item in audits if item["id"] == payload["auditId"])
|
||||
assert audit["action_id"] == "unraid.vm.start"
|
||||
assert audit["target_type"] == "virtual_machine"
|
||||
|
||||
|
||||
def test_vm_start_requires_explicit_confirmation(api_client):
|
||||
response = api_client.post(
|
||||
"/api/unraid/vms/vm-1/actions/start",
|
||||
json={"source": "manual", "confirm": False},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "confirm_required"
|
||||
|
||||
|
||||
def test_vm_pause_creates_executable_approval(api_client, monkeypatch):
|
||||
from backend.app.services import unraid_vm_service
|
||||
|
||||
inventory_calls = {"count": 0}
|
||||
|
||||
def fake_inventory(_self):
|
||||
inventory_calls["count"] += 1
|
||||
state = "RUNNING" if inventory_calls["count"] == 1 else "PAUSED"
|
||||
return {"status": "ok", "vms": [{"id": "vm-2", "name": "Workload", "state": state}]}
|
||||
|
||||
monkeypatch.setattr(unraid_vm_service.UnraidConnector, "get_vm_summary", fake_inventory)
|
||||
monkeypatch.setattr(
|
||||
unraid_vm_service.UnraidConnector,
|
||||
"mutate_vm",
|
||||
lambda _self, vm_id, action: {
|
||||
"success": vm_id == "vm-2" and action == "pause",
|
||||
"status": "ok",
|
||||
"message": "vm_pause_accepted",
|
||||
},
|
||||
)
|
||||
|
||||
requested = api_client.post(
|
||||
"/api/unraid/vms/vm-2/actions/pause",
|
||||
json={"source": "manual", "confirm": True},
|
||||
).json()
|
||||
assert requested["decision"] == "approval_required"
|
||||
assert requested["approvalId"].startswith("appr-")
|
||||
|
||||
approval_id = requested["approvalId"]
|
||||
approved = api_client.patch(f"/api/approvals/{approval_id}", json={"status": "approved"})
|
||||
assert approved.status_code == 200
|
||||
|
||||
executed = api_client.post(f"/api/approvals/{approval_id}/execute")
|
||||
assert executed.status_code == 200
|
||||
payload = executed.json()
|
||||
assert payload["success"] is True
|
||||
assert payload["approvalStatus"] == "executed"
|
||||
assert payload["after_state"] == {"state": "PAUSED", "verified": True}
|
||||
|
||||
|
||||
def test_vm_stop_is_blocked_without_calling_connector(api_client, monkeypatch):
|
||||
from backend.app.services import unraid_vm_service
|
||||
|
||||
monkeypatch.setattr(
|
||||
unraid_vm_service.UnraidConnector,
|
||||
"mutate_vm",
|
||||
lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("blocked action reached connector")),
|
||||
)
|
||||
|
||||
response = api_client.post(
|
||||
"/api/unraid/vms/vm-3/actions/stop",
|
||||
json={"source": "manual", "confirm": True},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload["decision"] == "blocked"
|
||||
assert payload["result"] == "blocked"
|
||||
assert "force-stop" in payload["message"]
|
||||
@@ -0,0 +1,274 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client(tmp_path, monkeypatch):
|
||||
db_path = tmp_path / "widefrog_test.db"
|
||||
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
||||
|
||||
from backend.app.main import app
|
||||
from backend.app.db import session
|
||||
from backend.app.db.init_db import init_db
|
||||
|
||||
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
||||
session.engine = engine
|
||||
SQLModel.metadata.drop_all(engine)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
init_db()
|
||||
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
|
||||
|
||||
def _set_widefrog_config(url: str | None = "http://widefrog:1207", api_key: str | None = None):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
previous = dict(settings.apis)
|
||||
settings.apis['WIDEFROG_URL'] = url
|
||||
settings.apis['WIDEFROG_API_KEY'] = api_key
|
||||
settings.apis['WIDEFROG_VERIFY_TLS'] = False
|
||||
settings.apis['WIDEFROG_TIMEOUT_SECONDS'] = 10
|
||||
return previous
|
||||
|
||||
|
||||
def _restore_config(previous):
|
||||
from backend.app.core.config import settings
|
||||
|
||||
settings.apis.clear()
|
||||
settings.apis.update(previous)
|
||||
|
||||
|
||||
def test_widefrog_connector_missing_config_reports_required_env(api_client):
|
||||
response = api_client.post('/api/connectors/widefrog/test')
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload['success'] is False
|
||||
assert payload['status'] == 'missing_config'
|
||||
assert payload['missing_config'] is True
|
||||
assert payload['required_env_vars'] == ['WIDEFROG_URL']
|
||||
assert payload['secretRef'] == 'WIDEFROG_API_KEY'
|
||||
|
||||
|
||||
def test_widefrog_health_success_normalizes_response(monkeypatch):
|
||||
from backend.app.connectors.widefrog import WidefrogConnector
|
||||
|
||||
previous = _set_widefrog_config(api_key='WIDEFROG_SECRET_TOKEN')
|
||||
|
||||
def fake_request(self, method, path, **_kwargs):
|
||||
assert method == 'GET'
|
||||
assert path == '/api/health'
|
||||
return {'status': 'ok', 'version': '1.2.3', 'checks': [{'name': 'db', 'status': 'ok'}]}
|
||||
|
||||
monkeypatch.setattr(WidefrogConnector, '_request_json', fake_request, raising=True)
|
||||
|
||||
payload = WidefrogConnector().health()
|
||||
|
||||
assert payload['success'] is True
|
||||
assert payload['status'] == 'connected'
|
||||
assert payload['version'] == '1.2.3'
|
||||
assert payload['port'] == 1207
|
||||
assert payload['checks'] == [{'name': 'db', 'status': 'ok'}]
|
||||
assert 'WIDEFROG_SECRET_TOKEN' not in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_widefrog_unreachable_redacts_secret(monkeypatch):
|
||||
from backend.app.connectors.widefrog import WidefrogConnector
|
||||
|
||||
previous = _set_widefrog_config(api_key='WIDEFROG_SECRET_TOKEN')
|
||||
|
||||
def fail_request(self, method, path, **_kwargs):
|
||||
raise RuntimeError('api_key=WIDEFROG_SECRET_TOKEN connection refused')
|
||||
|
||||
monkeypatch.setattr(WidefrogConnector, '_request_json', fail_request, raising=True)
|
||||
|
||||
payload = WidefrogConnector().health()
|
||||
|
||||
assert payload['success'] is False
|
||||
assert payload['status'] in {'unreachable', 'error'}
|
||||
assert 'WIDEFROG_SECRET_TOKEN' not in str(payload)
|
||||
assert '[REDACTED]' in str(payload)
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_widefrog_queue_unsupported_is_not_fake_success(monkeypatch):
|
||||
from backend.app.connectors.widefrog import WidefrogConnector
|
||||
|
||||
previous = _set_widefrog_config()
|
||||
|
||||
def fake_request(self, method, path, **_kwargs):
|
||||
raise RuntimeError('unsupported_endpoint')
|
||||
|
||||
monkeypatch.setattr(WidefrogConnector, '_request_json', fake_request, raising=True)
|
||||
|
||||
payload = WidefrogConnector().queue_status()
|
||||
|
||||
assert payload['success'] is False
|
||||
assert payload['status'] in {'unsupported', 'unavailable'}
|
||||
assert payload['total'] == 0
|
||||
assert payload['items'] == []
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_widefrog_queue_success_normalizes_summary(monkeypatch):
|
||||
from backend.app.connectors.widefrog import WidefrogConnector
|
||||
|
||||
previous = _set_widefrog_config()
|
||||
|
||||
def fake_request(self, method, path, **_kwargs):
|
||||
assert path == '/api/queue'
|
||||
return {
|
||||
'items': [
|
||||
{'id': 'a', 'title': 'active job', 'status': 'active'},
|
||||
{'id': 'q', 'title': 'queued job', 'status': 'queued'},
|
||||
{'id': 'c', 'title': 'completed job', 'status': 'completed'},
|
||||
{'id': 'f', 'title': 'failed job', 'status': 'failed'},
|
||||
]
|
||||
}
|
||||
|
||||
monkeypatch.setattr(WidefrogConnector, '_request_json', fake_request, raising=True)
|
||||
|
||||
payload = WidefrogConnector().queue_status()
|
||||
|
||||
assert payload['success'] is True
|
||||
assert payload['status'] == 'ok'
|
||||
assert payload['total'] == 4
|
||||
assert payload['active'] == 1
|
||||
assert payload['queued'] == 1
|
||||
assert payload['completed'] == 1
|
||||
assert payload['failed'] == 1
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_widefrog_failed_jobs_success_normalizes_items(monkeypatch):
|
||||
from backend.app.connectors.widefrog import WidefrogConnector
|
||||
|
||||
previous = _set_widefrog_config()
|
||||
|
||||
monkeypatch.setattr(
|
||||
WidefrogConnector,
|
||||
'_request_json',
|
||||
lambda self, method, path, **_kwargs: {'items': [{'id': 'f1', 'title': 'failed', 'status': 'failed'}]},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
payload = WidefrogConnector().failed_jobs()
|
||||
|
||||
assert payload['success'] is True
|
||||
assert payload['failed'] == 1
|
||||
assert payload['items'][0]['id'] == 'f1'
|
||||
|
||||
_restore_config(previous)
|
||||
|
||||
|
||||
def test_widefrog_retry_failed_voice_requires_approval_without_connector_call(api_client, monkeypatch):
|
||||
def fail_retry(*_args, **_kwargs):
|
||||
raise AssertionError('connector should not run before approval')
|
||||
|
||||
monkeypatch.setattr('backend.app.connectors.widefrog.WidefrogConnector.retry_failed', fail_retry, raising=True)
|
||||
|
||||
response = api_client.post('/api/apps/widefrog/retry-failed', json={'source': 'voice'})
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload['success'] is False
|
||||
assert payload['decision'] == 'approval_required'
|
||||
assert payload['approvalId'].startswith('appr-')
|
||||
assert payload['auditId'].startswith('audit-')
|
||||
|
||||
|
||||
def test_widefrog_retry_failed_allowed_creates_job_and_audit(api_client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
'backend.app.connectors.widefrog.WidefrogConnector.retry_failed',
|
||||
lambda self, payload=None: {'success': True, 'status': 'ok', 'message': 'retry_started', 'data': {'retried': 2}},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
response = api_client.post('/api/apps/widefrog/retry-failed', json={'source': 'manual'})
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
assert payload['success'] is True
|
||||
assert payload['decision'] == 'allowed'
|
||||
assert payload['status'] == 'success'
|
||||
assert payload['jobId'].startswith('job-')
|
||||
assert payload['auditId'].startswith('audit-')
|
||||
assert payload['data']['retried'] == 2
|
||||
|
||||
|
||||
def test_widefrog_retry_failed_unsupported_is_failed_not_fake_success(api_client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
'backend.app.connectors.widefrog.WidefrogConnector.retry_failed',
|
||||
lambda self, payload=None: {'success': False, 'status': 'unsupported', 'message': 'unsupported_endpoint'},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
response = api_client.post('/api/apps/widefrog/retry-failed', json={'source': 'manual'})
|
||||
|
||||
payload = response.json()
|
||||
assert payload['success'] is False
|
||||
assert payload['decision'] == 'failed'
|
||||
assert payload['status'] == 'failed'
|
||||
assert payload['message'] == 'unsupported_endpoint'
|
||||
assert payload['jobId'].startswith('job-')
|
||||
assert payload['auditId'].startswith('audit-')
|
||||
|
||||
|
||||
def test_widefrog_scan_library_allowed_and_unsupported_paths(api_client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
'backend.app.connectors.widefrog.WidefrogConnector.scan_library',
|
||||
lambda self, payload=None: {'success': True, 'status': 'ok', 'message': 'scan_started', 'data': {'accepted': True}},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
success = api_client.post('/api/apps/widefrog/scan-library', json={'source': 'manual'}).json()
|
||||
assert success['success'] is True
|
||||
assert success['decision'] == 'allowed'
|
||||
assert success['jobId'].startswith('job-')
|
||||
|
||||
monkeypatch.setattr(
|
||||
'backend.app.connectors.widefrog.WidefrogConnector.scan_library',
|
||||
lambda self, payload=None: {'success': False, 'status': 'unsupported', 'message': 'unsupported_endpoint'},
|
||||
raising=True,
|
||||
)
|
||||
|
||||
failed = api_client.post('/api/apps/widefrog/scan-library', json={'source': 'manual'}).json()
|
||||
assert failed['success'] is False
|
||||
assert failed['decision'] == 'failed'
|
||||
assert failed['message'] == 'unsupported_endpoint'
|
||||
|
||||
|
||||
def test_widefrog_destructive_actions_blocked(api_client):
|
||||
for action in [
|
||||
'widefrog.delete_job',
|
||||
'widefrog.delete_media',
|
||||
'widefrog.clear_library',
|
||||
'widefrog.clear_queue',
|
||||
'widefrog.reset_database',
|
||||
'widefrog.delete_downloads',
|
||||
'widefrog.prune',
|
||||
]:
|
||||
payload = api_client.post('/api/policies/evaluate', json={'action': action, 'target': 'widefrog', 'source': 'manual'}).json()
|
||||
assert payload['decision'] == 'blocked'
|
||||
|
||||
|
||||
def test_widefrog_inventory_classification_has_connector_and_port():
|
||||
from backend.app.services.app_classifier import classify_container
|
||||
|
||||
result = classify_container({'name': 'widefrog', 'image': 'widefrog/app:latest', 'ports': ['1207:1207']})
|
||||
|
||||
assert result['app_type'] == 'widefrog'
|
||||
assert result['category'] == 'media'
|
||||
assert result['connector'] == 'widefrog'
|
||||
assert result['policy_profile'] == 'media_custom_app'
|
||||
assert result['healthcheck_suggestion'] == {'port': 1207}
|
||||
@@ -0,0 +1,34 @@
|
||||
from backend.app.services.action_execution import (
|
||||
action_execution_profile,
|
||||
approval_execution_supported,
|
||||
mcp_execution_supported,
|
||||
)
|
||||
from backend.app.services.action_registry import (
|
||||
ACTION_RISK,
|
||||
ACTIONS,
|
||||
action_default_target,
|
||||
)
|
||||
from backend.app.services.policy_engine import evaluate_policy
|
||||
|
||||
|
||||
def test_workmesh_promotion_is_fixed_target_human_approved_and_not_mcp_executable():
|
||||
action = "deployment.target.promote_workmesh"
|
||||
|
||||
assert action in ACTIONS
|
||||
assert ACTION_RISK[action] == "high"
|
||||
assert action_default_target(action) == "jens/workmesh"
|
||||
assert evaluate_policy(None, action, "jens/workmesh", "manual")["decision"] == "approval_required"
|
||||
assert approval_execution_supported(action)
|
||||
assert not mcp_execution_supported(action)
|
||||
assert action_execution_profile(action, "approval_required", "operator") == {
|
||||
"executionMode": "approval_then_execute",
|
||||
"executionSupported": True,
|
||||
"commandExecutionSupported": False,
|
||||
"executionSurface": "/approvals",
|
||||
}
|
||||
|
||||
|
||||
def test_workmesh_promotion_action_does_not_generalize_to_another_target():
|
||||
# The product executor resolves its fixed target from persisted preflight
|
||||
# state; callers cannot alter it through the shared action registry.
|
||||
assert action_default_target("deployment.target.promote_workmesh") == "jens/workmesh"
|
||||
Reference in new issue
Block a user