from pathlib import Path from fastapi.testclient import TestClient from sqlmodel import SQLModel, Session, create_engine def _reset_db(tmp_path, monkeypatch): db_path = tmp_path / "ai_remediation_test.db" monkeypatch.setenv("APPOPS_DB_PATH", str(db_path)) monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token") from backend.app.core.config import settings from backend.app.db import session from backend.app.db.init_db import init_db previous = { "apis": dict(settings.apis), "environment": settings.environment, "secret_key": settings.secret_key, "secret_key_source": settings.secret_key_source, } for key in settings.apis: settings.apis[key] = None if key.endswith(("_URL", "_API_KEY", "_TOKEN", "_SECRET", "_PASSWORD")) else settings.apis[key] settings.environment = "development" settings.secret_key = "test-ai-remediation-key" settings.secret_key_source = "env" engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False}) session.engine = engine SQLModel.metadata.drop_all(engine) SQLModel.metadata.create_all(engine) init_db() return previous def _restore_config(previous): from backend.app.core.config import settings settings.apis.clear() settings.apis.update(previous["apis"]) settings.environment = previous["environment"] settings.secret_key = previous["secret_key"] settings.secret_key_source = previous["secret_key_source"] def _client(tmp_path, monkeypatch): previous = _reset_db(tmp_path, monkeypatch) from backend.app.main import app return TestClient(app), previous def _npm_exposure(): return { "success": True, "status": "ok", "totalProxyHosts": 2, "enabledProxyHosts": 2, "publicHosts": 2, "hostsWithoutCertificate": 1, "hostsWithExpiredCertificate": 0, "hostsWithExpiringCertificate": 1, "hostsWithoutAccessList": 1, "hostsWithoutSslForce": 1, "hostsWithWebsocket": 1, "warnings": [ {"type": "host_without_certificate", "severity": "high", "title": "open.example.com has no certificate", "evidence": ["open.example.com"], "suggestedAction": "review_certificate"}, {"type": "certificate_expiring_soon", "severity": "warning", "title": "plex.example.com certificate expires soon", "evidence": ["12"], "suggestedAction": "review_certificate_in_npm"}, ], } def _auth_coverage(): return { "success": True, "status": "ok", "summary": {"totalPublicHosts": 2, "protectedHosts": 1, "likelyProtectedHosts": 0, "unprotectedHosts": 1, "npmStatus": "ok"}, "publicHosts": [ {"hostId": 1, "domains": ["plex.example.com"], "target": "http://plex:32400", "coverageStatus": "protected", "authentikMatches": ["plex.example.com"]}, {"hostId": 2, "domains": ["open.example.com"], "target": "http://unknown:8080", "coverageStatus": "unprotected", "authentikMatches": []}, ], } def _seed_signals(monkeypatch): monkeypatch.setattr("backend.app.connectors.nginx_proxy_manager.NginxProxyManagerConnector.exposure_summary", lambda self: _npm_exposure(), raising=True) monkeypatch.setattr("backend.app.connectors.nginx_proxy_manager.NginxProxyManagerConnector.proxy_hosts", lambda self: {"success": True, "items": [{"id": 2, "domainNames": ["open.example.com"], "enabled": True}]}, raising=True) monkeypatch.setattr("backend.app.connectors.authentik.AuthentikConnector.auth_coverage", lambda self: _auth_coverage(), raising=True) monkeypatch.setattr("backend.app.connectors.authentik.AuthentikConnector.security_warnings", lambda self: {"success": True, "items": []}, raising=True) monkeypatch.setattr("backend.app.connectors.authentik.AuthentikConnector.outpost_health", lambda self: {"success": True, "available": True, "embeddedExpected": True, "items": []}, raising=True) monkeypatch.setattr( "backend.app.services.media_pipeline_service.issues", lambda session: { "success": True, "issues": [ {"id": "deluge-stalled", "severity": "warning", "title": "Deluge stalled downloads", "description": "2 stalled downloads", "affectedConnectors": ["deluge"], "evidence": ["stalled=2"]}, {"id": "widefrog-failed", "severity": "warning", "title": "Widefrog failed jobs", "description": "1 failed job", "affectedConnectors": ["widefrog"], "evidence": ["failed=1"]}, ], }, raising=False, ) def test_generate_plans_from_security_findings_and_media(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) _seed_signals(monkeypatch) with client: payload = client.post("/api/ai/remediation/plans/generate").json() plan_ids = {plan["id"] for plan in payload["plans"]} assert "rem-ai-sec-auth-public-host-without-authentik" in plan_ids assert "rem-ai-sec-npm-certificate-expiring-soon" in plan_ids assert "rem-media-deluge-stalled" in plan_ids assert "rem-media-widefrog-failed" in plan_ids assert all(plan["executable"] is False for plan in payload["plans"]) assert all(step["canExecute"] is False for plan in payload["plans"] for step in plan["steps"]) _restore_config(previous) def test_generated_media_security_plan_uses_media_steps_not_inventory(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) _seed_signals(monkeypatch) monkeypatch.setattr( "backend.app.services.media_pipeline_service.summary", lambda session=None: {"success": True, "status": "partial", "message": "Media pipeline needs review."}, raising=False, ) with client: payload = client.post("/api/ai/remediation/plans/generate").json() plans = [ item for item in payload["plans"] if item["sourceId"] == "ai-sec-media-pipeline-issue" ] assert plans for plan in plans: step_ids = {step["id"] for step in plan["steps"]} assert plan["title"] == "Review media pipeline issue" assert "run-diagnose" in step_ids assert "review-connectors" in step_ids assert "mark-protected" not in step_ids assert any(check["endpoint"] == "/api/media-pipeline/summary" for check in plan["verificationChecks"]) _restore_config(previous) def test_generate_plans_deduplicates_recommendations_covered_by_findings(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) _seed_signals(monkeypatch) with client: payload = client.get("/api/ai/remediation/plans").json() source_ids = [plan["sourceId"] for plan in payload["plans"]] assert "ai-sec-auth-public-host-without-authentik" in source_ids finding_source_ids = {source_id for source_id in source_ids if source_id.startswith("ai-sec-")} duplicate_recommendations = [ source_id for source_id in source_ids if source_id.startswith("rec-") and source_id.removeprefix("rec-") in finding_source_ids ] assert duplicate_recommendations == [] _restore_config(previous) def test_generate_missing_secret_plan_has_no_secret_values(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) secret = "AUTHENTIK_TOKEN_VALUE" from backend.app.db.models import Connector from backend.app.db.session import engine with Session(engine) as session: connector = session.get(Connector, "authentik") connector.secret_ref = "AUTHENTIK_TOKEN" session.add(connector) session.commit() monkeypatch.setattr( "backend.app.connectors.authentik.AuthentikConnector.security_warnings", lambda self: {"success": True, "items": [{"type": "token_leak", "severity": "critical", "title": f"leaked {secret}", "evidence": [secret]}]}, raising=True, ) with client: finding_payload = client.get("/api/ai/security/findings?scope=secrets").json() finding_id = next(item["id"] for item in finding_payload if item["category"] == "connector_config") plan = client.post(f"/api/ai/remediation/plans/from-finding/{finding_id}").json() assert "secret" in plan["title"].lower() assert secret not in str(plan) assert "[REDACTED]" not in plan["affectedResources"] assert any(step["type"] == "configure_secret" for step in plan["steps"]) configure_step = next(step for step in plan["steps"] if step["id"] == "configure-secret") test_step = next(step for step in plan["steps"] if step["id"] == "test-connector") secret_owner = plan["affectedResources"][0].removesuffix("_API_KEY").removesuffix("_TOKEN").lower() assert configure_step["href"] == f"/setup?step=connectors&connector={secret_owner}" assert test_step["href"] == f"/setup?step=connectors&connector={secret_owner}" assert plan["executable"] is False _restore_config(previous) def test_generate_plans_builds_security_review_once(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) from backend.app.services import ai_remediation_planner calls = {"count": 0} original_generate_review = ai_remediation_planner.ai_security_reviewer.generate_review def counted_generate_review(session, scope="all"): calls["count"] += 1 return original_generate_review(session, scope) monkeypatch.setattr(ai_remediation_planner.ai_security_reviewer, "generate_review", counted_generate_review) monkeypatch.setattr("backend.app.services.media_pipeline_service.issues", lambda session: {"success": True, "issues": []}, raising=False) with client: payload = client.post("/api/ai/remediation/plans/generate").json() assert payload["status"] == "ok" assert calls["count"] == 1 _restore_config(previous) def test_dry_run_evaluates_policy_without_execution(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) _seed_signals(monkeypatch) with client: plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-auth-public-host-without-authentik").json() dry_run = client.post(f"/api/ai/remediation/plans/{plan['id']}/dry-run").json() assert dry_run["executed"] is False assert dry_run["status"] in {"pass", "warning", "blocked"} assert any(step["policyDecision"] == "blocked" for step in dry_run["stepResults"]) assert any(item["actionId"].startswith("authentik.") for item in dry_run["blockedActions"]) _restore_config(previous) def test_auth_coverage_plan_is_host_aware_and_non_executing(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) _seed_signals(monkeypatch) with client: plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-auth-public-host-without-authentik").json() assert plan["title"] == "Add identity coverage for 1 public host" assert "open.example.com" in plan["description"] assert plan["affectedResources"] == ["open.example.com"] assert any("host-by-host" in step["description"] for step in plan["steps"]) assert any(step["id"] == "manual-auth-change" and step["canExecute"] is False for step in plan["steps"]) assert any(check["endpoint"] == "/api/apps/authentik/auth-coverage" for check in plan["verificationChecks"]) assert any(item["actionId"] == "authentik.create_application" for item in plan["approvalRequirements"]) assert plan["executable"] is False _restore_config(previous) def test_media_pipeline_security_finding_uses_media_plan_not_inventory(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) _seed_signals(monkeypatch) with client: plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-media-pipeline-issue").json() assert plan["title"] == "Review media pipeline issue" assert plan["affectedResources"] == ["media_pipeline"] assert any(step["href"] == "/media-pipeline" for step in plan["steps"]) assert not any(step["href"] == "/inventory" for step in plan["steps"]) assert any(check["endpoint"] == "/api/media-pipeline/summary" for check in plan["verificationChecks"]) assert plan["executable"] is False _restore_config(previous) def test_ssl_force_finding_has_specific_npm_review_plan(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) with client: plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-npm-host-without-ssl-force").json() assert plan["title"] == "Review NPM SSL enforcement" assert "Force SSL" in plan["description"] assert any(step["actionId"] == "npm.proxy_hosts" for step in plan["steps"]) assert any(step["id"] == "manual-ssl-change" and step["canExecute"] is False for step in plan["steps"]) assert any(check["endpoint"] == "/api/apps/nginx-proxy-manager/proxy-hosts" for check in plan["verificationChecks"]) _restore_config(previous) def test_authentik_outpost_finding_has_specific_review_plan(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) with client: plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-auth-outpost-unavailable").json() assert plan["title"] == "Review Authentik outpost health" assert any(step["actionId"] == "authentik.outpost_health" for step in plan["steps"]) assert any(step["id"] == "manual-outpost-review" and step["canExecute"] is False for step in plan["steps"]) assert any(check["endpoint"] == "/api/apps/authentik/outposts/health" for check in plan["verificationChecks"]) _restore_config(previous) def test_approval_proposal_is_draft_only_and_lifecycle_audited(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) _seed_signals(monkeypatch) with client: plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-policy-unraid-mover-voice-allowed").json() reviewed = client.post(f"/api/ai/remediation/plans/{plan['id']}/review").json() proposal = client.post(f"/api/ai/remediation/plans/{plan['id']}/create-approval-proposal").json() dismissed = client.post(f"/api/ai/remediation/plans/{plan['id']}/dismiss").json() approvals = client.get("/api/approvals").json() audit = client.get("/api/audit").json() assert reviewed["status"] == "reviewed" assert proposal["created"] is True assert proposal["executed"] is False assert proposal["approvalId"] is None assert dismissed["status"] == "dismissed" assert approvals == [] assert any(event["action_id"] == "ai.remediation.approval_proposal" for event in audit) _restore_config(previous) def test_routes_return_404_for_unknown_plan(tmp_path, monkeypatch): client, previous = _client(tmp_path, monkeypatch) with client: response = client.get("/api/ai/remediation/plans/rem-missing") assert response.status_code == 404 _restore_config(previous)