346 lines
15 KiB
Python
346 lines
15 KiB
Python
from pathlib import Path
|
|
|
|
from fastapi.testclient import TestClient
|
|
from sqlmodel import SQLModel, Session, create_engine
|
|
|
|
|
|
def _reset_db(tmp_path, monkeypatch):
|
|
db_path = tmp_path / "ai_remediation_test.db"
|
|
monkeypatch.setenv("APPOPS_DB_PATH", str(db_path))
|
|
monkeypatch.setenv("APP_OPS_VOICE_TOKEN", "test-voice-token")
|
|
|
|
from backend.app.core.config import settings
|
|
from backend.app.db import session
|
|
from backend.app.db.init_db import init_db
|
|
|
|
previous = {
|
|
"apis": dict(settings.apis),
|
|
"environment": settings.environment,
|
|
"secret_key": settings.secret_key,
|
|
"secret_key_source": settings.secret_key_source,
|
|
}
|
|
for key in settings.apis:
|
|
settings.apis[key] = None if key.endswith(("_URL", "_API_KEY", "_TOKEN", "_SECRET", "_PASSWORD")) else settings.apis[key]
|
|
settings.environment = "development"
|
|
settings.secret_key = "test-ai-remediation-key"
|
|
settings.secret_key_source = "env"
|
|
|
|
engine = create_engine(f"sqlite:///{Path(db_path)}", connect_args={"check_same_thread": False})
|
|
session.engine = engine
|
|
SQLModel.metadata.drop_all(engine)
|
|
SQLModel.metadata.create_all(engine)
|
|
init_db()
|
|
return previous
|
|
|
|
|
|
def _restore_config(previous):
|
|
from backend.app.core.config import settings
|
|
|
|
settings.apis.clear()
|
|
settings.apis.update(previous["apis"])
|
|
settings.environment = previous["environment"]
|
|
settings.secret_key = previous["secret_key"]
|
|
settings.secret_key_source = previous["secret_key_source"]
|
|
|
|
|
|
def _client(tmp_path, monkeypatch):
|
|
previous = _reset_db(tmp_path, monkeypatch)
|
|
from backend.app.main import app
|
|
|
|
return TestClient(app), previous
|
|
|
|
|
|
def _npm_exposure():
|
|
return {
|
|
"success": True,
|
|
"status": "ok",
|
|
"totalProxyHosts": 2,
|
|
"enabledProxyHosts": 2,
|
|
"publicHosts": 2,
|
|
"hostsWithoutCertificate": 1,
|
|
"hostsWithExpiredCertificate": 0,
|
|
"hostsWithExpiringCertificate": 1,
|
|
"hostsWithoutAccessList": 1,
|
|
"hostsWithoutSslForce": 1,
|
|
"hostsWithWebsocket": 1,
|
|
"warnings": [
|
|
{"type": "host_without_certificate", "severity": "high", "title": "open.example.com has no certificate", "evidence": ["open.example.com"], "suggestedAction": "review_certificate"},
|
|
{"type": "certificate_expiring_soon", "severity": "warning", "title": "plex.example.com certificate expires soon", "evidence": ["12"], "suggestedAction": "review_certificate_in_npm"},
|
|
],
|
|
}
|
|
|
|
|
|
def _auth_coverage():
|
|
return {
|
|
"success": True,
|
|
"status": "ok",
|
|
"summary": {"totalPublicHosts": 2, "protectedHosts": 1, "likelyProtectedHosts": 0, "unprotectedHosts": 1, "npmStatus": "ok"},
|
|
"publicHosts": [
|
|
{"hostId": 1, "domains": ["plex.example.com"], "target": "http://plex:32400", "coverageStatus": "protected", "authentikMatches": ["plex.example.com"]},
|
|
{"hostId": 2, "domains": ["open.example.com"], "target": "http://unknown:8080", "coverageStatus": "unprotected", "authentikMatches": []},
|
|
],
|
|
}
|
|
|
|
|
|
def _seed_signals(monkeypatch):
|
|
monkeypatch.setattr("backend.app.connectors.nginx_proxy_manager.NginxProxyManagerConnector.exposure_summary", lambda self: _npm_exposure(), raising=True)
|
|
monkeypatch.setattr("backend.app.connectors.nginx_proxy_manager.NginxProxyManagerConnector.proxy_hosts", lambda self: {"success": True, "items": [{"id": 2, "domainNames": ["open.example.com"], "enabled": True}]}, raising=True)
|
|
monkeypatch.setattr("backend.app.connectors.authentik.AuthentikConnector.auth_coverage", lambda self: _auth_coverage(), raising=True)
|
|
monkeypatch.setattr("backend.app.connectors.authentik.AuthentikConnector.security_warnings", lambda self: {"success": True, "items": []}, raising=True)
|
|
monkeypatch.setattr("backend.app.connectors.authentik.AuthentikConnector.outpost_health", lambda self: {"success": True, "available": True, "embeddedExpected": True, "items": []}, raising=True)
|
|
monkeypatch.setattr(
|
|
"backend.app.services.media_pipeline_service.issues",
|
|
lambda session: {
|
|
"success": True,
|
|
"issues": [
|
|
{"id": "deluge-stalled", "severity": "warning", "title": "Deluge stalled downloads", "description": "2 stalled downloads", "affectedConnectors": ["deluge"], "evidence": ["stalled=2"]},
|
|
{"id": "widefrog-failed", "severity": "warning", "title": "Widefrog failed jobs", "description": "1 failed job", "affectedConnectors": ["widefrog"], "evidence": ["failed=1"]},
|
|
],
|
|
},
|
|
raising=False,
|
|
)
|
|
|
|
|
|
def test_generate_plans_from_security_findings_and_media(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
_seed_signals(monkeypatch)
|
|
|
|
with client:
|
|
payload = client.post("/api/ai/remediation/plans/generate").json()
|
|
|
|
plan_ids = {plan["id"] for plan in payload["plans"]}
|
|
assert "rem-ai-sec-auth-public-host-without-authentik" in plan_ids
|
|
assert "rem-ai-sec-npm-certificate-expiring-soon" in plan_ids
|
|
assert "rem-media-deluge-stalled" in plan_ids
|
|
assert "rem-media-widefrog-failed" in plan_ids
|
|
assert all(plan["executable"] is False for plan in payload["plans"])
|
|
assert all(step["canExecute"] is False for plan in payload["plans"] for step in plan["steps"])
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_generated_media_security_plan_uses_media_steps_not_inventory(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
_seed_signals(monkeypatch)
|
|
monkeypatch.setattr(
|
|
"backend.app.services.media_pipeline_service.summary",
|
|
lambda session=None: {"success": True, "status": "partial", "message": "Media pipeline needs review."},
|
|
raising=False,
|
|
)
|
|
|
|
with client:
|
|
payload = client.post("/api/ai/remediation/plans/generate").json()
|
|
|
|
plans = [
|
|
item for item in payload["plans"]
|
|
if item["sourceId"] == "ai-sec-media-pipeline-issue"
|
|
]
|
|
|
|
assert plans
|
|
for plan in plans:
|
|
step_ids = {step["id"] for step in plan["steps"]}
|
|
assert plan["title"] == "Review media pipeline issue"
|
|
assert "run-diagnose" in step_ids
|
|
assert "review-connectors" in step_ids
|
|
assert "mark-protected" not in step_ids
|
|
assert any(check["endpoint"] == "/api/media-pipeline/summary" for check in plan["verificationChecks"])
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_generate_plans_deduplicates_recommendations_covered_by_findings(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
_seed_signals(monkeypatch)
|
|
|
|
with client:
|
|
payload = client.get("/api/ai/remediation/plans").json()
|
|
|
|
source_ids = [plan["sourceId"] for plan in payload["plans"]]
|
|
assert "ai-sec-auth-public-host-without-authentik" in source_ids
|
|
finding_source_ids = {source_id for source_id in source_ids if source_id.startswith("ai-sec-")}
|
|
duplicate_recommendations = [
|
|
source_id
|
|
for source_id in source_ids
|
|
if source_id.startswith("rec-") and source_id.removeprefix("rec-") in finding_source_ids
|
|
]
|
|
assert duplicate_recommendations == []
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_generate_missing_secret_plan_has_no_secret_values(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
secret = "AUTHENTIK_TOKEN_VALUE"
|
|
|
|
from backend.app.db.models import Connector
|
|
from backend.app.db.session import engine
|
|
|
|
with Session(engine) as session:
|
|
connector = session.get(Connector, "authentik")
|
|
connector.secret_ref = "AUTHENTIK_TOKEN"
|
|
session.add(connector)
|
|
session.commit()
|
|
|
|
monkeypatch.setattr(
|
|
"backend.app.connectors.authentik.AuthentikConnector.security_warnings",
|
|
lambda self: {"success": True, "items": [{"type": "token_leak", "severity": "critical", "title": f"leaked {secret}", "evidence": [secret]}]},
|
|
raising=True,
|
|
)
|
|
|
|
with client:
|
|
finding_payload = client.get("/api/ai/security/findings?scope=secrets").json()
|
|
finding_id = next(item["id"] for item in finding_payload if item["category"] == "connector_config")
|
|
plan = client.post(f"/api/ai/remediation/plans/from-finding/{finding_id}").json()
|
|
|
|
assert "secret" in plan["title"].lower()
|
|
assert secret not in str(plan)
|
|
assert "[REDACTED]" not in plan["affectedResources"]
|
|
assert any(step["type"] == "configure_secret" for step in plan["steps"])
|
|
configure_step = next(step for step in plan["steps"] if step["id"] == "configure-secret")
|
|
test_step = next(step for step in plan["steps"] if step["id"] == "test-connector")
|
|
secret_owner = plan["affectedResources"][0].removesuffix("_API_KEY").removesuffix("_TOKEN").lower()
|
|
assert configure_step["href"] == f"/setup?step=connectors&connector={secret_owner}"
|
|
assert test_step["href"] == f"/setup?step=connectors&connector={secret_owner}"
|
|
assert plan["executable"] is False
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_generate_plans_builds_security_review_once(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
|
|
from backend.app.services import ai_remediation_planner
|
|
|
|
calls = {"count": 0}
|
|
original_generate_review = ai_remediation_planner.ai_security_reviewer.generate_review
|
|
|
|
def counted_generate_review(session, scope="all"):
|
|
calls["count"] += 1
|
|
return original_generate_review(session, scope)
|
|
|
|
monkeypatch.setattr(ai_remediation_planner.ai_security_reviewer, "generate_review", counted_generate_review)
|
|
monkeypatch.setattr("backend.app.services.media_pipeline_service.issues", lambda session: {"success": True, "issues": []}, raising=False)
|
|
|
|
with client:
|
|
payload = client.post("/api/ai/remediation/plans/generate").json()
|
|
|
|
assert payload["status"] == "ok"
|
|
assert calls["count"] == 1
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_dry_run_evaluates_policy_without_execution(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
_seed_signals(monkeypatch)
|
|
|
|
with client:
|
|
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-auth-public-host-without-authentik").json()
|
|
dry_run = client.post(f"/api/ai/remediation/plans/{plan['id']}/dry-run").json()
|
|
|
|
assert dry_run["executed"] is False
|
|
assert dry_run["status"] in {"pass", "warning", "blocked"}
|
|
assert any(step["policyDecision"] == "blocked" for step in dry_run["stepResults"])
|
|
assert any(item["actionId"].startswith("authentik.") for item in dry_run["blockedActions"])
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_auth_coverage_plan_is_host_aware_and_non_executing(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
_seed_signals(monkeypatch)
|
|
|
|
with client:
|
|
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-auth-public-host-without-authentik").json()
|
|
|
|
assert plan["title"] == "Add identity coverage for 1 public host"
|
|
assert "open.example.com" in plan["description"]
|
|
assert plan["affectedResources"] == ["open.example.com"]
|
|
assert any("host-by-host" in step["description"] for step in plan["steps"])
|
|
assert any(step["id"] == "manual-auth-change" and step["canExecute"] is False for step in plan["steps"])
|
|
assert any(check["endpoint"] == "/api/apps/authentik/auth-coverage" for check in plan["verificationChecks"])
|
|
assert any(item["actionId"] == "authentik.create_application" for item in plan["approvalRequirements"])
|
|
assert plan["executable"] is False
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_media_pipeline_security_finding_uses_media_plan_not_inventory(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
_seed_signals(monkeypatch)
|
|
|
|
with client:
|
|
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-media-pipeline-issue").json()
|
|
|
|
assert plan["title"] == "Review media pipeline issue"
|
|
assert plan["affectedResources"] == ["media_pipeline"]
|
|
assert any(step["href"] == "/media-pipeline" for step in plan["steps"])
|
|
assert not any(step["href"] == "/inventory" for step in plan["steps"])
|
|
assert any(check["endpoint"] == "/api/media-pipeline/summary" for check in plan["verificationChecks"])
|
|
assert plan["executable"] is False
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_ssl_force_finding_has_specific_npm_review_plan(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
|
|
with client:
|
|
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-npm-host-without-ssl-force").json()
|
|
|
|
assert plan["title"] == "Review NPM SSL enforcement"
|
|
assert "Force SSL" in plan["description"]
|
|
assert any(step["actionId"] == "npm.proxy_hosts" for step in plan["steps"])
|
|
assert any(step["id"] == "manual-ssl-change" and step["canExecute"] is False for step in plan["steps"])
|
|
assert any(check["endpoint"] == "/api/apps/nginx-proxy-manager/proxy-hosts" for check in plan["verificationChecks"])
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_authentik_outpost_finding_has_specific_review_plan(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
|
|
with client:
|
|
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-auth-outpost-unavailable").json()
|
|
|
|
assert plan["title"] == "Review Authentik outpost health"
|
|
assert any(step["actionId"] == "authentik.outpost_health" for step in plan["steps"])
|
|
assert any(step["id"] == "manual-outpost-review" and step["canExecute"] is False for step in plan["steps"])
|
|
assert any(check["endpoint"] == "/api/apps/authentik/outposts/health" for check in plan["verificationChecks"])
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_approval_proposal_is_draft_only_and_lifecycle_audited(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
_seed_signals(monkeypatch)
|
|
|
|
with client:
|
|
plan = client.post("/api/ai/remediation/plans/from-finding/ai-sec-policy-unraid-mover-voice-allowed").json()
|
|
reviewed = client.post(f"/api/ai/remediation/plans/{plan['id']}/review").json()
|
|
proposal = client.post(f"/api/ai/remediation/plans/{plan['id']}/create-approval-proposal").json()
|
|
dismissed = client.post(f"/api/ai/remediation/plans/{plan['id']}/dismiss").json()
|
|
approvals = client.get("/api/approvals").json()
|
|
audit = client.get("/api/audit").json()
|
|
|
|
assert reviewed["status"] == "reviewed"
|
|
assert proposal["created"] is True
|
|
assert proposal["executed"] is False
|
|
assert proposal["approvalId"] is None
|
|
assert dismissed["status"] == "dismissed"
|
|
assert approvals == []
|
|
assert any(event["action_id"] == "ai.remediation.approval_proposal" for event in audit)
|
|
|
|
_restore_config(previous)
|
|
|
|
|
|
def test_routes_return_404_for_unknown_plan(tmp_path, monkeypatch):
|
|
client, previous = _client(tmp_path, monkeypatch)
|
|
|
|
with client:
|
|
response = client.get("/api/ai/remediation/plans/rem-missing")
|
|
|
|
assert response.status_code == 404
|
|
|
|
_restore_config(previous)
|