Initial curated RepoPulse source from 2524ccc3c1c27739996dccb2b7992989f88cd26d

backend/app/main.py

@@ -0,0 +1,491 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import hmac
+import json
+import os
+from contextlib import asynccontextmanager
+from pathlib import Path
+from threading import BoundedSemaphore, Lock
+
+from fastapi import FastAPI, HTTPException, Query, Request, Response, status
+from fastapi.concurrency import run_in_threadpool
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+
+from .acquisition import GitAcquirer
+from .gitea import GiteaClient
+from .jobs import ScanJobRunner
+from .models import (
+    RepositoryDetail,
+    RepositoryScanRequest,
+    RepositorySummary,
+    RepositorySyncResult,
+    ScanComparison,
+    ScanJob,
+    ScanReport,
+    ScanRequest,
+    ScanSummary,
+    ScanTrigger,
+    TrendPoint,
+    WebhookResponse,
+)
+from .reporting import content_disposition, render_html_report, safe_report_filename
+from .scanner import scan_repository
+from .store import MaintenanceModeError, QueueFullError, ScanStore
+
+VERSION = "1.5.0"
+store = ScanStore()
+SCAN_ROOT = os.getenv("REPOPULSE_SCAN_ROOT")
+
+
+def _positive_int_env(name: str, default: int) -> int:
+    try:
+        return max(1, int(os.getenv(name, str(default))))
+    except ValueError:
+        return default
+
+
+# The durable queue has one real worker. Keep direct legacy path scans at the
+# same capacity rather than advertising concurrency that cannot be delivered.
+REQUESTED_MAX_CONCURRENT_SCANS = _positive_int_env("REPOPULSE_MAX_CONCURRENT_SCANS", 1)
+MAX_CONCURRENT_SCANS = 1
+SCAN_SLOTS = BoundedSemaphore(MAX_CONCURRENT_SCANS)
+gitea_client = GiteaClient()
+repository_acquirer = GitAcquirer()
+job_runner: ScanJobRunner | None = None
+_api_mutation_lock = Lock()
+_MAINTENANCE_ENTER_PATH = "/api/v1/operator/maintenance/enter"
+_MAINTENANCE_EXIT_PATH = "/api/v1/operator/maintenance/exit"
+
+
+def _get_runner() -> ScanJobRunner:
+    global job_runner
+    if job_runner is None or job_runner.store is not store:
+        job_runner = ScanJobRunner(
+            store,
+            gitea=gitea_client,
+            acquirer=repository_acquirer,
+        )
+    return job_runner
+
+
+@asynccontextmanager
+async def lifespan(_app: FastAPI):
+    runner = _get_runner()
+    runner.start()
+    try:
+        yield
+    finally:
+        runner.stop()
+
+
+app = FastAPI(title="RepoPulse API", version=VERSION, lifespan=lifespan)
+
+
+@app.middleware("http")
+async def maintenance_mutation_gate(request: Request, call_next):
+    """Serialize external API writes with maintenance transitions and reject writes while paused."""
+
+    if not request.url.path.startswith("/api/v1/") or request.method in {"GET", "HEAD", "OPTIONS"}:
+        return await call_next(request)
+    if request.url.path in {_MAINTENANCE_ENTER_PATH, _MAINTENANCE_EXIT_PATH}:
+        return await call_next(request)
+
+    await run_in_threadpool(_api_mutation_lock.acquire)
+    try:
+        if store.maintenance_status()["enabled"]:
+            return JSONResponse(
+                status_code=status.HTTP_423_LOCKED,
+                content={"detail": "RepoPulse is in maintenance mode"},
+                headers={"Retry-After": "30"},
+            )
+        return await call_next(request)
+    finally:
+        _api_mutation_lock.release()
+
+
+def _maintenance_snapshot() -> dict[str, object]:
+    mode = store.maintenance_status()
+    stats = store.queue_stats()
+    queued = int(stats["queued"])
+    active = int(stats["active"])
+    return {
+        **mode,
+        "queue_depth": queued,
+        "active_jobs": active,
+        "active_drained": active == 0,
+        "drained": queued == 0 and active == 0,
+    }
+
+
+def _require_mutation_authorization(request: Request, *, strict: bool = False) -> None:
+    """Use an operator token when configured; deployment perimeter remains explicit otherwise."""
+
+    token = os.getenv("REPOPULSE_OPERATOR_TOKEN", "")
+    if strict:
+        token = os.getenv("REPOPULSE_MAINTENANCE_TOKEN", "") or token
+    if not token:
+        if strict:
+            raise HTTPException(status_code=503, detail="Operator token is not configured")
+        return
+    supplied = request.headers.get("x-repopulse-token", "")
+    if not hmac.compare_digest(supplied, token):
+        raise HTTPException(status_code=401, detail="Operator authorization is required")
+
+
+def _resolve_scan_path(raw_path: str) -> Path:
+    supplied = Path(raw_path).expanduser()
+    if SCAN_ROOT:
+        root = Path(SCAN_ROOT).expanduser().resolve()
+        candidate = (root / supplied).resolve() if not supplied.is_absolute() else supplied.resolve()
+        if candidate != root and root not in candidate.parents:
+            raise ValueError(f"Path must be inside configured scan root {root}")
+        return candidate
+    return supplied.resolve()
+
+
+@app.get("/health")
+def health() -> dict[str, str]:
+    recovery = store.database_recovery_status()
+    status = "degraded" if recovery.get("status") == "degraded_recovered" else "ok"
+    return {"status": status}
+
+
+@app.get("/api/v1/meta")
+def meta() -> dict[str, object]:
+    stats = store.queue_stats()
+    repository_states = store.repository_state_counts()
+    last_sync = store.get_state("last_gitea_sync")
+    parsed_sync: object = None
+    if last_sync:
+        try:
+            parsed_sync = json.loads(last_sync)
+        except json.JSONDecodeError:
+            parsed_sync = last_sync
+    return {
+        "name": "RepoPulse",
+        "version": VERSION,
+        "stage": "v1.5.0",
+        "scan_root_restricted": bool(SCAN_ROOT),
+        "scan_root": SCAN_ROOT,
+        "max_concurrent_scans": MAX_CONCURRENT_SCANS,
+        "requested_max_concurrent_scans": REQUESTED_MAX_CONCURRENT_SCANS,
+        "repository_count": store.repository_count(),
+        "repository_states": repository_states,
+        "current_repositories": repository_states.get("current", 0),
+        "stale_repositories": repository_states.get("stale", 0),
+        "queue_depth": stats["queued"],
+        "active_jobs": stats["active"],
+        "completed_jobs": stats["completed"],
+        "failed_jobs": stats["failed"],
+        "worker_heartbeat": stats["worker_heartbeat"],
+        "gitea": {
+            "configured": gitea_client.configured,
+            "url": gitea_client.base_url or None,
+            "last_sync": parsed_sync,
+        },
+        "storage": store.storage_stats(),
+        "database_recovery": store.database_recovery_status(),
+        "maintenance": _maintenance_snapshot(),
+        "mutation_authentication": "operator_token" if os.getenv("REPOPULSE_OPERATOR_TOKEN") else "perimeter_required",
+    }
+
+
+@app.get("/api/v1/operator/maintenance")
+def get_maintenance_status(request: Request) -> dict[str, object]:
+    _require_mutation_authorization(request, strict=True)
+    return _maintenance_snapshot()
+
+
+@app.post("/api/v1/operator/maintenance/enter")
+def enter_maintenance(request: Request) -> dict[str, object]:
+    _require_mutation_authorization(request, strict=True)
+    with _api_mutation_lock:
+        if store.maintenance_status()["enabled"]:
+            raise HTTPException(status_code=423, detail="RepoPulse is already in maintenance mode")
+        _get_runner().set_maintenance_mode(True)
+    return _maintenance_snapshot()
+
+
+@app.post("/api/v1/operator/maintenance/exit")
+def exit_maintenance(request: Request) -> dict[str, object]:
+    _require_mutation_authorization(request, strict=True)
+    with _api_mutation_lock:
+        _get_runner().set_maintenance_mode(False)
+    return _maintenance_snapshot()
+
+
+@app.post("/api/v1/scans", response_model=ScanReport, status_code=status.HTTP_201_CREATED)
+async def create_scan(payload: ScanRequest, request: Request) -> ScanReport:
+    _require_mutation_authorization(request)
+    try:
+        repository_path = _resolve_scan_path(payload.path)
+    except ValueError as exc:
+        raise HTTPException(status_code=422, detail=str(exc)) from exc
+
+    if not SCAN_SLOTS.acquire(blocking=False):
+        raise HTTPException(
+            status_code=429,
+            detail="Repository scan capacity is currently in use; retry after the active scan completes",
+            headers={"Retry-After": "2"},
+        )
+
+    try:
+        report = await asyncio.to_thread(scan_repository, str(repository_path))
+    except ValueError as exc:
+        raise HTTPException(status_code=422, detail=str(exc)) from exc
+    except PermissionError as exc:
+        raise HTTPException(status_code=403, detail="Repository path cannot be read") from exc
+    except OSError as exc:
+        raise HTTPException(status_code=422, detail="Repository path cannot be scanned") from exc
+    finally:
+        SCAN_SLOTS.release()
+    return store.save(report)
+
+
+@app.get("/api/v1/repositories", response_model=list[RepositorySummary])
+def list_repositories(
+    limit: int = Query(default=100, ge=1, le=1000),
+    offset: int = Query(default=0, ge=0, le=100_000),
+) -> list[RepositorySummary]:
+    return store.list_repositories(limit, offset)
+
+
+@app.post("/api/v1/repositories/sync", response_model=RepositorySyncResult)
+async def sync_repositories(request: Request) -> RepositorySyncResult:
+    _require_mutation_authorization(request)
+    if not gitea_client.configured:
+        raise HTTPException(status_code=503, detail="Gitea connector is not configured")
+    try:
+        result = await asyncio.to_thread(_get_runner().sync_now)
+    except MaintenanceModeError as exc:
+        raise HTTPException(status_code=423, detail=str(exc), headers={"Retry-After": "30"}) from exc
+    if result.errors and result.branch_failures == 0:
+        raise HTTPException(status_code=502, detail=result.errors[0])
+    return result
+
+
+@app.get("/api/v1/repositories/{repository_id}", response_model=RepositoryDetail)
+def get_repository(repository_id: str) -> RepositoryDetail:
+    repository = store.get_repository_detail(repository_id)
+    if repository is None:
+        raise HTTPException(status_code=404, detail="Repository not found")
+    return repository
+
+
+@app.get("/api/v1/repositories/{repository_id}/scans", response_model=list[ScanSummary])
+def list_repository_scans(
+    repository_id: str,
+    limit: int = Query(default=50, ge=1, le=200),
+) -> list[ScanSummary]:
+    if store.get_repository(repository_id) is None:
+        raise HTTPException(status_code=404, detail="Repository not found")
+    return store.repository_scans(repository_id, limit)
+
+
+@app.get("/api/v1/repositories/{repository_id}/trend", response_model=list[TrendPoint])
+def repository_trend(
+    repository_id: str,
+    limit: int = Query(default=50, ge=1, le=200),
+) -> list[TrendPoint]:
+    if store.get_repository(repository_id) is None:
+        raise HTTPException(status_code=404, detail="Repository not found")
+    return store.trend(repository_id, limit)
+
+
+@app.post("/api/v1/repositories/{repository_id}/scans", response_model=ScanJob, status_code=202)
+async def create_repository_scan(
+    request: Request,
+    repository_id: str,
+    payload: RepositoryScanRequest | None = None,
+) -> ScanJob:
+    _require_mutation_authorization(request)
+    repository = store.get_repository(repository_id)
+    if repository is None:
+        raise HTTPException(status_code=404, detail="Repository not found")
+    if repository.archived:
+        raise HTTPException(status_code=409, detail="Archived repositories are not scanned")
+    if not repository.available:
+        raise HTTPException(status_code=409, detail="Repository is no longer available to the Gitea connector")
+    if repository.provider == "gitea" and not gitea_client.configured:
+        raise HTTPException(status_code=503, detail="Gitea connector is not configured")
+    try:
+        return _get_runner().enqueue(repository, force=bool(payload and payload.force))
+    except MaintenanceModeError as exc:
+        raise HTTPException(status_code=423, detail=str(exc), headers={"Retry-After": "30"}) from exc
+    except QueueFullError as exc:
+        raise HTTPException(
+            status_code=429,
+            detail=str(exc),
+            headers={"Retry-After": "5"},
+        ) from exc
+
+
+@app.get("/api/v1/scan-jobs/{job_id}", response_model=ScanJob)
+def get_scan_job(job_id: str) -> ScanJob:
+    job = store.get_job(job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail="Scan job not found")
+    return job
+
+
+@app.get("/api/v1/repositories/{repository_id}/scans/{scan_id}", response_model=ScanReport)
+def get_repository_scan(repository_id: str, scan_id: str) -> ScanReport:
+    report = store.get(scan_id)
+    if report is None or report.source.repository_id != repository_id:
+        raise HTTPException(status_code=404, detail="Repository scan not found")
+    return report
+
+
+@app.post("/api/v1/webhooks/gitea", response_model=WebhookResponse)
+async def gitea_webhook(request: Request) -> WebhookResponse:
+    secret = os.getenv("REPOPULSE_GITEA_WEBHOOK_SECRET", "")
+    if not secret:
+        raise HTTPException(status_code=404, detail="Gitea webhook integration is not configured")
+    body = await request.body()
+    if len(body) > 2_000_000:
+        raise HTTPException(status_code=413, detail="Webhook payload is too large")
+    signature = request.headers.get("x-gitea-signature", "").removeprefix("sha256=")
+    expected = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
+    if not signature or not hmac.compare_digest(signature.casefold(), expected.casefold()):
+        raise HTTPException(status_code=401, detail="Invalid webhook signature")
+    try:
+        payload = json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise HTTPException(status_code=400, detail="Webhook payload is invalid JSON") from exc
+    if not isinstance(payload, dict):
+        raise HTTPException(status_code=400, detail="Webhook payload must be an object")
+    repository_payload = payload.get("repository")
+    if not isinstance(repository_payload, dict):
+        raise HTTPException(status_code=400, detail="Webhook repository is missing")
+    raw_id = repository_payload.get("id")
+    repository_id = f"gitea:{raw_id}" if str(raw_id).isdigit() else None
+    repository = store.get_repository(repository_id) if repository_id else None
+    if repository is None:
+        return WebhookResponse(
+            accepted=False,
+            message="Repository is not present in the synchronized catalog",
+        )
+    event = request.headers.get("x-gitea-event", "push").casefold()
+    if event != "push":
+        return WebhookResponse(
+            accepted=True,
+            repository_id=repository.id,
+            message="Non-push webhook acknowledged without scheduling a scan",
+        )
+    ref = payload.get("ref")
+    if ref != f"refs/heads/{repository.default_branch}":
+        return WebhookResponse(
+            accepted=True,
+            repository_id=repository.id,
+            message="Non-default branch push acknowledged without scheduling a scan",
+        )
+    after = payload.get("after")
+    if (
+        not isinstance(after, str)
+        or not 7 <= len(after) <= 64
+        or not all(char in "0123456789abcdefABCDEF" for char in after)
+        or not set(after) - {"0"}
+    ):
+        return WebhookResponse(
+            accepted=True,
+            repository_id=repository.id,
+            message="Push acknowledged without a usable commit revision",
+        )
+    if not gitea_client.configured:
+        raise HTTPException(status_code=503, detail="Gitea connector is not configured")
+    updated = repository.model_copy(update={"latest_revision": after, "sync_error": None})
+    store.upsert_repository(updated)
+    try:
+        job = _get_runner().enqueue(updated, trigger=ScanTrigger.WEBHOOK)
+    except MaintenanceModeError as exc:
+        raise HTTPException(status_code=423, detail=str(exc), headers={"Retry-After": "30"}) from exc
+    except QueueFullError as exc:
+        raise HTTPException(status_code=429, detail=str(exc), headers={"Retry-After": "5"}) from exc
+    return WebhookResponse(
+        accepted=True,
+        repository_id=repository.id,
+        job_id=job.id,
+        message="Push accepted and scan scheduled",
+    )
+
+
+@app.post("/api/v1/operator/database-recovery/acknowledge", status_code=status.HTTP_204_NO_CONTENT)
+def acknowledge_database_recovery(request: Request) -> Response:
+    _require_mutation_authorization(request, strict=True)
+    store.acknowledge_database_recovery()
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@app.get("/api/v1/scans", response_model=list[ScanReport])
+def list_scans(limit: int = Query(default=20, ge=1, le=100)) -> list[ScanReport]:
+    return store.recent(limit)
+
+
+@app.get("/api/v1/scans/{scan_id}", response_model=ScanReport)
+def get_scan(scan_id: str) -> ScanReport:
+    report = store.get(scan_id)
+    if report is None:
+        raise HTTPException(status_code=404, detail="Scan not found")
+    return report
+
+
+@app.get("/api/v1/scans/{scan_id}/comparison", response_model=ScanComparison)
+def get_scan_comparison(scan_id: str) -> ScanComparison:
+    report = store.get(scan_id)
+    if report is None:
+        raise HTTPException(status_code=404, detail="Scan not found")
+    return store.compare(report)
+
+
+@app.get("/api/v1/scans/{scan_id}/delta")
+def get_scan_delta(scan_id: str) -> dict[str, int | None]:
+    report = store.get(scan_id)
+    if report is None:
+        raise HTTPException(status_code=404, detail="Scan not found")
+    previous = store.previous_for_repository(
+        report.source.repository_id or report.repository_path,
+        report.id,
+    )
+    return {
+        "previous_health_score": previous.health_score if previous else None,
+        "health_score_delta": report.health_score - previous.health_score if previous else None,
+        "previous_finding_count": len(previous.findings) if previous else None,
+        "finding_count_delta": len(report.findings) - len(previous.findings) if previous else None,
+    }
+
+
+@app.get("/api/v1/trend", response_model=list[TrendPoint])
+def trend(path: str, limit: int = Query(default=30, ge=1, le=100)) -> list[TrendPoint]:
+    try:
+        resolved = _resolve_scan_path(path)
+    except ValueError as exc:
+        raise HTTPException(status_code=422, detail=str(exc)) from exc
+    return store.trend(str(resolved), limit)
+
+
+@app.get("/api/v1/scans/{scan_id}/export")
+def export_scan(scan_id: str, format: str = Query(default="html", pattern="^(html|json)$")) -> Response:
+    report = store.get(scan_id)
+    if report is None:
+        raise HTTPException(status_code=404, detail="Scan not found")
+    if format == "json":
+        filename = safe_report_filename(report.repository_name, "json")
+        return Response(
+            report.model_dump_json(indent=2),
+            media_type="application/json",
+            headers={"Content-Disposition": content_disposition(filename)},
+        )
+    filename = safe_report_filename(report.repository_name, "html")
+    return Response(
+        render_html_report(report),
+        media_type="text/html",
+        headers={"Content-Disposition": content_disposition(filename)},
+    )
+
+
+static_dir = Path(os.getenv("REPOPULSE_STATIC_DIR", "/app/static"))
+if static_dir.is_dir():
+    app.mount("/", StaticFiles(directory=static_dir, html=True), name="frontend")