Publish curated MobilityOps source snapshot

backend/app/core/config.py

@@ -0,0 +1,128 @@
+from functools import lru_cache
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+# The visible product name is fixed and never translated or configured per-deployment --
+# see docs/fleet-ops-correction/current-gap-audit.md section 1. Internal identifiers
+# (package name, Compose project, database name, repository) intentionally remain
+# "mobilityops"; this constant is only for user-facing surfaces (e.g. the OpenAPI title).
+PRODUCT_NAME = "Fleet Ops"
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    mobilityops_env: str = "development"
+    mobilityops_demo_mode: bool = True
+    database_url: str = "postgresql+psycopg://mobilityops:mobilityops@db:5432/mobilityops"
+    knowledge_provider: str = "demo"
+    ragcore_base_url: str = "http://ragcore-api:8000"
+    ragcore_tenant: str = "northstar-mobility-demo"
+    ragcore_workspace: str = "mobilityops"
+    ragcore_collection: str = "internal-procedures"
+    ragcore_api_token: str = ""
+    ragcore_space_id: str = ""
+    ragcore_http_timeout_seconds: float = 5.0
+    ragcore_answers_circuit_breaker_seconds: float = 60.0
+    # Search fallback is only labelled grounded above this explicit retrieval threshold.
+    # RAGcore's fused score is reciprocal-rank based (top ranks are ~1/61), so this
+    # accepts only leading results while still rejecting absent and low-ranked evidence.
+    ragcore_min_search_score: float = 0.016
+    n8n_webhook_url: str = "http://n8n:5678/webhook/mobilityops-return"
+    n8n_webhook_trigger_token: str = "replace-me-n8n-webhook-trigger-token"
+    n8n_callback_token: str = "replace-me-n8n-callback-token"
+    n8n_dispatch_enabled: bool = True
+    n8n_dispatch_interval_seconds: float = 3.0
+    # The synchronous n8n workflow performs two bounded, retried callbacks before it
+    # acknowledges an event. Keep this above that complete workflow budget, while the
+    # delivery lease remains the wider crash-recovery boundary (enforced by the contract
+    # check in scripts/check-contracts.py).
+    n8n_http_timeout_seconds: float = 15.0
+    n8n_max_attempts: int = 5
+    n8n_delivery_lease_seconds: float = 120.0
+    app_secret: str = "replace-in-production"
+    session_cookie_name: str = "mobilityops_session"
+    session_ttl_seconds: int = 60 * 60 * 8
+    session_cookie_secure: bool = False
+    seed_dir: str = "/app/seed"
+    knowledge_dir: str = "/app/knowledge/procedures"
+    mcp_hub_service_token: str = "replace-me-mcp-hub-token"
+    mcp_hub_registration_enabled: bool = False
+    # MCP Hub's own registration is catalog-driven on the Hub side (the Hub reconciles
+    # its catalog into the gateway; Fleet Ops never pushes a registration call), so
+    # these are only used for an honest reachability health check, not self-registration.
+    mcp_hub_base_url: str = ""
+    mcp_provider_id: str = "fleet-ops"
+    cors_allow_origins: str = "http://localhost:1228"
+    demo_organization_name: str = "Northstar Mobility"
+    demo_timezone: str = "Europe/Brussels"
+    demo_allow_reset: bool = True
+    demo_reset_cooldown_seconds: int = 60
+    mcp_hub_health_cache_seconds: int = 60
+    initial_admin_email: str = ""
+    initial_admin_password: str = ""
+    initial_admin_display_name: str = "Operations Manager"
+    mobilityops_public_url: str = "http://localhost:1228"
+    oidc_enabled: bool = False
+    oidc_provider_name: str = "Organisatieaccount"
+    oidc_issuer_url: str = ""
+    oidc_client_id: str = ""
+    oidc_client_secret: str = ""
+    oidc_redirect_uri: str = ""
+    oidc_allowed_email_domains: str = ""
+    oidc_auto_provision: bool = True
+    oidc_default_role: str = "rental_employee"
+    log_level: str = "INFO"
+    # Failed password logins per client IP before a temporary 429 (0 disables).
+    login_max_failures: int = 10
+    login_failure_window_seconds: int = 900
+    knowledge_max_requests: int = 30
+    knowledge_rate_limit_window_seconds: int = 60
+    metrics_bearer_token: str = ""
+    privacy_minimum_booking_retention_days: int = 30
+    privacy_audit_retention_days: int = 2555
+    privacy_audit_export_max_rows: int = 10000
+
+
+# Secrets that guard *inbound* trust (session cookies, service callbacks). Running
+# production with any of these at their placeholder value means forged sessions or
+# unauthenticated writes, so startup refuses.
+INSECURE_DEFAULT_SECRETS: tuple[tuple[str, str], ...] = (
+    ("app_secret", "replace-in-production"),
+    ("n8n_callback_token", "replace-me-n8n-callback-token"),
+    ("mcp_hub_service_token", "replace-me-mcp-hub-token"),
+)
+
+
+def insecure_default_secrets(settings: "Settings") -> list[str]:
+    """Return the names of secret settings that still carry their placeholder value.
+
+    MCP routes are always mounted, independently of the Hub reachability-status flag, so
+    their inbound token must always be non-placeholder in production.
+    """
+    insecure: list[str] = []
+    for name, placeholder in INSECURE_DEFAULT_SECRETS:
+        value = getattr(settings, name)
+        if not value or value == placeholder or value.startswith("replace-me"):
+            insecure.append(name)
+    return insecure
+
+
+@lru_cache
+def get_settings() -> Settings:
+    settings = Settings()
+    if settings.mobilityops_env.lower() == "production":
+        insecure = insecure_default_secrets(settings)
+        if insecure:
+            # Refuse to boot rather than run production with forgeable session cookies
+            # or guessable service tokens. Development/test/demo keep the defaults.
+            raise RuntimeError(
+                "Refusing to start in production with placeholder secrets: "
+                + ", ".join(insecure)
+                + ". Set real values in the environment (see .env.example)."
+            )
+        if not settings.mobilityops_public_url.lower().startswith("https://"):
+            raise RuntimeError("Production MOBILITYOPS_PUBLIC_URL must use HTTPS.")
+        if not settings.session_cookie_secure:
+            raise RuntimeError("Production SESSION_COOKIE_SECURE must be true.")
+    return settings

backend/app/core/db.py

@@ -0,0 +1,54 @@
+from collections.abc import Generator
+
+from sqlalchemy import create_engine, event, func, select
+from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
+
+from app.core.config import get_settings
+
+settings = get_settings()
+
+engine = create_engine(settings.database_url, pool_pre_ping=True, future=True)
+SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, future=True)
+
+# Every SQLAlchemy transaction participates in a shared database-wide barrier. Normal
+# reads/writes coexist; the short demo reset takes the exclusive form so it can never
+# interleave deletes/inserts with an API request, scanner, callback, or dispatcher cycle.
+DEMO_DATA_BARRIER_LOCK_ID = 5_344_725_149_212_793_901
+_EXCLUSIVE_RESET_INFO_KEY = "mobilityops_demo_reset_exclusive"
+
+
+@event.listens_for(Session, "after_begin")
+def _acquire_demo_data_barrier(session: Session, _transaction, connection) -> None:
+    if connection.dialect.name != "postgresql":
+        return
+    lock = (
+        func.pg_advisory_xact_lock(DEMO_DATA_BARRIER_LOCK_ID)
+        if session.info.get(_EXCLUSIVE_RESET_INFO_KEY)
+        else func.pg_advisory_xact_lock_shared(DEMO_DATA_BARRIER_LOCK_ID)
+    )
+    connection.execute(select(lock))
+
+
+def begin_exclusive_demo_reset(db: Session) -> None:
+    """Make the session's next transaction the exclusive side of the reset barrier."""
+    if db.in_transaction():
+        # Auth normally already read the user under a shared barrier. End that read-only
+        # transaction before requesting exclusive; in-place lock upgrades can deadlock.
+        db.rollback()
+    db.info[_EXCLUSIVE_RESET_INFO_KEY] = True
+
+
+def end_exclusive_demo_reset(db: Session) -> None:
+    db.info.pop(_EXCLUSIVE_RESET_INFO_KEY, None)
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+def get_db() -> Generator[Session, None, None]:
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

backend/app/core/errors.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+
+class AppError(Exception):
+    def __init__(
+        self,
+        code: str,
+        message: str,
+        status_code: int = 400,
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+        self.status_code = status_code
+        self.details = details or {}
+        self.correlation_id = str(uuid.uuid4())
+
+
+def error_body(code: str, message: str, correlation_id: str, details: dict[str, Any]) -> dict:
+    return {
+        "error": {
+            "code": code,
+            "message": message,
+            "correlation_id": correlation_id,
+            "details": details,
+        }
+    }

backend/app/core/observability.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import json
+import logging
+import time
+import uuid
+from contextvars import ContextVar
+from datetime import UTC, datetime
+
+from fastapi import Request
+from prometheus_client import Counter, Gauge, Histogram
+
+correlation_id_context: ContextVar[str] = ContextVar("correlation_id", default="")
+
+HTTP_REQUESTS = Counter(
+    "mobilityops_http_requests_total",
+    "Completed MobilityOps HTTP requests.",
+    ("method", "route", "status"),
+)
+HTTP_DURATION = Histogram(
+    "mobilityops_http_request_duration_seconds",
+    "MobilityOps HTTP request duration.",
+    ("method", "route"),
+    buckets=(0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10),
+)
+HTTP_IN_PROGRESS = Gauge(
+    "mobilityops_http_requests_in_progress",
+    "MobilityOps HTTP requests currently executing.",
+)
+OUTBOX_EVENTS = Gauge(
+    "mobilityops_outbox_events",
+    "Persisted outbox events by state and scenario type.",
+    ("scenario", "status"),
+)
+DATABASE_READY = Gauge(
+    "mobilityops_database_ready",
+    "Whether the canonical PostgreSQL database answered the most recent readiness probe.",
+)
+KNOWLEDGE_PROVIDER_REQUESTS = Counter(
+    "mobilityops_knowledge_provider_requests_total",
+    "RAGcore adapter requests by stage and outcome.",
+    ("stage", "outcome"),
+)
+KNOWLEDGE_RETRIEVAL_SCORE = Histogram(
+    "mobilityops_knowledge_retrieval_score",
+    "Observed RAGcore fused/rerank retrieval scores.",
+    buckets=(0.005, 0.01, 0.015, 0.016, 0.0162, 0.0164, 0.02, 0.05, 0.1, 0.5, 1.0),
+)
+
+
+class JsonFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, object] = {
+            "timestamp": datetime.now(UTC).isoformat(),
+            "level": record.levelname.lower(),
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        correlation_id = correlation_id_context.get()
+        if correlation_id:
+            payload["correlation_id"] = correlation_id
+        for key in ("method", "path", "status_code", "duration_ms", "client_ip"):
+            value = getattr(record, key, None)
+            if value is not None:
+                payload[key] = value
+        if record.exc_info:
+            payload["exception"] = self.formatException(record.exc_info)
+        return json.dumps(payload, separators=(",", ":"), default=str)
+
+
+def configure_logging(level: str) -> None:
+    handler = logging.StreamHandler()
+    handler.setFormatter(JsonFormatter())
+    root = logging.getLogger()
+    root.handlers = [handler]
+    root.setLevel(level.upper())
+
+
+def correlation_id_for(request: Request) -> str:
+    candidate = request.headers.get("X-Correlation-Id", "").strip()
+    try:
+        return str(uuid.UUID(candidate)) if candidate else str(uuid.uuid4())
+    except ValueError:
+        return str(uuid.uuid4())
+
+
+UNMATCHED_ROUTE_LABEL = "<unmatched>"
+
+
+def route_label(request: Request) -> str:
+    """Return the route *template* for metrics labels.
+
+    Unmatched paths (404 probes, scanners) must not become their own label value:
+    every distinct URL would otherwise create a new Prometheus time series and the
+    metric cardinality would grow without bound.
+    """
+    route = request.scope.get("route")
+    path = getattr(route, "path", None)
+    return str(path) if path else UNMATCHED_ROUTE_LABEL
+
+
+def request_started() -> float:
+    HTTP_IN_PROGRESS.inc()
+    return time.perf_counter()
+
+
+def request_finished(request: Request, status_code: int, started_at: float) -> float:
+    duration = time.perf_counter() - started_at
+    route = route_label(request)
+    HTTP_REQUESTS.labels(request.method, route, str(status_code)).inc()
+    HTTP_DURATION.labels(request.method, route).observe(duration)
+    HTTP_IN_PROGRESS.dec()
+    return duration

backend/app/core/ratelimit.py

@@ -0,0 +1,72 @@
+"""Small in-process failed-attempt limiter for credential endpoints.
+
+Fleet Ops runs as a single API process per deployment, so an in-memory sliding window
+is sufficient to blunt online password guessing (and the scrypt CPU amplification that
+comes with it) without adding Redis. Only *failed* attempts count, so legitimate users
+and the automated test suite are never throttled.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections import deque
+
+
+class FailedAttemptLimiter:
+    def __init__(self, *, max_failures: int, window_seconds: float) -> None:
+        self.max_failures = max_failures
+        self.window_seconds = window_seconds
+        self._failures: dict[str, deque[float]] = {}
+        self._lock = threading.Lock()
+
+    def _prune(self, key: str, now: float) -> deque[float]:
+        bucket = self._failures.setdefault(key, deque())
+        cutoff = now - self.window_seconds
+        while bucket and bucket[0] <= cutoff:
+            bucket.popleft()
+        if not bucket:
+            self._failures.pop(key, None)
+        return bucket
+
+    def retry_after_seconds(self, key: str) -> int:
+        """Return >0 seconds to wait when the key is currently blocked, else 0."""
+        now = time.monotonic()
+        with self._lock:
+            bucket = self._prune(key, now)
+            if len(bucket) < self.max_failures:
+                return 0
+            return max(1, int(bucket[0] + self.window_seconds - now + 0.999))
+
+    def record_failure(self, key: str) -> None:
+        now = time.monotonic()
+        with self._lock:
+            self._prune(key, now)
+            self._failures.setdefault(key, deque()).append(now)
+
+    def reset(self, key: str) -> None:
+        with self._lock:
+            self._failures.pop(key, None)
+
+
+class SlidingWindowLimiter:
+    """Thread-safe request limiter where every accepted request consumes capacity."""
+
+    def __init__(self, *, max_requests: int, window_seconds: float) -> None:
+        self.max_requests = max_requests
+        self.window_seconds = window_seconds
+        self._requests: dict[str, deque[float]] = {}
+        self._lock = threading.Lock()
+
+    def consume(self, key: str) -> int:
+        """Record an accepted request, or return the seconds until capacity is available."""
+        now = time.monotonic()
+        with self._lock:
+            bucket = self._requests.setdefault(key, deque())
+            cutoff = now - self.window_seconds
+            while bucket and bucket[0] <= cutoff:
+                bucket.popleft()
+            if len(bucket) >= self.max_requests:
+                return max(1, int(bucket[0] + self.window_seconds - now + 0.999))
+            bucket.append(now)
+            return 0

backend/app/core/security.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import json
+import os
+import time
+from dataclasses import dataclass
+
+from app.core.config import get_settings
+
+settings = get_settings()
+
+
+@dataclass(frozen=True)
+class SessionPayload:
+    user_id: str
+    public_ref: str
+    role: str
+    display_name: str
+    issued_at: int
+    session_id: str = ""
+
+
+def _sign(data: bytes) -> str:
+    digest = hmac.new(settings.app_secret.encode(), data, hashlib.sha256).digest()
+    return base64.urlsafe_b64encode(digest).decode().rstrip("=")
+
+
+def session_token_hash(token: str) -> str:
+    """Return a non-reversible identifier safe to persist for token revocation."""
+    return hashlib.sha256(token.encode()).hexdigest()
+
+
+def create_session_token(payload: SessionPayload) -> str:
+    body = json.dumps(payload.__dict__, separators=(",", ":")).encode()
+    encoded_body = base64.urlsafe_b64encode(body).decode().rstrip("=")
+    signature = _sign(encoded_body.encode())
+    return f"{encoded_body}.{signature}"
+
+
+def read_session_token(token: str) -> SessionPayload | None:
+    try:
+        encoded_body, signature = token.split(".", 1)
+    except ValueError:
+        return None
+    expected = _sign(encoded_body.encode())
+    if not hmac.compare_digest(expected, signature):
+        return None
+    padding = "=" * (-len(encoded_body) % 4)
+    try:
+        body = json.loads(base64.urlsafe_b64decode(encoded_body + padding))
+    except (ValueError, json.JSONDecodeError):
+        return None
+    payload = SessionPayload(**body)
+    if time.time() - payload.issued_at > settings.session_ttl_seconds:
+        return None
+    return payload
+
+
+def hash_password(password: str) -> str:
+    salt = os.urandom(16)
+    derived = hashlib.scrypt(password.encode(), salt=salt, n=2**14, r=8, p=1, dklen=32)
+    encoded_salt = base64.urlsafe_b64encode(salt).decode()
+    encoded_hash = base64.urlsafe_b64encode(derived).decode()
+    return f"scrypt$16384$8$1${encoded_salt}${encoded_hash}"
+
+
+def verify_password(password: str, encoded: str | None) -> bool:
+    if not encoded:
+        return False
+    try:
+        algorithm, n, r, p, salt, expected = encoded.split("$")
+        if algorithm != "scrypt":
+            return False
+        derived = hashlib.scrypt(
+            password.encode(),
+            salt=base64.urlsafe_b64decode(salt.encode()),
+            n=int(n),
+            r=int(r),
+            p=int(p),
+            dklen=32,
+        )
+        return hmac.compare_digest(derived, base64.urlsafe_b64decode(expected.encode()))
+    except (ValueError, TypeError):
+        return False