from __future__ import annotations from collections import defaultdict, deque from datetime import UTC, datetime import math from threading import Lock from typing import Any from sqlmodel import Session from ..core.time import utc_now from ..db.models import AppSetting _MAX_SAMPLES = 2_000 _PERSISTED_SAMPLES = 500 _SETTING_KEY = "observability:request_samples" _samples: deque[dict[str, Any]] = deque(maxlen=_MAX_SAMPLES) _lock = Lock() _source_identity: str | None = None def record_request(*, method: str, path: str, status: int, duration_ms: float) -> None: sample = { "method": method, "path": path, "status": status, "durationMs": round(max(0.0, duration_ms), 2), "observedAt": datetime.now(UTC).isoformat(), } with _lock: _samples.append(sample) def _percentile(values: list[float], percentile: float) -> float: if not values: return 0.0 rank = max(1, math.ceil(max(0.0, min(percentile, 1.0)) * len(values))) return values[min(len(values) - 1, rank - 1)] def request_metrics_snapshot() -> dict[str, Any]: with _lock: samples = list(_samples) grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) for sample in samples: grouped[(sample["method"], sample["path"])].append(sample) endpoints = [] for (method, path), items in grouped.items(): durations = sorted(float(item["durationMs"]) for item in items) endpoints.append({ "method": method, "path": path, "requests": len(items), "errors": sum(1 for item in items if int(item["status"]) >= 400), "averageMs": round(sum(durations) / len(durations), 2), "p95Ms": round(_percentile(durations, 0.95), 2), "maxMs": round(durations[-1], 2), }) endpoints.sort(key=lambda item: (item["p95Ms"], item["requests"]), reverse=True) return { "window": "process_and_persisted", "sampleLimit": _MAX_SAMPLES, "sampleCount": len(samples), "generatedAt": datetime.now(UTC).isoformat(), "slowEndpoints": endpoints[:30], "recentErrors": [sample for sample in samples if int(sample["status"]) >= 400][-20:], } def restore_request_metrics(session: Session) -> int: global _source_identity source_identity = str(session.get_bind().url) row = session.get(AppSetting, _SETTING_KEY) persisted = (row.value_json or {}).get("samples", []) if row else [] restored = 0 with _lock: if _source_identity != source_identity: _samples.clear() _source_identity = source_identity if _samples: return 0 for sample in persisted[-_PERSISTED_SAMPLES:]: if not isinstance(sample, dict): continue if not {"method", "path", "status", "durationMs", "observedAt"}.issubset(sample): continue _samples.append({ "method": str(sample["method"])[:12], "path": str(sample["path"])[:240], "status": int(sample["status"]), "durationMs": round(max(0.0, float(sample["durationMs"])), 2), "observedAt": str(sample["observedAt"])[:64], }) restored += 1 return restored def persist_request_metrics(session: Session) -> int: with _lock: samples = list(_samples)[-_PERSISTED_SAMPLES:] row = session.get(AppSetting, _SETTING_KEY) or AppSetting(key=_SETTING_KEY) row.value_json = {"samples": samples, "persistedAt": datetime.now(UTC).isoformat()} row.is_secret = False row.updated_at = utc_now() session.add(row) session.commit() return len(samples) def request_metrics_openmetrics() -> str: snapshot = request_metrics_snapshot() lines = [ "# HELP appops_http_requests_total Requests observed by method and route template.", "# TYPE appops_http_requests_total counter", "# HELP appops_http_request_duration_p95_milliseconds Nearest-rank p95 request duration.", "# TYPE appops_http_request_duration_p95_milliseconds gauge", ] for endpoint in snapshot["slowEndpoints"]: method = str(endpoint["method"]).replace('"', '') path = str(endpoint["path"]).replace('\\', '\\\\').replace('"', '\\"') labels = f'method="{method}",path="{path}"' lines.append(f'appops_http_requests_total{{{labels}}} {endpoint["requests"]}') lines.append(f'appops_http_request_errors_total{{{labels}}} {endpoint["errors"]}') lines.append(f'appops_http_request_duration_p95_milliseconds{{{labels}}} {endpoint["p95Ms"]}') lines.append(f'appops_http_request_samples {snapshot["sampleCount"]}') return "\n".join(lines) + "\n"