131 lines
4.7 KiB
Python
131 lines
4.7 KiB
Python
from __future__ import annotations
|
|
|
|
from collections import defaultdict, deque
|
|
from datetime import UTC, datetime
|
|
import math
|
|
from threading import Lock
|
|
from typing import Any
|
|
|
|
from sqlmodel import Session
|
|
|
|
from ..core.time import utc_now
|
|
from ..db.models import AppSetting
|
|
|
|
|
|
_MAX_SAMPLES = 2_000
|
|
_PERSISTED_SAMPLES = 500
|
|
_SETTING_KEY = "observability:request_samples"
|
|
_samples: deque[dict[str, Any]] = deque(maxlen=_MAX_SAMPLES)
|
|
_lock = Lock()
|
|
_source_identity: str | None = None
|
|
|
|
|
|
def record_request(*, method: str, path: str, status: int, duration_ms: float) -> None:
|
|
sample = {
|
|
"method": method,
|
|
"path": path,
|
|
"status": status,
|
|
"durationMs": round(max(0.0, duration_ms), 2),
|
|
"observedAt": datetime.now(UTC).isoformat(),
|
|
}
|
|
with _lock:
|
|
_samples.append(sample)
|
|
|
|
|
|
def _percentile(values: list[float], percentile: float) -> float:
|
|
if not values:
|
|
return 0.0
|
|
rank = max(1, math.ceil(max(0.0, min(percentile, 1.0)) * len(values)))
|
|
return values[min(len(values) - 1, rank - 1)]
|
|
|
|
|
|
def request_metrics_snapshot() -> dict[str, Any]:
|
|
with _lock:
|
|
samples = list(_samples)
|
|
|
|
grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
|
|
for sample in samples:
|
|
grouped[(sample["method"], sample["path"])].append(sample)
|
|
|
|
endpoints = []
|
|
for (method, path), items in grouped.items():
|
|
durations = sorted(float(item["durationMs"]) for item in items)
|
|
endpoints.append({
|
|
"method": method,
|
|
"path": path,
|
|
"requests": len(items),
|
|
"errors": sum(1 for item in items if int(item["status"]) >= 400),
|
|
"averageMs": round(sum(durations) / len(durations), 2),
|
|
"p95Ms": round(_percentile(durations, 0.95), 2),
|
|
"maxMs": round(durations[-1], 2),
|
|
})
|
|
|
|
endpoints.sort(key=lambda item: (item["p95Ms"], item["requests"]), reverse=True)
|
|
return {
|
|
"window": "process_and_persisted",
|
|
"sampleLimit": _MAX_SAMPLES,
|
|
"sampleCount": len(samples),
|
|
"generatedAt": datetime.now(UTC).isoformat(),
|
|
"slowEndpoints": endpoints[:30],
|
|
"recentErrors": [sample for sample in samples if int(sample["status"]) >= 400][-20:],
|
|
}
|
|
|
|
|
|
def restore_request_metrics(session: Session) -> int:
|
|
global _source_identity
|
|
source_identity = str(session.get_bind().url)
|
|
row = session.get(AppSetting, _SETTING_KEY)
|
|
persisted = (row.value_json or {}).get("samples", []) if row else []
|
|
restored = 0
|
|
with _lock:
|
|
if _source_identity != source_identity:
|
|
_samples.clear()
|
|
_source_identity = source_identity
|
|
if _samples:
|
|
return 0
|
|
for sample in persisted[-_PERSISTED_SAMPLES:]:
|
|
if not isinstance(sample, dict):
|
|
continue
|
|
if not {"method", "path", "status", "durationMs", "observedAt"}.issubset(sample):
|
|
continue
|
|
_samples.append({
|
|
"method": str(sample["method"])[:12],
|
|
"path": str(sample["path"])[:240],
|
|
"status": int(sample["status"]),
|
|
"durationMs": round(max(0.0, float(sample["durationMs"])), 2),
|
|
"observedAt": str(sample["observedAt"])[:64],
|
|
})
|
|
restored += 1
|
|
return restored
|
|
|
|
|
|
def persist_request_metrics(session: Session) -> int:
|
|
with _lock:
|
|
samples = list(_samples)[-_PERSISTED_SAMPLES:]
|
|
row = session.get(AppSetting, _SETTING_KEY) or AppSetting(key=_SETTING_KEY)
|
|
row.value_json = {"samples": samples, "persistedAt": datetime.now(UTC).isoformat()}
|
|
row.is_secret = False
|
|
row.updated_at = utc_now()
|
|
session.add(row)
|
|
session.commit()
|
|
return len(samples)
|
|
|
|
|
|
def request_metrics_openmetrics() -> str:
|
|
snapshot = request_metrics_snapshot()
|
|
lines = [
|
|
"# HELP appops_http_requests_total Requests observed by method and route template.",
|
|
"# TYPE appops_http_requests_total counter",
|
|
"# HELP appops_http_request_duration_p95_milliseconds Nearest-rank p95 request duration.",
|
|
"# TYPE appops_http_request_duration_p95_milliseconds gauge",
|
|
]
|
|
for endpoint in snapshot["slowEndpoints"]:
|
|
method = str(endpoint["method"]).replace('"', '')
|
|
path = str(endpoint["path"]).replace('\\', '\\\\').replace('"', '\\"')
|
|
labels = f'method="{method}",path="{path}"'
|
|
lines.append(f'appops_http_requests_total{{{labels}}} {endpoint["requests"]}')
|
|
lines.append(f'appops_http_request_errors_total{{{labels}}} {endpoint["errors"]}')
|
|
lines.append(f'appops_http_request_duration_p95_milliseconds{{{labels}}} {endpoint["p95Ms"]}')
|
|
lines.append(f'appops_http_request_samples {snapshot["sampleCount"]}')
|
|
return "\n".join(lines) + "\n"
|