Publish curated Unraid-AppOps-Gateway source from 034c9f844d2455850ba37b31379b7edaa786583e

backend/app/api/routes_observability.py

@@ -1,16 +1,19 @@
 from __future__ import annotations
 
-from concurrent.futures import ThreadPoolExecutor
-from datetime import UTC, datetime
 import hashlib
 import json
+import math
 import os
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from threading import Lock
 from typing import Any
 
 from fastapi import APIRouter, Depends
 from sqlmodel import Session, select
 
 from ..connectors.authentik import AuthentikConnector
+from ..connectors.gpu_metrics import GpuMetricsConnector
 from ..connectors.nginx_proxy_manager import NginxProxyManagerConnector
 from ..connectors.portainer import PortainerConnector
 from ..connectors.unraid import UnraidConnector
@@ -19,18 +22,23 @@ from ..core.redaction import redact_message
 from ..core.time import utc_now
 from ..db.models import AppSetting, AuditEvent, Connector, JobRun, Policy, Secret
 from ..db.session import get_session
-from ..services.incident_service import list_incidents as list_live_incidents, update_incident_status as update_live_incident_status
 from ..services.app_registry import list_active_apps
+from ..services.incident_service import list_incidents as list_live_incidents
+from ..services.incident_service import (
+    update_incident_status as update_live_incident_status,
+)
 from ..services.setup_service import apply_stored_connector_settings
+from ..services.exposure_preferences import get_exposure_preferences, runtime_nginx_access_list_checks_enabled
 
 router = APIRouter(prefix='/api')
 
-NETWORK_EXPOSURE_CACHE_SCHEMA_VERSION = 1
+NETWORK_EXPOSURE_CACHE_SCHEMA_VERSION = 2
 NETWORK_EXPOSURE_CACHE_KEY = 'network_exposure:cache'
 NETWORK_EXPOSURE_CACHE_TTL_SECONDS = 60
 NETWORK_EXPOSURE_CACHE_MAX_STALE_SECONDS = 300
-STATISTICS_HISTORY_MAX_POINTS = 48
+STATISTICS_HISTORY_MAX_POINTS = 361
 _STATISTICS_HISTORY: list[dict[str, Any]] = []
+_STATISTICS_HISTORY_LOCK = Lock()
 
 
 @router.get('/incidents')
@@ -90,40 +98,55 @@ def operations_map(session: Session = Depends(get_session)):
     }
 
 
-def _nullable_int(value: Any) -> int | None:
+def _nullable_percent(value: Any) -> float | None:
     if value is None or isinstance(value, bool):
         return None
-    if isinstance(value, int | float):
-        return int(round(float(value)))
     if isinstance(value, str):
-        stripped = value.strip().strip('%')
-        if not stripped:
-            return None
-        try:
-            return int(round(float(stripped)))
-        except ValueError:
-            return None
-    return None
+        value = value.strip().strip('%')
+    try:
+        number = float(value)
+    except (ValueError, TypeError, OverflowError):
+        return None
+    return round(number, 2) if math.isfinite(number) and 0 <= number <= 100 else None
 
 
-def _record_statistics_sample(generated_at: str, cpu_usage: int | None, ram_usage: int | None) -> list[dict[str, Any]]:
-    if cpu_usage is None and ram_usage is None:
+def _record_statistics_sample(generated_at: str, cpu_usage: float | None, ram_usage: float | None, gpu_usage: float | None) -> list[dict[str, Any]]:
+    sample = {'time': generated_at, 'cpu': cpu_usage, 'ram': ram_usage, 'gpu': gpu_usage}
+    sampled_at = datetime.fromisoformat(generated_at.replace('Z', '+00:00'))
+    with _STATISTICS_HISTORY_LOCK:
+        if _STATISTICS_HISTORY:
+            previous_at = datetime.fromisoformat(_STATISTICS_HISTORY[-1]['time'].replace('Z', '+00:00'))
+            elapsed = (sampled_at - previous_at).total_seconds()
+            if elapsed >= 0 and int(sampled_at.timestamp() // 5) == int(previous_at.timestamp() // 5):
+                # Multiple viewers refresh the latest real measurement, rather than exhausting the window.
+                _STATISTICS_HISTORY[-1] = sample
+            elif elapsed >= 0:
+                _STATISTICS_HISTORY.append(sample)
+        else:
+            _STATISTICS_HISTORY.append(sample)
+        _STATISTICS_HISTORY[:] = [
+            item for item in _STATISTICS_HISTORY
+            if (sampled_at - datetime.fromisoformat(item['time'].replace('Z', '+00:00'))).total_seconds() <= 1800
+        ][-STATISTICS_HISTORY_MAX_POINTS:]
         return list(_STATISTICS_HISTORY)
 
-    _STATISTICS_HISTORY.append({'time': generated_at, 'cpu': cpu_usage, 'ram': ram_usage})
-    if len(_STATISTICS_HISTORY) > STATISTICS_HISTORY_MAX_POINTS:
-        del _STATISTICS_HISTORY[:-STATISTICS_HISTORY_MAX_POINTS]
-    return list(_STATISTICS_HISTORY)
-
 
 @router.get('/statistics/summary')
 def statistics_summary(session: Session = Depends(get_session)):
     apply_stored_connector_settings(session)
-    generated_at = datetime.now(UTC).replace(microsecond=0).isoformat().replace('+00:00', 'Z')
-    metrics = UnraidConnector().get_system_metrics()
-    cpu_usage = _nullable_int(metrics.get('cpuUsage'))
-    ram_usage = _nullable_int(metrics.get('ramUsage'))
-    samples = _record_statistics_sample(generated_at, cpu_usage, ram_usage)
+    gpu_connector = GpuMetricsConnector(snapshot_url=settings.gpu_metrics_url, verify_tls=settings.verify_tls)
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        unraid_future = executor.submit(UnraidConnector().get_system_metrics)
+        gpu_future = executor.submit(gpu_connector.get_metrics)
+        metrics = unraid_future.result()
+        gpu = gpu_future.result()
+    generated_at = datetime.now(UTC).isoformat(timespec='milliseconds').replace('+00:00', 'Z')
+    cpu_usage = _nullable_percent(metrics.get('cpuUsage'))
+    ram_usage = _nullable_percent(metrics.get('ramUsage'))
+    gpu_values = [_nullable_percent(device.get('utilizationPercent')) for device in gpu.get('devices', [])]
+    gpu_usage = max((value for value in gpu_values if value is not None), default=None)
+    gpu = {**gpu, 'receivedAt': generated_at}
+    samples = _record_statistics_sample(generated_at, cpu_usage, ram_usage, gpu_usage)
     source_warnings = []
     if metrics.get('status') != 'ok':
         source_warnings.append({
@@ -132,6 +155,11 @@ def statistics_summary(session: Session = Depends(get_session)):
             'status': metrics.get('status', 'unknown'),
             'reason': redact_message(str(metrics.get('reason', 'metrics_unavailable')))[:240],
         })
+    if gpu.get('status') != 'ok':
+        source_warnings.append({
+            'source': 'gpu', 'surface': 'metrics', 'status': gpu.get('status', 'unknown'),
+            'reason': redact_message(str(gpu.get('reason', 'gpu_metrics_unavailable')))[:240],
+        })
 
     audit_events = session.exec(select(AuditEvent).order_by(AuditEvent.timestamp.desc())).all()
     jobs = session.exec(select(JobRun).order_by(JobRun.started_at.desc())).all()
@@ -148,17 +176,23 @@ def statistics_summary(session: Session = Depends(get_session)):
         for event in audit_events
         if event.policy_decision == 'blocked' or event.result == 'blocked'
     )
-    voice_action_count = sum(1 for event in audit_events if event.source == 'voice' or event.identity == 'voice')
+    voice_action_count = sum(
+        1 for event in audit_events
+        if event.source == 'voice' or event.identity == 'voice'
+        or (event.redacted_metadata or {}).get('requestSource') == 'voice'
+    )
     mttr_hours = None
     if durations_ms:
         mttr_hours = round((sum(durations_ms) / len(durations_ms)) / 1000 / 60 / 60, 2)
 
     return {
         'generatedAt': generated_at,
-        'sources': {'unraid': metrics.get('status', 'unknown'), 'audit': 'ok', 'jobs': 'ok'},
+        'sources': {'unraid': metrics.get('status', 'unknown'), 'gpu': gpu.get('status', 'unknown'), 'audit': 'ok', 'jobs': 'ok'},
         'sourceWarnings': source_warnings,
         'cpuHistory': [{'time': sample['time'], 'value': sample.get('cpu')} for sample in samples],
         'ramHistory': [{'time': sample['time'], 'value': sample.get('ram')} for sample in samples],
+        'gpuHistory': [{'time': sample['time'], 'value': sample.get('gpu')} for sample in samples],
+        'gpu': gpu,
         'restartsByApp': [
             {'appId': app_id, 'count': count}
             for app_id, count in sorted(restarts.items(), key=lambda item: (-item[1], item[0]))
@@ -401,6 +435,7 @@ def _network_exposure_signature(session: Session) -> str:
     ]
     signature_payload = {
         'schemaVersion': NETWORK_EXPOSURE_CACHE_SCHEMA_VERSION,
+        'exposurePreferences': get_exposure_preferences(session).model_dump(),
         'connectors': sorted(connectors, key=lambda item: item['id']),
         'secrets': sorted(secret_rows, key=lambda item: item['id']),
     }
@@ -446,6 +481,7 @@ def _store_persistent_network_exposure(session: Session, signature: str, payload
 
 
 def _build_network_exposure() -> list[dict]:
+    access_list_checks_enabled = runtime_nginx_access_list_checks_enabled()
     with ThreadPoolExecutor(max_workers=3) as executor:
         hosts_future = executor.submit(lambda: NginxProxyManagerConnector().proxy_hosts())
         certs_future = executor.submit(lambda: NginxProxyManagerConnector().certificates())
@@ -479,7 +515,7 @@ def _build_network_exposure() -> list[dict]:
         ssl_forced = bool(host.get('sslForced'))
         risk_reasons: list[str] = []
         if enabled and not protected:
-            risk_reasons.append('No NPM access list or Authentik match')
+            risk_reasons.append('No NPM access list or Authentik match' if access_list_checks_enabled else 'No Authentik or other verified access protection')
         if enabled and not has_certificate:
             risk_reasons.append('No certificate assigned')
         if certificate_expired: