Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d

backend/app/services/maintenance_scheduler.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import UTC, datetime
+from threading import Lock
+from time import monotonic
+from time import perf_counter
+from typing import Callable
+
+from sqlmodel import Session
+
+from ..api.routes_observability import network_exposure
+from ..core.config import settings
+from ..db import session as db_session
+from ..mcp_server import persist_tool_metrics, run_read_only_canary
+from . import media_pipeline_service, notification_service, safety_center_service
+from .connector_health import refresh_connector_statuses
+from .request_metrics import persist_request_metrics
+from .runtime_service import verify_latest_backup_drill
+
+
+logger = logging.getLogger(__name__)
+_STATUS_LOCK = Lock()
+_TASK_STATUS: dict[str, dict[str, object]] = {}
+
+
+def maintenance_status() -> dict[str, object]:
+    with _STATUS_LOCK:
+        tasks = {name: dict(status) for name, status in _TASK_STATUS.items()}
+    return {
+        "status": "running",
+        "tasks": tasks,
+        "intervalsSeconds": {
+            "notificationEvaluation": settings.notification_interval_seconds,
+            "connectorHealth": settings.connector_health_interval_seconds,
+            "observabilityPersistence": settings.observability_persist_interval_seconds,
+            "backupRestoreDrill": settings.restore_drill_interval_seconds,
+            "mcpCanary": settings.mcp_canary_interval_seconds if settings.mcp_enabled else 0,
+        },
+    }
+
+
+def warm_operational_caches() -> None:
+    with Session(db_session.engine) as session:
+        safety_center_service.get_dashboard(session)
+        media_pipeline_service.summary(session)
+        network_exposure(False, session)
+
+
+def evaluate_notifications_and_retries() -> None:
+    with Session(db_session.engine) as session:
+        notification_service.retry_failed_deliveries(session)
+        notification_service.evaluate_source(session, "all")
+
+
+def reconcile_connector_health() -> None:
+    with Session(db_session.engine) as session:
+        refresh_connector_statuses(session)
+
+
+def persist_observability() -> None:
+    with Session(db_session.engine) as session:
+        persist_request_metrics(session)
+        persist_tool_metrics(session)
+
+
+def run_restore_drill() -> None:
+    with Session(db_session.engine) as session:
+        verify_latest_backup_drill(session)
+
+
+async def _safe_thread(label: str, function: Callable[[], object]) -> None:
+    started = perf_counter()
+    try:
+        await asyncio.to_thread(function)
+    except asyncio.CancelledError:
+        raise
+    except Exception:  # noqa: BLE001 - maintenance failures must not stop the API.
+        logger.exception("Scheduled %s failed", label)
+        state = {"status": "failed", "error": "scheduled_task_failed"}
+    else:
+        state = {"status": "ok"}
+    state.update(
+        {
+            "lastRunAt": datetime.now(UTC).isoformat(),
+            "durationMs": round((perf_counter() - started) * 1000, 2),
+        }
+    )
+    with _STATUS_LOCK:
+        _TASK_STATUS[label] = state
+
+
+async def run_periodic_maintenance() -> None:
+    await asyncio.sleep(2)
+    if settings.connector_health_interval_seconds:
+        await _safe_thread("connector health reconciliation", reconcile_connector_health)
+    await _safe_thread("operational cache warmup", warm_operational_caches)
+
+    now = monotonic()
+    next_notification = now + min(15, settings.notification_interval_seconds) if settings.notification_interval_seconds else float("inf")
+    next_connector_health = now + settings.connector_health_interval_seconds if settings.connector_health_interval_seconds else float("inf")
+    next_observability = now + min(10, settings.observability_persist_interval_seconds) if settings.observability_persist_interval_seconds else float("inf")
+    next_restore = now + min(60, settings.restore_drill_interval_seconds) if settings.restore_drill_interval_seconds else float("inf")
+    next_canary = now + min(10, settings.mcp_canary_interval_seconds) if settings.mcp_enabled and settings.mcp_canary_interval_seconds else float("inf")
+
+    while True:
+        now = monotonic()
+        if now >= next_notification:
+            await _safe_thread("notification evaluation", evaluate_notifications_and_retries)
+            next_notification = monotonic() + settings.notification_interval_seconds
+        if now >= next_connector_health:
+            await _safe_thread("connector health reconciliation", reconcile_connector_health)
+            next_connector_health = monotonic() + settings.connector_health_interval_seconds
+        if now >= next_canary:
+            await _safe_thread("MCP read-only canary", run_read_only_canary)
+            next_canary = monotonic() + settings.mcp_canary_interval_seconds
+        if now >= next_observability:
+            await _safe_thread("observability persistence", persist_observability)
+            next_observability = monotonic() + settings.observability_persist_interval_seconds
+        if now >= next_restore:
+            await _safe_thread("backup restore drill", run_restore_drill)
+            next_restore = monotonic() + settings.restore_drill_interval_seconds
+        await asyncio.sleep(5)