from __future__ import annotations import asyncio import logging from datetime import UTC, datetime from threading import Lock from time import monotonic from time import perf_counter from typing import Callable from sqlmodel import Session from ..api.routes_observability import network_exposure from ..core.config import settings from ..db import session as db_session from ..mcp_server import persist_tool_metrics, run_read_only_canary from . import media_pipeline_service, notification_service, safety_center_service from .connector_health import refresh_connector_statuses from .request_metrics import persist_request_metrics from .runtime_service import verify_latest_backup_drill logger = logging.getLogger(__name__) _STATUS_LOCK = Lock() _TASK_STATUS: dict[str, dict[str, object]] = {} def maintenance_status() -> dict[str, object]: with _STATUS_LOCK: tasks = {name: dict(status) for name, status in _TASK_STATUS.items()} return { "status": "running", "tasks": tasks, "intervalsSeconds": { "notificationEvaluation": settings.notification_interval_seconds, "connectorHealth": settings.connector_health_interval_seconds, "observabilityPersistence": settings.observability_persist_interval_seconds, "backupRestoreDrill": settings.restore_drill_interval_seconds, "mcpCanary": settings.mcp_canary_interval_seconds if settings.mcp_enabled else 0, }, } def warm_operational_caches() -> None: with Session(db_session.engine) as session: safety_center_service.get_dashboard(session) media_pipeline_service.summary(session) network_exposure(False, session) def evaluate_notifications_and_retries() -> None: with Session(db_session.engine) as session: notification_service.retry_failed_deliveries(session) notification_service.evaluate_source(session, "all") def reconcile_connector_health() -> None: with Session(db_session.engine) as session: refresh_connector_statuses(session) def persist_observability() -> None: with Session(db_session.engine) as session: persist_request_metrics(session) persist_tool_metrics(session) def run_restore_drill() -> None: with Session(db_session.engine) as session: verify_latest_backup_drill(session) async def _safe_thread(label: str, function: Callable[[], object]) -> None: started = perf_counter() try: await asyncio.to_thread(function) except asyncio.CancelledError: raise except Exception: # noqa: BLE001 - maintenance failures must not stop the API. logger.exception("Scheduled %s failed", label) state = {"status": "failed", "error": "scheduled_task_failed"} else: state = {"status": "ok"} state.update( { "lastRunAt": datetime.now(UTC).isoformat(), "durationMs": round((perf_counter() - started) * 1000, 2), } ) with _STATUS_LOCK: _TASK_STATUS[label] = state async def run_periodic_maintenance() -> None: await asyncio.sleep(2) if settings.connector_health_interval_seconds: await _safe_thread("connector health reconciliation", reconcile_connector_health) await _safe_thread("operational cache warmup", warm_operational_caches) now = monotonic() next_notification = now + min(15, settings.notification_interval_seconds) if settings.notification_interval_seconds else float("inf") next_connector_health = now + settings.connector_health_interval_seconds if settings.connector_health_interval_seconds else float("inf") next_observability = now + min(10, settings.observability_persist_interval_seconds) if settings.observability_persist_interval_seconds else float("inf") next_restore = now + min(60, settings.restore_drill_interval_seconds) if settings.restore_drill_interval_seconds else float("inf") next_canary = now + min(10, settings.mcp_canary_interval_seconds) if settings.mcp_enabled and settings.mcp_canary_interval_seconds else float("inf") while True: now = monotonic() if now >= next_notification: await _safe_thread("notification evaluation", evaluate_notifications_and_retries) next_notification = monotonic() + settings.notification_interval_seconds if now >= next_connector_health: await _safe_thread("connector health reconciliation", reconcile_connector_health) next_connector_health = monotonic() + settings.connector_health_interval_seconds if now >= next_canary: await _safe_thread("MCP read-only canary", run_read_only_canary) next_canary = monotonic() + settings.mcp_canary_interval_seconds if now >= next_observability: await _safe_thread("observability persistence", persist_observability) next_observability = monotonic() + settings.observability_persist_interval_seconds if now >= next_restore: await _safe_thread("backup restore drill", run_restore_drill) next_restore = monotonic() + settings.restore_drill_interval_seconds await asyncio.sleep(5)