125 lines
5.1 KiB
Python
125 lines
5.1 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from datetime import UTC, datetime
|
|
from threading import Lock
|
|
from time import monotonic
|
|
from time import perf_counter
|
|
from typing import Callable
|
|
|
|
from sqlmodel import Session
|
|
|
|
from ..api.routes_observability import network_exposure
|
|
from ..core.config import settings
|
|
from ..db import session as db_session
|
|
from ..mcp_server import persist_tool_metrics, run_read_only_canary
|
|
from . import media_pipeline_service, notification_service, safety_center_service
|
|
from .connector_health import refresh_connector_statuses
|
|
from .request_metrics import persist_request_metrics
|
|
from .runtime_service import verify_latest_backup_drill
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
_STATUS_LOCK = Lock()
|
|
_TASK_STATUS: dict[str, dict[str, object]] = {}
|
|
|
|
|
|
def maintenance_status() -> dict[str, object]:
|
|
with _STATUS_LOCK:
|
|
tasks = {name: dict(status) for name, status in _TASK_STATUS.items()}
|
|
return {
|
|
"status": "running",
|
|
"tasks": tasks,
|
|
"intervalsSeconds": {
|
|
"notificationEvaluation": settings.notification_interval_seconds,
|
|
"connectorHealth": settings.connector_health_interval_seconds,
|
|
"observabilityPersistence": settings.observability_persist_interval_seconds,
|
|
"backupRestoreDrill": settings.restore_drill_interval_seconds,
|
|
"mcpCanary": settings.mcp_canary_interval_seconds if settings.mcp_enabled else 0,
|
|
},
|
|
}
|
|
|
|
|
|
def warm_operational_caches() -> None:
|
|
with Session(db_session.engine) as session:
|
|
safety_center_service.get_dashboard(session)
|
|
media_pipeline_service.summary(session)
|
|
network_exposure(False, session)
|
|
|
|
|
|
def evaluate_notifications_and_retries() -> None:
|
|
with Session(db_session.engine) as session:
|
|
notification_service.retry_failed_deliveries(session)
|
|
notification_service.evaluate_source(session, "all")
|
|
|
|
|
|
def reconcile_connector_health() -> None:
|
|
with Session(db_session.engine) as session:
|
|
refresh_connector_statuses(session)
|
|
|
|
|
|
def persist_observability() -> None:
|
|
with Session(db_session.engine) as session:
|
|
persist_request_metrics(session)
|
|
persist_tool_metrics(session)
|
|
|
|
|
|
def run_restore_drill() -> None:
|
|
with Session(db_session.engine) as session:
|
|
verify_latest_backup_drill(session)
|
|
|
|
|
|
async def _safe_thread(label: str, function: Callable[[], object]) -> None:
|
|
started = perf_counter()
|
|
try:
|
|
await asyncio.to_thread(function)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception: # noqa: BLE001 - maintenance failures must not stop the API.
|
|
logger.exception("Scheduled %s failed", label)
|
|
state = {"status": "failed", "error": "scheduled_task_failed"}
|
|
else:
|
|
state = {"status": "ok"}
|
|
state.update(
|
|
{
|
|
"lastRunAt": datetime.now(UTC).isoformat(),
|
|
"durationMs": round((perf_counter() - started) * 1000, 2),
|
|
}
|
|
)
|
|
with _STATUS_LOCK:
|
|
_TASK_STATUS[label] = state
|
|
|
|
|
|
async def run_periodic_maintenance() -> None:
|
|
await asyncio.sleep(2)
|
|
if settings.connector_health_interval_seconds:
|
|
await _safe_thread("connector health reconciliation", reconcile_connector_health)
|
|
await _safe_thread("operational cache warmup", warm_operational_caches)
|
|
|
|
now = monotonic()
|
|
next_notification = now + min(15, settings.notification_interval_seconds) if settings.notification_interval_seconds else float("inf")
|
|
next_connector_health = now + settings.connector_health_interval_seconds if settings.connector_health_interval_seconds else float("inf")
|
|
next_observability = now + min(10, settings.observability_persist_interval_seconds) if settings.observability_persist_interval_seconds else float("inf")
|
|
next_restore = now + min(60, settings.restore_drill_interval_seconds) if settings.restore_drill_interval_seconds else float("inf")
|
|
next_canary = now + min(10, settings.mcp_canary_interval_seconds) if settings.mcp_enabled and settings.mcp_canary_interval_seconds else float("inf")
|
|
|
|
while True:
|
|
now = monotonic()
|
|
if now >= next_notification:
|
|
await _safe_thread("notification evaluation", evaluate_notifications_and_retries)
|
|
next_notification = monotonic() + settings.notification_interval_seconds
|
|
if now >= next_connector_health:
|
|
await _safe_thread("connector health reconciliation", reconcile_connector_health)
|
|
next_connector_health = monotonic() + settings.connector_health_interval_seconds
|
|
if now >= next_canary:
|
|
await _safe_thread("MCP read-only canary", run_read_only_canary)
|
|
next_canary = monotonic() + settings.mcp_canary_interval_seconds
|
|
if now >= next_observability:
|
|
await _safe_thread("observability persistence", persist_observability)
|
|
next_observability = monotonic() + settings.observability_persist_interval_seconds
|
|
if now >= next_restore:
|
|
await _safe_thread("backup restore drill", run_restore_drill)
|
|
next_restore = monotonic() + settings.restore_drill_interval_seconds
|
|
await asyncio.sleep(5)
|