"""Recurring, side-effect-bounded infrastructure health observation.""" from __future__ import annotations import os import threading from dataclasses import dataclass from datetime import timedelta from uuid import uuid4 from .docker_discovery import appops_inventory, observe_container_health from .docker_provider import docker_provider_mode from .evidence import appops_health_status, utc_now from .models import ( EvidenceProvider, EvidenceScope, HealthEvidence, HealthStatus, ProviderHealth, ProviderStatus, ) from .store import Store def observation_ttl(interval_seconds: int) -> timedelta: """Provider observation TTL with room for one delayed refresh cycle.""" return timedelta(seconds=max(150, interval_seconds * 3)) def _appops_status(value: str | None) -> HealthStatus | None: return appops_health_status(value) def _docker_status(state: str, health: str | None) -> HealthStatus | None: if (health or "").casefold() in {"unhealthy", "down"} or state.casefold() in {"exited", "dead", "stopped"}: return HealthStatus.DOWN if (health or "").casefold() == "degraded" or state.casefold() == "restarting": return HealthStatus.DEGRADED if (health or "").casefold() == "healthy": return HealthStatus.HEALTHY return None @dataclass(frozen=True) class ObservationResult: docker_observations: int = 0 appops_observations: int = 0 skipped: bool = False class ObservationManager: """Refresh provider evidence only; discovery and reconciliation stay manual.""" def __init__(self, store: Store, *, interval_seconds: int) -> None: self.store = store self.interval_seconds = interval_seconds self._lock = threading.Lock() @staticmethod def _docker_host() -> str: return os.getenv("SERVICEMAP_DOCKER_HOST", "unix:///var/run/docker.sock") @staticmethod def _appops_url() -> str | None: return os.getenv("SERVICEMAP_APPOPS_URL") def refresh(self) -> ObservationResult: if self.store.demo_active() or not self._lock.acquire(blocking=False): return ObservationResult(skipped=True) try: now = utc_now() valid_until = now + observation_ttl(self.interval_seconds) services_by_ref = { service.source_ref: service for service in self.store.list_services() if service.source_ref } persisted_app_links = self.store.application_evidence_links(EvidenceProvider.APPOPS) appops = appops_inventory(self._appops_url()) if appops.available: self.store.set_provider_status( ProviderStatus( provider=EvidenceProvider.APPOPS, status=ProviderHealth.HEALTHY, observed_at=now, valid_until=valid_until, detail=appops.detail, metadata={"mapping_count": len(appops.by_container)}, ) ) else: self.store.set_provider_status( ProviderStatus( provider=EvidenceProvider.APPOPS, status=ProviderHealth.UNAVAILABLE, observed_at=now, valid_until=valid_until, detail=appops.detail, ) ) linked_apps: dict[str, set[str]] = { app_id: set(service_ids) for app_id, service_ids in persisted_app_links.items() } docker_count = 0 try: docker_observations = observe_container_health(self._docker_host()) if ( docker_provider_mode() == "broker" and not docker_observations and any(ref.startswith("docker:") for ref in services_by_ref) ): raise ValueError("Docker broker returned no containers for configured services") self.store.set_provider_status( ProviderStatus( provider=EvidenceProvider.DOCKER, status=ProviderHealth.HEALTHY, observed_at=now, valid_until=valid_until, detail=f"{len(docker_observations)} containers observed", metadata={"container_count": len(docker_observations)}, ) ) for observation in docker_observations: service = services_by_ref.get(observation.stable_source_ref) if service is None: continue docker_count += 1 appops_info = appops.by_container.get(observation.name.casefold()) if appops_info: linked_apps.setdefault(appops_info.app_id, set()).add(service.id) status = _docker_status(observation.state, observation.health) if status is None: continue self.store.record_health_evidence( HealthEvidence( id=str(uuid4()), subject_type="service", subject_id=service.id, service_id=service.id, provider=EvidenceProvider.DOCKER, provider_ref=observation.stable_source_ref, status=status, scope=EvidenceScope.COMPONENT, confidence=0.95 if observation.healthcheck_present else 0.8, observed_at=now, valid_until=valid_until, received_at=now, summary="Docker healthcheck" if observation.healthcheck_present else "Docker runtime state", metadata={ "container_id": observation.container_id, "state": observation.state, "health": observation.health, }, ) ) except (OSError, RuntimeError, ValueError) as exc: self.store.set_provider_status( ProviderStatus( provider=EvidenceProvider.DOCKER, status=ProviderHealth.UNAVAILABLE, observed_at=now, valid_until=valid_until, detail=str(exc)[:500], ) ) appops_count = 0 if appops.available: for app_id, service_ids in linked_apps.items(): info = next( (item for item in appops.by_container.values() if item.app_id == app_id), None, ) status = _appops_status(info.health if info else None) if info is None or status is None or not service_ids: continue appops_count += 1 self.store.record_health_evidence( HealthEvidence( id=str(uuid4()), subject_type="application", subject_id=f"appops:{app_id}", provider=EvidenceProvider.APPOPS, provider_ref=app_id, status=status, scope=EvidenceScope.APPLICATION, confidence=info.discovery_confidence if info.discovery_confidence is not None else 0.5, observed_at=now, valid_until=valid_until, received_at=now, summary="AppOps application health; never used as component direct health", metadata={ "app_id": app_id, "category": info.category, "policy_profile": info.policy_profile, "app_type": info.app_type, "discovery_confidence": info.discovery_confidence, }, ), service_ids=sorted(service_ids), ) return ObservationResult(docker_observations=docker_count, appops_observations=appops_count) finally: self._lock.release()