ServiceMap-Public/backend/app/observation.py

"""Recurring, side-effect-bounded infrastructure health observation."""

from __future__ import annotations

import os
import threading
from dataclasses import dataclass
from datetime import timedelta
from uuid import uuid4

from .docker_discovery import appops_inventory, observe_container_health
from .docker_provider import docker_provider_mode
from .evidence import appops_health_status, utc_now
from .models import (
    EvidenceProvider,
    EvidenceScope,
    HealthEvidence,
    HealthStatus,
    ProviderHealth,
    ProviderStatus,
)
from .store import Store


def observation_ttl(interval_seconds: int) -> timedelta:
    """Provider observation TTL with room for one delayed refresh cycle."""

    return timedelta(seconds=max(150, interval_seconds * 3))


def _appops_status(value: str | None) -> HealthStatus | None:
    return appops_health_status(value)


def _docker_status(state: str, health: str | None) -> HealthStatus | None:
    if (health or "").casefold() in {"unhealthy", "down"} or state.casefold() in {"exited", "dead", "stopped"}:
        return HealthStatus.DOWN
    if (health or "").casefold() == "degraded" or state.casefold() == "restarting":
        return HealthStatus.DEGRADED
    if (health or "").casefold() == "healthy":
        return HealthStatus.HEALTHY
    return None


@dataclass(frozen=True)
class ObservationResult:
    docker_observations: int = 0
    appops_observations: int = 0
    skipped: bool = False


class ObservationManager:
    """Refresh provider evidence only; discovery and reconciliation stay manual."""

    def __init__(self, store: Store, *, interval_seconds: int) -> None:
        self.store = store
        self.interval_seconds = interval_seconds
        self._lock = threading.Lock()

    @staticmethod
    def _docker_host() -> str:
        return os.getenv("SERVICEMAP_DOCKER_HOST", "unix:///var/run/docker.sock")

    @staticmethod
    def _appops_url() -> str | None:
        return os.getenv("SERVICEMAP_APPOPS_URL")

    def refresh(self) -> ObservationResult:
        if self.store.demo_active() or not self._lock.acquire(blocking=False):
            return ObservationResult(skipped=True)
        try:
            now = utc_now()
            valid_until = now + observation_ttl(self.interval_seconds)
            services_by_ref = {
                service.source_ref: service
                for service in self.store.list_services()
                if service.source_ref
            }
            persisted_app_links = self.store.application_evidence_links(EvidenceProvider.APPOPS)
            appops = appops_inventory(self._appops_url())
            if appops.available:
                self.store.set_provider_status(
                    ProviderStatus(
                        provider=EvidenceProvider.APPOPS,
                        status=ProviderHealth.HEALTHY,
                        observed_at=now,
                        valid_until=valid_until,
                        detail=appops.detail,
                        metadata={"mapping_count": len(appops.by_container)},
                    )
                )
            else:
                self.store.set_provider_status(
                    ProviderStatus(
                        provider=EvidenceProvider.APPOPS,
                        status=ProviderHealth.UNAVAILABLE,
                        observed_at=now,
                        valid_until=valid_until,
                        detail=appops.detail,
                    )
                )

            linked_apps: dict[str, set[str]] = {
                app_id: set(service_ids) for app_id, service_ids in persisted_app_links.items()
            }
            docker_count = 0
            try:
                docker_observations = observe_container_health(self._docker_host())
                if (
                    docker_provider_mode() == "broker"
                    and not docker_observations
                    and any(ref.startswith("docker:") for ref in services_by_ref)
                ):
                    raise ValueError("Docker broker returned no containers for configured services")
                self.store.set_provider_status(
                    ProviderStatus(
                        provider=EvidenceProvider.DOCKER,
                        status=ProviderHealth.HEALTHY,
                        observed_at=now,
                        valid_until=valid_until,
                        detail=f"{len(docker_observations)} containers observed",
                        metadata={"container_count": len(docker_observations)},
                    )
                )
                for observation in docker_observations:
                    service = services_by_ref.get(observation.stable_source_ref)
                    if service is None:
                        continue
                    docker_count += 1
                    appops_info = appops.by_container.get(observation.name.casefold())
                    if appops_info:
                        linked_apps.setdefault(appops_info.app_id, set()).add(service.id)
                    status = _docker_status(observation.state, observation.health)
                    if status is None:
                        continue
                    self.store.record_health_evidence(
                        HealthEvidence(
                            id=str(uuid4()),
                            subject_type="service",
                            subject_id=service.id,
                            service_id=service.id,
                            provider=EvidenceProvider.DOCKER,
                            provider_ref=observation.stable_source_ref,
                            status=status,
                            scope=EvidenceScope.COMPONENT,
                            confidence=0.95 if observation.healthcheck_present else 0.8,
                            observed_at=now,
                            valid_until=valid_until,
                            received_at=now,
                            summary="Docker healthcheck" if observation.healthcheck_present else "Docker runtime state",
                            metadata={
                                "container_id": observation.container_id,
                                "state": observation.state,
                                "health": observation.health,
                            },
                        )
                    )
            except (OSError, RuntimeError, ValueError) as exc:
                self.store.set_provider_status(
                    ProviderStatus(
                        provider=EvidenceProvider.DOCKER,
                        status=ProviderHealth.UNAVAILABLE,
                        observed_at=now,
                        valid_until=valid_until,
                        detail=str(exc)[:500],
                    )
                )

            appops_count = 0
            if appops.available:
                for app_id, service_ids in linked_apps.items():
                    info = next(
                        (item for item in appops.by_container.values() if item.app_id == app_id),
                        None,
                    )
                    status = _appops_status(info.health if info else None)
                    if info is None or status is None or not service_ids:
                        continue
                    appops_count += 1
                    self.store.record_health_evidence(
                        HealthEvidence(
                            id=str(uuid4()),
                            subject_type="application",
                            subject_id=f"appops:{app_id}",
                            provider=EvidenceProvider.APPOPS,
                            provider_ref=app_id,
                            status=status,
                            scope=EvidenceScope.APPLICATION,
                            confidence=info.discovery_confidence if info.discovery_confidence is not None else 0.5,
                            observed_at=now,
                            valid_until=valid_until,
                            received_at=now,
                            summary="AppOps application health; never used as component direct health",
                            metadata={
                                "app_id": app_id,
                                "category": info.category,
                                "policy_profile": info.policy_profile,
                                "app_type": info.app_type,
                                "discovery_confidence": info.discovery_confidence,
                            },
                        ),
                        service_ids=sorted(service_ids),
                    )
            return ObservationResult(docker_observations=docker_count, appops_observations=appops_count)
        finally:
            self._lock.release()