ServiceMap-Public/backend/app/incidents.py

from __future__ import annotations

from datetime import UTC, datetime
from uuid import uuid4

from .health import dependency_status, effective_status
from .models import (
    Criticality,
    HealthStatus,
    IncidentCandidate,
    IncidentOverview,
    IncidentRecord,
    IncidentStatus,
    Service,
)
from .store import Store


def _reverse_dependencies(services: list[Service]) -> dict[str, list[str]]:
    reverse: dict[str, list[str]] = {service.id: [] for service in services}
    for service in services:
        for dependency_id in service.depends_on:
            reverse.setdefault(dependency_id, []).append(service.id)
    return reverse


def affected_service_ids(root_id: str, services: list[Service]) -> list[str]:
    reverse = _reverse_dependencies(services)
    seen: set[str] = set()
    queue = list(reverse.get(root_id, []))
    while queue:
        current = queue.pop(0)
        if current in seen or current == root_id:
            continue
        seen.add(current)
        queue.extend(reverse.get(current, []))
    return sorted(seen)


def incident_overview(store: Store) -> IncidentOverview:
    services = store.list_services()
    by_id = {service.id: service for service in services}
    effective = {service.id: effective_status(service, store) for service in services}
    dependency = {service.id: dependency_status(service, store) for service in services}

    down_ids = sorted(service_id for service_id, state in effective.items() if state == HealthStatus.DOWN)
    degraded_ids = sorted(
        service_id for service_id, state in effective.items() if state in {HealthStatus.DEGRADED, HealthStatus.RECOVERING}
    )

    candidates: list[IncidentCandidate] = []
    for service in services:
        state = effective[service.id]
        if state != HealthStatus.DOWN or service.maintenance_enabled:
            continue
        down_dependencies = [
            dependency_id
            for dependency_id in service.depends_on
            if effective.get(dependency_id) == HealthStatus.DOWN
        ]
        if down_dependencies:
            continue
        affected_ids = affected_service_ids(service.id, services)
        affected_names = [by_id[item].name for item in affected_ids if item in by_id]
        reason = "Service is down and no configured dependency is also down"
        if dependency[service.id] == HealthStatus.UNKNOWN:
            reason += "; at least one dependency has unknown health"
        candidates.append(
            IncidentCandidate(
                service_id=service.id,
                service_name=service.name,
                status=state,
                reason=reason,
                affected_service_ids=affected_ids,
                affected_service_names=affected_names,
            )
        )

    candidates.sort(key=lambda item: (-len(item.affected_service_ids), item.service_name.lower()))
    return IncidentOverview(
        generated_at=datetime.now(UTC),
        active=bool(down_ids or degraded_ids),
        candidates=candidates,
        degraded_service_ids=degraded_ids,
        down_service_ids=down_ids,
        incidents=[],
    )


def _severity(service: Service) -> Criticality:
    return service.criticality


def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recovered; incident resolved") -> list[IncidentRecord]:
    """Reconcile the current graph with the durable incident journal.

    A root service has at most one active incident. Repeated probe cycles only
    update its affected set; a later independent outage gets a new id.
    """

    overview = incident_overview(store)
    services = {service.id: service for service in store.list_services()}
    now = datetime.now(UTC)
    active: dict[str, IncidentRecord] = {}
    for incident in sorted(
        store.list_incidents(active_only=True),
        key=lambda item: (item.detected_at, item.id),
    ):
        if incident.root_service_id in active:
            store.upsert_incident(incident.model_copy(update={
                "status": IncidentStatus.RESOLVED,
                "resolved_at": now,
                "latest_update_at": now,
            }))
            store.append_incident_event(
                incident.id,
                "incident.duplicate_resolved",
                "Duplicate active incident closed; the earliest incident remains active",
                incident.root_service_id,
            )
            continue
        active[incident.root_service_id] = incident
    for candidate in overview.candidates:
        service = services.get(candidate.service_id)
        if service is None:
            continue
        incident = active.get(candidate.service_id)
        if incident is None:
            incident = IncidentRecord(
                id=str(uuid4()),
                status=IncidentStatus.OPEN,
                root_service_id=candidate.service_id,
                root_service_name=candidate.service_name,
                started_at=now,
                detected_at=now,
                title=f"{candidate.service_name} is down",
                root_cause_reason=candidate.reason,
                affected_service_ids=candidate.affected_service_ids,
                affected_service_names=candidate.affected_service_names,
                severity=_severity(service),
                latest_update_at=now,
            )
            store.upsert_incident(incident)
            store.append_incident_event(
                incident.id,
                "incident.opened",
                f"Incident opened: {candidate.service_name} is the likely root cause",
                candidate.service_id,
            )
            active[candidate.service_id] = incident
        elif incident.affected_service_ids != candidate.affected_service_ids:
            updated = incident.model_copy(
                update={
                    "affected_service_ids": candidate.affected_service_ids,
                    "affected_service_names": candidate.affected_service_names,
                    "latest_update_at": now,
                    "root_cause_reason": candidate.reason,
                }
            )
            store.upsert_incident(updated)
            store.append_incident_event(incident.id, "incident.updated", "Downstream impact recalculated", candidate.service_id)

    for root_id, incident in list(active.items()):
        if any(candidate.service_id == root_id for candidate in overview.candidates):
            continue
        root = services.get(root_id)
        if (
            (root is None or store.service_health(root_id) not in {
                HealthStatus.DOWN,
                HealthStatus.DEGRADED,
                HealthStatus.RECOVERING,
            })
            and incident.status != IncidentStatus.RESOLVED
        ):
            resolved = incident.model_copy(
                update={"status": IncidentStatus.RESOLVED, "resolved_at": now, "latest_update_at": now}
            )
            store.upsert_incident(resolved)
            store.append_incident_event(incident.id, "incident.resolved", resolution_reason, root_id)
    return store.list_incidents(active_only=False)