Initial curated ServiceMap source from 7a0a61232d05ba9097432abef6f7f4102a64d0b8

backend/app/incidents.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import uuid4
+
+from .health import dependency_status, effective_status
+from .models import (
+    Criticality,
+    HealthStatus,
+    IncidentCandidate,
+    IncidentOverview,
+    IncidentRecord,
+    IncidentStatus,
+    Service,
+)
+from .store import Store
+
+
+def _reverse_dependencies(services: list[Service]) -> dict[str, list[str]]:
+    reverse: dict[str, list[str]] = {service.id: [] for service in services}
+    for service in services:
+        for dependency_id in service.depends_on:
+            reverse.setdefault(dependency_id, []).append(service.id)
+    return reverse
+
+
+def affected_service_ids(root_id: str, services: list[Service]) -> list[str]:
+    reverse = _reverse_dependencies(services)
+    seen: set[str] = set()
+    queue = list(reverse.get(root_id, []))
+    while queue:
+        current = queue.pop(0)
+        if current in seen or current == root_id:
+            continue
+        seen.add(current)
+        queue.extend(reverse.get(current, []))
+    return sorted(seen)
+
+
+def incident_overview(store: Store) -> IncidentOverview:
+    services = store.list_services()
+    by_id = {service.id: service for service in services}
+    effective = {service.id: effective_status(service, store) for service in services}
+    dependency = {service.id: dependency_status(service, store) for service in services}
+
+    down_ids = sorted(service_id for service_id, state in effective.items() if state == HealthStatus.DOWN)
+    degraded_ids = sorted(
+        service_id for service_id, state in effective.items() if state in {HealthStatus.DEGRADED, HealthStatus.RECOVERING}
+    )
+
+    candidates: list[IncidentCandidate] = []
+    for service in services:
+        state = effective[service.id]
+        if state != HealthStatus.DOWN or service.maintenance_enabled:
+            continue
+        down_dependencies = [
+            dependency_id
+            for dependency_id in service.depends_on
+            if effective.get(dependency_id) == HealthStatus.DOWN
+        ]
+        if down_dependencies:
+            continue
+        affected_ids = affected_service_ids(service.id, services)
+        affected_names = [by_id[item].name for item in affected_ids if item in by_id]
+        reason = "Service is down and no configured dependency is also down"
+        if dependency[service.id] == HealthStatus.UNKNOWN:
+            reason += "; at least one dependency has unknown health"
+        candidates.append(
+            IncidentCandidate(
+                service_id=service.id,
+                service_name=service.name,
+                status=state,
+                reason=reason,
+                affected_service_ids=affected_ids,
+                affected_service_names=affected_names,
+            )
+        )
+
+    candidates.sort(key=lambda item: (-len(item.affected_service_ids), item.service_name.lower()))
+    return IncidentOverview(
+        generated_at=datetime.now(UTC),
+        active=bool(down_ids or degraded_ids),
+        candidates=candidates,
+        degraded_service_ids=degraded_ids,
+        down_service_ids=down_ids,
+        incidents=[],
+    )
+
+
+def _severity(service: Service) -> Criticality:
+    return service.criticality
+
+
+def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recovered; incident resolved") -> list[IncidentRecord]:
+    """Reconcile the current graph with the durable incident journal.
+
+    A root service has at most one active incident. Repeated probe cycles only
+    update its affected set; a later independent outage gets a new id.
+    """
+
+    overview = incident_overview(store)
+    services = {service.id: service for service in store.list_services()}
+    now = datetime.now(UTC)
+    active: dict[str, IncidentRecord] = {}
+    for incident in sorted(
+        store.list_incidents(active_only=True),
+        key=lambda item: (item.detected_at, item.id),
+    ):
+        if incident.root_service_id in active:
+            store.upsert_incident(incident.model_copy(update={
+                "status": IncidentStatus.RESOLVED,
+                "resolved_at": now,
+                "latest_update_at": now,
+            }))
+            store.append_incident_event(
+                incident.id,
+                "incident.duplicate_resolved",
+                "Duplicate active incident closed; the earliest incident remains active",
+                incident.root_service_id,
+            )
+            continue
+        active[incident.root_service_id] = incident
+    for candidate in overview.candidates:
+        service = services.get(candidate.service_id)
+        if service is None:
+            continue
+        incident = active.get(candidate.service_id)
+        if incident is None:
+            incident = IncidentRecord(
+                id=str(uuid4()),
+                status=IncidentStatus.OPEN,
+                root_service_id=candidate.service_id,
+                root_service_name=candidate.service_name,
+                started_at=now,
+                detected_at=now,
+                title=f"{candidate.service_name} is down",
+                root_cause_reason=candidate.reason,
+                affected_service_ids=candidate.affected_service_ids,
+                affected_service_names=candidate.affected_service_names,
+                severity=_severity(service),
+                latest_update_at=now,
+            )
+            store.upsert_incident(incident)
+            store.append_incident_event(
+                incident.id,
+                "incident.opened",
+                f"Incident opened: {candidate.service_name} is the likely root cause",
+                candidate.service_id,
+            )
+            active[candidate.service_id] = incident
+        elif incident.affected_service_ids != candidate.affected_service_ids:
+            updated = incident.model_copy(
+                update={
+                    "affected_service_ids": candidate.affected_service_ids,
+                    "affected_service_names": candidate.affected_service_names,
+                    "latest_update_at": now,
+                    "root_cause_reason": candidate.reason,
+                }
+            )
+            store.upsert_incident(updated)
+            store.append_incident_event(incident.id, "incident.updated", "Downstream impact recalculated", candidate.service_id)
+
+    for root_id, incident in list(active.items()):
+        if any(candidate.service_id == root_id for candidate in overview.candidates):
+            continue
+        root = services.get(root_id)
+        if (
+            (root is None or store.service_health(root_id) not in {
+                HealthStatus.DOWN,
+                HealthStatus.DEGRADED,
+                HealthStatus.RECOVERING,
+            })
+            and incident.status != IncidentStatus.RESOLVED
+        ):
+            resolved = incident.model_copy(
+                update={"status": IncidentStatus.RESOLVED, "resolved_at": now, "latest_update_at": now}
+            )
+            store.upsert_incident(resolved)
+            store.append_incident_event(incident.id, "incident.resolved", resolution_reason, root_id)
+    return store.list_incidents(active_only=False)