from __future__ import annotations from datetime import UTC, datetime from uuid import uuid4 from .health import dependency_status, effective_status from .models import ( Criticality, HealthStatus, IncidentCandidate, IncidentOverview, IncidentRecord, IncidentStatus, Service, ) from .store import Store def _reverse_dependencies(services: list[Service]) -> dict[str, list[str]]: reverse: dict[str, list[str]] = {service.id: [] for service in services} for service in services: for dependency_id in service.depends_on: reverse.setdefault(dependency_id, []).append(service.id) return reverse def affected_service_ids(root_id: str, services: list[Service]) -> list[str]: reverse = _reverse_dependencies(services) seen: set[str] = set() queue = list(reverse.get(root_id, [])) while queue: current = queue.pop(0) if current in seen or current == root_id: continue seen.add(current) queue.extend(reverse.get(current, [])) return sorted(seen) def incident_overview(store: Store) -> IncidentOverview: services = store.list_services() by_id = {service.id: service for service in services} effective = {service.id: effective_status(service, store) for service in services} dependency = {service.id: dependency_status(service, store) for service in services} down_ids = sorted(service_id for service_id, state in effective.items() if state == HealthStatus.DOWN) degraded_ids = sorted( service_id for service_id, state in effective.items() if state in {HealthStatus.DEGRADED, HealthStatus.RECOVERING} ) candidates: list[IncidentCandidate] = [] for service in services: state = effective[service.id] if state != HealthStatus.DOWN or service.maintenance_enabled: continue down_dependencies = [ dependency_id for dependency_id in service.depends_on if effective.get(dependency_id) == HealthStatus.DOWN ] if down_dependencies: continue affected_ids = affected_service_ids(service.id, services) affected_names = [by_id[item].name for item in affected_ids if item in by_id] reason = "Service is down and no configured dependency is also down" if dependency[service.id] == HealthStatus.UNKNOWN: reason += "; at least one dependency has unknown health" candidates.append( IncidentCandidate( service_id=service.id, service_name=service.name, status=state, reason=reason, affected_service_ids=affected_ids, affected_service_names=affected_names, ) ) candidates.sort(key=lambda item: (-len(item.affected_service_ids), item.service_name.lower())) return IncidentOverview( generated_at=datetime.now(UTC), active=bool(down_ids or degraded_ids), candidates=candidates, degraded_service_ids=degraded_ids, down_service_ids=down_ids, incidents=[], ) def _severity(service: Service) -> Criticality: return service.criticality def sync_incidents(store: Store, *, resolution_reason: str = "Root cause recovered; incident resolved") -> list[IncidentRecord]: """Reconcile the current graph with the durable incident journal. A root service has at most one active incident. Repeated probe cycles only update its affected set; a later independent outage gets a new id. """ overview = incident_overview(store) services = {service.id: service for service in store.list_services()} now = datetime.now(UTC) active: dict[str, IncidentRecord] = {} for incident in sorted( store.list_incidents(active_only=True), key=lambda item: (item.detected_at, item.id), ): if incident.root_service_id in active: store.upsert_incident(incident.model_copy(update={ "status": IncidentStatus.RESOLVED, "resolved_at": now, "latest_update_at": now, })) store.append_incident_event( incident.id, "incident.duplicate_resolved", "Duplicate active incident closed; the earliest incident remains active", incident.root_service_id, ) continue active[incident.root_service_id] = incident for candidate in overview.candidates: service = services.get(candidate.service_id) if service is None: continue incident = active.get(candidate.service_id) if incident is None: incident = IncidentRecord( id=str(uuid4()), status=IncidentStatus.OPEN, root_service_id=candidate.service_id, root_service_name=candidate.service_name, started_at=now, detected_at=now, title=f"{candidate.service_name} is down", root_cause_reason=candidate.reason, affected_service_ids=candidate.affected_service_ids, affected_service_names=candidate.affected_service_names, severity=_severity(service), latest_update_at=now, ) store.upsert_incident(incident) store.append_incident_event( incident.id, "incident.opened", f"Incident opened: {candidate.service_name} is the likely root cause", candidate.service_id, ) active[candidate.service_id] = incident elif incident.affected_service_ids != candidate.affected_service_ids: updated = incident.model_copy( update={ "affected_service_ids": candidate.affected_service_ids, "affected_service_names": candidate.affected_service_names, "latest_update_at": now, "root_cause_reason": candidate.reason, } ) store.upsert_incident(updated) store.append_incident_event(incident.id, "incident.updated", "Downstream impact recalculated", candidate.service_id) for root_id, incident in list(active.items()): if any(candidate.service_id == root_id for candidate in overview.candidates): continue root = services.get(root_id) if ( (root is None or store.service_health(root_id) not in { HealthStatus.DOWN, HealthStatus.DEGRADED, HealthStatus.RECOVERING, }) and incident.status != IncidentStatus.RESOLVED ): resolved = incident.model_copy( update={"status": IncidentStatus.RESOLVED, "resolved_at": now, "latest_update_at": now} ) store.upsert_incident(resolved) store.append_incident_event(incident.id, "incident.resolved", resolution_reason, root_id) return store.list_incidents(active_only=False)