"""Deterministic semantic topology derived from the canonical service graph. The discovery store remains the source of truth. This module deliberately does not persist a second topology: a domain, application and component view can therefore evolve without risking probe history or an operator's graph. """ from __future__ import annotations import hashlib import re from collections import defaultdict from datetime import UTC, datetime from .evidence import with_freshness from .models import ( ApplicationHealthResolution, ComponentRole, Criticality, EvidenceFreshness, EvidenceScope, HealthResolutionReason, HealthStatus, LogicalRelation, SemanticApplication, SemanticComponent, SemanticDomain, SemanticTopology, Service, ServiceSnapshot, ) _ROLE_BY_CATEGORY = { "database": ComponentRole.DATABASE, "cache": ComponentRole.CACHE, "proxy": ComponentRole.PROXY, "background_worker": ComponentRole.WORKER, "storage": ComponentRole.STORAGE, "monitoring": ComponentRole.MONITORING, } _ROLE_HINTS = ( (ComponentRole.DATABASE, ("postgres", "mariadb", "mysql", "mongo", "influx", "database", "db")), (ComponentRole.CACHE, ("redis", "memcache", "cache")), (ComponentRole.PROXY, ("proxy", "nginx", "traefik", "caddy", "swag", "haproxy", "cloudflare")), (ComponentRole.WORKER, ("worker", "runner", "queue", "task")), (ComponentRole.STORAGE, ("minio", "storage", "s3", "volume", "files")), (ComponentRole.MONITORING, ("grafana", "prometheus", "zabbix", "netdata", "loki", "uptime")), (ComponentRole.AGENT, ("agent", "exporter", "sidecar")), (ComponentRole.SCHEDULER, ("scheduler", "cron", "schedule")), (ComponentRole.FRONTEND, ("frontend", "webui", "dashboard", "ui")), (ComponentRole.API, ("api", "backend", "server")), ) _DOMAIN_HINTS = ( ("Edge", ("proxy", "nginx", "traefik", "caddy", "swag", "cloudflare", "authelia", "auth")), ("Media", ("plex", "jellyfin", "sonarr", "radarr", "lidarr", "readarr", "bazarr", "media", "tautulli")), ("Dev", ("gitea", "git", "code", "dev", "ci", "runner")), ("Monitoring", ("grafana", "prometheus", "zabbix", "netdata", "loki", "uptime", "monitor")), ("AI", ("ollama", "open webui", "comfy", "ai", "llm")), ("Storage", ("postgres", "mariadb", "mysql", "mongo", "redis", "minio", "storage", "nextcloud", "file")), ("Automation", ("n8n", "worker", "scheduler", "cron", "automation", "arr")), ("Productivity", ("book", "paperless", "vault", "wiki", "project", "calendar", "office")), ) def _slug(prefix: str, value: str) -> str: normalized = re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-") or "unknown" digest = hashlib.sha1(value.casefold().encode("utf-8")).hexdigest()[:8] return f"{prefix}:{normalized[:48]}-{digest}" def component_role(service: Service) -> ComponentRole: if service.component_role_locked or service.component_role != ComponentRole.UNKNOWN: return service.component_role category_role = _ROLE_BY_CATEGORY.get(service.category.value) if category_role: return category_role haystack = " ".join(filter(None, (service.name, service.container_image, service.source_ref, service.description))).casefold() for role, hints in _ROLE_HINTS: if any(hint in haystack for hint in hints): return role # A monitoring gap says nothing about the component's workload role. Keep # it unknown until source metadata, a deterministic hint or an operator # override supplies evidence. return ComponentRole.UNKNOWN def application_name(service: Service) -> str: """Use stable provenance before falling back to a standalone application. Category is intentionally *not* used as a shared application name. Doing so was the old "Independent ยท Category" compaction bug: unrelated services looked like one stack. Categories still inform domain derivation below. """ if service.application: return service.application.strip() if service.compose_project: return service.compose_project.replace("_", " ").replace("-", " ").title() ref = service.source_ref or "" compose = re.match(r"docker:compose:([^:]+):", ref, flags=re.IGNORECASE) if compose: return compose.group(1).replace("_", " ").replace("-", " ").title() name = re.sub(r"(?:[-_ ](?:server|web|api|db|database|redis|worker|frontend|backend))$", "", service.name, flags=re.IGNORECASE).strip() return name or service.name def application_identity(service: Service) -> tuple[str, str]: """Return a stable grouping key plus its operator-facing display name. Compose provenance wins over a generic display label. This keeps two unrelated ``PostgreSQL`` containers from being merged merely because an aggregate provider happens to use the same name for both applications. Explicit operator grouping remains authoritative. """ name = application_name(service) if service.application_locked and service.application: return f"operator:{service.application.casefold().strip()}", name if service.compose_project: return f"compose:{service.compose_project.casefold().strip()}", name ref = service.source_ref or "" compose = re.match(r"docker:compose:([^:]+):", ref, flags=re.IGNORECASE) if compose: return f"compose:{compose.group(1).casefold()}", name if ref: return f"source:{ref.casefold()}", name if service.application: return f"application:{service.application.casefold().strip()}", name return f"service:{service.id}", name def domain_name(service: Service, role: ComponentRole, application: str) -> str: if service.domain: return service.domain.strip() haystack = " ".join(filter(None, (application, service.name, service.container_image, service.category.value))).casefold() for domain, hints in _DOMAIN_HINTS: if any(hint in haystack for hint in hints): return domain if role in {ComponentRole.DATABASE, ComponentRole.CACHE, ComponentRole.STORAGE}: return "Storage" if role in {ComponentRole.PROXY}: return "Edge" if role in {ComponentRole.MONITORING, ComponentRole.AGENT}: return "Monitoring" return "Infrastructure" if service.category.value in {"infrastructure", "system"} else "Productivity" def _application_resolution( members: list[ServiceSnapshot], roles: dict[str, ComponentRole] ) -> ApplicationHealthResolution: """Resolve application health without promoting aggregate evidence to a component. A fresh, high-confidence application observation can answer the health of its application even where individual components are intentionally not probed. It can never change a component's direct status. Conversely a concrete component failure always outweighs a weak aggregate healthy claim, which preserves conflict visibility instead of masking incidents. """ resolved = [ item for item in members if item.direct_status not in {HealthStatus.UNKNOWN, HealthStatus.UNMONITORED} ] unresolved = len(members) - len(resolved) coverage = round((len(resolved) / len(members)) * 100, 1) if members else 0.0 application_evidence = { item.id: with_freshness(item) for snapshot in members for item in snapshot.health_evidence if item.active and item.scope == EvidenceScope.APPLICATION } fresh_claims = [ item for item in application_evidence.values() if item.freshness == EvidenceFreshness.FRESH ] selected = max( fresh_claims, key=lambda item: (item.confidence, item.observed_at, item.received_at, item.id), default=None, ) direct_down = [item for item in members if item.direct_status == HealthStatus.DOWN] impacted = [ item for item in members if item.effective_status in {HealthStatus.DOWN, HealthStatus.DEGRADED, HealthStatus.RECOVERING} and item.direct_status != HealthStatus.DOWN ] conflicting_aggregate = bool( selected and ( (selected.status == HealthStatus.HEALTHY and direct_down) or (selected.status in {HealthStatus.DOWN, HealthStatus.DEGRADED} and resolved) ) ) common = { "selected_evidence_id": selected.id if selected else None, "selected_provider": selected.provider if selected else None, "evidence_freshness": selected.freshness if selected else None, "conflict": conflicting_aggregate, "resolved_components": len(resolved), "unresolved_components": unresolved, "component_coverage_percent": coverage, } if direct_down: primary = [ item for item in direct_down if item.service.criticality == Criticality.CRITICAL or roles[item.service.id] in {ComponentRole.FRONTEND, ComponentRole.API, ComponentRole.PROXY} ] status = HealthStatus.DOWN if len(members) == 1 or len(direct_down) == len(resolved) or primary else HealthStatus.DEGRADED names = ", ".join(item.service.name for item in direct_down[:3]) return ApplicationHealthResolution( status=status, confidence=max((item.health_resolution.confidence or 1.0 for item in direct_down), default=None), explanation=f"{names} has current component failure evidence; aggregate health does not override it.", reason_code=HealthResolutionReason.COMPONENT_FAILURE, **common, ) if impacted: names = ", ".join(item.service.name for item in impacted[:3]) return ApplicationHealthResolution( status=HealthStatus.DEGRADED, confidence=selected.confidence if selected else None, explanation=f"{names} is impacted through a dependency health signal.", reason_code=HealthResolutionReason.COMPONENT_IMPACT, **common, ) if selected and selected.confidence >= 0.7 and selected.status not in {HealthStatus.UNKNOWN, HealthStatus.UNMONITORED}: return ApplicationHealthResolution( status=selected.status, confidence=selected.confidence, explanation=( f"{selected.provider.value} application observation selected " f"(confidence {selected.confidence:.2f}); component coverage is {coverage:.0f}% ." ), reason_code=HealthResolutionReason.SELECTED_APPLICATION_EVIDENCE, **common, ) if resolved: status = HealthStatus.HEALTHY if any(item.direct_status in {HealthStatus.DEGRADED, HealthStatus.RECOVERING} for item in resolved): status = HealthStatus.DEGRADED return ApplicationHealthResolution( status=status, explanation=f"{len(resolved)} of {len(members)} components have current direct health evidence.", reason_code=HealthResolutionReason.COMPONENT_COVERAGE, **common, ) stale = list(application_evidence.values()) if stale: reason = ( HealthResolutionReason.STALE_EVIDENCE if any(item.freshness == EvidenceFreshness.STALE for item in stale) else HealthResolutionReason.EXPIRED_EVIDENCE ) return ApplicationHealthResolution( status=HealthStatus.UNKNOWN, explanation="Application evidence is no longer current; waiting for a provider refresh.", reason_code=reason, **common, ) if members and all(item.direct_status == HealthStatus.UNMONITORED for item in members): return ApplicationHealthResolution( status=HealthStatus.UNMONITORED, explanation="No active health signal is configured for this application.", reason_code=HealthResolutionReason.NO_APPLICATION_EVIDENCE, **common, ) return ApplicationHealthResolution( status=HealthStatus.UNKNOWN, explanation="No current application or component health evidence is available.", reason_code=HealthResolutionReason.NO_APPLICATION_EVIDENCE, **common, ) def build_semantic_topology( snapshots: list[ServiceSnapshot], relations: list[LogicalRelation], root_service_ids: set[str] | None = None, affected_service_ids: set[str] | None = None, ) -> SemanticTopology: root_service_ids = root_service_ids or set() affected_service_ids = affected_service_ids or set() snapshot_by_id = {item.service.id: item for item in snapshots} roles = {item.service.id: component_role(item.service) for item in snapshots} apps_by_key: dict[str, list[ServiceSnapshot]] = defaultdict(list) app_names: dict[str, str] = {} app_domain_names: dict[str, str] = {} component_app_ids: dict[str, str] = {} components: list[SemanticComponent] = [] dependent_ids: dict[str, list[str]] = defaultdict(list) for snapshot in snapshots: for dependency_id in snapshot.service.depends_on: dependent_ids[dependency_id].append(snapshot.service.id) for snapshot in snapshots: application_key, application = application_identity(snapshot.service) apps_by_key[application_key].append(snapshot) app_names[application_key] = application role = roles[snapshot.service.id] app_domain_names.setdefault(application_key, domain_name(snapshot.service, role, application)) component_app_ids[snapshot.service.id] = _slug("app", application_key) applications: list[SemanticApplication] = [] application_by_id: dict[str, SemanticApplication] = {} domain_name_by_id: dict[str, str] = {} for application_key, members in sorted(apps_by_key.items(), key=lambda item: app_names[item[0]].casefold()): name = app_names[application_key] app_id = _slug("app", application_key) domain = app_domain_names[application_key] domain_id = _slug("domain", domain) domain_name_by_id[domain_id] = domain resolution = _application_resolution(members, roles) unmonitored = sum(item.direct_status == HealthStatus.UNMONITORED for item in members) app = SemanticApplication( id=app_id, name=name, domain_id=domain_id, component_ids=[item.service.id for item in members], component_count=len(members), roles=sorted({roles[item.service.id] for item in members}, key=lambda role: role.value), status=resolution.status, status_explanation=resolution.explanation, health_resolution=resolution, monitored_components=resolution.resolved_components, unmonitored_components=unmonitored, resolved_components=resolution.resolved_components, unresolved_components=resolution.unresolved_components, component_coverage_percent=resolution.component_coverage_percent, root_cause=any(item.service.id in root_service_ids for item in members), affected=any(item.service.id in affected_service_ids for item in members), ) applications.append(app) application_by_id[app_id] = app for snapshot in snapshots: app_id = component_app_ids[snapshot.service.id] app = application_by_id[app_id] components.append(SemanticComponent( service_id=snapshot.service.id, name=snapshot.service.name, kind=snapshot.service.kind, application_id=app_id, domain_id=app.domain_id, role=roles[snapshot.service.id], direct_status=snapshot.direct_status, effective_status=snapshot.effective_status, unknown_reason=( snapshot.health_resolution.reason_code if snapshot.direct_status == HealthStatus.UNKNOWN else None ), runtime_state=snapshot.service.runtime_state, runtime_health=snapshot.service.runtime_health, monitoring_state=snapshot.service.monitoring_state, latest_latency_ms=snapshot.latest_probe.latency_ms if snapshot.latest_probe else None, dependency_ids=[item for item in snapshot.service.depends_on if item in snapshot_by_id], dependent_ids=dependent_ids.get(snapshot.service.id, []), root_cause=snapshot.service.id in root_service_ids, affected=snapshot.service.id in affected_service_ids, )) app_dependencies: dict[str, set[str]] = defaultdict(set) app_dependents: dict[str, set[str]] = defaultdict(set) for relation in relations: if not relation.active or not relation.target_service_id: continue source_app = component_app_ids.get(relation.source_service_id) target_app = component_app_ids.get(relation.target_service_id) if source_app and target_app and source_app != target_app: app_dependencies[source_app].add(target_app) app_dependents[target_app].add(source_app) for app in applications: app.dependency_application_ids = sorted(app_dependencies[app.id]) app.dependent_application_ids = sorted(app_dependents[app.id]) apps_by_domain: dict[str, list[SemanticApplication]] = defaultdict(list) domain_names: dict[str, str] = {} for app in applications: apps_by_domain[app.domain_id].append(app) domain_names[app.domain_id] = domain_name_by_id[app.domain_id] domains: list[SemanticDomain] = [] for domain_id, members in sorted(apps_by_domain.items(), key=lambda item: domain_names[item[0]].casefold()): down = [item for item in members if item.status == HealthStatus.DOWN] degraded = [ item for item in members if item.status in {HealthStatus.DEGRADED, HealthStatus.RECOVERING} ] healthy = [item for item in members if item.status == HealthStatus.HEALTHY] resolved = [ item for item in members if item.status not in {HealthStatus.UNKNOWN, HealthStatus.UNMONITORED} ] if down: status = HealthStatus.DOWN elif degraded: status = HealthStatus.DEGRADED elif healthy: status = HealthStatus.HEALTHY elif members and all(item.status == HealthStatus.UNMONITORED for item in members): status = HealthStatus.UNMONITORED else: status = HealthStatus.UNKNOWN domains.append(SemanticDomain( id=domain_id, name=domain_names[domain_id], application_ids=[item.id for item in members], application_count=len(members), status=status, resolved_application_count=len(resolved), unresolved_application_count=len(members) - len(resolved), application_coverage_percent=round((len(resolved) / len(members)) * 100, 1) if members else 0, incident_count=sum(item.root_cause for item in members), root_cause=any(item.root_cause for item in members), affected=any(item.affected for item in members), )) coverage: dict[str, int] = defaultdict(int) for snapshot in snapshots: coverage[snapshot.service.monitoring_state.value] += 1 return SemanticTopology( generated_at=datetime.now(UTC), domains=domains, applications=applications, components=components, relations=relations, monitoring_coverage=dict(coverage), )