430 lines
19 KiB
Python
430 lines
19 KiB
Python
"""Deterministic semantic topology derived from the canonical service graph.
|
|
|
|
The discovery store remains the source of truth. This module deliberately
|
|
does not persist a second topology: a domain, application and component view
|
|
can therefore evolve without risking probe history or an operator's graph.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import re
|
|
from collections import defaultdict
|
|
from datetime import UTC, datetime
|
|
|
|
from .evidence import with_freshness
|
|
from .models import (
|
|
ApplicationHealthResolution,
|
|
ComponentRole,
|
|
Criticality,
|
|
EvidenceFreshness,
|
|
EvidenceScope,
|
|
HealthResolutionReason,
|
|
HealthStatus,
|
|
LogicalRelation,
|
|
SemanticApplication,
|
|
SemanticComponent,
|
|
SemanticDomain,
|
|
SemanticTopology,
|
|
Service,
|
|
ServiceSnapshot,
|
|
)
|
|
|
|
_ROLE_BY_CATEGORY = {
|
|
"database": ComponentRole.DATABASE,
|
|
"cache": ComponentRole.CACHE,
|
|
"proxy": ComponentRole.PROXY,
|
|
"background_worker": ComponentRole.WORKER,
|
|
"storage": ComponentRole.STORAGE,
|
|
"monitoring": ComponentRole.MONITORING,
|
|
}
|
|
_ROLE_HINTS = (
|
|
(ComponentRole.DATABASE, ("postgres", "mariadb", "mysql", "mongo", "influx", "database", "db")),
|
|
(ComponentRole.CACHE, ("redis", "memcache", "cache")),
|
|
(ComponentRole.PROXY, ("proxy", "nginx", "traefik", "caddy", "swag", "haproxy", "cloudflare")),
|
|
(ComponentRole.WORKER, ("worker", "runner", "queue", "task")),
|
|
(ComponentRole.STORAGE, ("minio", "storage", "s3", "volume", "files")),
|
|
(ComponentRole.MONITORING, ("grafana", "prometheus", "zabbix", "netdata", "loki", "uptime")),
|
|
(ComponentRole.AGENT, ("agent", "exporter", "sidecar")),
|
|
(ComponentRole.SCHEDULER, ("scheduler", "cron", "schedule")),
|
|
(ComponentRole.FRONTEND, ("frontend", "webui", "dashboard", "ui")),
|
|
(ComponentRole.API, ("api", "backend", "server")),
|
|
)
|
|
_DOMAIN_HINTS = (
|
|
("Edge", ("proxy", "nginx", "traefik", "caddy", "swag", "cloudflare", "authelia", "auth")),
|
|
("Media", ("plex", "jellyfin", "sonarr", "radarr", "lidarr", "readarr", "bazarr", "media", "tautulli")),
|
|
("Dev", ("gitea", "git", "code", "dev", "ci", "runner")),
|
|
("Monitoring", ("grafana", "prometheus", "zabbix", "netdata", "loki", "uptime", "monitor")),
|
|
("AI", ("ollama", "open webui", "comfy", "ai", "llm")),
|
|
("Storage", ("postgres", "mariadb", "mysql", "mongo", "redis", "minio", "storage", "nextcloud", "file")),
|
|
("Automation", ("n8n", "worker", "scheduler", "cron", "automation", "arr")),
|
|
("Productivity", ("book", "paperless", "vault", "wiki", "project", "calendar", "office")),
|
|
)
|
|
def _slug(prefix: str, value: str) -> str:
|
|
normalized = re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-") or "unknown"
|
|
digest = hashlib.sha1(value.casefold().encode("utf-8")).hexdigest()[:8]
|
|
return f"{prefix}:{normalized[:48]}-{digest}"
|
|
|
|
|
|
def component_role(service: Service) -> ComponentRole:
|
|
if service.component_role_locked or service.component_role != ComponentRole.UNKNOWN:
|
|
return service.component_role
|
|
category_role = _ROLE_BY_CATEGORY.get(service.category.value)
|
|
if category_role:
|
|
return category_role
|
|
haystack = " ".join(filter(None, (service.name, service.container_image, service.source_ref, service.description))).casefold()
|
|
for role, hints in _ROLE_HINTS:
|
|
if any(hint in haystack for hint in hints):
|
|
return role
|
|
# A monitoring gap says nothing about the component's workload role. Keep
|
|
# it unknown until source metadata, a deterministic hint or an operator
|
|
# override supplies evidence.
|
|
return ComponentRole.UNKNOWN
|
|
|
|
|
|
def application_name(service: Service) -> str:
|
|
"""Use stable provenance before falling back to a standalone application.
|
|
|
|
Category is intentionally *not* used as a shared application name. Doing
|
|
so was the old "Independent · Category" compaction bug: unrelated services
|
|
looked like one stack. Categories still inform domain derivation below.
|
|
"""
|
|
|
|
if service.application:
|
|
return service.application.strip()
|
|
if service.compose_project:
|
|
return service.compose_project.replace("_", " ").replace("-", " ").title()
|
|
ref = service.source_ref or ""
|
|
compose = re.match(r"docker:compose:([^:]+):", ref, flags=re.IGNORECASE)
|
|
if compose:
|
|
return compose.group(1).replace("_", " ").replace("-", " ").title()
|
|
name = re.sub(r"(?:[-_ ](?:server|web|api|db|database|redis|worker|frontend|backend))$", "", service.name, flags=re.IGNORECASE).strip()
|
|
return name or service.name
|
|
|
|
|
|
def application_identity(service: Service) -> tuple[str, str]:
|
|
"""Return a stable grouping key plus its operator-facing display name.
|
|
|
|
Compose provenance wins over a generic display label. This keeps two
|
|
unrelated ``PostgreSQL`` containers from being merged merely because an
|
|
aggregate provider happens to use the same name for both applications.
|
|
Explicit operator grouping remains authoritative.
|
|
"""
|
|
|
|
name = application_name(service)
|
|
if service.application_locked and service.application:
|
|
return f"operator:{service.application.casefold().strip()}", name
|
|
if service.compose_project:
|
|
return f"compose:{service.compose_project.casefold().strip()}", name
|
|
ref = service.source_ref or ""
|
|
compose = re.match(r"docker:compose:([^:]+):", ref, flags=re.IGNORECASE)
|
|
if compose:
|
|
return f"compose:{compose.group(1).casefold()}", name
|
|
if ref:
|
|
return f"source:{ref.casefold()}", name
|
|
if service.application:
|
|
return f"application:{service.application.casefold().strip()}", name
|
|
return f"service:{service.id}", name
|
|
|
|
|
|
def domain_name(service: Service, role: ComponentRole, application: str) -> str:
|
|
if service.domain:
|
|
return service.domain.strip()
|
|
haystack = " ".join(filter(None, (application, service.name, service.container_image, service.category.value))).casefold()
|
|
for domain, hints in _DOMAIN_HINTS:
|
|
if any(hint in haystack for hint in hints):
|
|
return domain
|
|
if role in {ComponentRole.DATABASE, ComponentRole.CACHE, ComponentRole.STORAGE}:
|
|
return "Storage"
|
|
if role in {ComponentRole.PROXY}:
|
|
return "Edge"
|
|
if role in {ComponentRole.MONITORING, ComponentRole.AGENT}:
|
|
return "Monitoring"
|
|
return "Infrastructure" if service.category.value in {"infrastructure", "system"} else "Productivity"
|
|
|
|
|
|
def _application_resolution(
|
|
members: list[ServiceSnapshot], roles: dict[str, ComponentRole]
|
|
) -> ApplicationHealthResolution:
|
|
"""Resolve application health without promoting aggregate evidence to a component.
|
|
|
|
A fresh, high-confidence application observation can answer the health of
|
|
its application even where individual components are intentionally not
|
|
probed. It can never change a component's direct status. Conversely a
|
|
concrete component failure always outweighs a weak aggregate healthy
|
|
claim, which preserves conflict visibility instead of masking incidents.
|
|
"""
|
|
|
|
resolved = [
|
|
item
|
|
for item in members
|
|
if item.direct_status not in {HealthStatus.UNKNOWN, HealthStatus.UNMONITORED}
|
|
]
|
|
unresolved = len(members) - len(resolved)
|
|
coverage = round((len(resolved) / len(members)) * 100, 1) if members else 0.0
|
|
application_evidence = {
|
|
item.id: with_freshness(item)
|
|
for snapshot in members
|
|
for item in snapshot.health_evidence
|
|
if item.active and item.scope == EvidenceScope.APPLICATION
|
|
}
|
|
fresh_claims = [
|
|
item
|
|
for item in application_evidence.values()
|
|
if item.freshness == EvidenceFreshness.FRESH
|
|
]
|
|
selected = max(
|
|
fresh_claims,
|
|
key=lambda item: (item.confidence, item.observed_at, item.received_at, item.id),
|
|
default=None,
|
|
)
|
|
|
|
direct_down = [item for item in members if item.direct_status == HealthStatus.DOWN]
|
|
impacted = [
|
|
item
|
|
for item in members
|
|
if item.effective_status in {HealthStatus.DOWN, HealthStatus.DEGRADED, HealthStatus.RECOVERING}
|
|
and item.direct_status != HealthStatus.DOWN
|
|
]
|
|
conflicting_aggregate = bool(
|
|
selected
|
|
and (
|
|
(selected.status == HealthStatus.HEALTHY and direct_down)
|
|
or (selected.status in {HealthStatus.DOWN, HealthStatus.DEGRADED} and resolved)
|
|
)
|
|
)
|
|
common = {
|
|
"selected_evidence_id": selected.id if selected else None,
|
|
"selected_provider": selected.provider if selected else None,
|
|
"evidence_freshness": selected.freshness if selected else None,
|
|
"conflict": conflicting_aggregate,
|
|
"resolved_components": len(resolved),
|
|
"unresolved_components": unresolved,
|
|
"component_coverage_percent": coverage,
|
|
}
|
|
if direct_down:
|
|
primary = [
|
|
item
|
|
for item in direct_down
|
|
if item.service.criticality == Criticality.CRITICAL
|
|
or roles[item.service.id] in {ComponentRole.FRONTEND, ComponentRole.API, ComponentRole.PROXY}
|
|
]
|
|
status = HealthStatus.DOWN if len(members) == 1 or len(direct_down) == len(resolved) or primary else HealthStatus.DEGRADED
|
|
names = ", ".join(item.service.name for item in direct_down[:3])
|
|
return ApplicationHealthResolution(
|
|
status=status,
|
|
confidence=max((item.health_resolution.confidence or 1.0 for item in direct_down), default=None),
|
|
explanation=f"{names} has current component failure evidence; aggregate health does not override it.",
|
|
reason_code=HealthResolutionReason.COMPONENT_FAILURE,
|
|
**common,
|
|
)
|
|
if impacted:
|
|
names = ", ".join(item.service.name for item in impacted[:3])
|
|
return ApplicationHealthResolution(
|
|
status=HealthStatus.DEGRADED,
|
|
confidence=selected.confidence if selected else None,
|
|
explanation=f"{names} is impacted through a dependency health signal.",
|
|
reason_code=HealthResolutionReason.COMPONENT_IMPACT,
|
|
**common,
|
|
)
|
|
if selected and selected.confidence >= 0.7 and selected.status not in {HealthStatus.UNKNOWN, HealthStatus.UNMONITORED}:
|
|
return ApplicationHealthResolution(
|
|
status=selected.status,
|
|
confidence=selected.confidence,
|
|
explanation=(
|
|
f"{selected.provider.value} application observation selected "
|
|
f"(confidence {selected.confidence:.2f}); component coverage is {coverage:.0f}% ."
|
|
),
|
|
reason_code=HealthResolutionReason.SELECTED_APPLICATION_EVIDENCE,
|
|
**common,
|
|
)
|
|
if resolved:
|
|
status = HealthStatus.HEALTHY
|
|
if any(item.direct_status in {HealthStatus.DEGRADED, HealthStatus.RECOVERING} for item in resolved):
|
|
status = HealthStatus.DEGRADED
|
|
return ApplicationHealthResolution(
|
|
status=status,
|
|
explanation=f"{len(resolved)} of {len(members)} components have current direct health evidence.",
|
|
reason_code=HealthResolutionReason.COMPONENT_COVERAGE,
|
|
**common,
|
|
)
|
|
stale = list(application_evidence.values())
|
|
if stale:
|
|
reason = (
|
|
HealthResolutionReason.STALE_EVIDENCE
|
|
if any(item.freshness == EvidenceFreshness.STALE for item in stale)
|
|
else HealthResolutionReason.EXPIRED_EVIDENCE
|
|
)
|
|
return ApplicationHealthResolution(
|
|
status=HealthStatus.UNKNOWN,
|
|
explanation="Application evidence is no longer current; waiting for a provider refresh.",
|
|
reason_code=reason,
|
|
**common,
|
|
)
|
|
if members and all(item.direct_status == HealthStatus.UNMONITORED for item in members):
|
|
return ApplicationHealthResolution(
|
|
status=HealthStatus.UNMONITORED,
|
|
explanation="No active health signal is configured for this application.",
|
|
reason_code=HealthResolutionReason.NO_APPLICATION_EVIDENCE,
|
|
**common,
|
|
)
|
|
return ApplicationHealthResolution(
|
|
status=HealthStatus.UNKNOWN,
|
|
explanation="No current application or component health evidence is available.",
|
|
reason_code=HealthResolutionReason.NO_APPLICATION_EVIDENCE,
|
|
**common,
|
|
)
|
|
|
|
|
|
def build_semantic_topology(
|
|
snapshots: list[ServiceSnapshot],
|
|
relations: list[LogicalRelation],
|
|
root_service_ids: set[str] | None = None,
|
|
affected_service_ids: set[str] | None = None,
|
|
) -> SemanticTopology:
|
|
root_service_ids = root_service_ids or set()
|
|
affected_service_ids = affected_service_ids or set()
|
|
snapshot_by_id = {item.service.id: item for item in snapshots}
|
|
roles = {item.service.id: component_role(item.service) for item in snapshots}
|
|
apps_by_key: dict[str, list[ServiceSnapshot]] = defaultdict(list)
|
|
app_names: dict[str, str] = {}
|
|
app_domain_names: dict[str, str] = {}
|
|
component_app_ids: dict[str, str] = {}
|
|
components: list[SemanticComponent] = []
|
|
dependent_ids: dict[str, list[str]] = defaultdict(list)
|
|
for snapshot in snapshots:
|
|
for dependency_id in snapshot.service.depends_on:
|
|
dependent_ids[dependency_id].append(snapshot.service.id)
|
|
for snapshot in snapshots:
|
|
application_key, application = application_identity(snapshot.service)
|
|
apps_by_key[application_key].append(snapshot)
|
|
app_names[application_key] = application
|
|
role = roles[snapshot.service.id]
|
|
app_domain_names.setdefault(application_key, domain_name(snapshot.service, role, application))
|
|
component_app_ids[snapshot.service.id] = _slug("app", application_key)
|
|
|
|
applications: list[SemanticApplication] = []
|
|
application_by_id: dict[str, SemanticApplication] = {}
|
|
domain_name_by_id: dict[str, str] = {}
|
|
for application_key, members in sorted(apps_by_key.items(), key=lambda item: app_names[item[0]].casefold()):
|
|
name = app_names[application_key]
|
|
app_id = _slug("app", application_key)
|
|
domain = app_domain_names[application_key]
|
|
domain_id = _slug("domain", domain)
|
|
domain_name_by_id[domain_id] = domain
|
|
resolution = _application_resolution(members, roles)
|
|
unmonitored = sum(item.direct_status == HealthStatus.UNMONITORED for item in members)
|
|
app = SemanticApplication(
|
|
id=app_id,
|
|
name=name,
|
|
domain_id=domain_id,
|
|
component_ids=[item.service.id for item in members],
|
|
component_count=len(members),
|
|
roles=sorted({roles[item.service.id] for item in members}, key=lambda role: role.value),
|
|
status=resolution.status,
|
|
status_explanation=resolution.explanation,
|
|
health_resolution=resolution,
|
|
monitored_components=resolution.resolved_components,
|
|
unmonitored_components=unmonitored,
|
|
resolved_components=resolution.resolved_components,
|
|
unresolved_components=resolution.unresolved_components,
|
|
component_coverage_percent=resolution.component_coverage_percent,
|
|
root_cause=any(item.service.id in root_service_ids for item in members),
|
|
affected=any(item.service.id in affected_service_ids for item in members),
|
|
)
|
|
applications.append(app)
|
|
application_by_id[app_id] = app
|
|
|
|
for snapshot in snapshots:
|
|
app_id = component_app_ids[snapshot.service.id]
|
|
app = application_by_id[app_id]
|
|
components.append(SemanticComponent(
|
|
service_id=snapshot.service.id,
|
|
name=snapshot.service.name,
|
|
kind=snapshot.service.kind,
|
|
application_id=app_id,
|
|
domain_id=app.domain_id,
|
|
role=roles[snapshot.service.id],
|
|
direct_status=snapshot.direct_status,
|
|
effective_status=snapshot.effective_status,
|
|
unknown_reason=(
|
|
snapshot.health_resolution.reason_code
|
|
if snapshot.direct_status == HealthStatus.UNKNOWN
|
|
else None
|
|
),
|
|
runtime_state=snapshot.service.runtime_state,
|
|
runtime_health=snapshot.service.runtime_health,
|
|
monitoring_state=snapshot.service.monitoring_state,
|
|
latest_latency_ms=snapshot.latest_probe.latency_ms if snapshot.latest_probe else None,
|
|
dependency_ids=[item for item in snapshot.service.depends_on if item in snapshot_by_id],
|
|
dependent_ids=dependent_ids.get(snapshot.service.id, []),
|
|
root_cause=snapshot.service.id in root_service_ids,
|
|
affected=snapshot.service.id in affected_service_ids,
|
|
))
|
|
|
|
app_dependencies: dict[str, set[str]] = defaultdict(set)
|
|
app_dependents: dict[str, set[str]] = defaultdict(set)
|
|
for relation in relations:
|
|
if not relation.active or not relation.target_service_id:
|
|
continue
|
|
source_app = component_app_ids.get(relation.source_service_id)
|
|
target_app = component_app_ids.get(relation.target_service_id)
|
|
if source_app and target_app and source_app != target_app:
|
|
app_dependencies[source_app].add(target_app)
|
|
app_dependents[target_app].add(source_app)
|
|
for app in applications:
|
|
app.dependency_application_ids = sorted(app_dependencies[app.id])
|
|
app.dependent_application_ids = sorted(app_dependents[app.id])
|
|
|
|
apps_by_domain: dict[str, list[SemanticApplication]] = defaultdict(list)
|
|
domain_names: dict[str, str] = {}
|
|
for app in applications:
|
|
apps_by_domain[app.domain_id].append(app)
|
|
domain_names[app.domain_id] = domain_name_by_id[app.domain_id]
|
|
domains: list[SemanticDomain] = []
|
|
for domain_id, members in sorted(apps_by_domain.items(), key=lambda item: domain_names[item[0]].casefold()):
|
|
down = [item for item in members if item.status == HealthStatus.DOWN]
|
|
degraded = [
|
|
item for item in members if item.status in {HealthStatus.DEGRADED, HealthStatus.RECOVERING}
|
|
]
|
|
healthy = [item for item in members if item.status == HealthStatus.HEALTHY]
|
|
resolved = [
|
|
item
|
|
for item in members
|
|
if item.status not in {HealthStatus.UNKNOWN, HealthStatus.UNMONITORED}
|
|
]
|
|
if down:
|
|
status = HealthStatus.DOWN
|
|
elif degraded:
|
|
status = HealthStatus.DEGRADED
|
|
elif healthy:
|
|
status = HealthStatus.HEALTHY
|
|
elif members and all(item.status == HealthStatus.UNMONITORED for item in members):
|
|
status = HealthStatus.UNMONITORED
|
|
else:
|
|
status = HealthStatus.UNKNOWN
|
|
domains.append(SemanticDomain(
|
|
id=domain_id,
|
|
name=domain_names[domain_id],
|
|
application_ids=[item.id for item in members],
|
|
application_count=len(members),
|
|
status=status,
|
|
resolved_application_count=len(resolved),
|
|
unresolved_application_count=len(members) - len(resolved),
|
|
application_coverage_percent=round((len(resolved) / len(members)) * 100, 1) if members else 0,
|
|
incident_count=sum(item.root_cause for item in members),
|
|
root_cause=any(item.root_cause for item in members),
|
|
affected=any(item.affected for item in members),
|
|
))
|
|
coverage: dict[str, int] = defaultdict(int)
|
|
for snapshot in snapshots:
|
|
coverage[snapshot.service.monitoring_state.value] += 1
|
|
return SemanticTopology(
|
|
generated_at=datetime.now(UTC),
|
|
domains=domains,
|
|
applications=applications,
|
|
components=components,
|
|
relations=relations,
|
|
monitoring_coverage=dict(coverage),
|
|
)
|