Initial curated ServiceMap source from 7a0a61232d05ba9097432abef6f7f4102a64d0b8
This commit is contained in:
commit
105251b9c8
96 files changed
+23587
No files matched your search
@@ -0,0 +1,682 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import docker
|
||||
import httpx
|
||||
from docker.errors import DockerException
|
||||
|
||||
from .docker_provider import DockerProviderError, docker_client_for_inventory, docker_provider_mode
|
||||
from .models import (
|
||||
DiscoveryCategory,
|
||||
DiscoveryConfidence,
|
||||
DiscoveryLifecycle,
|
||||
DiscoveryVisibility,
|
||||
DockerCandidate,
|
||||
DockerDiscoveryResult,
|
||||
DockerPort,
|
||||
DockerSuggestion,
|
||||
EvidenceScope,
|
||||
ProbeEndpointCandidate,
|
||||
ProbeEndpointSource,
|
||||
ProbeReachability,
|
||||
Service,
|
||||
ServiceKind,
|
||||
)
|
||||
|
||||
ALLOWED_SCHEMES = {"unix", "tcp", "http", "https"}
|
||||
_RELATION_ENV = re.compile(r"(URL|HOST|HOSTNAME|ADDRESS|ENDPOINT|DSN|DATABASE|REDIS|POSTGRES|MYSQL|MONGO|BROKER|UPSTREAM|S3|STORAGE)", re.IGNORECASE)
|
||||
_SECRET_ENV = re.compile(r"(PASSWORD|PASSWD|SECRET|TOKEN|API[_-]?KEY|PRIVATE[_-]?KEY|CREDENTIAL|AUTHORIZATION|BEARER)", re.IGNORECASE)
|
||||
_SECRET_LABEL = re.compile(
|
||||
r"(PASSWORD|PASSWD|SECRET|TOKEN|(?:API|ACCESS|PRIVATE|SIGNING|ENCRYPTION)[_-]?KEY|"
|
||||
r"CREDENTIAL|AUTH(?:ORIZATION)?|BEARER|COOKIE)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_HTTP_CONTAINER_PORTS = {
|
||||
80, 443, 3000, 3001, 5000, 6060, 8000, 8080, 8081, 8181, 8188,
|
||||
8265, 8989, 9000, 9443, 9494, 11434,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppOpsInfo:
|
||||
app_id: str
|
||||
category: str | None
|
||||
criticality: str | None
|
||||
health: str | None
|
||||
display_name: str | None
|
||||
discovery_confidence: float | None = None
|
||||
policy_profile: str | None = None
|
||||
app_type: str | None = None
|
||||
reasons: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppOpsInventory:
|
||||
by_container: dict[str, AppOpsInfo]
|
||||
available: bool
|
||||
detail: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DockerHealthObservation:
|
||||
"""The small, health-only subset needed by the recurring observer."""
|
||||
|
||||
container_id: str
|
||||
name: str
|
||||
stable_source_ref: str
|
||||
state: str
|
||||
health: str | None
|
||||
healthcheck_present: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProxyRoute:
|
||||
domains: tuple[str, ...]
|
||||
forward_host: str
|
||||
forward_port: int
|
||||
forward_scheme: str
|
||||
|
||||
|
||||
def _normal_name(value: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-")
|
||||
|
||||
|
||||
def stable_source_ref(name: str, labels: dict[str, str]) -> str:
|
||||
project = labels.get("com.docker.compose.project", "").strip()
|
||||
service = labels.get("com.docker.compose.service", "").strip()
|
||||
if project and service:
|
||||
return f"docker:compose:{_normal_name(project)}:{_normal_name(service)}"
|
||||
return f"docker:name:{_normal_name(name)}"
|
||||
|
||||
|
||||
def _legacy_source_ref(container_id: str, image_name: str) -> str:
|
||||
return f"docker:{container_id[:12]}@{image_name}"
|
||||
|
||||
|
||||
def _safe_labels(labels: dict[str, str]) -> dict[str, str]:
|
||||
prefixes = ("servicemap.", "com.docker.compose.", "net.unraid.docker.", "org.opencontainers.image.")
|
||||
return {
|
||||
key: value
|
||||
for key, value in labels.items()
|
||||
if key.startswith(prefixes) and not _SECRET_LABEL.search(key)
|
||||
}
|
||||
|
||||
|
||||
def _classify(name: str, image: str, labels: dict[str, str], appops: AppOpsInfo | None = None) -> DiscoveryCategory:
|
||||
explicit = labels.get("servicemap.category", "").strip().casefold()
|
||||
if explicit in {item.value for item in DiscoveryCategory}:
|
||||
return DiscoveryCategory(explicit)
|
||||
# AppOps category is application metadata. It is helpful only when its
|
||||
# discovery confidence is high enough; a low-confidence registry row must
|
||||
# not override deterministic Docker or operator classification.
|
||||
if appops and appops.category and (appops.discovery_confidence or 0) >= 0.75:
|
||||
mapped = {
|
||||
"database": DiscoveryCategory.DATABASE,
|
||||
"database_admin": DiscoveryCategory.DATABASE,
|
||||
"cache": DiscoveryCategory.CACHE,
|
||||
"network": DiscoveryCategory.INFRASTRUCTURE,
|
||||
"monitoring": DiscoveryCategory.MONITORING,
|
||||
"media": DiscoveryCategory.APPLICATION,
|
||||
"development": DiscoveryCategory.DEVELOPMENT,
|
||||
"storage": DiscoveryCategory.STORAGE,
|
||||
"infrastructure": DiscoveryCategory.INFRASTRUCTURE,
|
||||
}.get(appops.category.casefold())
|
||||
if mapped:
|
||||
return mapped
|
||||
value = f"{name} {image}".casefold()
|
||||
if any(token in value for token in ("postgres", "mariadb", "mysql", "mongo", "qdrant")):
|
||||
return DiscoveryCategory.DATABASE
|
||||
if "redis" in value:
|
||||
return DiscoveryCategory.CACHE
|
||||
if any(token in value for token in ("nginx", "traefik", "caddy", "proxy")):
|
||||
return DiscoveryCategory.PROXY
|
||||
if any(token in value for token in ("prometheus", "grafana", "zabbix", "netdata", "cadvisor", "loki", "alloy")):
|
||||
return DiscoveryCategory.MONITORING
|
||||
if any(token in value for token in ("worker", "agent", "runner", "backup", "scheduler", "node")):
|
||||
return DiscoveryCategory.BACKGROUND_WORKER
|
||||
if any(token in value for token in ("portainer", "dockdeck", "unraid", "watchtower", "appops", "gitea")):
|
||||
return DiscoveryCategory.INFRASTRUCTURE
|
||||
if any(token in value for token in ("dev", "forge", "lab", "builder", "runner")):
|
||||
return DiscoveryCategory.DEVELOPMENT
|
||||
return DiscoveryCategory.APPLICATION
|
||||
|
||||
|
||||
def _visibility(name: str, state: str, category: DiscoveryCategory, labels: dict[str, str]) -> DiscoveryVisibility:
|
||||
if labels.get("servicemap.ignore", "false").casefold() == "true":
|
||||
return DiscoveryVisibility.IGNORED
|
||||
if state != "running":
|
||||
return DiscoveryVisibility.HIDDEN
|
||||
if category == DiscoveryCategory.SYSTEM:
|
||||
return DiscoveryVisibility.IGNORED
|
||||
if labels.get("servicemap.visible", "true").casefold() == "false":
|
||||
return DiscoveryVisibility.HIDDEN
|
||||
return DiscoveryVisibility.VISIBLE
|
||||
|
||||
|
||||
def _kind_and_endpoint(
|
||||
name: str,
|
||||
ports: list[DockerPort],
|
||||
labels: dict[str, str],
|
||||
probe_host: str | None = None,
|
||||
) -> tuple[ServiceKind, str | None, str | None, int | None, list[str]]:
|
||||
"""Select only an endpoint ServiceMap can reach from its own network.
|
||||
|
||||
Docker's embedded DNS is scoped to a container network. A name such as
|
||||
``postgres`` is useful discovery evidence but cannot become an active
|
||||
probe merely because it exists in another project's network.
|
||||
"""
|
||||
probe_host = (probe_host or os.getenv("SERVICEMAP_PROBE_HOST") or "").strip()
|
||||
explicit_kind = labels.get("servicemap.probe", "").casefold()
|
||||
explicit_port = labels.get("servicemap.port", "")
|
||||
tcp_ports = [item for item in ports if item.container_port.endswith("/tcp")]
|
||||
|
||||
def container_port(item: DockerPort) -> int | None:
|
||||
raw = item.container_port.split("/", 1)[0]
|
||||
return int(raw) if raw.isdigit() else None
|
||||
|
||||
explicit_value = int(explicit_port) if explicit_port.isdigit() else None
|
||||
selected = next((item for item in tcp_ports if container_port(item) == explicit_value), None)
|
||||
if selected is None:
|
||||
published = [item for item in tcp_ports if item.host_port is not None]
|
||||
# The mapped host port is arbitrary. Select using the container's
|
||||
# listener port so 8080 -> 8070 remains an HTTP candidate.
|
||||
selected = next((item for item in published if container_port(item) in _HTTP_CONTAINER_PORTS), None)
|
||||
selected = selected or (published[0] if published else (tcp_ports[0] if tcp_ports else None))
|
||||
container_listener_port = container_port(selected) if selected else None
|
||||
host_port = selected.host_port if selected else None
|
||||
port = explicit_value or container_listener_port
|
||||
evidence = ["container metadata"]
|
||||
if explicit_kind in {"http", "tcp", "dns", "tls"}:
|
||||
kind = ServiceKind(explicit_kind)
|
||||
evidence.append("servicemap.probe label")
|
||||
elif port in _HTTP_CONTAINER_PORTS:
|
||||
kind = ServiceKind.HTTP
|
||||
evidence.append("HTTP-like container port")
|
||||
elif port:
|
||||
kind = ServiceKind.TCP
|
||||
evidence.append("published TCP port")
|
||||
else:
|
||||
return ServiceKind.TCP, None, name, None, evidence
|
||||
if kind == ServiceKind.HTTP:
|
||||
protocol = "https" if port in {443, 8443, 9443} else "http"
|
||||
if host_port is not None and probe_host:
|
||||
# HTTP services are represented by a URL in the canonical model;
|
||||
# carrying the published port alongside it creates an invalid
|
||||
# mixed endpoint payload during reconciliation.
|
||||
return kind, f"{protocol}://{probe_host}:{host_port}", None, None, evidence + ["host-published endpoint"]
|
||||
return kind, None, name, port, evidence + ["internal container endpoint candidate"]
|
||||
if kind == ServiceKind.DNS:
|
||||
return kind, None, name, None, evidence
|
||||
if host_port is not None and probe_host:
|
||||
return kind, None, probe_host, host_port, evidence + ["host-published endpoint"]
|
||||
return kind, None, name, port, evidence + ["internal container endpoint candidate"]
|
||||
|
||||
|
||||
def _preflight_candidate(candidate: ProbeEndpointCandidate) -> ProbeEndpointCandidate:
|
||||
"""Verify only host-published endpoints before allowing active probes.
|
||||
|
||||
Discovery never probes Docker-internal DNS names and never scans paths: the
|
||||
endpoint must already come from a Docker port mapping or explicit label.
|
||||
"""
|
||||
if candidate.source not in {
|
||||
ProbeEndpointSource.PUBLISHED_PORT,
|
||||
ProbeEndpointSource.EXPLICIT_LABEL,
|
||||
}:
|
||||
return candidate
|
||||
parsed = urlparse(candidate.url) if candidate.url else None
|
||||
hostname = parsed.hostname if parsed else candidate.host
|
||||
port = parsed.port if parsed else candidate.port
|
||||
if not hostname or not port:
|
||||
return candidate.model_copy(update={"preflight_detail": "No host-published endpoint to preflight"})
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
with socket.create_connection((hostname, port), timeout=0.75):
|
||||
pass
|
||||
elapsed = round((time.perf_counter() - started) * 1000, 1)
|
||||
if candidate.kind != ServiceKind.HTTP or not candidate.url:
|
||||
return candidate.model_copy(update={
|
||||
"reachability": ProbeReachability.VERIFIED,
|
||||
"primary": True,
|
||||
"preflight_detail": "TCP connect succeeded",
|
||||
"preflight_latency_ms": elapsed,
|
||||
})
|
||||
with httpx.Client(timeout=1.25, follow_redirects=False) as client:
|
||||
response = client.request("HEAD", candidate.url)
|
||||
if response.status_code == 405:
|
||||
response = client.request("GET", candidate.url)
|
||||
return candidate.model_copy(update={
|
||||
"reachability": ProbeReachability.VERIFIED,
|
||||
"primary": True,
|
||||
"preflight_detail": f"HTTP {response.status_code} responded",
|
||||
"preflight_latency_ms": elapsed,
|
||||
})
|
||||
except (OSError, httpx.HTTPError) as exc:
|
||||
return candidate.model_copy(update={
|
||||
"reachability": ProbeReachability.UNVERIFIED,
|
||||
"primary": False,
|
||||
"preflight_detail": f"Preflight failed: {type(exc).__name__}",
|
||||
})
|
||||
|
||||
|
||||
def _probe_candidates(
|
||||
suggestion: DockerSuggestion | None,
|
||||
*,
|
||||
healthcheck_present: bool,
|
||||
) -> list[ProbeEndpointCandidate]:
|
||||
candidates: list[ProbeEndpointCandidate] = []
|
||||
if suggestion:
|
||||
source = ProbeEndpointSource.EXPLICIT_LABEL if "servicemap.probe label" in suggestion.evidence else ProbeEndpointSource.PUBLISHED_PORT
|
||||
reachability = ProbeReachability.UNVERIFIED if "host-published endpoint" in suggestion.evidence else ProbeReachability.INTERNAL_ONLY
|
||||
candidates.append(
|
||||
ProbeEndpointCandidate(
|
||||
source=source if reachability == ProbeReachability.UNVERIFIED else ProbeEndpointSource.CONTAINER_INTERNAL,
|
||||
kind=suggestion.kind,
|
||||
url=suggestion.url,
|
||||
host=suggestion.host,
|
||||
port=suggestion.port,
|
||||
reachability=reachability,
|
||||
confidence=DiscoveryConfidence.MEDIUM if reachability == ProbeReachability.UNVERIFIED else DiscoveryConfidence.LOW,
|
||||
evidence="; ".join(suggestion.evidence),
|
||||
)
|
||||
)
|
||||
if healthcheck_present:
|
||||
candidates.append(
|
||||
ProbeEndpointCandidate(
|
||||
source=ProbeEndpointSource.DOCKER_HEALTHCHECK,
|
||||
kind=suggestion.kind if suggestion else ServiceKind.TCP,
|
||||
reachability=ProbeReachability.VERIFIED,
|
||||
confidence=DiscoveryConfidence.HIGH,
|
||||
evidence="Docker healthcheck present (observed without a network probe)",
|
||||
)
|
||||
)
|
||||
return candidates
|
||||
|
||||
|
||||
def safe_environment_evidence(env: list[str]) -> list[str]:
|
||||
evidence: list[str] = []
|
||||
for entry in env:
|
||||
name, separator, value = entry.partition("=")
|
||||
if not separator or not value or _SECRET_ENV.search(name) or not _RELATION_ENV.search(name):
|
||||
continue
|
||||
try:
|
||||
parsed = urlparse(value if "://" in value else f"//{value}")
|
||||
hostname = parsed.hostname
|
||||
except ValueError:
|
||||
# Docker environments frequently contain comma-separated host
|
||||
# lists or malformed IPv6-like values. Discovery evidence is
|
||||
# best-effort; one invalid value must not abort the complete scan.
|
||||
continue
|
||||
if not hostname or hostname.casefold() in {"localhost", "127.0.0.1", "0.0.0.0"}:
|
||||
continue
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError:
|
||||
# A parseable hostname with a non-numeric suffix is still useful
|
||||
# relation evidence; omit only the unsafe port component.
|
||||
port = None
|
||||
suffix = f":{port}" if port else ""
|
||||
evidence.append(f"{name} hostname={hostname}{suffix}")
|
||||
return sorted(set(evidence))
|
||||
|
||||
|
||||
def _appops_inventory(appops_url: str | None) -> AppOpsInventory:
|
||||
if not appops_url:
|
||||
return AppOpsInventory({}, False, "AppOps URL is not configured")
|
||||
base = appops_url.rstrip("/")
|
||||
try:
|
||||
mappings_response = httpx.get(f"{base}/api/inventory/mappings", timeout=5)
|
||||
apps_response = httpx.get(f"{base}/api/inventory/apps", timeout=5)
|
||||
mappings_response.raise_for_status()
|
||||
apps_response.raise_for_status()
|
||||
mappings = mappings_response.json()
|
||||
apps = apps_response.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
return AppOpsInventory({}, False, f"AppOps inventory unavailable: {type(exc).__name__}")
|
||||
app_by_id = {str(item.get("id")): item for item in apps if isinstance(item, dict)}
|
||||
result: dict[str, AppOpsInfo] = {}
|
||||
for mapping in mappings if isinstance(mappings, list) else []:
|
||||
if not isinstance(mapping, dict):
|
||||
continue
|
||||
container = str(mapping.get("containerName") or "")
|
||||
app_id = str(mapping.get("appId") or "")
|
||||
app = app_by_id.get(app_id, {})
|
||||
try:
|
||||
confidence = float(app.get("discovery_confidence")) if app.get("discovery_confidence") is not None else None
|
||||
except (TypeError, ValueError):
|
||||
confidence = None
|
||||
if container:
|
||||
result[container.casefold()] = AppOpsInfo(
|
||||
app_id=app_id,
|
||||
category=str(app.get("category")) if app.get("category") else None,
|
||||
criticality=str(app.get("criticality")) if app.get("criticality") else None,
|
||||
health=str(app.get("health_status")) if app.get("health_status") else None,
|
||||
display_name=str(app.get("display_name")) if app.get("display_name") else None,
|
||||
discovery_confidence=max(0.0, min(confidence, 1.0)) if confidence is not None else None,
|
||||
policy_profile=str(app.get("policy_profile")) if app.get("policy_profile") else None,
|
||||
app_type=str(app.get("app_type")) if app.get("app_type") else None,
|
||||
reasons=tuple(str(item) for item in (app.get("discovery_reasons") or [])[:3]),
|
||||
)
|
||||
return AppOpsInventory(result, True, f"{len(result)} container mappings loaded")
|
||||
|
||||
|
||||
def appops_inventory(appops_url: str | None) -> AppOpsInventory:
|
||||
"""Read AppOps health inventory without triggering Docker discovery."""
|
||||
|
||||
return _appops_inventory(appops_url)
|
||||
|
||||
|
||||
def observe_container_health(docker_host: str) -> list[DockerHealthObservation]:
|
||||
"""Read bounded Docker runtime/health facts without discovery side effects.
|
||||
|
||||
This deliberately avoids endpoint preflight, relationship inference,
|
||||
suggestion persistence, reconciliation and snapshot capture. It is safe
|
||||
to run on the observation cadence because it cannot rewrite topology or
|
||||
operator-owned configuration.
|
||||
"""
|
||||
|
||||
parsed = urlparse(docker_host)
|
||||
if parsed.scheme not in ALLOWED_SCHEMES:
|
||||
raise ValueError("docker_host must use unix://, tcp://, http:// or https://")
|
||||
client = None
|
||||
try:
|
||||
client = (
|
||||
docker.DockerClient(base_url=docker_host, timeout=5)
|
||||
if docker_provider_mode() == "socket"
|
||||
else docker_client_for_inventory(docker_host)
|
||||
)
|
||||
client.ping()
|
||||
observations: list[DockerHealthObservation] = []
|
||||
for container in client.containers.list(all=True):
|
||||
attrs = container.attrs or {}
|
||||
state = attrs.get("State", {}) or {}
|
||||
config = attrs.get("Config", {}) or {}
|
||||
raw_labels = {str(key): str(value) for key, value in (config.get("Labels", {}) or {}).items()}
|
||||
name = str(getattr(container, "name", "unknown")).lstrip("/")
|
||||
observations.append(
|
||||
DockerHealthObservation(
|
||||
container_id=container.id[:12],
|
||||
name=name,
|
||||
stable_source_ref=stable_source_ref(name, raw_labels),
|
||||
state=str(state.get("Status", getattr(container, "status", "unknown"))),
|
||||
health=(
|
||||
str(state.get("Health", {}).get("Status"))
|
||||
if isinstance(state.get("Health"), dict) and state.get("Health", {}).get("Status") is not None
|
||||
else None
|
||||
),
|
||||
healthcheck_present=bool(config.get("Healthcheck")),
|
||||
)
|
||||
)
|
||||
return observations
|
||||
except (DockerException, DockerProviderError) as exc:
|
||||
raise RuntimeError(f"Docker observation unavailable: {type(exc).__name__}") from exc
|
||||
finally:
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
|
||||
|
||||
def _appops_proxy_routes(appops_url: str | None) -> list[ProxyRoute]:
|
||||
if not appops_url:
|
||||
return []
|
||||
try:
|
||||
payload = httpx.get(f"{appops_url.rstrip('/')}/api/apps/npm/proxy-hosts", timeout=5).json()
|
||||
except (httpx.HTTPError, ValueError):
|
||||
return []
|
||||
items = payload.get("items", []) if isinstance(payload, dict) else payload
|
||||
routes: list[ProxyRoute] = []
|
||||
for item in items if isinstance(items, list) else []:
|
||||
if not isinstance(item, dict) or item.get("enabled") is False:
|
||||
continue
|
||||
domains = tuple(sorted({str(domain).strip() for domain in item.get("domainNames", []) if str(domain).strip()}))
|
||||
host = str(item.get("forwardHost") or "").strip()
|
||||
try:
|
||||
port = int(item.get("forwardPort"))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
scheme = str(item.get("forwardScheme") or "http").casefold()
|
||||
if not domains or not host or not 1 <= port <= 65535 or scheme not in {"http", "https"}:
|
||||
continue
|
||||
routes.append(ProxyRoute(domains=domains, forward_host=host, forward_port=port, forward_scheme=scheme))
|
||||
return routes
|
||||
|
||||
|
||||
def _suggestions(
|
||||
name: str,
|
||||
networks: list[str],
|
||||
ports: list[DockerPort],
|
||||
labels: dict[str, str],
|
||||
*,
|
||||
image: str,
|
||||
state: str,
|
||||
appops: AppOpsInfo | None = None,
|
||||
stable_ref: str,
|
||||
probe_host: str | None = None,
|
||||
) -> list[DockerSuggestion]:
|
||||
if labels.get("servicemap.enabled", "true").casefold() == "false":
|
||||
return []
|
||||
kind, url, host, port, evidence = _kind_and_endpoint(name, ports, labels, probe_host)
|
||||
depends = [item.strip() for item in labels.get("servicemap.depends_on", "").split(",") if item.strip()]
|
||||
compose_depends = labels.get("com.docker.compose.depends_on", "")
|
||||
if compose_depends:
|
||||
depends.extend(item.split(":", 1)[0].strip() for item in compose_depends.split(",") if item.strip())
|
||||
evidence.append("Compose depends_on metadata")
|
||||
if depends:
|
||||
evidence.append("explicit dependency metadata")
|
||||
if appops:
|
||||
confidence = f" ({appops.discovery_confidence:.2f})" if appops.discovery_confidence is not None else ""
|
||||
evidence.append(f"AppOps application mapping{confidence}; application scope only")
|
||||
evidence.extend(appops.reasons)
|
||||
category = _classify(name, image, labels, appops)
|
||||
return [
|
||||
DockerSuggestion(
|
||||
name=labels.get("servicemap.name", appops.display_name if appops and appops.display_name else name),
|
||||
kind=kind,
|
||||
host=host,
|
||||
port=port,
|
||||
url=url,
|
||||
group=labels.get("servicemap.group") or (f"Compose · {labels.get('com.docker.compose.project')}" if labels.get("com.docker.compose.project") else category.value.replace("_", " ").title()),
|
||||
evidence=sorted(set(evidence)),
|
||||
confidence="high" if labels.get("servicemap.probe") or labels.get("servicemap.name") or compose_depends or (appops and (appops.discovery_confidence or 0) >= 0.75) else "medium",
|
||||
depends_on_names=sorted(set(depends)),
|
||||
stable_source_ref=stable_ref,
|
||||
category=category,
|
||||
visibility=_visibility(name, state, category, labels),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def discover_containers(
|
||||
docker_host: str,
|
||||
existing_services: list[Service] | None = None,
|
||||
*,
|
||||
include_stopped: bool = True,
|
||||
appops_url: str | None = None,
|
||||
probe_host: str | None = None,
|
||||
) -> DockerDiscoveryResult:
|
||||
parsed = urlparse(docker_host)
|
||||
if parsed.scheme not in ALLOWED_SCHEMES:
|
||||
raise ValueError("docker_host must use unix://, tcp://, http:// or https://")
|
||||
|
||||
# Keep the direct client import at this boundary so existing deployments
|
||||
# and deterministic SDK-contract tests retain the socket behaviour. Broker
|
||||
# mode is explicit and never falls back here.
|
||||
client = docker.DockerClient(base_url=docker_host, timeout=5) if docker_provider_mode() == "socket" else docker_client_for_inventory(docker_host)
|
||||
try:
|
||||
client.ping()
|
||||
containers = client.containers.list(all=include_stopped)
|
||||
existing_services = existing_services or []
|
||||
appops_inventory = _appops_inventory(appops_url)
|
||||
appops = appops_inventory.by_container
|
||||
configured_probe_host = (probe_host or os.getenv("SERVICEMAP_PROBE_HOST") or "").strip() or None
|
||||
proxy_routes = _appops_proxy_routes(appops_url)
|
||||
candidates: list[DockerCandidate] = []
|
||||
discovered_refs: set[str] = set()
|
||||
for container in containers:
|
||||
attrs = container.attrs or {}
|
||||
state = attrs.get("State", {}) or {}
|
||||
config = attrs.get("Config", {}) or {}
|
||||
network_settings = attrs.get("NetworkSettings", {}) or {}
|
||||
raw_ports = network_settings.get("Ports", {}) or {}
|
||||
ports: list[DockerPort] = []
|
||||
for container_port, bindings in raw_ports.items():
|
||||
if not bindings:
|
||||
ports.append(DockerPort(container_port=container_port))
|
||||
continue
|
||||
for binding in bindings:
|
||||
raw_host_port = binding.get("HostPort")
|
||||
host_port = str(raw_host_port) if raw_host_port is not None else ""
|
||||
ports.append(
|
||||
DockerPort(
|
||||
container_port=container_port,
|
||||
host_ip=binding.get("HostIp") or None,
|
||||
host_port=int(host_port) if host_port.isdigit() else None,
|
||||
)
|
||||
)
|
||||
|
||||
image_tags = getattr(container.image, "tags", []) or []
|
||||
image_name = image_tags[0] if image_tags else str(config.get("Image", "unknown"))
|
||||
name = str(getattr(container, "name", "unknown")).lstrip("/")
|
||||
health = state.get("Health", {}).get("Status") if isinstance(state.get("Health"), dict) else None
|
||||
state_name = str(state.get("Status", getattr(container, "status", "unknown")))
|
||||
networks = sorted((network_settings.get("Networks", {}) or {}).keys())
|
||||
raw_labels = {str(key): str(value) for key, value in (config.get("Labels", {}) or {}).items()}
|
||||
labels = _safe_labels(raw_labels)
|
||||
stable_ref = stable_source_ref(name, raw_labels)
|
||||
legacy_ref = _legacy_source_ref(container.id, image_name)
|
||||
discovered_refs.add(stable_ref)
|
||||
matching_source = next(
|
||||
(
|
||||
service
|
||||
for service in existing_services
|
||||
if service.source_ref in {stable_ref, legacy_ref}
|
||||
or (service.source_ref and service.source_ref.removeprefix("docker:").startswith(container.id[:12]))
|
||||
),
|
||||
None,
|
||||
)
|
||||
matching_name = next((service for service in existing_services if service.name.casefold() == name.casefold()), None)
|
||||
appops_info = appops.get(name.casefold())
|
||||
category = _classify(name, image_name, raw_labels, appops_info)
|
||||
visibility = _visibility(name, state_name, category, raw_labels)
|
||||
if matching_source is not None:
|
||||
import_state = "already imported" if matching_source.source_ref in {stable_ref, legacy_ref} else "changed"
|
||||
elif matching_name is not None:
|
||||
import_state = "already imported"
|
||||
else:
|
||||
import_state = "new"
|
||||
suggestions = _suggestions(
|
||||
name,
|
||||
networks,
|
||||
ports,
|
||||
raw_labels,
|
||||
image=image_name,
|
||||
state=state_name,
|
||||
appops=appops_info,
|
||||
stable_ref=stable_ref,
|
||||
probe_host=configured_probe_host,
|
||||
)
|
||||
for suggestion in suggestions:
|
||||
suggestion.source_type = "docker"
|
||||
suggestion.source_ref = legacy_ref
|
||||
env_evidence = safe_environment_evidence([str(item) for item in (config.get("Env") or [])])
|
||||
probe_candidates = _probe_candidates(
|
||||
suggestions[0] if suggestions else None,
|
||||
healthcheck_present=bool(config.get("Healthcheck")),
|
||||
)
|
||||
if state_name == "running":
|
||||
probe_candidates = [_preflight_candidate(item) for item in probe_candidates]
|
||||
candidates.append(
|
||||
DockerCandidate(
|
||||
container_id=container.id[:12],
|
||||
name=name,
|
||||
image=image_name,
|
||||
state=state_name,
|
||||
health=health,
|
||||
networks=networks,
|
||||
ports=ports,
|
||||
labels=labels,
|
||||
import_state=import_state,
|
||||
suggestions=suggestions,
|
||||
stable_source_ref=stable_ref,
|
||||
compose_project=raw_labels.get("com.docker.compose.project"),
|
||||
compose_service=raw_labels.get("com.docker.compose.service"),
|
||||
category=category,
|
||||
visibility=visibility,
|
||||
lifecycle=DiscoveryLifecycle.ACTIVE if state_name == "running" else DiscoveryLifecycle.STALE,
|
||||
healthcheck_present=bool(config.get("Healthcheck")),
|
||||
environment_evidence=env_evidence,
|
||||
proxy_route_evidence=[],
|
||||
appops_app_id=appops_info.app_id if appops_info else None,
|
||||
appops_category=appops_info.category if appops_info else None,
|
||||
appops_criticality=appops_info.criticality if appops_info else None,
|
||||
appops_health=appops_info.health if appops_info else None,
|
||||
appops_scope=EvidenceScope.APPLICATION if appops_info else None,
|
||||
appops_discovery_confidence=appops_info.discovery_confidence if appops_info else None,
|
||||
appops_policy_profile=appops_info.policy_profile if appops_info else None,
|
||||
appops_app_type=appops_info.app_type if appops_info else None,
|
||||
probe_candidates=probe_candidates,
|
||||
)
|
||||
)
|
||||
|
||||
for candidate in candidates:
|
||||
candidate.dependency_suggestions = sorted(
|
||||
other.name
|
||||
for other in candidates
|
||||
if other.name != candidate.name and set(candidate.networks).intersection(other.networks)
|
||||
)
|
||||
|
||||
# Reverse-proxy metadata is useful evidence, but it is never turned
|
||||
# into a health dependency automatically. Persist only domains and the
|
||||
# sanitized upstream tuple; AppOps metadata such as certificate or
|
||||
# Let's Encrypt fields is intentionally discarded.
|
||||
by_name = {candidate.name.casefold(): candidate for candidate in candidates}
|
||||
by_published_port = {
|
||||
port.host_port: candidate
|
||||
for candidate in candidates
|
||||
for port in candidate.ports
|
||||
if port.host_port is not None
|
||||
}
|
||||
for route in proxy_routes:
|
||||
target = by_name.get(route.forward_host.casefold()) or by_published_port.get(route.forward_port)
|
||||
if target:
|
||||
target.proxy_route_evidence.append(
|
||||
f"Reverse proxy route: {', '.join(route.domains)} -> {route.forward_scheme}://{route.forward_host}:{route.forward_port}"
|
||||
)
|
||||
|
||||
for service in existing_services:
|
||||
if service.source_type != "docker" or not service.source_ref:
|
||||
continue
|
||||
if service.source_ref not in discovered_refs and service.lifecycle not in {DiscoveryLifecycle.RETIRED, DiscoveryLifecycle.IGNORED}:
|
||||
candidates.append(
|
||||
DockerCandidate(
|
||||
container_id=service.source_ref.removeprefix("docker:")[:12],
|
||||
name=service.name,
|
||||
image=service.container_image or "not found",
|
||||
state="missing",
|
||||
health=None,
|
||||
networks=[],
|
||||
ports=[],
|
||||
import_state="missing",
|
||||
stable_source_ref=service.source_ref,
|
||||
category=service.category,
|
||||
visibility=service.visibility,
|
||||
lifecycle=DiscoveryLifecycle.MISSING,
|
||||
)
|
||||
)
|
||||
candidates.sort(key=lambda item: item.name.casefold())
|
||||
return DockerDiscoveryResult(
|
||||
docker_host=docker_host,
|
||||
candidates=candidates,
|
||||
proxy_routes=len(proxy_routes),
|
||||
appops_available=appops_inventory.available,
|
||||
appops_detail=appops_inventory.detail,
|
||||
)
|
||||
except (DockerException, DockerProviderError) as exc:
|
||||
raise ValueError(f"Docker discovery failed: {exc}") from exc
|
||||
finally:
|
||||
client.close()
|
||||
Reference in new issue
Block a user