Initial curated ServiceMap source from 7a0a61232d05ba9097432abef6f7f4102a64d0b8
This commit is contained in:
Public Source Publisher committed
commit
105251b9c8
96 files changed
+23587
No files matched your search
@@ -0,0 +1,345 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
import pytest
|
||||
|
||||
from app import docker_discovery
|
||||
from app.demo import build_demo_records
|
||||
from app.discovery import DiscoveryManager
|
||||
from app.discovery_store import DiscoveryStore
|
||||
from app.models import (
|
||||
Criticality,
|
||||
DiscoveryConfidence,
|
||||
DiscoveryLifecycle,
|
||||
DiscoveryReconcileRequest,
|
||||
DiscoveryRunRequest,
|
||||
DiscoveryRunStatus,
|
||||
DockerCandidate,
|
||||
DockerPort,
|
||||
DockerSuggestion,
|
||||
ProbeEndpointCandidate,
|
||||
ProbeEndpointSource,
|
||||
ProbeReachability,
|
||||
RelationEvidence,
|
||||
RelationType,
|
||||
ServiceCreate,
|
||||
ServiceKind,
|
||||
ServiceUpdate,
|
||||
SuggestionStatus,
|
||||
TopologySuggestion,
|
||||
)
|
||||
from app.store import Store
|
||||
|
||||
|
||||
class _Image:
|
||||
def __init__(self, tag: str) -> None:
|
||||
self.tags = [tag]
|
||||
|
||||
|
||||
class _Container:
|
||||
def __init__(self, name: str, container_id: str, image: str, service: str, *, env: list[str] | None = None) -> None:
|
||||
self.name = name
|
||||
self.id = container_id
|
||||
self.status = "running"
|
||||
self.image = _Image(image)
|
||||
self.attrs = {
|
||||
"State": {"Status": "running", "Health": {"Status": "healthy"}},
|
||||
"Config": {
|
||||
"Image": image,
|
||||
"Env": env or [],
|
||||
"Healthcheck": {"Test": ["CMD", "true"]},
|
||||
"Labels": {
|
||||
"com.docker.compose.project": "northstar",
|
||||
"com.docker.compose.service": service,
|
||||
"com.docker.compose.depends_on": "postgres:service_healthy:false" if service == "api" else "",
|
||||
},
|
||||
},
|
||||
"NetworkSettings": {
|
||||
"Networks": {"northstar_default": {}},
|
||||
"Ports": {"8000/tcp": [{"HostIp": "127.0.0.1", "HostPort": "18080"}]} if service == "api" else {"5432/tcp": [{"HostIp": "127.0.0.1", "HostPort": "15432"}]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class _DockerClient:
|
||||
containers: ClassVar[_DockerClient]
|
||||
current: ClassVar[list[_Container]] = []
|
||||
|
||||
def __init__(self, **_: object) -> None:
|
||||
self.containers = self
|
||||
|
||||
def ping(self) -> bool:
|
||||
return True
|
||||
|
||||
def list(self, all: bool) -> list[_Container]:
|
||||
assert all is False
|
||||
return self.current
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _manager(tmp_path: Path) -> DiscoveryManager:
|
||||
store = Store(str(tmp_path / "servicemap.db"))
|
||||
return DiscoveryManager(store, DiscoveryStore(store))
|
||||
|
||||
|
||||
def test_malformed_multi_host_environment_does_not_abort_discovery() -> None:
|
||||
evidence = docker_discovery.safe_environment_evidence(
|
||||
["UPSTREAM_HOSTS=chimera:6967,localhost:1221,127.0.0.1:1221"]
|
||||
)
|
||||
|
||||
assert evidence == ["UPSTREAM_HOSTS hostname=chimera"]
|
||||
assert docker_discovery.safe_environment_evidence(["BIND_ADDRESS=http://[invalid:8000"]) == []
|
||||
|
||||
|
||||
def test_internal_http_candidate_is_imported_without_an_unsafe_probe_target(tmp_path: Path) -> None:
|
||||
manager = _manager(tmp_path)
|
||||
candidate = DockerCandidate(
|
||||
container_id="a" * 64,
|
||||
name="internal-web",
|
||||
image="example/web:1",
|
||||
state="running",
|
||||
networks=[],
|
||||
ports=[DockerPort(container_port="80/tcp")],
|
||||
stable_source_ref="docker:name:internal-web",
|
||||
suggestions=[
|
||||
DockerSuggestion(
|
||||
name="internal-web",
|
||||
kind=ServiceKind.HTTP,
|
||||
host="internal-web",
|
||||
port=80,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
payload = manager._payload_for_candidate(candidate)
|
||||
|
||||
assert payload.kind == ServiceKind.HTTP
|
||||
assert payload.monitoring_enabled is False
|
||||
assert payload.url is None
|
||||
assert payload.host is None
|
||||
assert payload.port is None
|
||||
|
||||
published = candidate.model_copy(
|
||||
update={
|
||||
"name": "published-web",
|
||||
"suggestions": [
|
||||
DockerSuggestion(
|
||||
name="published-web",
|
||||
kind=ServiceKind.HTTP,
|
||||
url="http://127.0.0.1:9494",
|
||||
port=9494,
|
||||
)
|
||||
],
|
||||
"probe_candidates": [
|
||||
ProbeEndpointCandidate(
|
||||
source=ProbeEndpointSource.PUBLISHED_PORT,
|
||||
kind=ServiceKind.HTTP,
|
||||
url="http://127.0.0.1:9494",
|
||||
reachability=ProbeReachability.VERIFIED,
|
||||
primary=True,
|
||||
)
|
||||
],
|
||||
}
|
||||
)
|
||||
published_payload = manager._payload_for_candidate(published)
|
||||
|
||||
assert published_payload.url == "http://127.0.0.1:9494"
|
||||
assert published_payload.host is None
|
||||
assert published_payload.port is None
|
||||
|
||||
|
||||
def test_new_discovery_run_closes_stale_active_run(tmp_path: Path) -> None:
|
||||
manager = _manager(tmp_path)
|
||||
first = manager.discovery_store.start_run()
|
||||
second = manager.discovery_store.start_run()
|
||||
|
||||
assert manager.discovery_store.get_run(first.id).status == DiscoveryRunStatus.FAILED
|
||||
assert manager.discovery_store.get_run(second.id).status == DiscoveryRunStatus.DISCOVERING
|
||||
|
||||
|
||||
def test_repeated_suggestion_moves_to_latest_discovery_run(tmp_path: Path) -> None:
|
||||
manager = _manager(tmp_path)
|
||||
first = manager.discovery_store.start_run()
|
||||
now = datetime.now(UTC)
|
||||
suggestion = TopologySuggestion(
|
||||
id="suggestion-first",
|
||||
run_id=first.id,
|
||||
suggestion_type="service",
|
||||
proposed_name="example",
|
||||
confidence=DiscoveryConfidence.HIGH,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
saved_first = manager.discovery_store.save_suggestion(suggestion, "same-fingerprint")
|
||||
second = manager.discovery_store.start_run()
|
||||
saved_second = manager.discovery_store.save_suggestion(
|
||||
suggestion.model_copy(
|
||||
update={
|
||||
"id": "suggestion-second",
|
||||
"run_id": second.id,
|
||||
"status": SuggestionStatus.OPEN,
|
||||
"updated_at": datetime.now(UTC),
|
||||
}
|
||||
),
|
||||
"same-fingerprint",
|
||||
)
|
||||
|
||||
assert saved_first.id == saved_second.id == "suggestion-first"
|
||||
assert saved_second.run_id == second.id
|
||||
|
||||
|
||||
def test_discovery_sanitizes_environment_and_reconciles_high_confidence_edges(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_DockerClient.current = [
|
||||
_Container(
|
||||
"api",
|
||||
"a" * 64,
|
||||
"northstar/api:1",
|
||||
"api",
|
||||
env=["DATABASE_URL=postgres://user:supersecret@postgres:5432/app", "API_TOKEN=never-store"],
|
||||
),
|
||||
_Container("postgres", "b" * 64, "postgres:17", "postgres"),
|
||||
]
|
||||
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
||||
manager = _manager(tmp_path)
|
||||
|
||||
result = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
api = next(item for item in result.candidates if item.name == "api")
|
||||
assert api.stable_source_ref == "docker:compose:northstar:api"
|
||||
assert any("hostname=postgres:5432" in item for item in api.environment_evidence)
|
||||
assert all("supersecret" not in item and "user" not in item for item in api.environment_evidence)
|
||||
assert all("API_TOKEN" not in item for item in api.environment_evidence)
|
||||
assert any(item.confidence == DiscoveryConfidence.HIGH for item in result.suggestions if item.suggestion_type == "relation")
|
||||
|
||||
reconciled = manager.reconcile(DiscoveryReconcileRequest(run_id=result.run.id, apply_high_confidence=True))
|
||||
assert reconciled.created == 2
|
||||
assert reconciled.accepted_relations >= 1
|
||||
services = manager.store.list_services()
|
||||
api_service = next(item for item in services if item.source_ref == "docker:compose:northstar:api")
|
||||
postgres_service = next(item for item in services if item.source_ref == "docker:compose:northstar:postgres")
|
||||
assert postgres_service.id in api_service.depends_on
|
||||
|
||||
edge = next(item for item in manager.discovery_store.list_evidence() if item.source_service_id == api_service.id and item.target_service_id == postgres_service.id)
|
||||
manager.edge_action(edge.id, "reject")
|
||||
repeated = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
manager.reconcile(DiscoveryReconcileRequest(run_id=repeated.run.id, apply_high_confidence=True))
|
||||
api_after_rejection = next(item for item in manager.store.list_services() if item.id == api_service.id)
|
||||
assert postgres_service.id not in api_after_rejection.depends_on
|
||||
assert manager.discovery_store.get_evidence(edge.id) is not None
|
||||
assert not manager.discovery_store.get_evidence(edge.id).active
|
||||
|
||||
serialized = (tmp_path / "servicemap.db").read_bytes()
|
||||
assert b"supersecret" not in serialized
|
||||
assert b"API_TOKEN" not in serialized
|
||||
|
||||
|
||||
def test_stable_identity_survives_container_recreation_without_duplicate_service(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_DockerClient.current = [_Container("api", "a" * 64, "northstar/api:1", "api")]
|
||||
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
||||
manager = _manager(tmp_path)
|
||||
first = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
manager.reconcile(DiscoveryReconcileRequest(run_id=first.run.id, apply_high_confidence=True))
|
||||
|
||||
_DockerClient.current = [_Container("api", "c" * 64, "northstar/api:2", "api")]
|
||||
second = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
assert second.run.diff.changed == 1
|
||||
manager.reconcile(DiscoveryReconcileRequest(run_id=second.run.id, apply_high_confidence=True))
|
||||
services = [item for item in manager.store.list_services() if item.source_ref == "docker:compose:northstar:api"]
|
||||
assert len(services) == 1
|
||||
assert services[0].container_image == "northstar/api:2"
|
||||
|
||||
|
||||
def test_missing_container_is_retained_and_becomes_stale(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_DockerClient.current = [_Container("api", "a" * 64, "northstar/api:1", "api")]
|
||||
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
||||
manager = _manager(tmp_path)
|
||||
first = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
manager.reconcile(DiscoveryReconcileRequest(run_id=first.run.id, apply_high_confidence=True))
|
||||
|
||||
_DockerClient.current = []
|
||||
missing = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
assert missing.run.diff.missing == 1
|
||||
reconciled = manager.reconcile(DiscoveryReconcileRequest(run_id=missing.run.id))
|
||||
assert reconciled.missing == 1
|
||||
service = next(item for item in manager.store.list_services() if item.source_ref == "docker:compose:northstar:api")
|
||||
assert service.lifecycle == DiscoveryLifecycle.MISSING
|
||||
assert service.id in {item.id for item in manager.store.list_services()}
|
||||
|
||||
|
||||
def test_missing_identity_progresses_to_stale_then_retired_without_delete(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_DockerClient.current = [_Container("api", "a" * 64, "northstar/api:1", "api")]
|
||||
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
||||
manager = _manager(tmp_path)
|
||||
first = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
manager.reconcile(DiscoveryReconcileRequest(run_id=first.run.id, apply_high_confidence=True))
|
||||
_DockerClient.current = []
|
||||
lifecycle: list[DiscoveryLifecycle] = []
|
||||
for _ in range(3):
|
||||
run = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
manager.reconcile(DiscoveryReconcileRequest(run_id=run.run.id))
|
||||
lifecycle.append(next(item for item in manager.store.list_services() if item.source_ref == "docker:compose:northstar:api").lifecycle)
|
||||
assert lifecycle == [DiscoveryLifecycle.MISSING, DiscoveryLifecycle.STALE, DiscoveryLifecycle.RETIRED]
|
||||
assert len(manager.store.list_services()) == 1
|
||||
|
||||
|
||||
def test_low_confidence_network_hint_is_evidence_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_DockerClient.current = [
|
||||
_Container("frontend", "a" * 64, "northstar/frontend:1", "frontend"),
|
||||
_Container("backend", "b" * 64, "northstar/backend:1", "backend"),
|
||||
]
|
||||
for item in _DockerClient.current:
|
||||
item.attrs["Config"]["Labels"]["com.docker.compose.depends_on"] = ""
|
||||
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
||||
manager = _manager(tmp_path)
|
||||
run = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
manager.reconcile(DiscoveryReconcileRequest(run_id=run.run.id, apply_high_confidence=True))
|
||||
services = manager.store.list_services()
|
||||
assert all(not service.depends_on for service in services)
|
||||
assert any(item.confidence == DiscoveryConfidence.LOW for item in run.suggestions if item.suggestion_type == "relation")
|
||||
|
||||
|
||||
def test_manual_group_override_survives_rediscovery(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_DockerClient.current = [_Container("api", "a" * 64, "northstar/api:1", "api")]
|
||||
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
||||
manager = _manager(tmp_path)
|
||||
first = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
manager.reconcile(DiscoveryReconcileRequest(run_id=first.run.id, apply_high_confidence=True))
|
||||
before = next(item for item in manager.store.list_services() if item.source_ref == "docker:compose:northstar:api")
|
||||
payload = ServiceUpdate.model_validate(before.model_dump())
|
||||
payload.group = "Manual ownership"
|
||||
after = manager.store.update_service(before.id, payload)
|
||||
assert after is not None
|
||||
manager.record_service_override(before, after)
|
||||
_DockerClient.current = [_Container("api", "c" * 64, "northstar/api:2", "api")]
|
||||
second = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
||||
manager.reconcile(DiscoveryReconcileRequest(run_id=second.run.id, apply_high_confidence=True))
|
||||
updated = next(item for item in manager.store.list_services() if item.source_ref == "docker:compose:northstar:api")
|
||||
assert updated.group == "Manual ownership"
|
||||
assert updated.container_image == "northstar/api:2"
|
||||
|
||||
|
||||
def test_demo_exit_restores_discovery_evidence(tmp_path: Path) -> None:
|
||||
store = Store(str(tmp_path / "servicemap.db"))
|
||||
discovery = DiscoveryStore(store)
|
||||
database = store.create_service(ServiceCreate(name="Database", kind=ServiceKind.TCP, host="db", port=5432, category="database", criticality=Criticality.HIGH))
|
||||
application = store.create_service(ServiceCreate(name="Application", kind=ServiceKind.HTTP, url="https://app.local", depends_on=[database.id]))
|
||||
source = discovery.save_source(source_type="docker", source_id="docker:compose:stack:app", source_name="app", lifecycle=DiscoveryLifecycle.ACTIVE, metadata={"image": "app:1"})
|
||||
discovery.attach_source(application.id, source.id)
|
||||
now = datetime.now(UTC)
|
||||
discovery.save_evidence(RelationEvidence(id="evidence-1", source_service_id=application.id, target_service_id=database.id, relation_type=RelationType.DEPENDS_ON, evidence_type="manual", confidence=DiscoveryConfidence.CONFIRMED, explanation="test evidence", first_seen_at=now, last_seen_at=now))
|
||||
demo_services, demo_probes = build_demo_records()
|
||||
store.activate_demo(demo_services, demo_probes)
|
||||
assert store.demo_active()
|
||||
assert {item.name for item in store.list_services()} == {item.name for item in demo_services}
|
||||
assert store.exit_demo()
|
||||
restored = store.list_services()
|
||||
assert {item.name for item in restored} == {"Database", "Application"}
|
||||
assert discovery.get_evidence("evidence-1") is not None
|
||||
Reference in new issue
Block a user