346 lines
15 KiB
Python
346 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import ClassVar
|
|
|
|
import pytest
|
|
|
|
from app import docker_discovery
|
|
from app.demo import build_demo_records
|
|
from app.discovery import DiscoveryManager
|
|
from app.discovery_store import DiscoveryStore
|
|
from app.models import (
|
|
Criticality,
|
|
DiscoveryConfidence,
|
|
DiscoveryLifecycle,
|
|
DiscoveryReconcileRequest,
|
|
DiscoveryRunRequest,
|
|
DiscoveryRunStatus,
|
|
DockerCandidate,
|
|
DockerPort,
|
|
DockerSuggestion,
|
|
ProbeEndpointCandidate,
|
|
ProbeEndpointSource,
|
|
ProbeReachability,
|
|
RelationEvidence,
|
|
RelationType,
|
|
ServiceCreate,
|
|
ServiceKind,
|
|
ServiceUpdate,
|
|
SuggestionStatus,
|
|
TopologySuggestion,
|
|
)
|
|
from app.store import Store
|
|
|
|
|
|
class _Image:
|
|
def __init__(self, tag: str) -> None:
|
|
self.tags = [tag]
|
|
|
|
|
|
class _Container:
|
|
def __init__(self, name: str, container_id: str, image: str, service: str, *, env: list[str] | None = None) -> None:
|
|
self.name = name
|
|
self.id = container_id
|
|
self.status = "running"
|
|
self.image = _Image(image)
|
|
self.attrs = {
|
|
"State": {"Status": "running", "Health": {"Status": "healthy"}},
|
|
"Config": {
|
|
"Image": image,
|
|
"Env": env or [],
|
|
"Healthcheck": {"Test": ["CMD", "true"]},
|
|
"Labels": {
|
|
"com.docker.compose.project": "northstar",
|
|
"com.docker.compose.service": service,
|
|
"com.docker.compose.depends_on": "postgres:service_healthy:false" if service == "api" else "",
|
|
},
|
|
},
|
|
"NetworkSettings": {
|
|
"Networks": {"northstar_default": {}},
|
|
"Ports": {"8000/tcp": [{"HostIp": "127.0.0.1", "HostPort": "18080"}]} if service == "api" else {"5432/tcp": [{"HostIp": "127.0.0.1", "HostPort": "15432"}]},
|
|
},
|
|
}
|
|
|
|
|
|
class _DockerClient:
|
|
containers: ClassVar[_DockerClient]
|
|
current: ClassVar[list[_Container]] = []
|
|
|
|
def __init__(self, **_: object) -> None:
|
|
self.containers = self
|
|
|
|
def ping(self) -> bool:
|
|
return True
|
|
|
|
def list(self, all: bool) -> list[_Container]:
|
|
assert all is False
|
|
return self.current
|
|
|
|
def close(self) -> None:
|
|
return None
|
|
|
|
|
|
def _manager(tmp_path: Path) -> DiscoveryManager:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
return DiscoveryManager(store, DiscoveryStore(store))
|
|
|
|
|
|
def test_malformed_multi_host_environment_does_not_abort_discovery() -> None:
|
|
evidence = docker_discovery.safe_environment_evidence(
|
|
["UPSTREAM_HOSTS=chimera:6967,localhost:1221,127.0.0.1:1221"]
|
|
)
|
|
|
|
assert evidence == ["UPSTREAM_HOSTS hostname=chimera"]
|
|
assert docker_discovery.safe_environment_evidence(["BIND_ADDRESS=http://[invalid:8000"]) == []
|
|
|
|
|
|
def test_internal_http_candidate_is_imported_without_an_unsafe_probe_target(tmp_path: Path) -> None:
|
|
manager = _manager(tmp_path)
|
|
candidate = DockerCandidate(
|
|
container_id="a" * 64,
|
|
name="internal-web",
|
|
image="example/web:1",
|
|
state="running",
|
|
networks=[],
|
|
ports=[DockerPort(container_port="80/tcp")],
|
|
stable_source_ref="docker:name:internal-web",
|
|
suggestions=[
|
|
DockerSuggestion(
|
|
name="internal-web",
|
|
kind=ServiceKind.HTTP,
|
|
host="internal-web",
|
|
port=80,
|
|
)
|
|
],
|
|
)
|
|
|
|
payload = manager._payload_for_candidate(candidate)
|
|
|
|
assert payload.kind == ServiceKind.HTTP
|
|
assert payload.monitoring_enabled is False
|
|
assert payload.url is None
|
|
assert payload.host is None
|
|
assert payload.port is None
|
|
|
|
published = candidate.model_copy(
|
|
update={
|
|
"name": "published-web",
|
|
"suggestions": [
|
|
DockerSuggestion(
|
|
name="published-web",
|
|
kind=ServiceKind.HTTP,
|
|
url="http://127.0.0.1:9494",
|
|
port=9494,
|
|
)
|
|
],
|
|
"probe_candidates": [
|
|
ProbeEndpointCandidate(
|
|
source=ProbeEndpointSource.PUBLISHED_PORT,
|
|
kind=ServiceKind.HTTP,
|
|
url="http://127.0.0.1:9494",
|
|
reachability=ProbeReachability.VERIFIED,
|
|
primary=True,
|
|
)
|
|
],
|
|
}
|
|
)
|
|
published_payload = manager._payload_for_candidate(published)
|
|
|
|
assert published_payload.url == "http://127.0.0.1:9494"
|
|
assert published_payload.host is None
|
|
assert published_payload.port is None
|
|
|
|
|
|
def test_new_discovery_run_closes_stale_active_run(tmp_path: Path) -> None:
|
|
manager = _manager(tmp_path)
|
|
first = manager.discovery_store.start_run()
|
|
second = manager.discovery_store.start_run()
|
|
|
|
assert manager.discovery_store.get_run(first.id).status == DiscoveryRunStatus.FAILED
|
|
assert manager.discovery_store.get_run(second.id).status == DiscoveryRunStatus.DISCOVERING
|
|
|
|
|
|
def test_repeated_suggestion_moves_to_latest_discovery_run(tmp_path: Path) -> None:
|
|
manager = _manager(tmp_path)
|
|
first = manager.discovery_store.start_run()
|
|
now = datetime.now(UTC)
|
|
suggestion = TopologySuggestion(
|
|
id="suggestion-first",
|
|
run_id=first.id,
|
|
suggestion_type="service",
|
|
proposed_name="example",
|
|
confidence=DiscoveryConfidence.HIGH,
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
saved_first = manager.discovery_store.save_suggestion(suggestion, "same-fingerprint")
|
|
second = manager.discovery_store.start_run()
|
|
saved_second = manager.discovery_store.save_suggestion(
|
|
suggestion.model_copy(
|
|
update={
|
|
"id": "suggestion-second",
|
|
"run_id": second.id,
|
|
"status": SuggestionStatus.OPEN,
|
|
"updated_at": datetime.now(UTC),
|
|
}
|
|
),
|
|
"same-fingerprint",
|
|
)
|
|
|
|
assert saved_first.id == saved_second.id == "suggestion-first"
|
|
assert saved_second.run_id == second.id
|
|
|
|
|
|
def test_discovery_sanitizes_environment_and_reconciles_high_confidence_edges(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
_DockerClient.current = [
|
|
_Container(
|
|
"api",
|
|
"a" * 64,
|
|
"northstar/api:1",
|
|
"api",
|
|
env=["DATABASE_URL=postgres://user:supersecret@postgres:5432/app", "API_TOKEN=never-store"],
|
|
),
|
|
_Container("postgres", "b" * 64, "postgres:17", "postgres"),
|
|
]
|
|
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
|
manager = _manager(tmp_path)
|
|
|
|
result = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
api = next(item for item in result.candidates if item.name == "api")
|
|
assert api.stable_source_ref == "docker:compose:northstar:api"
|
|
assert any("hostname=postgres:5432" in item for item in api.environment_evidence)
|
|
assert all("supersecret" not in item and "user" not in item for item in api.environment_evidence)
|
|
assert all("API_TOKEN" not in item for item in api.environment_evidence)
|
|
assert any(item.confidence == DiscoveryConfidence.HIGH for item in result.suggestions if item.suggestion_type == "relation")
|
|
|
|
reconciled = manager.reconcile(DiscoveryReconcileRequest(run_id=result.run.id, apply_high_confidence=True))
|
|
assert reconciled.created == 2
|
|
assert reconciled.accepted_relations >= 1
|
|
services = manager.store.list_services()
|
|
api_service = next(item for item in services if item.source_ref == "docker:compose:northstar:api")
|
|
postgres_service = next(item for item in services if item.source_ref == "docker:compose:northstar:postgres")
|
|
assert postgres_service.id in api_service.depends_on
|
|
|
|
edge = next(item for item in manager.discovery_store.list_evidence() if item.source_service_id == api_service.id and item.target_service_id == postgres_service.id)
|
|
manager.edge_action(edge.id, "reject")
|
|
repeated = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
manager.reconcile(DiscoveryReconcileRequest(run_id=repeated.run.id, apply_high_confidence=True))
|
|
api_after_rejection = next(item for item in manager.store.list_services() if item.id == api_service.id)
|
|
assert postgres_service.id not in api_after_rejection.depends_on
|
|
assert manager.discovery_store.get_evidence(edge.id) is not None
|
|
assert not manager.discovery_store.get_evidence(edge.id).active
|
|
|
|
serialized = (tmp_path / "servicemap.db").read_bytes()
|
|
assert b"supersecret" not in serialized
|
|
assert b"API_TOKEN" not in serialized
|
|
|
|
|
|
def test_stable_identity_survives_container_recreation_without_duplicate_service(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
_DockerClient.current = [_Container("api", "a" * 64, "northstar/api:1", "api")]
|
|
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
|
manager = _manager(tmp_path)
|
|
first = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
manager.reconcile(DiscoveryReconcileRequest(run_id=first.run.id, apply_high_confidence=True))
|
|
|
|
_DockerClient.current = [_Container("api", "c" * 64, "northstar/api:2", "api")]
|
|
second = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
assert second.run.diff.changed == 1
|
|
manager.reconcile(DiscoveryReconcileRequest(run_id=second.run.id, apply_high_confidence=True))
|
|
services = [item for item in manager.store.list_services() if item.source_ref == "docker:compose:northstar:api"]
|
|
assert len(services) == 1
|
|
assert services[0].container_image == "northstar/api:2"
|
|
|
|
|
|
def test_missing_container_is_retained_and_becomes_stale(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_DockerClient.current = [_Container("api", "a" * 64, "northstar/api:1", "api")]
|
|
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
|
manager = _manager(tmp_path)
|
|
first = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
manager.reconcile(DiscoveryReconcileRequest(run_id=first.run.id, apply_high_confidence=True))
|
|
|
|
_DockerClient.current = []
|
|
missing = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
assert missing.run.diff.missing == 1
|
|
reconciled = manager.reconcile(DiscoveryReconcileRequest(run_id=missing.run.id))
|
|
assert reconciled.missing == 1
|
|
service = next(item for item in manager.store.list_services() if item.source_ref == "docker:compose:northstar:api")
|
|
assert service.lifecycle == DiscoveryLifecycle.MISSING
|
|
assert service.id in {item.id for item in manager.store.list_services()}
|
|
|
|
|
|
def test_missing_identity_progresses_to_stale_then_retired_without_delete(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_DockerClient.current = [_Container("api", "a" * 64, "northstar/api:1", "api")]
|
|
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
|
manager = _manager(tmp_path)
|
|
first = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
manager.reconcile(DiscoveryReconcileRequest(run_id=first.run.id, apply_high_confidence=True))
|
|
_DockerClient.current = []
|
|
lifecycle: list[DiscoveryLifecycle] = []
|
|
for _ in range(3):
|
|
run = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
manager.reconcile(DiscoveryReconcileRequest(run_id=run.run.id))
|
|
lifecycle.append(next(item for item in manager.store.list_services() if item.source_ref == "docker:compose:northstar:api").lifecycle)
|
|
assert lifecycle == [DiscoveryLifecycle.MISSING, DiscoveryLifecycle.STALE, DiscoveryLifecycle.RETIRED]
|
|
assert len(manager.store.list_services()) == 1
|
|
|
|
|
|
def test_low_confidence_network_hint_is_evidence_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_DockerClient.current = [
|
|
_Container("frontend", "a" * 64, "northstar/frontend:1", "frontend"),
|
|
_Container("backend", "b" * 64, "northstar/backend:1", "backend"),
|
|
]
|
|
for item in _DockerClient.current:
|
|
item.attrs["Config"]["Labels"]["com.docker.compose.depends_on"] = ""
|
|
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
|
manager = _manager(tmp_path)
|
|
run = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
manager.reconcile(DiscoveryReconcileRequest(run_id=run.run.id, apply_high_confidence=True))
|
|
services = manager.store.list_services()
|
|
assert all(not service.depends_on for service in services)
|
|
assert any(item.confidence == DiscoveryConfidence.LOW for item in run.suggestions if item.suggestion_type == "relation")
|
|
|
|
|
|
def test_manual_group_override_survives_rediscovery(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_DockerClient.current = [_Container("api", "a" * 64, "northstar/api:1", "api")]
|
|
monkeypatch.setattr(docker_discovery.docker, "DockerClient", lambda **kwargs: _DockerClient(**kwargs))
|
|
manager = _manager(tmp_path)
|
|
first = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
manager.reconcile(DiscoveryReconcileRequest(run_id=first.run.id, apply_high_confidence=True))
|
|
before = next(item for item in manager.store.list_services() if item.source_ref == "docker:compose:northstar:api")
|
|
payload = ServiceUpdate.model_validate(before.model_dump())
|
|
payload.group = "Manual ownership"
|
|
after = manager.store.update_service(before.id, payload)
|
|
assert after is not None
|
|
manager.record_service_override(before, after)
|
|
_DockerClient.current = [_Container("api", "c" * 64, "northstar/api:2", "api")]
|
|
second = manager.run(DiscoveryRunRequest(docker_host="tcp://docker.local:2375"))
|
|
manager.reconcile(DiscoveryReconcileRequest(run_id=second.run.id, apply_high_confidence=True))
|
|
updated = next(item for item in manager.store.list_services() if item.source_ref == "docker:compose:northstar:api")
|
|
assert updated.group == "Manual ownership"
|
|
assert updated.container_image == "northstar/api:2"
|
|
|
|
|
|
def test_demo_exit_restores_discovery_evidence(tmp_path: Path) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
discovery = DiscoveryStore(store)
|
|
database = store.create_service(ServiceCreate(name="Database", kind=ServiceKind.TCP, host="db", port=5432, category="database", criticality=Criticality.HIGH))
|
|
application = store.create_service(ServiceCreate(name="Application", kind=ServiceKind.HTTP, url="https://app.local", depends_on=[database.id]))
|
|
source = discovery.save_source(source_type="docker", source_id="docker:compose:stack:app", source_name="app", lifecycle=DiscoveryLifecycle.ACTIVE, metadata={"image": "app:1"})
|
|
discovery.attach_source(application.id, source.id)
|
|
now = datetime.now(UTC)
|
|
discovery.save_evidence(RelationEvidence(id="evidence-1", source_service_id=application.id, target_service_id=database.id, relation_type=RelationType.DEPENDS_ON, evidence_type="manual", confidence=DiscoveryConfidence.CONFIRMED, explanation="test evidence", first_seen_at=now, last_seen_at=now))
|
|
demo_services, demo_probes = build_demo_records()
|
|
store.activate_demo(demo_services, demo_probes)
|
|
assert store.demo_active()
|
|
assert {item.name for item in store.list_services()} == {item.name for item in demo_services}
|
|
assert store.exit_demo()
|
|
restored = store.list_services()
|
|
assert {item.name for item in restored} == {"Database", "Application"}
|
|
assert discovery.get_evidence("evidence-1") is not None
|