348 lines
15 KiB
Python
348 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from app import observation as observation_module
|
|
from app import store as store_module
|
|
from app.docker_discovery import AppOpsInfo, AppOpsInventory, DockerHealthObservation
|
|
from app.evidence import freshness_for
|
|
from app.models import (
|
|
EvidenceProvider,
|
|
EvidenceScope,
|
|
HealthEvidence,
|
|
HealthResolutionReason,
|
|
HealthStatus,
|
|
ServiceCreate,
|
|
ServiceKind,
|
|
ServiceSnapshot,
|
|
)
|
|
from app.observation import ObservationManager, observation_ttl
|
|
from app.semantic import build_semantic_topology
|
|
from app.store import Store
|
|
|
|
|
|
def _service(identifier: str, name: str, *, application: str | None = "Northstar", source_ref: str | None = None):
|
|
return ServiceCreate(name=name, kind=ServiceKind.HTTP, url=f"https://{identifier}.test", application=application, source_ref=source_ref)
|
|
|
|
|
|
def _application_evidence(
|
|
evidence_id: str,
|
|
*,
|
|
status: HealthStatus = HealthStatus.HEALTHY,
|
|
confidence: float = 0.9,
|
|
observed_at: datetime | None = None,
|
|
valid_for: timedelta = timedelta(minutes=3),
|
|
app_id: str = "northstar",
|
|
) -> HealthEvidence:
|
|
now = observed_at or datetime.now(UTC)
|
|
return HealthEvidence(
|
|
id=evidence_id,
|
|
subject_type="application",
|
|
subject_id=f"appops:{app_id}",
|
|
provider=EvidenceProvider.APPOPS,
|
|
provider_ref=app_id,
|
|
status=status,
|
|
scope=EvidenceScope.APPLICATION,
|
|
confidence=confidence,
|
|
observed_at=now,
|
|
valid_until=now + valid_for,
|
|
received_at=now,
|
|
summary="AppOps aggregate health",
|
|
)
|
|
|
|
|
|
def _component_evidence(evidence_id: str, service_id: str, status: HealthStatus) -> HealthEvidence:
|
|
now = datetime.now(UTC)
|
|
return HealthEvidence(
|
|
id=evidence_id,
|
|
subject_type="service",
|
|
subject_id=service_id,
|
|
service_id=service_id,
|
|
provider=EvidenceProvider.DOCKER,
|
|
provider_ref=service_id,
|
|
status=status,
|
|
scope=EvidenceScope.COMPONENT,
|
|
confidence=0.95,
|
|
observed_at=now,
|
|
valid_until=now + timedelta(minutes=3),
|
|
received_at=now,
|
|
summary="Docker healthcheck",
|
|
)
|
|
|
|
|
|
def _snapshot(service, *, direct: HealthStatus, effective: HealthStatus | None = None, evidence: list[HealthEvidence] | None = None) -> ServiceSnapshot:
|
|
return ServiceSnapshot(
|
|
service=service,
|
|
direct_status=direct,
|
|
dependency_status=effective or direct,
|
|
effective_status=effective or direct,
|
|
health_evidence=evidence or [],
|
|
)
|
|
|
|
|
|
def test_appops_health_resolves_application_without_promoting_component_scope(tmp_path) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
api = store.create_service(_service("api", "Northstar API"))
|
|
worker = store.create_service(_service("worker", "Northstar Worker"))
|
|
aggregate = _application_evidence("appops-northstar")
|
|
store.record_health_evidence(aggregate, service_ids=[api.id, worker.id])
|
|
|
|
snapshots = [
|
|
_snapshot(api, direct=HealthStatus.UNKNOWN, evidence=store.health_evidence_for_service(api.id)),
|
|
_snapshot(worker, direct=HealthStatus.UNKNOWN, evidence=store.health_evidence_for_service(worker.id)),
|
|
]
|
|
topology = build_semantic_topology(snapshots, [])
|
|
application = topology.applications[0]
|
|
|
|
assert application.status == HealthStatus.HEALTHY
|
|
assert application.health_resolution.reason_code == HealthResolutionReason.SELECTED_APPLICATION_EVIDENCE
|
|
assert application.health_resolution.confidence == 0.9
|
|
assert all(component.direct_status == HealthStatus.UNKNOWN for component in topology.components)
|
|
assert all(component.unknown_reason is not None for component in topology.components)
|
|
|
|
|
|
def test_component_failure_overrides_healthy_aggregate_and_exposes_conflict(tmp_path) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
database = store.create_service(_service("db", "Northstar PostgreSQL"))
|
|
api = store.create_service(_service("api", "Northstar API"))
|
|
aggregate = _application_evidence("appops-northstar")
|
|
store.record_health_evidence(aggregate, service_ids=[database.id, api.id])
|
|
store.record_health_evidence(_component_evidence("db-down", database.id, HealthStatus.DOWN))
|
|
|
|
topology = build_semantic_topology(
|
|
[
|
|
_snapshot(database, direct=HealthStatus.DOWN, evidence=store.health_evidence_for_service(database.id)),
|
|
_snapshot(api, direct=HealthStatus.UNKNOWN, effective=HealthStatus.DEGRADED, evidence=store.health_evidence_for_service(api.id)),
|
|
],
|
|
[],
|
|
)
|
|
application = topology.applications[0]
|
|
|
|
assert application.status in {HealthStatus.DOWN, HealthStatus.DEGRADED}
|
|
assert application.health_resolution.reason_code == HealthResolutionReason.COMPONENT_FAILURE
|
|
assert application.health_resolution.conflict is True
|
|
|
|
|
|
def test_application_and_domain_coverage_treat_partial_unknown_as_coverage_not_failure() -> None:
|
|
now = datetime.now(UTC)
|
|
from app.models import Service
|
|
|
|
healthy = Service(id="healthy", **_service("healthy", "Known", application="App A").model_dump(), created_at=now, updated_at=now)
|
|
unknown = Service(id="unknown", **_service("unknown", "Unresolved", application="App A").model_dump(), created_at=now, updated_at=now)
|
|
topology = build_semantic_topology(
|
|
[_snapshot(healthy, direct=HealthStatus.HEALTHY), _snapshot(unknown, direct=HealthStatus.UNKNOWN)],
|
|
[],
|
|
)
|
|
application = topology.applications[0]
|
|
domain = topology.domains[0]
|
|
|
|
assert application.status == HealthStatus.HEALTHY
|
|
assert application.resolved_components == 1
|
|
assert application.unresolved_components == 1
|
|
assert application.component_coverage_percent == 50
|
|
assert domain.status == HealthStatus.HEALTHY
|
|
assert domain.application_coverage_percent == 100
|
|
|
|
|
|
def test_missing_stale_and_expired_application_evidence_have_structured_reasons() -> None:
|
|
now = datetime.now(UTC)
|
|
from app.models import Service
|
|
|
|
service = Service(id="app", **_service("app", "No signal").model_dump(), created_at=now, updated_at=now)
|
|
no_evidence = build_semantic_topology([_snapshot(service, direct=HealthStatus.UNKNOWN)], []).applications[0]
|
|
assert no_evidence.health_resolution.reason_code == HealthResolutionReason.NO_APPLICATION_EVIDENCE
|
|
|
|
stale_evidence = _application_evidence(
|
|
"stale", observed_at=now - timedelta(minutes=2, seconds=30), valid_for=timedelta(minutes=2)
|
|
)
|
|
stale = build_semantic_topology([_snapshot(service, direct=HealthStatus.UNKNOWN, evidence=[stale_evidence])], []).applications[0]
|
|
assert stale.health_resolution.reason_code == HealthResolutionReason.STALE_EVIDENCE
|
|
|
|
expired_evidence = _application_evidence(
|
|
"expired", observed_at=now - timedelta(minutes=8), valid_for=timedelta(minutes=2)
|
|
)
|
|
expired = build_semantic_topology([_snapshot(service, direct=HealthStatus.UNKNOWN, evidence=[expired_evidence])], []).applications[0]
|
|
assert expired.health_resolution.reason_code == HealthResolutionReason.EXPIRED_EVIDENCE
|
|
|
|
|
|
def test_same_named_postgres_sources_do_not_share_application_or_appops_claim(tmp_path) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
first = store.create_service(_service("one", "PostgreSQL one", application="PostgreSQL", source_ref="docker:name:postgres-one"))
|
|
second = store.create_service(_service("two", "PostgreSQL two", application="PostgreSQL", source_ref="docker:name:postgres-two"))
|
|
store.record_health_evidence(_application_evidence("first", status=HealthStatus.HEALTHY, app_id="postgres-one"), service_ids=[first.id])
|
|
store.record_health_evidence(_application_evidence("second", status=HealthStatus.DOWN, app_id="postgres-two"), service_ids=[second.id])
|
|
|
|
topology = build_semantic_topology(
|
|
[
|
|
_snapshot(first, direct=HealthStatus.UNKNOWN, evidence=store.health_evidence_for_service(first.id)),
|
|
_snapshot(second, direct=HealthStatus.UNKNOWN, evidence=store.health_evidence_for_service(second.id)),
|
|
],
|
|
[],
|
|
)
|
|
assert len(topology.applications) == 2
|
|
assert sorted(item.status for item in topology.applications) == [HealthStatus.DOWN, HealthStatus.HEALTHY]
|
|
|
|
|
|
def test_observation_refresh_renews_evidence_without_duplicates_and_uses_bounded_ttl(tmp_path, monkeypatch) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
service = store.create_service(_service("api", "Northstar API", source_ref="docker:name:northstar-api"))
|
|
first_now = datetime(2026, 1, 1, tzinfo=UTC)
|
|
current = {"value": first_now}
|
|
monkeypatch.setattr(observation_module, "utc_now", lambda: current["value"])
|
|
monkeypatch.setattr(
|
|
observation_module,
|
|
"appops_inventory",
|
|
lambda _: AppOpsInventory(
|
|
{"northstar-api": AppOpsInfo("northstar", None, None, "healthy", "Northstar", 0.9)},
|
|
True,
|
|
"1 mapping loaded",
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
observation_module,
|
|
"observe_container_health",
|
|
lambda _: [DockerHealthObservation("abc", "northstar-api", "docker:name:northstar-api", "running", "healthy", True)],
|
|
)
|
|
events: list[dict[str, object]] = []
|
|
monkeypatch.setattr(store_module.event_bus, "publish", lambda event: events.append(event))
|
|
manager = ObservationManager(store, interval_seconds=60)
|
|
first = manager.refresh()
|
|
events.clear()
|
|
current["value"] = first_now + timedelta(minutes=20)
|
|
second = manager.refresh()
|
|
|
|
evidence = store.health_evidence_for_service(service.id)
|
|
active = [item for item in evidence if item.active]
|
|
assert first.docker_observations == second.docker_observations == 1
|
|
assert observation_ttl(60) == timedelta(seconds=180)
|
|
assert len(active) == 2
|
|
assert all(item.valid_until - item.observed_at == timedelta(seconds=180) for item in active)
|
|
assert all(freshness_for(item, now=current["value"]).value == "fresh" for item in active)
|
|
assert store.provider_statuses()[0].valid_until - store.provider_statuses()[0].observed_at == timedelta(seconds=180)
|
|
assert events == []
|
|
|
|
|
|
def test_appops_warning_is_degraded_application_evidence_only(tmp_path, monkeypatch) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
service = store.create_service(
|
|
_service("postgres", "PostgreSQL", source_ref="docker:name:postgresql15")
|
|
)
|
|
monkeypatch.setattr(
|
|
observation_module,
|
|
"appops_inventory",
|
|
lambda _: AppOpsInventory(
|
|
{"postgresql15": AppOpsInfo("postgres", "database", "critical", "warning", "PostgreSQL", 1.0)},
|
|
True,
|
|
"1 mapping loaded",
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
observation_module,
|
|
"observe_container_health",
|
|
lambda _: [
|
|
DockerHealthObservation(
|
|
"abc", "postgresql15", "docker:name:postgresql15", "running", None, False
|
|
)
|
|
],
|
|
)
|
|
|
|
result = ObservationManager(store, interval_seconds=60).refresh()
|
|
evidence = store.health_evidence_for_service(service.id)
|
|
topology = build_semantic_topology(
|
|
[_snapshot(service, direct=HealthStatus.UNKNOWN, evidence=evidence)], []
|
|
)
|
|
|
|
assert result.appops_observations == 1
|
|
assert len(evidence) == 1
|
|
assert evidence[0].scope == EvidenceScope.APPLICATION
|
|
assert evidence[0].status == HealthStatus.DEGRADED
|
|
assert topology.applications[0].status == HealthStatus.DEGRADED
|
|
assert topology.components[0].direct_status == HealthStatus.UNKNOWN
|
|
|
|
|
|
def test_loaded_evidence_reports_expired_freshness(tmp_path) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
service = store.create_service(_service("api", "Northstar API"))
|
|
old = datetime.now(UTC) - timedelta(minutes=10)
|
|
store.record_health_evidence(
|
|
_application_evidence("old-claim", observed_at=old, valid_for=timedelta(minutes=2)),
|
|
service_ids=[service.id],
|
|
)
|
|
|
|
assert store.health_evidence_for_service(service.id)[0].freshness.value == "expired"
|
|
assert store.health_evidence_for_services([service.id])[service.id][0].freshness.value == "expired"
|
|
|
|
|
|
def test_empty_broker_inventory_is_unavailable_with_existing_docker_services(
|
|
tmp_path, monkeypatch
|
|
) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
service = store.create_service(
|
|
_service("api", "Northstar API", source_ref="docker:name:northstar-api")
|
|
)
|
|
store.record_health_evidence(
|
|
_component_evidence("existing-docker-claim", service.id, HealthStatus.HEALTHY),
|
|
service_ids=[service.id],
|
|
)
|
|
monkeypatch.setenv("SERVICEMAP_DOCKER_PROVIDER", "broker")
|
|
monkeypatch.setattr(
|
|
observation_module,
|
|
"appops_inventory",
|
|
lambda _: AppOpsInventory({}, False, "AppOps unavailable"),
|
|
)
|
|
monkeypatch.setattr(observation_module, "observe_container_health", lambda _: [])
|
|
|
|
result = ObservationManager(store, interval_seconds=60).refresh()
|
|
|
|
assert result.docker_observations == 0
|
|
assert next(
|
|
item for item in store.provider_statuses() if item.provider == EvidenceProvider.DOCKER
|
|
).status.value == "unavailable"
|
|
assert [item.id for item in store.health_evidence_for_service(service.id)] == [
|
|
"existing-docker-claim"
|
|
]
|
|
|
|
|
|
def test_single_provider_outage_ages_only_its_claim_and_recovers_without_duplicates(tmp_path, monkeypatch) -> None:
|
|
store = Store(str(tmp_path / "servicemap.db"))
|
|
service = store.create_service(_service("api", "Northstar API", source_ref="docker:name:northstar-api"))
|
|
now = datetime(2026, 1, 1, tzinfo=UTC)
|
|
current = {"value": now}
|
|
mode = {"available": True}
|
|
monkeypatch.setattr(observation_module, "utc_now", lambda: current["value"])
|
|
monkeypatch.setattr(
|
|
observation_module,
|
|
"appops_inventory",
|
|
lambda _: (
|
|
AppOpsInventory(
|
|
{"northstar-api": AppOpsInfo("northstar", None, None, "healthy", "Northstar", 0.9)},
|
|
True,
|
|
"1 mapping loaded",
|
|
)
|
|
if mode["available"]
|
|
else AppOpsInventory({}, False, "AppOps inventory unavailable: ConnectError")
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
observation_module,
|
|
"observe_container_health",
|
|
lambda _: [DockerHealthObservation("abc", "northstar-api", "docker:name:northstar-api", "running", "healthy", True)],
|
|
)
|
|
manager = ObservationManager(store, interval_seconds=60)
|
|
manager.refresh()
|
|
mode["available"] = False
|
|
current["value"] += timedelta(seconds=60)
|
|
manager.refresh()
|
|
|
|
evidence_during_outage = store.health_evidence_for_service(service.id)
|
|
assert any(item.provider == EvidenceProvider.APPOPS and freshness_for(item, now=current["value"]).value == "fresh" for item in evidence_during_outage)
|
|
assert any(item.provider == EvidenceProvider.DOCKER and freshness_for(item, now=current["value"]).value == "fresh" for item in evidence_during_outage)
|
|
assert next(item for item in store.provider_statuses() if item.provider == EvidenceProvider.APPOPS).status.value == "unavailable"
|
|
|
|
mode["available"] = True
|
|
current["value"] += timedelta(seconds=60)
|
|
manager.refresh()
|
|
active = [item for item in store.health_evidence_for_service(service.id) if item.active]
|
|
assert len(active) == 2
|
|
assert next(item for item in store.provider_statuses() if item.provider == EvidenceProvider.APPOPS).status.value == "healthy"
|