from __future__ import annotations from datetime import UTC, datetime, timedelta from app import observation as observation_module from app import store as store_module from app.docker_discovery import AppOpsInfo, AppOpsInventory, DockerHealthObservation from app.evidence import freshness_for from app.models import ( EvidenceProvider, EvidenceScope, HealthEvidence, HealthResolutionReason, HealthStatus, ServiceCreate, ServiceKind, ServiceSnapshot, ) from app.observation import ObservationManager, observation_ttl from app.semantic import build_semantic_topology from app.store import Store def _service(identifier: str, name: str, *, application: str | None = "Northstar", source_ref: str | None = None): return ServiceCreate(name=name, kind=ServiceKind.HTTP, url=f"https://{identifier}.test", application=application, source_ref=source_ref) def _application_evidence( evidence_id: str, *, status: HealthStatus = HealthStatus.HEALTHY, confidence: float = 0.9, observed_at: datetime | None = None, valid_for: timedelta = timedelta(minutes=3), app_id: str = "northstar", ) -> HealthEvidence: now = observed_at or datetime.now(UTC) return HealthEvidence( id=evidence_id, subject_type="application", subject_id=f"appops:{app_id}", provider=EvidenceProvider.APPOPS, provider_ref=app_id, status=status, scope=EvidenceScope.APPLICATION, confidence=confidence, observed_at=now, valid_until=now + valid_for, received_at=now, summary="AppOps aggregate health", ) def _component_evidence(evidence_id: str, service_id: str, status: HealthStatus) -> HealthEvidence: now = datetime.now(UTC) return HealthEvidence( id=evidence_id, subject_type="service", subject_id=service_id, service_id=service_id, provider=EvidenceProvider.DOCKER, provider_ref=service_id, status=status, scope=EvidenceScope.COMPONENT, confidence=0.95, observed_at=now, valid_until=now + timedelta(minutes=3), received_at=now, summary="Docker healthcheck", ) def _snapshot(service, *, direct: HealthStatus, effective: HealthStatus | None = None, evidence: list[HealthEvidence] | None = None) -> ServiceSnapshot: return ServiceSnapshot( service=service, direct_status=direct, dependency_status=effective or direct, effective_status=effective or direct, health_evidence=evidence or [], ) def test_appops_health_resolves_application_without_promoting_component_scope(tmp_path) -> None: store = Store(str(tmp_path / "servicemap.db")) api = store.create_service(_service("api", "Northstar API")) worker = store.create_service(_service("worker", "Northstar Worker")) aggregate = _application_evidence("appops-northstar") store.record_health_evidence(aggregate, service_ids=[api.id, worker.id]) snapshots = [ _snapshot(api, direct=HealthStatus.UNKNOWN, evidence=store.health_evidence_for_service(api.id)), _snapshot(worker, direct=HealthStatus.UNKNOWN, evidence=store.health_evidence_for_service(worker.id)), ] topology = build_semantic_topology(snapshots, []) application = topology.applications[0] assert application.status == HealthStatus.HEALTHY assert application.health_resolution.reason_code == HealthResolutionReason.SELECTED_APPLICATION_EVIDENCE assert application.health_resolution.confidence == 0.9 assert all(component.direct_status == HealthStatus.UNKNOWN for component in topology.components) assert all(component.unknown_reason is not None for component in topology.components) def test_component_failure_overrides_healthy_aggregate_and_exposes_conflict(tmp_path) -> None: store = Store(str(tmp_path / "servicemap.db")) database = store.create_service(_service("db", "Northstar PostgreSQL")) api = store.create_service(_service("api", "Northstar API")) aggregate = _application_evidence("appops-northstar") store.record_health_evidence(aggregate, service_ids=[database.id, api.id]) store.record_health_evidence(_component_evidence("db-down", database.id, HealthStatus.DOWN)) topology = build_semantic_topology( [ _snapshot(database, direct=HealthStatus.DOWN, evidence=store.health_evidence_for_service(database.id)), _snapshot(api, direct=HealthStatus.UNKNOWN, effective=HealthStatus.DEGRADED, evidence=store.health_evidence_for_service(api.id)), ], [], ) application = topology.applications[0] assert application.status in {HealthStatus.DOWN, HealthStatus.DEGRADED} assert application.health_resolution.reason_code == HealthResolutionReason.COMPONENT_FAILURE assert application.health_resolution.conflict is True def test_application_and_domain_coverage_treat_partial_unknown_as_coverage_not_failure() -> None: now = datetime.now(UTC) from app.models import Service healthy = Service(id="healthy", **_service("healthy", "Known", application="App A").model_dump(), created_at=now, updated_at=now) unknown = Service(id="unknown", **_service("unknown", "Unresolved", application="App A").model_dump(), created_at=now, updated_at=now) topology = build_semantic_topology( [_snapshot(healthy, direct=HealthStatus.HEALTHY), _snapshot(unknown, direct=HealthStatus.UNKNOWN)], [], ) application = topology.applications[0] domain = topology.domains[0] assert application.status == HealthStatus.HEALTHY assert application.resolved_components == 1 assert application.unresolved_components == 1 assert application.component_coverage_percent == 50 assert domain.status == HealthStatus.HEALTHY assert domain.application_coverage_percent == 100 def test_missing_stale_and_expired_application_evidence_have_structured_reasons() -> None: now = datetime.now(UTC) from app.models import Service service = Service(id="app", **_service("app", "No signal").model_dump(), created_at=now, updated_at=now) no_evidence = build_semantic_topology([_snapshot(service, direct=HealthStatus.UNKNOWN)], []).applications[0] assert no_evidence.health_resolution.reason_code == HealthResolutionReason.NO_APPLICATION_EVIDENCE stale_evidence = _application_evidence( "stale", observed_at=now - timedelta(minutes=2, seconds=30), valid_for=timedelta(minutes=2) ) stale = build_semantic_topology([_snapshot(service, direct=HealthStatus.UNKNOWN, evidence=[stale_evidence])], []).applications[0] assert stale.health_resolution.reason_code == HealthResolutionReason.STALE_EVIDENCE expired_evidence = _application_evidence( "expired", observed_at=now - timedelta(minutes=8), valid_for=timedelta(minutes=2) ) expired = build_semantic_topology([_snapshot(service, direct=HealthStatus.UNKNOWN, evidence=[expired_evidence])], []).applications[0] assert expired.health_resolution.reason_code == HealthResolutionReason.EXPIRED_EVIDENCE def test_same_named_postgres_sources_do_not_share_application_or_appops_claim(tmp_path) -> None: store = Store(str(tmp_path / "servicemap.db")) first = store.create_service(_service("one", "PostgreSQL one", application="PostgreSQL", source_ref="docker:name:postgres-one")) second = store.create_service(_service("two", "PostgreSQL two", application="PostgreSQL", source_ref="docker:name:postgres-two")) store.record_health_evidence(_application_evidence("first", status=HealthStatus.HEALTHY, app_id="postgres-one"), service_ids=[first.id]) store.record_health_evidence(_application_evidence("second", status=HealthStatus.DOWN, app_id="postgres-two"), service_ids=[second.id]) topology = build_semantic_topology( [ _snapshot(first, direct=HealthStatus.UNKNOWN, evidence=store.health_evidence_for_service(first.id)), _snapshot(second, direct=HealthStatus.UNKNOWN, evidence=store.health_evidence_for_service(second.id)), ], [], ) assert len(topology.applications) == 2 assert sorted(item.status for item in topology.applications) == [HealthStatus.DOWN, HealthStatus.HEALTHY] def test_observation_refresh_renews_evidence_without_duplicates_and_uses_bounded_ttl(tmp_path, monkeypatch) -> None: store = Store(str(tmp_path / "servicemap.db")) service = store.create_service(_service("api", "Northstar API", source_ref="docker:name:northstar-api")) first_now = datetime(2026, 1, 1, tzinfo=UTC) current = {"value": first_now} monkeypatch.setattr(observation_module, "utc_now", lambda: current["value"]) monkeypatch.setattr( observation_module, "appops_inventory", lambda _: AppOpsInventory( {"northstar-api": AppOpsInfo("northstar", None, None, "healthy", "Northstar", 0.9)}, True, "1 mapping loaded", ), ) monkeypatch.setattr( observation_module, "observe_container_health", lambda _: [DockerHealthObservation("abc", "northstar-api", "docker:name:northstar-api", "running", "healthy", True)], ) events: list[dict[str, object]] = [] monkeypatch.setattr(store_module.event_bus, "publish", lambda event: events.append(event)) manager = ObservationManager(store, interval_seconds=60) first = manager.refresh() events.clear() current["value"] = first_now + timedelta(minutes=20) second = manager.refresh() evidence = store.health_evidence_for_service(service.id) active = [item for item in evidence if item.active] assert first.docker_observations == second.docker_observations == 1 assert observation_ttl(60) == timedelta(seconds=180) assert len(active) == 2 assert all(item.valid_until - item.observed_at == timedelta(seconds=180) for item in active) assert all(freshness_for(item, now=current["value"]).value == "fresh" for item in active) assert store.provider_statuses()[0].valid_until - store.provider_statuses()[0].observed_at == timedelta(seconds=180) assert events == [] def test_appops_warning_is_degraded_application_evidence_only(tmp_path, monkeypatch) -> None: store = Store(str(tmp_path / "servicemap.db")) service = store.create_service( _service("postgres", "PostgreSQL", source_ref="docker:name:postgresql15") ) monkeypatch.setattr( observation_module, "appops_inventory", lambda _: AppOpsInventory( {"postgresql15": AppOpsInfo("postgres", "database", "critical", "warning", "PostgreSQL", 1.0)}, True, "1 mapping loaded", ), ) monkeypatch.setattr( observation_module, "observe_container_health", lambda _: [ DockerHealthObservation( "abc", "postgresql15", "docker:name:postgresql15", "running", None, False ) ], ) result = ObservationManager(store, interval_seconds=60).refresh() evidence = store.health_evidence_for_service(service.id) topology = build_semantic_topology( [_snapshot(service, direct=HealthStatus.UNKNOWN, evidence=evidence)], [] ) assert result.appops_observations == 1 assert len(evidence) == 1 assert evidence[0].scope == EvidenceScope.APPLICATION assert evidence[0].status == HealthStatus.DEGRADED assert topology.applications[0].status == HealthStatus.DEGRADED assert topology.components[0].direct_status == HealthStatus.UNKNOWN def test_loaded_evidence_reports_expired_freshness(tmp_path) -> None: store = Store(str(tmp_path / "servicemap.db")) service = store.create_service(_service("api", "Northstar API")) old = datetime.now(UTC) - timedelta(minutes=10) store.record_health_evidence( _application_evidence("old-claim", observed_at=old, valid_for=timedelta(minutes=2)), service_ids=[service.id], ) assert store.health_evidence_for_service(service.id)[0].freshness.value == "expired" assert store.health_evidence_for_services([service.id])[service.id][0].freshness.value == "expired" def test_empty_broker_inventory_is_unavailable_with_existing_docker_services( tmp_path, monkeypatch ) -> None: store = Store(str(tmp_path / "servicemap.db")) service = store.create_service( _service("api", "Northstar API", source_ref="docker:name:northstar-api") ) store.record_health_evidence( _component_evidence("existing-docker-claim", service.id, HealthStatus.HEALTHY), service_ids=[service.id], ) monkeypatch.setenv("SERVICEMAP_DOCKER_PROVIDER", "broker") monkeypatch.setattr( observation_module, "appops_inventory", lambda _: AppOpsInventory({}, False, "AppOps unavailable"), ) monkeypatch.setattr(observation_module, "observe_container_health", lambda _: []) result = ObservationManager(store, interval_seconds=60).refresh() assert result.docker_observations == 0 assert next( item for item in store.provider_statuses() if item.provider == EvidenceProvider.DOCKER ).status.value == "unavailable" assert [item.id for item in store.health_evidence_for_service(service.id)] == [ "existing-docker-claim" ] def test_single_provider_outage_ages_only_its_claim_and_recovers_without_duplicates(tmp_path, monkeypatch) -> None: store = Store(str(tmp_path / "servicemap.db")) service = store.create_service(_service("api", "Northstar API", source_ref="docker:name:northstar-api")) now = datetime(2026, 1, 1, tzinfo=UTC) current = {"value": now} mode = {"available": True} monkeypatch.setattr(observation_module, "utc_now", lambda: current["value"]) monkeypatch.setattr( observation_module, "appops_inventory", lambda _: ( AppOpsInventory( {"northstar-api": AppOpsInfo("northstar", None, None, "healthy", "Northstar", 0.9)}, True, "1 mapping loaded", ) if mode["available"] else AppOpsInventory({}, False, "AppOps inventory unavailable: ConnectError") ), ) monkeypatch.setattr( observation_module, "observe_container_health", lambda _: [DockerHealthObservation("abc", "northstar-api", "docker:name:northstar-api", "running", "healthy", True)], ) manager = ObservationManager(store, interval_seconds=60) manager.refresh() mode["available"] = False current["value"] += timedelta(seconds=60) manager.refresh() evidence_during_outage = store.health_evidence_for_service(service.id) assert any(item.provider == EvidenceProvider.APPOPS and freshness_for(item, now=current["value"]).value == "fresh" for item in evidence_during_outage) assert any(item.provider == EvidenceProvider.DOCKER and freshness_for(item, now=current["value"]).value == "fresh" for item in evidence_during_outage) assert next(item for item in store.provider_statuses() if item.provider == EvidenceProvider.APPOPS).status.value == "unavailable" mode["available"] = True current["value"] += timedelta(seconds=60) manager.refresh() active = [item for item in store.health_evidence_for_service(service.id) if item.active] assert len(active) == 2 assert next(item for item in store.provider_statuses() if item.provider == EvidenceProvider.APPOPS).status.value == "healthy"