2013 lines
101 KiB
Python
2013 lines
101 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from .events import event_bus
|
|
from .evidence import resolve_health, with_freshness
|
|
from .models import (
|
|
AvailabilitySummary,
|
|
ConfigImportResult,
|
|
ConfigService,
|
|
EventRecord,
|
|
EvidenceProvider,
|
|
EvidenceScope,
|
|
HealthEvidence,
|
|
HealthResolution,
|
|
HealthStatus,
|
|
IncidentEvent,
|
|
IncidentRecord,
|
|
IncidentStatus,
|
|
MonitoringState,
|
|
ProbeResult,
|
|
ProviderStatus,
|
|
Service,
|
|
ServiceCreate,
|
|
ServiceUpdate,
|
|
)
|
|
|
|
SERVICE_COLUMNS = (
|
|
"id, name, kind, url, host, port, depends_on, description, tags, \"group\", "
|
|
"environment, owner, enabled, monitoring_enabled, monitoring_state, probe_source, probe_reachability, probe_confidence, "
|
|
"runtime_state, runtime_health, application, domain, component_role, application_locked, domain_locked, component_role_locked, criticality, category, visibility, lifecycle, "
|
|
"source_type, source_ref, first_seen_at, last_seen_at, last_discovery_status, container_image, "
|
|
"compose_project, layout_x, layout_y, layout_pinned, "
|
|
"interval_seconds, timeout_seconds, retries, failure_threshold, recovery_threshold, "
|
|
"expected_status_min, expected_status_max, availability_target, maintenance_enabled, "
|
|
"maintenance_reason, state, failure_streak, recovery_streak, flapping, flap_count, "
|
|
"last_state_change_at, created_at, updated_at"
|
|
)
|
|
|
|
|
|
def _now() -> datetime:
|
|
return datetime.now(UTC)
|
|
|
|
|
|
def _iso(value: datetime | None) -> str | None:
|
|
return value.isoformat() if value is not None else None
|
|
|
|
|
|
def _bool(value: object, default: bool = False) -> bool:
|
|
if value is None:
|
|
return default
|
|
return bool(int(value)) if isinstance(value, (int, str)) and str(value).isdigit() else bool(value)
|
|
|
|
|
|
class Store:
|
|
def __init__(self, db_path: str | None = None) -> None:
|
|
raw_path = db_path or os.getenv("SERVICEMAP_DB", "data/servicemap.db")
|
|
self.path = Path(raw_path)
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
self._init_schema()
|
|
|
|
def _connect(self) -> sqlite3.Connection:
|
|
connection = sqlite3.connect(self.path, timeout=5.0)
|
|
connection.row_factory = sqlite3.Row
|
|
connection.execute("PRAGMA foreign_keys = ON")
|
|
connection.execute("PRAGMA busy_timeout = 5000")
|
|
return connection
|
|
|
|
def _init_schema(self) -> None:
|
|
with self._connect() as db:
|
|
db.execute("PRAGMA journal_mode = WAL")
|
|
db.execute("PRAGMA synchronous = NORMAL")
|
|
db.executescript(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS services (
|
|
id TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL UNIQUE,
|
|
kind TEXT NOT NULL,
|
|
url TEXT,
|
|
host TEXT,
|
|
port INTEGER,
|
|
depends_on TEXT NOT NULL DEFAULT '[]',
|
|
description TEXT,
|
|
tags TEXT NOT NULL DEFAULT '[]',
|
|
"group" TEXT,
|
|
environment TEXT,
|
|
owner TEXT,
|
|
enabled INTEGER NOT NULL DEFAULT 1,
|
|
monitoring_enabled INTEGER NOT NULL DEFAULT 1,
|
|
monitoring_state TEXT NOT NULL DEFAULT 'active_probe',
|
|
probe_source TEXT NOT NULL DEFAULT 'manual',
|
|
probe_reachability TEXT NOT NULL DEFAULT 'verified',
|
|
probe_confidence TEXT NOT NULL DEFAULT 'confirmed',
|
|
runtime_state TEXT,
|
|
runtime_health TEXT,
|
|
application TEXT,
|
|
domain TEXT,
|
|
component_role TEXT NOT NULL DEFAULT 'unknown',
|
|
application_locked INTEGER NOT NULL DEFAULT 0,
|
|
domain_locked INTEGER NOT NULL DEFAULT 0,
|
|
component_role_locked INTEGER NOT NULL DEFAULT 0,
|
|
criticality TEXT NOT NULL DEFAULT 'medium',
|
|
category TEXT NOT NULL DEFAULT 'unknown',
|
|
visibility TEXT NOT NULL DEFAULT 'visible',
|
|
lifecycle TEXT NOT NULL DEFAULT 'active',
|
|
source_type TEXT,
|
|
source_ref TEXT,
|
|
first_seen_at TEXT,
|
|
last_seen_at TEXT,
|
|
last_discovery_status TEXT,
|
|
container_image TEXT,
|
|
compose_project TEXT,
|
|
layout_x REAL,
|
|
layout_y REAL,
|
|
layout_pinned INTEGER NOT NULL DEFAULT 0,
|
|
interval_seconds INTEGER,
|
|
timeout_seconds REAL,
|
|
retries INTEGER,
|
|
failure_threshold INTEGER NOT NULL DEFAULT 1,
|
|
recovery_threshold INTEGER NOT NULL DEFAULT 1,
|
|
expected_status_min INTEGER,
|
|
expected_status_max INTEGER,
|
|
availability_target REAL,
|
|
maintenance_enabled INTEGER NOT NULL DEFAULT 0,
|
|
maintenance_reason TEXT,
|
|
state TEXT NOT NULL DEFAULT 'unknown',
|
|
failure_streak INTEGER NOT NULL DEFAULT 0,
|
|
recovery_streak INTEGER NOT NULL DEFAULT 0,
|
|
flapping INTEGER NOT NULL DEFAULT 0,
|
|
flap_count INTEGER NOT NULL DEFAULT 0,
|
|
last_state_change_at TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS probe_results (
|
|
id TEXT PRIMARY KEY,
|
|
service_id TEXT NOT NULL,
|
|
status TEXT NOT NULL,
|
|
latency_ms REAL,
|
|
detail TEXT NOT NULL,
|
|
checked_at TEXT NOT NULL,
|
|
trigger TEXT NOT NULL DEFAULT 'manual',
|
|
metadata TEXT NOT NULL DEFAULT '{}',
|
|
FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_probe_service_time
|
|
ON probe_results(service_id, checked_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_probe_time
|
|
ON probe_results(checked_at DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS events (
|
|
id TEXT PRIMARY KEY,
|
|
event_type TEXT NOT NULL,
|
|
service_id TEXT,
|
|
incident_id TEXT,
|
|
from_status TEXT,
|
|
to_status TEXT,
|
|
message TEXT NOT NULL,
|
|
metadata TEXT NOT NULL DEFAULT '{}',
|
|
created_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_events_time ON events(created_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_events_service_time ON events(service_id, created_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_events_incident_time ON events(incident_id, created_at ASC);
|
|
|
|
CREATE TABLE IF NOT EXISTS incidents (
|
|
id TEXT PRIMARY KEY,
|
|
status TEXT NOT NULL,
|
|
root_service_id TEXT NOT NULL,
|
|
started_at TEXT NOT NULL,
|
|
detected_at TEXT NOT NULL,
|
|
acknowledged_at TEXT,
|
|
resolved_at TEXT,
|
|
title TEXT NOT NULL,
|
|
root_cause_reason TEXT NOT NULL,
|
|
affected_service_ids TEXT NOT NULL DEFAULT '[]',
|
|
severity TEXT NOT NULL DEFAULT 'medium',
|
|
latest_update_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_incidents_status ON incidents(status, latest_update_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_incidents_root ON incidents(root_service_id, started_at DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS demo_sessions (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
snapshot_services TEXT NOT NULL,
|
|
snapshot_probes TEXT NOT NULL,
|
|
snapshot_events TEXT NOT NULL DEFAULT '[]',
|
|
snapshot_incidents TEXT NOT NULL DEFAULT '[]',
|
|
snapshot_discovery TEXT NOT NULL DEFAULT '{}',
|
|
snapshot_health_evidence TEXT NOT NULL DEFAULT '{}',
|
|
demo_service_ids TEXT NOT NULL,
|
|
started_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS health_evidence (
|
|
id TEXT PRIMARY KEY,
|
|
subject_type TEXT NOT NULL,
|
|
subject_id TEXT NOT NULL,
|
|
service_id TEXT,
|
|
provider TEXT NOT NULL,
|
|
provider_ref TEXT,
|
|
status TEXT NOT NULL,
|
|
scope TEXT NOT NULL,
|
|
confidence REAL NOT NULL,
|
|
observed_at TEXT NOT NULL,
|
|
valid_until TEXT NOT NULL,
|
|
received_at TEXT NOT NULL,
|
|
summary TEXT NOT NULL DEFAULT '',
|
|
metadata TEXT NOT NULL DEFAULT '{}',
|
|
active INTEGER NOT NULL DEFAULT 1,
|
|
superseded_by TEXT,
|
|
FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_health_evidence_subject
|
|
ON health_evidence(subject_type, subject_id, provider, active, observed_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_health_evidence_service
|
|
ON health_evidence(service_id, active, observed_at DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS health_evidence_service_links (
|
|
evidence_id TEXT NOT NULL,
|
|
service_id TEXT NOT NULL,
|
|
PRIMARY KEY(evidence_id, service_id),
|
|
FOREIGN KEY(evidence_id) REFERENCES health_evidence(id) ON DELETE CASCADE,
|
|
FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_health_evidence_link_service
|
|
ON health_evidence_service_links(service_id, evidence_id);
|
|
|
|
CREATE TABLE IF NOT EXISTS provider_status (
|
|
provider TEXT PRIMARY KEY,
|
|
status TEXT NOT NULL,
|
|
observed_at TEXT NOT NULL,
|
|
valid_until TEXT NOT NULL,
|
|
detail TEXT NOT NULL DEFAULT '',
|
|
metadata TEXT NOT NULL DEFAULT '{}'
|
|
);
|
|
"""
|
|
)
|
|
self._migrate_columns(db)
|
|
self._backfill_health_evidence(db)
|
|
|
|
@staticmethod
|
|
def _migrate_columns(db: sqlite3.Connection) -> None:
|
|
columns = {row["name"] for row in db.execute("PRAGMA table_info(services)").fetchall()}
|
|
additions = {
|
|
"updated_at": "TEXT",
|
|
"description": "TEXT",
|
|
"tags": "TEXT NOT NULL DEFAULT '[]'",
|
|
"group": "TEXT",
|
|
"environment": "TEXT",
|
|
"owner": "TEXT",
|
|
"enabled": "INTEGER NOT NULL DEFAULT 1",
|
|
"monitoring_enabled": "INTEGER NOT NULL DEFAULT 1",
|
|
"monitoring_state": "TEXT NOT NULL DEFAULT 'active_probe'",
|
|
"probe_source": "TEXT NOT NULL DEFAULT 'manual'",
|
|
"probe_reachability": "TEXT NOT NULL DEFAULT 'verified'",
|
|
"probe_confidence": "TEXT NOT NULL DEFAULT 'confirmed'",
|
|
"runtime_state": "TEXT",
|
|
"runtime_health": "TEXT",
|
|
"application": "TEXT",
|
|
"domain": "TEXT",
|
|
"component_role": "TEXT NOT NULL DEFAULT 'unknown'",
|
|
"application_locked": "INTEGER NOT NULL DEFAULT 0",
|
|
"domain_locked": "INTEGER NOT NULL DEFAULT 0",
|
|
"component_role_locked": "INTEGER NOT NULL DEFAULT 0",
|
|
"criticality": "TEXT NOT NULL DEFAULT 'medium'",
|
|
"category": "TEXT NOT NULL DEFAULT 'unknown'",
|
|
"visibility": "TEXT NOT NULL DEFAULT 'visible'",
|
|
"lifecycle": "TEXT NOT NULL DEFAULT 'active'",
|
|
"source_type": "TEXT",
|
|
"source_ref": "TEXT",
|
|
"first_seen_at": "TEXT",
|
|
"last_seen_at": "TEXT",
|
|
"last_discovery_status": "TEXT",
|
|
"container_image": "TEXT",
|
|
"compose_project": "TEXT",
|
|
"layout_x": "REAL",
|
|
"layout_y": "REAL",
|
|
"layout_pinned": "INTEGER NOT NULL DEFAULT 0",
|
|
"interval_seconds": "INTEGER",
|
|
"timeout_seconds": "REAL",
|
|
"retries": "INTEGER",
|
|
"failure_threshold": "INTEGER NOT NULL DEFAULT 1",
|
|
"recovery_threshold": "INTEGER NOT NULL DEFAULT 1",
|
|
"expected_status_min": "INTEGER",
|
|
"expected_status_max": "INTEGER",
|
|
"availability_target": "REAL",
|
|
"maintenance_enabled": "INTEGER NOT NULL DEFAULT 0",
|
|
"maintenance_reason": "TEXT",
|
|
"state": "TEXT NOT NULL DEFAULT 'unknown'",
|
|
"failure_streak": "INTEGER NOT NULL DEFAULT 0",
|
|
"recovery_streak": "INTEGER NOT NULL DEFAULT 0",
|
|
"flapping": "INTEGER NOT NULL DEFAULT 0",
|
|
"flap_count": "INTEGER NOT NULL DEFAULT 0",
|
|
"last_state_change_at": "TEXT",
|
|
}
|
|
for name, definition in additions.items():
|
|
if name not in columns:
|
|
quoted = f'"{name}"' if name == "group" else name
|
|
db.execute(f"ALTER TABLE services ADD COLUMN {quoted} {definition}")
|
|
db.execute("UPDATE services SET updated_at = created_at WHERE updated_at IS NULL")
|
|
|
|
probe_columns = {row["name"] for row in db.execute("PRAGMA table_info(probe_results)").fetchall()}
|
|
for name, definition in {"trigger": "TEXT NOT NULL DEFAULT 'manual'", "metadata": "TEXT NOT NULL DEFAULT '{}'"}.items():
|
|
if name not in probe_columns:
|
|
db.execute(f"ALTER TABLE probe_results ADD COLUMN {name} {definition}")
|
|
|
|
# v1.0.1 derived health directly from the latest probe. Backfill the
|
|
# durable v1.1 state column so an in-place migration does not make an
|
|
# existing healthy/down service appear unknown until the next cycle.
|
|
db.execute(
|
|
"""
|
|
UPDATE services
|
|
SET state = COALESCE(
|
|
(SELECT latest.status FROM probe_results latest
|
|
WHERE latest.service_id = services.id
|
|
ORDER BY latest.checked_at DESC, latest.rowid DESC LIMIT 1),
|
|
state
|
|
),
|
|
last_state_change_at = COALESCE(
|
|
last_state_change_at,
|
|
(SELECT latest.checked_at FROM probe_results latest
|
|
WHERE latest.service_id = services.id
|
|
ORDER BY latest.checked_at DESC, latest.rowid DESC LIMIT 1)
|
|
)
|
|
WHERE state = 'unknown'
|
|
AND EXISTS (SELECT 1 FROM probe_results existing WHERE existing.service_id = services.id)
|
|
"""
|
|
)
|
|
|
|
demo_columns = {row["name"] for row in db.execute("PRAGMA table_info(demo_sessions)").fetchall()}
|
|
for name in ("snapshot_events", "snapshot_incidents", "snapshot_discovery", "snapshot_health_evidence"):
|
|
if name not in demo_columns:
|
|
default = "'{}'" if name in {"snapshot_discovery", "snapshot_health_evidence"} else "'[]'"
|
|
db.execute(f"ALTER TABLE demo_sessions ADD COLUMN {name} TEXT NOT NULL DEFAULT {default}")
|
|
|
|
@staticmethod
|
|
def _backfill_health_evidence(db: sqlite3.Connection) -> None:
|
|
"""Preserve legitimate recent v1.5 observations during the v1.6 migration.
|
|
|
|
This is a one-time projection of existing probe/runtime facts, not a
|
|
fabricated monitoring result. Historic observations stay expired and
|
|
visible in provenance; only their original interval makes them current.
|
|
AppOps rows are deliberately excluded because pre-v1.6 persisted an
|
|
application-scoped status as component runtime health.
|
|
"""
|
|
|
|
if db.execute("SELECT 1 FROM health_evidence LIMIT 1").fetchone():
|
|
return
|
|
latest = db.execute(
|
|
"""
|
|
SELECT p.* FROM probe_results p
|
|
WHERE p.rowid = (
|
|
SELECT candidate.rowid FROM probe_results candidate
|
|
WHERE candidate.service_id = p.service_id
|
|
ORDER BY candidate.checked_at DESC, candidate.rowid DESC LIMIT 1
|
|
)
|
|
"""
|
|
).fetchall()
|
|
intervals = {
|
|
row["id"]: max(60, int(row["interval_seconds"] or 60) * 2)
|
|
for row in db.execute("SELECT id, interval_seconds FROM services").fetchall()
|
|
}
|
|
for row in latest:
|
|
observed = datetime.fromisoformat(row["checked_at"])
|
|
ttl = timedelta(seconds=intervals.get(row["service_id"], 120))
|
|
evidence_id = f"migration-probe-{row['id']}"
|
|
db.execute(
|
|
"""
|
|
INSERT INTO health_evidence (
|
|
id, subject_type, subject_id, service_id, provider, provider_ref, status, scope,
|
|
confidence, observed_at, valid_until, received_at, summary, metadata, active
|
|
) VALUES (?, 'service', ?, ?, 'active_probe', ?, ?, 'component', 1.0, ?, ?, ?, ?, ?, 1)
|
|
""",
|
|
(
|
|
evidence_id,
|
|
row["service_id"],
|
|
row["service_id"],
|
|
row["id"],
|
|
row["status"],
|
|
observed.isoformat(),
|
|
(observed + ttl).isoformat(),
|
|
observed.isoformat(),
|
|
"Migrated latest active probe result",
|
|
row["metadata"] or "{}",
|
|
),
|
|
)
|
|
db.execute(
|
|
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
|
|
(evidence_id, row["service_id"]),
|
|
)
|
|
|
|
runtime_rows = db.execute(
|
|
"""
|
|
SELECT id, runtime_state, runtime_health, updated_at, monitoring_state
|
|
FROM services
|
|
WHERE COALESCE(monitoring_state, '') != 'appops_observation'
|
|
AND (runtime_state IS NOT NULL OR runtime_health IS NOT NULL)
|
|
"""
|
|
).fetchall()
|
|
for row in runtime_rows:
|
|
observed = datetime.fromisoformat(row["updated_at"] or _now().isoformat())
|
|
runtime_state = str(row["runtime_state"] or "").casefold()
|
|
runtime_health = str(row["runtime_health"] or "").casefold()
|
|
if runtime_health in {"unhealthy", "down"} or runtime_state in {"exited", "dead", "stopped"}:
|
|
status = HealthStatus.DOWN
|
|
elif runtime_health == "degraded" or runtime_state == "restarting":
|
|
status = HealthStatus.DEGRADED
|
|
elif runtime_health == "healthy":
|
|
status = HealthStatus.HEALTHY
|
|
else:
|
|
continue
|
|
evidence_id = f"migration-runtime-{row['id']}"
|
|
db.execute(
|
|
"""
|
|
INSERT INTO health_evidence (
|
|
id, subject_type, subject_id, service_id, provider, provider_ref, status, scope,
|
|
confidence, observed_at, valid_until, received_at, summary, metadata, active
|
|
) VALUES (?, 'service', ?, ?, 'docker', 'legacy-runtime', ?, 'component', 0.7, ?, ?, ?, ?, ?, 1)
|
|
""",
|
|
(
|
|
evidence_id,
|
|
row["id"],
|
|
row["id"],
|
|
status.value,
|
|
observed.isoformat(),
|
|
(observed + timedelta(minutes=10)).isoformat(),
|
|
observed.isoformat(),
|
|
"Migrated Docker runtime observation",
|
|
json.dumps({"runtime_state": row["runtime_state"], "runtime_health": row["runtime_health"]}),
|
|
),
|
|
)
|
|
db.execute(
|
|
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
|
|
(evidence_id, row["id"]),
|
|
)
|
|
|
|
@staticmethod
|
|
def _json_list(value: object, fallback: list[str] | None = None) -> list[str]:
|
|
if isinstance(value, list):
|
|
return [str(item) for item in value]
|
|
if isinstance(value, str):
|
|
try:
|
|
parsed = json.loads(value)
|
|
return [str(item) for item in parsed] if isinstance(parsed, list) else (fallback or [])
|
|
except json.JSONDecodeError:
|
|
return fallback or []
|
|
return fallback or []
|
|
|
|
@staticmethod
|
|
def _json_object(value: object) -> dict[str, object]:
|
|
if isinstance(value, dict):
|
|
return value
|
|
if isinstance(value, str):
|
|
try:
|
|
parsed = json.loads(value)
|
|
return parsed if isinstance(parsed, dict) else {}
|
|
except json.JSONDecodeError:
|
|
return {}
|
|
return {}
|
|
|
|
@classmethod
|
|
def _row_to_service(cls, row: sqlite3.Row | dict[str, Any]) -> Service:
|
|
created_at = datetime.fromisoformat(str(row["created_at"]))
|
|
raw_updated = row.get("updated_at") if isinstance(row, dict) else row["updated_at"]
|
|
raw_state = row.get("state", "unknown") if isinstance(row, dict) else row["state"]
|
|
try:
|
|
state = HealthStatus(raw_state or "unknown")
|
|
except ValueError:
|
|
state = HealthStatus.UNKNOWN
|
|
return Service(
|
|
id=row["id"],
|
|
name=row["name"],
|
|
kind=row["kind"],
|
|
url=row["url"],
|
|
host=row["host"],
|
|
port=row["port"],
|
|
depends_on=cls._json_list(row["depends_on"]),
|
|
description=row.get("description") if isinstance(row, dict) else row["description"],
|
|
tags=cls._json_list(row.get("tags", "[]") if isinstance(row, dict) else row["tags"]),
|
|
group=row.get("group") if isinstance(row, dict) else row["group"],
|
|
environment=row.get("environment") if isinstance(row, dict) else row["environment"],
|
|
owner=row.get("owner") if isinstance(row, dict) else row["owner"],
|
|
enabled=_bool(row.get("enabled", 1) if isinstance(row, dict) else row["enabled"], True),
|
|
monitoring_enabled=_bool(
|
|
row.get("monitoring_enabled", 1) if isinstance(row, dict) else row["monitoring_enabled"],
|
|
True,
|
|
),
|
|
monitoring_state=(row.get("monitoring_state", "active_probe") if isinstance(row, dict) else row["monitoring_state"]) or "active_probe",
|
|
probe_source=(row.get("probe_source", "manual") if isinstance(row, dict) else row["probe_source"]) or "manual",
|
|
probe_reachability=(row.get("probe_reachability", "verified") if isinstance(row, dict) else row["probe_reachability"]) or "verified",
|
|
probe_confidence=(row.get("probe_confidence", "confirmed") if isinstance(row, dict) else row["probe_confidence"]) or "confirmed",
|
|
runtime_state=row.get("runtime_state") if isinstance(row, dict) else row["runtime_state"],
|
|
runtime_health=row.get("runtime_health") if isinstance(row, dict) else row["runtime_health"],
|
|
application=row.get("application") if isinstance(row, dict) else row["application"],
|
|
domain=row.get("domain") if isinstance(row, dict) else row["domain"],
|
|
component_role=(row.get("component_role", "unknown") if isinstance(row, dict) else row["component_role"]) or "unknown",
|
|
application_locked=_bool(row.get("application_locked", 0) if isinstance(row, dict) else row["application_locked"]),
|
|
domain_locked=_bool(row.get("domain_locked", 0) if isinstance(row, dict) else row["domain_locked"]),
|
|
component_role_locked=_bool(row.get("component_role_locked", 0) if isinstance(row, dict) else row["component_role_locked"]),
|
|
criticality=row.get("criticality", "medium") if isinstance(row, dict) else row["criticality"],
|
|
category=row.get("category", "unknown") if isinstance(row, dict) else row["category"],
|
|
visibility=row.get("visibility", "visible") if isinstance(row, dict) else row["visibility"],
|
|
lifecycle=row.get("lifecycle", "active") if isinstance(row, dict) else row["lifecycle"],
|
|
source_type=row.get("source_type") if isinstance(row, dict) else row["source_type"],
|
|
source_ref=row.get("source_ref") if isinstance(row, dict) else row["source_ref"],
|
|
first_seen_at=(
|
|
datetime.fromisoformat(row["first_seen_at"])
|
|
if (row.get("first_seen_at") if isinstance(row, dict) else row["first_seen_at"])
|
|
else None
|
|
),
|
|
last_seen_at=(
|
|
datetime.fromisoformat(row["last_seen_at"])
|
|
if (row.get("last_seen_at") if isinstance(row, dict) else row["last_seen_at"])
|
|
else None
|
|
),
|
|
last_discovery_status=(
|
|
row.get("last_discovery_status")
|
|
if isinstance(row, dict)
|
|
else row["last_discovery_status"]
|
|
),
|
|
container_image=row.get("container_image") if isinstance(row, dict) else row["container_image"],
|
|
compose_project=row.get("compose_project") if isinstance(row, dict) else row["compose_project"],
|
|
layout_x=row.get("layout_x") if isinstance(row, dict) else row["layout_x"],
|
|
layout_y=row.get("layout_y") if isinstance(row, dict) else row["layout_y"],
|
|
layout_pinned=_bool(
|
|
row.get("layout_pinned", 0) if isinstance(row, dict) else row["layout_pinned"]
|
|
),
|
|
interval_seconds=row.get("interval_seconds") if isinstance(row, dict) else row["interval_seconds"],
|
|
timeout_seconds=row.get("timeout_seconds") if isinstance(row, dict) else row["timeout_seconds"],
|
|
retries=row.get("retries") if isinstance(row, dict) else row["retries"],
|
|
failure_threshold=(row.get("failure_threshold", 1) if isinstance(row, dict) else row["failure_threshold"]) or 1,
|
|
recovery_threshold=(row.get("recovery_threshold", 1) if isinstance(row, dict) else row["recovery_threshold"]) or 1,
|
|
expected_status_min=row.get("expected_status_min") if isinstance(row, dict) else row["expected_status_min"],
|
|
expected_status_max=row.get("expected_status_max") if isinstance(row, dict) else row["expected_status_max"],
|
|
availability_target=row.get("availability_target") if isinstance(row, dict) else row["availability_target"],
|
|
maintenance_enabled=_bool(row.get("maintenance_enabled", 0) if isinstance(row, dict) else row["maintenance_enabled"]),
|
|
maintenance_reason=row.get("maintenance_reason") if isinstance(row, dict) else row["maintenance_reason"],
|
|
state=state,
|
|
failure_streak=(row.get("failure_streak", 0) if isinstance(row, dict) else row["failure_streak"]) or 0,
|
|
recovery_streak=(row.get("recovery_streak", 0) if isinstance(row, dict) else row["recovery_streak"]) or 0,
|
|
flapping=_bool(row.get("flapping", 0) if isinstance(row, dict) else row["flapping"]),
|
|
flap_count=(row.get("flap_count", 0) if isinstance(row, dict) else row["flap_count"]) or 0,
|
|
last_state_change_at=(
|
|
datetime.fromisoformat(row["last_state_change_at"])
|
|
if (row.get("last_state_change_at") if isinstance(row, dict) else row["last_state_change_at"])
|
|
else None
|
|
),
|
|
created_at=created_at,
|
|
updated_at=datetime.fromisoformat(str(raw_updated)) if raw_updated else created_at,
|
|
)
|
|
|
|
@classmethod
|
|
def _row_to_probe(cls, row: sqlite3.Row | dict[str, Any]) -> ProbeResult:
|
|
return ProbeResult(
|
|
id=row["id"],
|
|
service_id=row["service_id"],
|
|
status=HealthStatus(row["status"]),
|
|
latency_ms=row["latency_ms"],
|
|
detail=row["detail"],
|
|
checked_at=datetime.fromisoformat(row["checked_at"]),
|
|
trigger=(row.get("trigger", "manual") if isinstance(row, dict) else row["trigger"]) or "manual",
|
|
metadata=cls._json_object(row.get("metadata", "{}") if isinstance(row, dict) else row["metadata"]),
|
|
)
|
|
|
|
@classmethod
|
|
def _row_to_event(cls, row: sqlite3.Row | dict[str, Any]) -> EventRecord:
|
|
raw_from = row["from_status"]
|
|
raw_to = row["to_status"]
|
|
return EventRecord(
|
|
id=row["id"],
|
|
event_type=row["event_type"],
|
|
service_id=row["service_id"],
|
|
incident_id=row["incident_id"],
|
|
from_status=HealthStatus(raw_from) if raw_from else None,
|
|
to_status=HealthStatus(raw_to) if raw_to else None,
|
|
message=row["message"],
|
|
metadata=cls._json_object(row["metadata"]),
|
|
created_at=datetime.fromisoformat(row["created_at"]),
|
|
)
|
|
|
|
@classmethod
|
|
def _row_to_incident(cls, row: sqlite3.Row | dict[str, Any], services: dict[str, Service] | None = None) -> IncidentRecord:
|
|
services = services or {}
|
|
root = services.get(row["root_service_id"])
|
|
affected_ids = cls._json_list(row["affected_service_ids"])
|
|
return IncidentRecord(
|
|
id=row["id"],
|
|
status=IncidentStatus(row["status"]),
|
|
root_service_id=row["root_service_id"],
|
|
root_service_name=root.name if root else row["root_service_id"],
|
|
started_at=datetime.fromisoformat(row["started_at"]),
|
|
detected_at=datetime.fromisoformat(row["detected_at"]),
|
|
acknowledged_at=datetime.fromisoformat(row["acknowledged_at"]) if row["acknowledged_at"] else None,
|
|
resolved_at=datetime.fromisoformat(row["resolved_at"]) if row["resolved_at"] else None,
|
|
title=row["title"],
|
|
root_cause_reason=row["root_cause_reason"],
|
|
affected_service_ids=affected_ids,
|
|
affected_service_names=[services[item].name for item in affected_ids if item in services],
|
|
severity=row["severity"],
|
|
latest_update_at=datetime.fromisoformat(row["latest_update_at"]),
|
|
)
|
|
|
|
@staticmethod
|
|
def _service_params(service: Service) -> dict[str, Any]:
|
|
return {
|
|
"id": service.id,
|
|
"name": service.name,
|
|
"kind": service.kind.value,
|
|
"url": service.url,
|
|
"host": service.host,
|
|
"port": service.port,
|
|
"depends_on": json.dumps(service.depends_on),
|
|
"description": service.description,
|
|
"tags": json.dumps(service.tags),
|
|
"group": service.group,
|
|
"environment": service.environment,
|
|
"owner": service.owner,
|
|
"enabled": int(service.enabled),
|
|
"monitoring_enabled": int(service.monitoring_enabled),
|
|
"monitoring_state": service.monitoring_state.value,
|
|
"probe_source": service.probe_source.value,
|
|
"probe_reachability": service.probe_reachability.value,
|
|
"probe_confidence": service.probe_confidence.value,
|
|
"runtime_state": service.runtime_state,
|
|
"runtime_health": service.runtime_health,
|
|
"application": service.application,
|
|
"domain": service.domain,
|
|
"component_role": service.component_role.value,
|
|
"application_locked": int(service.application_locked),
|
|
"domain_locked": int(service.domain_locked),
|
|
"component_role_locked": int(service.component_role_locked),
|
|
"criticality": service.criticality.value,
|
|
"category": service.category.value,
|
|
"visibility": service.visibility.value,
|
|
"lifecycle": service.lifecycle.value,
|
|
"source_type": service.source_type,
|
|
"source_ref": service.source_ref,
|
|
"first_seen_at": _iso(service.first_seen_at),
|
|
"last_seen_at": _iso(service.last_seen_at),
|
|
"last_discovery_status": service.last_discovery_status,
|
|
"container_image": service.container_image,
|
|
"compose_project": service.compose_project,
|
|
"layout_x": service.layout_x,
|
|
"layout_y": service.layout_y,
|
|
"layout_pinned": int(service.layout_pinned),
|
|
"interval_seconds": service.interval_seconds,
|
|
"timeout_seconds": service.timeout_seconds,
|
|
"retries": service.retries,
|
|
"failure_threshold": service.failure_threshold,
|
|
"recovery_threshold": service.recovery_threshold,
|
|
"expected_status_min": service.expected_status_min,
|
|
"expected_status_max": service.expected_status_max,
|
|
"availability_target": service.availability_target,
|
|
"maintenance_enabled": int(service.maintenance_enabled),
|
|
"maintenance_reason": service.maintenance_reason,
|
|
"state": service.state.value,
|
|
"failure_streak": service.failure_streak,
|
|
"recovery_streak": service.recovery_streak,
|
|
"flapping": int(service.flapping),
|
|
"flap_count": service.flap_count,
|
|
"last_state_change_at": _iso(service.last_state_change_at),
|
|
"created_at": service.created_at.isoformat(),
|
|
"updated_at": service.updated_at.isoformat(),
|
|
}
|
|
|
|
@classmethod
|
|
def _insert_service(cls, db: sqlite3.Connection, service: Service) -> None:
|
|
params = cls._service_params(service)
|
|
db.execute(
|
|
f"INSERT INTO services ({SERVICE_COLUMNS}) VALUES ("
|
|
":id, :name, :kind, :url, :host, :port, :depends_on, :description, :tags, :group, "
|
|
":environment, :owner, :enabled, :monitoring_enabled, :monitoring_state, :probe_source, :probe_reachability, :probe_confidence, :runtime_state, :runtime_health, :application, :domain, :component_role, :application_locked, :domain_locked, :component_role_locked, :criticality, :category, :visibility, :lifecycle, "
|
|
":source_type, :source_ref, :first_seen_at, :last_seen_at, :last_discovery_status, :container_image, "
|
|
":compose_project, :layout_x, :layout_y, :layout_pinned, "
|
|
":interval_seconds, :timeout_seconds, :retries, :failure_threshold, :recovery_threshold, "
|
|
":expected_status_min, :expected_status_max, :availability_target, :maintenance_enabled, "
|
|
":maintenance_reason, :state, :failure_streak, :recovery_streak, :flapping, :flap_count, "
|
|
":last_state_change_at, :created_at, :updated_at)",
|
|
params,
|
|
)
|
|
|
|
@classmethod
|
|
def _upsert_service(cls, db: sqlite3.Connection, service: Service) -> None:
|
|
params = cls._service_params(service)
|
|
db.execute(
|
|
f"INSERT INTO services ({SERVICE_COLUMNS}) VALUES ("
|
|
":id, :name, :kind, :url, :host, :port, :depends_on, :description, :tags, :group, "
|
|
":environment, :owner, :enabled, :monitoring_enabled, :monitoring_state, :probe_source, :probe_reachability, :probe_confidence, :runtime_state, :runtime_health, :application, :domain, :component_role, :application_locked, :domain_locked, :component_role_locked, :criticality, :category, :visibility, :lifecycle, "
|
|
":source_type, :source_ref, :first_seen_at, :last_seen_at, :last_discovery_status, :container_image, "
|
|
":compose_project, :layout_x, :layout_y, :layout_pinned, "
|
|
":interval_seconds, :timeout_seconds, :retries, :failure_threshold, :recovery_threshold, "
|
|
":expected_status_min, :expected_status_max, :availability_target, :maintenance_enabled, "
|
|
":maintenance_reason, :state, :failure_streak, :recovery_streak, :flapping, :flap_count, "
|
|
":last_state_change_at, :created_at, :updated_at) "
|
|
"ON CONFLICT(id) DO UPDATE SET "
|
|
"name=excluded.name, kind=excluded.kind, url=excluded.url, host=excluded.host, port=excluded.port, "
|
|
"depends_on=excluded.depends_on, description=excluded.description, tags=excluded.tags, "
|
|
'"group"=excluded."group", environment=excluded.environment, owner=excluded.owner, '
|
|
"enabled=excluded.enabled, monitoring_enabled=excluded.monitoring_enabled, monitoring_state=excluded.monitoring_state, "
|
|
"probe_source=excluded.probe_source, probe_reachability=excluded.probe_reachability, probe_confidence=excluded.probe_confidence, "
|
|
"runtime_state=excluded.runtime_state, runtime_health=excluded.runtime_health, application=excluded.application, domain=excluded.domain, component_role=excluded.component_role, "
|
|
"application_locked=excluded.application_locked, domain_locked=excluded.domain_locked, component_role_locked=excluded.component_role_locked, criticality=excluded.criticality, "
|
|
"category=excluded.category, visibility=excluded.visibility, lifecycle=excluded.lifecycle, "
|
|
"source_type=excluded.source_type, source_ref=excluded.source_ref, first_seen_at=excluded.first_seen_at, "
|
|
"last_seen_at=excluded.last_seen_at, last_discovery_status=excluded.last_discovery_status, "
|
|
"container_image=excluded.container_image, compose_project=excluded.compose_project, "
|
|
"layout_x=excluded.layout_x, layout_y=excluded.layout_y, layout_pinned=excluded.layout_pinned, "
|
|
"interval_seconds=excluded.interval_seconds, "
|
|
"timeout_seconds=excluded.timeout_seconds, retries=excluded.retries, failure_threshold=excluded.failure_threshold, "
|
|
"recovery_threshold=excluded.recovery_threshold, expected_status_min=excluded.expected_status_min, "
|
|
"expected_status_max=excluded.expected_status_max, availability_target=excluded.availability_target, "
|
|
"maintenance_enabled=excluded.maintenance_enabled, maintenance_reason=excluded.maintenance_reason, "
|
|
"state=excluded.state, failure_streak=excluded.failure_streak, recovery_streak=excluded.recovery_streak, "
|
|
"flapping=excluded.flapping, flap_count=excluded.flap_count, last_state_change_at=excluded.last_state_change_at, "
|
|
"created_at=excluded.created_at, updated_at=excluded.updated_at",
|
|
params,
|
|
)
|
|
|
|
def list_services(self) -> list[Service]:
|
|
with self._connect() as db:
|
|
rows = db.execute(f"SELECT {SERVICE_COLUMNS} FROM services ORDER BY name COLLATE NOCASE").fetchall()
|
|
return [self._row_to_service(row) for row in rows]
|
|
|
|
def get_service(self, service_id: str) -> Service | None:
|
|
with self._connect() as db:
|
|
row = db.execute(f"SELECT {SERVICE_COLUMNS} FROM services WHERE id = ?", (service_id,)).fetchone()
|
|
return self._row_to_service(row) if row else None
|
|
|
|
def get_service_by_name(self, name: str) -> Service | None:
|
|
with self._connect() as db:
|
|
row = db.execute(
|
|
f"SELECT {SERVICE_COLUMNS} FROM services WHERE name = ? COLLATE NOCASE", (name,)
|
|
).fetchone()
|
|
return self._row_to_service(row) if row else None
|
|
|
|
def create_service(self, payload: ServiceCreate) -> Service:
|
|
now = _now()
|
|
service = Service(id=str(uuid4()), created_at=now, updated_at=now, **payload.model_dump())
|
|
with self._connect() as db:
|
|
self._insert_service(db, service)
|
|
self._insert_event_db(db, "topology.changed", None, None, None, None, f"Service {service.name} created", {})
|
|
event_bus.publish({"event_type": "topology.changed", "message": f"Service {service.name} created"})
|
|
return service
|
|
|
|
def update_service(self, service_id: str, payload: ServiceUpdate) -> Service | None:
|
|
existing = self.get_service(service_id)
|
|
if existing is None:
|
|
return None
|
|
updated = Service(
|
|
id=service_id,
|
|
created_at=existing.created_at,
|
|
updated_at=_now(),
|
|
state=existing.state,
|
|
failure_streak=existing.failure_streak,
|
|
recovery_streak=existing.recovery_streak,
|
|
flapping=existing.flapping,
|
|
flap_count=existing.flap_count,
|
|
last_state_change_at=existing.last_state_change_at,
|
|
**payload.model_dump(),
|
|
)
|
|
with self._connect() as db:
|
|
self._upsert_service(db, updated)
|
|
self._insert_event_db(db, "topology.changed", service_id, None, None, None, f"Service {updated.name} updated", {})
|
|
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service updated"})
|
|
return updated
|
|
|
|
def delete_service(self, service_id: str) -> bool:
|
|
with self._connect() as db:
|
|
row = db.execute("SELECT name FROM services WHERE id = ?", (service_id,)).fetchone()
|
|
if row is None:
|
|
return False
|
|
db.execute("DELETE FROM services WHERE id = ?", (service_id,))
|
|
rows = db.execute("SELECT id, depends_on FROM services").fetchall()
|
|
for item in rows:
|
|
dependencies = self._json_list(item["depends_on"])
|
|
if service_id in dependencies:
|
|
db.execute(
|
|
"UPDATE services SET depends_on = ?, updated_at = ? WHERE id = ?",
|
|
(json.dumps([value for value in dependencies if value != service_id]), _now().isoformat(), item["id"]),
|
|
)
|
|
db.execute("UPDATE incidents SET affected_service_ids = replace(affected_service_ids, ?, '') WHERE status != 'resolved'", (service_id,))
|
|
self._insert_event_db(db, "topology.changed", service_id, None, None, None, f"Service {row['name']} deleted", {})
|
|
event_bus.publish({"event_type": "topology.changed", "service_id": service_id, "message": "Service deleted"})
|
|
return True
|
|
|
|
def import_config_services(self, definitions: list[ConfigService], mode: str) -> ConfigImportResult:
|
|
if mode not in {"merge", "replace"}:
|
|
raise ValueError("Import mode must be merge or replace")
|
|
|
|
with self._connect() as db:
|
|
db.execute("BEGIN IMMEDIATE")
|
|
rows = db.execute(f"SELECT {SERVICE_COLUMNS} FROM services").fetchall()
|
|
existing = {row["name"].casefold(): self._row_to_service(row) for row in rows}
|
|
if mode == "replace":
|
|
db.execute("DELETE FROM incidents")
|
|
db.execute("DELETE FROM events")
|
|
db.execute("DELETE FROM services")
|
|
existing = {}
|
|
|
|
ids_by_name = {name: service.id for name, service in existing.items()}
|
|
for definition in definitions:
|
|
ids_by_name.setdefault(definition.name.casefold(), str(uuid4()))
|
|
for definition in definitions:
|
|
missing = [item for item in definition.depends_on if item.casefold() not in ids_by_name]
|
|
if missing:
|
|
raise ValueError(
|
|
f"Service {definition.name} references unknown dependencies: {', '.join(missing)}"
|
|
)
|
|
|
|
created = 0
|
|
updated = 0
|
|
now = _now()
|
|
for definition in definitions:
|
|
key = definition.name.casefold()
|
|
current = existing.get(key)
|
|
payload = definition.model_dump()
|
|
payload["depends_on"] = [ids_by_name[item.casefold()] for item in definition.depends_on]
|
|
if current is None:
|
|
service = Service(id=ids_by_name[key], created_at=now, updated_at=now, **payload)
|
|
created += 1
|
|
else:
|
|
service = Service(
|
|
id=current.id,
|
|
created_at=current.created_at,
|
|
updated_at=now,
|
|
state=current.state,
|
|
failure_streak=current.failure_streak,
|
|
recovery_streak=current.recovery_streak,
|
|
flapping=current.flapping,
|
|
flap_count=current.flap_count,
|
|
last_state_change_at=current.last_state_change_at,
|
|
**payload,
|
|
)
|
|
updated += 1
|
|
if current is None:
|
|
self._insert_service(db, service)
|
|
else:
|
|
self._upsert_service(db, service)
|
|
self._insert_event_db(db, "topology.changed", None, None, None, None, "Configuration imported", {"mode": mode})
|
|
event_bus.publish({"event_type": "topology.changed", "message": "Configuration imported"})
|
|
return ConfigImportResult(created=created, updated=updated, total=len(definitions))
|
|
|
|
def replace_all_services(self, definitions: list[ServiceCreate]) -> list[Service]:
|
|
with self._connect() as db:
|
|
db.execute("DELETE FROM services")
|
|
return [self.create_service(item) for item in definitions]
|
|
|
|
@staticmethod
|
|
def _row_to_health_evidence(row: sqlite3.Row) -> HealthEvidence:
|
|
return with_freshness(
|
|
HealthEvidence(
|
|
id=row["id"],
|
|
subject_type=row["subject_type"],
|
|
subject_id=row["subject_id"],
|
|
service_id=row["service_id"],
|
|
provider=row["provider"],
|
|
provider_ref=row["provider_ref"],
|
|
status=row["status"],
|
|
scope=row["scope"],
|
|
confidence=float(row["confidence"]),
|
|
observed_at=datetime.fromisoformat(row["observed_at"]),
|
|
valid_until=datetime.fromisoformat(row["valid_until"]),
|
|
received_at=datetime.fromisoformat(row["received_at"]),
|
|
summary=row["summary"] or "",
|
|
metadata=Store._json_object(row["metadata"]),
|
|
active=_bool(row["active"], True),
|
|
superseded_by=row["superseded_by"],
|
|
)
|
|
)
|
|
|
|
def record_health_evidence(
|
|
self,
|
|
evidence: HealthEvidence,
|
|
*,
|
|
service_ids: list[str] | None = None,
|
|
) -> HealthEvidence:
|
|
"""Store a current provider claim and retain superseded provenance.
|
|
|
|
Repeated discovery cycles renew the same active claim instead of
|
|
growing unbounded identical rows. A changed claim deactivates only
|
|
the prior claim from that provider/subject; other providers remain
|
|
independently auditable and can surface a conflict.
|
|
"""
|
|
|
|
links = sorted({*(service_ids or []), *({evidence.service_id} if evidence.service_id else set())})
|
|
with self._connect() as db:
|
|
prior = db.execute(
|
|
"""
|
|
SELECT * FROM health_evidence
|
|
WHERE subject_type = ? AND subject_id = ? AND provider = ?
|
|
AND COALESCE(provider_ref, '') = COALESCE(?, '') AND scope = ? AND active = 1
|
|
ORDER BY observed_at DESC, rowid DESC LIMIT 1
|
|
""",
|
|
(
|
|
evidence.subject_type,
|
|
evidence.subject_id,
|
|
evidence.provider.value,
|
|
evidence.provider_ref,
|
|
evidence.scope.value,
|
|
),
|
|
).fetchone()
|
|
saved = evidence
|
|
materially_changed = prior is None or any(
|
|
(
|
|
prior[field] != value
|
|
for field, value in (
|
|
("status", evidence.status.value),
|
|
("confidence", evidence.confidence),
|
|
("summary", evidence.summary),
|
|
("metadata", json.dumps(evidence.metadata, sort_keys=True)),
|
|
)
|
|
)
|
|
)
|
|
if prior is not None and prior["status"] == evidence.status.value:
|
|
saved = evidence.model_copy(update={"id": prior["id"], "active": True, "superseded_by": None})
|
|
db.execute(
|
|
"""
|
|
UPDATE health_evidence SET service_id = ?, confidence = ?, observed_at = ?, valid_until = ?,
|
|
received_at = ?, summary = ?, metadata = ?, active = 1, superseded_by = NULL
|
|
WHERE id = ?
|
|
""",
|
|
(
|
|
saved.service_id,
|
|
saved.confidence,
|
|
saved.observed_at.isoformat(),
|
|
saved.valid_until.isoformat(),
|
|
saved.received_at.isoformat(),
|
|
saved.summary,
|
|
json.dumps(saved.metadata, sort_keys=True),
|
|
saved.id,
|
|
),
|
|
)
|
|
else:
|
|
if prior is not None:
|
|
db.execute("UPDATE health_evidence SET active = 0, superseded_by = ? WHERE id = ?", (evidence.id, prior["id"]))
|
|
db.execute(
|
|
"""
|
|
INSERT INTO health_evidence (
|
|
id, subject_type, subject_id, service_id, provider, provider_ref, status, scope,
|
|
confidence, observed_at, valid_until, received_at, summary, metadata, active, superseded_by
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
saved.id,
|
|
saved.subject_type,
|
|
saved.subject_id,
|
|
saved.service_id,
|
|
saved.provider.value,
|
|
saved.provider_ref,
|
|
saved.status.value,
|
|
saved.scope.value,
|
|
saved.confidence,
|
|
saved.observed_at.isoformat(),
|
|
saved.valid_until.isoformat(),
|
|
saved.received_at.isoformat(),
|
|
saved.summary,
|
|
json.dumps(saved.metadata, sort_keys=True),
|
|
int(saved.active),
|
|
saved.superseded_by,
|
|
),
|
|
)
|
|
for service_id in links:
|
|
db.execute(
|
|
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
|
|
(saved.id, service_id),
|
|
)
|
|
if materially_changed:
|
|
event_bus.publish(
|
|
{
|
|
"event_type": "health.evidence_changed",
|
|
"service_id": evidence.service_id,
|
|
"provider": evidence.provider.value,
|
|
"status": evidence.status.value,
|
|
"message": "Health evidence changed",
|
|
}
|
|
)
|
|
return saved
|
|
|
|
def health_evidence_for_service(self, service_id: str, *, limit: int = 100) -> list[HealthEvidence]:
|
|
with self._connect() as db:
|
|
rows = db.execute(
|
|
"""
|
|
SELECT DISTINCT e.* FROM health_evidence e
|
|
LEFT JOIN health_evidence_service_links link ON link.evidence_id = e.id
|
|
WHERE e.service_id = ? OR link.service_id = ?
|
|
ORDER BY e.observed_at DESC, e.rowid DESC LIMIT ?
|
|
""",
|
|
(service_id, service_id, limit),
|
|
).fetchall()
|
|
return [self._row_to_health_evidence(row) for row in rows]
|
|
|
|
def health_evidence_for_services(
|
|
self, service_ids: list[str], *, limit_per_service: int = 100
|
|
) -> dict[str, list[HealthEvidence]]:
|
|
"""Fetch linked evidence for a topology in one bounded query.
|
|
|
|
A semantic topology can contain many applications. Resolving each
|
|
application with a separate evidence query creates an avoidable N+1
|
|
SQLite scan, so this method projects the evidence links once and then
|
|
groups them in memory.
|
|
"""
|
|
|
|
ids = sorted(set(service_ids))
|
|
result = {service_id: [] for service_id in ids}
|
|
if not ids:
|
|
return result
|
|
placeholders = ", ".join("?" for _ in ids)
|
|
with self._connect() as db:
|
|
rows = db.execute(
|
|
f"""
|
|
SELECT e.*, e.service_id AS direct_service_id, link.service_id AS linked_service_id
|
|
FROM health_evidence e
|
|
LEFT JOIN health_evidence_service_links link ON link.evidence_id = e.id
|
|
WHERE e.service_id IN ({placeholders}) OR link.service_id IN ({placeholders})
|
|
ORDER BY e.observed_at DESC, e.rowid DESC
|
|
""",
|
|
[*ids, *ids],
|
|
).fetchall()
|
|
seen: dict[str, set[str]] = {service_id: set() for service_id in ids}
|
|
for row in rows:
|
|
evidence = self._row_to_health_evidence(row)
|
|
for service_id in {row["direct_service_id"], row["linked_service_id"]}:
|
|
if (
|
|
service_id in result
|
|
and evidence.id not in seen[service_id]
|
|
and len(result[service_id]) < limit_per_service
|
|
):
|
|
result[service_id].append(evidence)
|
|
seen[service_id].add(evidence.id)
|
|
return result
|
|
|
|
def application_evidence_links(self, provider: EvidenceProvider) -> dict[str, set[str]]:
|
|
"""Return persisted application-claim links keyed by provider reference."""
|
|
|
|
with self._connect() as db:
|
|
rows = db.execute(
|
|
"""
|
|
SELECT e.provider_ref, e.subject_id, link.service_id
|
|
FROM health_evidence e
|
|
JOIN health_evidence_service_links link ON link.evidence_id = e.id
|
|
WHERE e.provider = ? AND e.scope = ?
|
|
""",
|
|
(provider.value, EvidenceScope.APPLICATION.value),
|
|
).fetchall()
|
|
result: dict[str, set[str]] = {}
|
|
for row in rows:
|
|
key = row["provider_ref"] or row["subject_id"]
|
|
if key:
|
|
result.setdefault(key, set()).add(row["service_id"])
|
|
return result
|
|
|
|
def health_resolution(self, service_id: str) -> HealthResolution:
|
|
return resolve_health(self.health_evidence_for_service(service_id))
|
|
|
|
def set_provider_status(self, value: ProviderStatus) -> ProviderStatus:
|
|
with self._connect() as db:
|
|
previous = db.execute(
|
|
"SELECT status, detail, metadata FROM provider_status WHERE provider = ?",
|
|
(value.provider.value,),
|
|
).fetchone()
|
|
db.execute(
|
|
"""
|
|
INSERT INTO provider_status (provider, status, observed_at, valid_until, detail, metadata)
|
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(provider) DO UPDATE SET status=excluded.status, observed_at=excluded.observed_at,
|
|
valid_until=excluded.valid_until, detail=excluded.detail, metadata=excluded.metadata
|
|
""",
|
|
(
|
|
value.provider.value,
|
|
value.status.value,
|
|
value.observed_at.isoformat(),
|
|
value.valid_until.isoformat(),
|
|
value.detail,
|
|
json.dumps(value.metadata, sort_keys=True),
|
|
),
|
|
)
|
|
if previous is None or any(
|
|
(
|
|
previous[field] != expected
|
|
for field, expected in (
|
|
("status", value.status.value),
|
|
("detail", value.detail),
|
|
("metadata", json.dumps(value.metadata, sort_keys=True)),
|
|
)
|
|
)
|
|
):
|
|
event_bus.publish(
|
|
{
|
|
"event_type": "provider.status_changed",
|
|
"provider": value.provider.value,
|
|
"status": value.status.value,
|
|
"message": "Provider availability changed",
|
|
}
|
|
)
|
|
return value
|
|
|
|
def provider_statuses(self) -> list[ProviderStatus]:
|
|
with self._connect() as db:
|
|
rows = db.execute("SELECT * FROM provider_status ORDER BY provider").fetchall()
|
|
return [
|
|
ProviderStatus(
|
|
provider=row["provider"],
|
|
status=row["status"],
|
|
observed_at=datetime.fromisoformat(row["observed_at"]),
|
|
valid_until=datetime.fromisoformat(row["valid_until"]),
|
|
detail=row["detail"] or "",
|
|
metadata=self._json_object(row["metadata"]),
|
|
)
|
|
for row in rows
|
|
]
|
|
|
|
@staticmethod
|
|
def _legacy_health_from_row(row: sqlite3.Row) -> HealthStatus:
|
|
"""Compatibility fallback for pre-v1.6 data without component evidence."""
|
|
|
|
runtime_state = str(row["runtime_state"] or "").casefold()
|
|
runtime_health = str(row["runtime_health"] or "").casefold()
|
|
if runtime_health == "unhealthy" or runtime_state in {"exited", "dead", "stopped"}:
|
|
return HealthStatus.DOWN
|
|
if runtime_state == "restarting":
|
|
return HealthStatus.DEGRADED
|
|
if not _bool(row["monitoring_enabled"], True):
|
|
if runtime_health == "healthy":
|
|
return HealthStatus.HEALTHY
|
|
if str(row["monitoring_state"] or "").casefold() in {"unmonitored", "observed"}:
|
|
return HealthStatus.UNMONITORED
|
|
return HealthStatus.UNKNOWN
|
|
try:
|
|
return HealthStatus(row["state"] or "unknown")
|
|
except ValueError:
|
|
return HealthStatus.UNKNOWN
|
|
|
|
def service_health(self, service_id: str) -> HealthStatus:
|
|
with self._connect() as db:
|
|
row = db.execute(
|
|
"SELECT state, maintenance_enabled, enabled, monitoring_enabled, monitoring_state, runtime_state, runtime_health "
|
|
"FROM services WHERE id = ?",
|
|
(service_id,),
|
|
).fetchone()
|
|
if row is None or not _bool(row["enabled"], True):
|
|
return HealthStatus.UNKNOWN
|
|
if _bool(row["maintenance_enabled"]):
|
|
return HealthStatus.MAINTENANCE
|
|
resolution = self.health_resolution(service_id)
|
|
if resolution.selected_evidence_id:
|
|
return resolution.status
|
|
# Once v1.6 evidence exists, an expired/stale claim must not silently
|
|
# keep a historical state alive. Legacy fallback is only for rows
|
|
# that predate evidence entirely. Old AppOps-only rows were always
|
|
# aggregate application observations, never component health.
|
|
if resolution.evidence:
|
|
return HealthStatus.UNKNOWN
|
|
if str(row["monitoring_state"] or "").casefold() == MonitoringState.APPOPS.value:
|
|
return HealthStatus.UNMONITORED
|
|
return self._legacy_health_from_row(row)
|
|
|
|
def service_healths(
|
|
self,
|
|
service_ids: list[str],
|
|
*,
|
|
evidence_by_service: dict[str, list[HealthEvidence]] | None = None,
|
|
) -> tuple[dict[str, HealthStatus], dict[str, HealthResolution]]:
|
|
"""Resolve direct health for a service set without per-service SQL reads."""
|
|
|
|
ids = sorted(set(service_ids))
|
|
if not ids:
|
|
return {}, {}
|
|
evidence_by_service = evidence_by_service or self.health_evidence_for_services(ids)
|
|
placeholders = ", ".join("?" for _ in ids)
|
|
with self._connect() as db:
|
|
rows = db.execute(
|
|
f"""
|
|
SELECT id, state, maintenance_enabled, enabled, monitoring_enabled,
|
|
monitoring_state, runtime_state, runtime_health
|
|
FROM services WHERE id IN ({placeholders})
|
|
""",
|
|
ids,
|
|
).fetchall()
|
|
status_by_id: dict[str, HealthStatus] = {}
|
|
resolutions: dict[str, HealthResolution] = {}
|
|
for row in rows:
|
|
service_id = row["id"]
|
|
resolution = resolve_health(evidence_by_service.get(service_id, []))
|
|
resolutions[service_id] = resolution
|
|
if not _bool(row["enabled"], True):
|
|
status_by_id[service_id] = HealthStatus.UNKNOWN
|
|
elif _bool(row["maintenance_enabled"]):
|
|
status_by_id[service_id] = HealthStatus.MAINTENANCE
|
|
elif resolution.selected_evidence_id:
|
|
status_by_id[service_id] = resolution.status
|
|
elif resolution.evidence:
|
|
status_by_id[service_id] = HealthStatus.UNKNOWN
|
|
elif str(row["monitoring_state"] or "").casefold() == MonitoringState.APPOPS.value:
|
|
status_by_id[service_id] = HealthStatus.UNMONITORED
|
|
else:
|
|
status_by_id[service_id] = self._legacy_health_from_row(row)
|
|
return status_by_id, resolutions
|
|
|
|
def set_maintenance(self, service_id: str, enabled: bool, reason: str | None = None) -> Service | None:
|
|
service = self.get_service(service_id)
|
|
if service is None:
|
|
return None
|
|
updated = service.model_copy(
|
|
update={
|
|
"maintenance_enabled": enabled,
|
|
"maintenance_reason": reason if enabled else None,
|
|
"state": HealthStatus.MAINTENANCE if enabled else HealthStatus.UNKNOWN,
|
|
"failure_streak": 0,
|
|
"recovery_streak": 0,
|
|
"updated_at": _now(),
|
|
}
|
|
)
|
|
with self._connect() as db:
|
|
self._upsert_service(db, updated)
|
|
self._insert_event_db(
|
|
db,
|
|
"service.maintenance_changed",
|
|
service_id,
|
|
None,
|
|
service.state.value,
|
|
HealthStatus.MAINTENANCE.value if enabled else HealthStatus.UNKNOWN.value,
|
|
f"Maintenance {'enabled' if enabled else 'disabled'} for {service.name}",
|
|
{"reason": reason} if reason else {},
|
|
)
|
|
event_bus.publish(
|
|
{
|
|
"event_type": "service.maintenance_changed",
|
|
"service_id": service_id,
|
|
"status": HealthStatus.MAINTENANCE.value if enabled else HealthStatus.UNKNOWN.value,
|
|
"message": f"Maintenance {'enabled' if enabled else 'disabled'}",
|
|
}
|
|
)
|
|
from .incidents import sync_incidents
|
|
|
|
sync_incidents(self)
|
|
return updated
|
|
|
|
def _stable_state(self, row: sqlite3.Row, raw_status: HealthStatus) -> tuple[HealthStatus, int, int]:
|
|
if not _bool(row["enabled"], True) or not _bool(row["monitoring_enabled"], True):
|
|
monitoring_state = str(row["monitoring_state"] or "").casefold()
|
|
return (HealthStatus.UNMONITORED if monitoring_state in {"unmonitored", "observed"} else HealthStatus.UNKNOWN), 0, 0
|
|
if _bool(row["maintenance_enabled"]):
|
|
return HealthStatus.MAINTENANCE, 0, 0
|
|
previous = HealthStatus(row["state"] or "unknown")
|
|
failure_streak = int(row["failure_streak"] or 0)
|
|
recovery_streak = int(row["recovery_streak"] or 0)
|
|
failure_threshold = max(1, int(row["failure_threshold"] or 1))
|
|
recovery_threshold = max(1, int(row["recovery_threshold"] or 1))
|
|
if raw_status in {HealthStatus.DOWN, HealthStatus.DEGRADED}:
|
|
failure_streak += 1
|
|
recovery_streak = 0
|
|
if failure_streak < failure_threshold:
|
|
if previous == HealthStatus.UNKNOWN:
|
|
return HealthStatus.UNKNOWN, failure_streak, recovery_streak
|
|
if previous == HealthStatus.RECOVERING:
|
|
return HealthStatus.RECOVERING, failure_streak, recovery_streak
|
|
return previous, failure_streak, recovery_streak
|
|
return raw_status, failure_streak, recovery_streak
|
|
if raw_status == HealthStatus.HEALTHY:
|
|
recovery_streak += 1
|
|
failure_streak = 0
|
|
if previous in {HealthStatus.DOWN, HealthStatus.DEGRADED, HealthStatus.RECOVERING} and recovery_streak < recovery_threshold:
|
|
return HealthStatus.RECOVERING, failure_streak, recovery_streak
|
|
return HealthStatus.HEALTHY, failure_streak, recovery_streak
|
|
return HealthStatus.UNMONITORED if raw_status == HealthStatus.UNMONITORED else HealthStatus.UNKNOWN, 0, 0
|
|
|
|
def set_runtime_observation(
|
|
self,
|
|
service_id: str,
|
|
*,
|
|
runtime_state: str | None,
|
|
runtime_health: str | None,
|
|
monitoring_state: MonitoringState | str,
|
|
provider: EvidenceProvider = EvidenceProvider.DOCKER,
|
|
provider_ref: str | None = None,
|
|
confidence: float = 0.9,
|
|
) -> Service | None:
|
|
"""Persist Docker runtime truth without fabricating a network probe."""
|
|
|
|
service = self.get_service(service_id)
|
|
if service is None:
|
|
return None
|
|
state_kind = MonitoringState(monitoring_state)
|
|
normalized_state = (runtime_state or "").casefold()
|
|
normalized_health = (runtime_health or "").casefold()
|
|
if service.maintenance_enabled:
|
|
status = HealthStatus.MAINTENANCE
|
|
elif normalized_health in {"unhealthy", "down"} or normalized_state in {"exited", "dead", "stopped"}:
|
|
status = HealthStatus.DOWN
|
|
elif normalized_health == "degraded" or normalized_state == "restarting":
|
|
status = HealthStatus.DEGRADED
|
|
elif normalized_health == "healthy":
|
|
status = HealthStatus.HEALTHY
|
|
elif state_kind in {
|
|
MonitoringState.UNMONITORED,
|
|
MonitoringState.OBSERVED,
|
|
MonitoringState.APPOPS,
|
|
}:
|
|
status = HealthStatus.UNMONITORED
|
|
else:
|
|
status = service.state
|
|
updated = service.model_copy(
|
|
update={
|
|
"runtime_state": runtime_state,
|
|
"runtime_health": runtime_health,
|
|
"monitoring_state": state_kind,
|
|
"state": status,
|
|
"failure_streak": 0 if status in {HealthStatus.UNMONITORED, HealthStatus.HEALTHY} else service.failure_streak,
|
|
"recovery_streak": 0 if status == HealthStatus.UNMONITORED else service.recovery_streak,
|
|
"last_state_change_at": _now() if service.state != status else service.last_state_change_at,
|
|
"updated_at": _now(),
|
|
}
|
|
)
|
|
with self._connect() as db:
|
|
self._upsert_service(db, updated)
|
|
if service.state != status:
|
|
self._insert_event_db(
|
|
db,
|
|
"service.runtime_observed",
|
|
service_id,
|
|
None,
|
|
service.state.value,
|
|
status.value,
|
|
f"Docker runtime observation updated for {service.name}",
|
|
{"runtime_state": runtime_state, "runtime_health": runtime_health, "monitoring_state": state_kind.value},
|
|
)
|
|
if service.state != status:
|
|
event_bus.publish({"event_type": "service.runtime_observed", "service_id": service_id, "status": status.value, "message": "Docker runtime observation updated"})
|
|
observed_at = _now()
|
|
self.record_health_evidence(
|
|
HealthEvidence(
|
|
id=str(uuid4()),
|
|
subject_type="service",
|
|
subject_id=service_id,
|
|
service_id=service_id,
|
|
provider=provider,
|
|
provider_ref=provider_ref or service.source_ref,
|
|
status=status,
|
|
scope=EvidenceScope.COMPONENT,
|
|
confidence=confidence,
|
|
observed_at=observed_at,
|
|
valid_until=observed_at + timedelta(seconds=max(60, int(service.interval_seconds or 60) * 2)),
|
|
received_at=observed_at,
|
|
summary="Docker runtime observation" if provider == EvidenceProvider.DOCKER else "Runtime observation",
|
|
metadata={"runtime_state": runtime_state, "runtime_health": runtime_health},
|
|
)
|
|
)
|
|
return updated
|
|
|
|
def save_probe(self, probe: ProbeResult) -> ProbeResult:
|
|
events: list[dict[str, Any]] = []
|
|
with self._connect() as db:
|
|
row = db.execute("SELECT * FROM services WHERE id = ?", (probe.service_id,)).fetchone()
|
|
if row is None:
|
|
return probe
|
|
db.execute(
|
|
"""
|
|
INSERT INTO probe_results (id, service_id, status, latency_ms, detail, checked_at, trigger, metadata)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
probe.id,
|
|
probe.service_id,
|
|
probe.status.value,
|
|
probe.latency_ms,
|
|
probe.detail,
|
|
probe.checked_at.isoformat(),
|
|
probe.trigger,
|
|
json.dumps(probe.metadata),
|
|
),
|
|
)
|
|
previous = HealthStatus(row["state"] or "unknown")
|
|
next_state, failure_streak, recovery_streak = self._stable_state(row, probe.status)
|
|
now = probe.checked_at
|
|
state_changed = next_state != previous
|
|
last_state_change = now.isoformat() if state_changed else row["last_state_change_at"]
|
|
if state_changed:
|
|
event = {
|
|
"event_type": "service.state_changed",
|
|
"service_id": probe.service_id,
|
|
"from_status": previous.value,
|
|
"to_status": next_state.value,
|
|
"message": f"Service changed from {previous.value} to {next_state.value}",
|
|
"metadata": {"trigger": probe.trigger, "probe_id": probe.id},
|
|
}
|
|
self._insert_event_db(
|
|
db,
|
|
event["event_type"],
|
|
probe.service_id,
|
|
None,
|
|
previous.value,
|
|
next_state.value,
|
|
event["message"],
|
|
event["metadata"],
|
|
created_at=now,
|
|
)
|
|
events.append(event)
|
|
since = (now - timedelta(minutes=30)).isoformat()
|
|
flap_count = int(
|
|
db.execute(
|
|
"SELECT COUNT(*) FROM events WHERE service_id = ? AND event_type = 'service.state_changed' AND created_at >= ?",
|
|
(probe.service_id, since),
|
|
).fetchone()[0]
|
|
)
|
|
flapping = flap_count >= int(os.getenv("SERVICEMAP_FLAP_THRESHOLD", "6"))
|
|
if flapping and not _bool(row["flapping"]):
|
|
self._insert_event_db(
|
|
db,
|
|
"flapping.detected",
|
|
probe.service_id,
|
|
None,
|
|
None,
|
|
None,
|
|
f"Flapping detected: {flap_count} state changes in 30 minutes",
|
|
{"state_changes": flap_count, "window_minutes": 30},
|
|
created_at=now,
|
|
)
|
|
events.append({"event_type": "flapping.detected", "service_id": probe.service_id, "message": "Flapping detected"})
|
|
db.execute(
|
|
"""
|
|
UPDATE services
|
|
SET state = ?, failure_streak = ?, recovery_streak = ?, flapping = ?, flap_count = ?,
|
|
last_state_change_at = ?, updated_at = ?
|
|
WHERE id = ?
|
|
""",
|
|
(
|
|
next_state.value,
|
|
failure_streak,
|
|
recovery_streak,
|
|
int(flapping),
|
|
flap_count,
|
|
last_state_change,
|
|
now.isoformat(),
|
|
probe.service_id,
|
|
),
|
|
)
|
|
event_bus.publish(
|
|
{
|
|
"event_type": "probe.completed",
|
|
"service_id": probe.service_id,
|
|
"status": probe.status.value,
|
|
"trigger": probe.trigger,
|
|
"message": "Probe completed",
|
|
}
|
|
)
|
|
for event in events:
|
|
event_bus.publish(event)
|
|
self.record_health_evidence(
|
|
HealthEvidence(
|
|
id=f"probe-{probe.id}",
|
|
subject_type="service",
|
|
subject_id=probe.service_id,
|
|
service_id=probe.service_id,
|
|
provider=EvidenceProvider.PROBE,
|
|
provider_ref="active-probe",
|
|
# The evidence mirrors the hysteresis-filtered service state,
|
|
# while the raw probe result remains intact in history.
|
|
status=next_state,
|
|
scope=EvidenceScope.COMPONENT,
|
|
confidence=1.0,
|
|
observed_at=probe.checked_at,
|
|
valid_until=probe.checked_at + timedelta(seconds=max(60, int(os.getenv("SERVICEMAP_PROBE_INTERVAL", "60")) * 2)),
|
|
received_at=_now(),
|
|
summary=probe.detail,
|
|
metadata={"probe_id": probe.id, "trigger": probe.trigger, "raw_status": probe.status.value, "latency_ms": probe.latency_ms, **probe.metadata},
|
|
)
|
|
)
|
|
from .incidents import sync_incidents
|
|
|
|
sync_incidents(self)
|
|
return probe
|
|
|
|
def latest_probe(self, service_id: str) -> ProbeResult | None:
|
|
with self._connect() as db:
|
|
row = db.execute(
|
|
"SELECT * FROM probe_results WHERE service_id = ? ORDER BY checked_at DESC, rowid DESC LIMIT 1",
|
|
(service_id,),
|
|
).fetchone()
|
|
return self._row_to_probe(row) if row else None
|
|
|
|
def latest_probes(self, service_ids: list[str] | None = None) -> dict[str, ProbeResult]:
|
|
with self._connect() as db:
|
|
if service_ids:
|
|
placeholders = ",".join("?" for _ in service_ids)
|
|
rows = db.execute(
|
|
f"""
|
|
SELECT p.* FROM probe_results p
|
|
WHERE p.service_id IN ({placeholders})
|
|
AND p.rowid = (
|
|
SELECT candidate.rowid FROM probe_results candidate
|
|
WHERE candidate.service_id = p.service_id
|
|
ORDER BY candidate.checked_at DESC, candidate.rowid DESC LIMIT 1
|
|
)
|
|
""",
|
|
service_ids,
|
|
).fetchall()
|
|
else:
|
|
rows = db.execute(
|
|
"""
|
|
SELECT p.* FROM probe_results p
|
|
WHERE p.rowid = (
|
|
SELECT candidate.rowid FROM probe_results candidate
|
|
WHERE candidate.service_id = p.service_id
|
|
ORDER BY candidate.checked_at DESC, candidate.rowid DESC LIMIT 1
|
|
)
|
|
"""
|
|
).fetchall()
|
|
return {row["service_id"]: self._row_to_probe(row) for row in rows}
|
|
|
|
def probe_history(self, service_id: str, limit: int = 50) -> list[ProbeResult]:
|
|
with self._connect() as db:
|
|
rows = db.execute(
|
|
"SELECT * FROM probe_results WHERE service_id = ? ORDER BY checked_at DESC, rowid DESC LIMIT ?",
|
|
(service_id, limit),
|
|
).fetchall()
|
|
return [self._row_to_probe(row) for row in rows]
|
|
|
|
def availability_summary(self, service_id: str, hours: int = 24) -> AvailabilitySummary:
|
|
hours = max(1, min(hours, 24 * 90))
|
|
since = _now() - timedelta(hours=hours)
|
|
with self._connect() as db:
|
|
row = db.execute(
|
|
"""
|
|
SELECT COUNT(*) AS total,
|
|
SUM(CASE WHEN status = 'healthy' THEN 1 ELSE 0 END) AS healthy,
|
|
SUM(CASE WHEN status = 'degraded' THEN 1 ELSE 0 END) AS degraded,
|
|
SUM(CASE WHEN status = 'down' THEN 1 ELSE 0 END) AS down_count,
|
|
AVG(latency_ms) AS avg_latency, MAX(checked_at) AS last_checked
|
|
FROM probe_results WHERE service_id = ? AND checked_at >= ?
|
|
""",
|
|
(service_id, since.isoformat()),
|
|
).fetchone()
|
|
service_row = db.execute("SELECT availability_target FROM services WHERE id = ?", (service_id,)).fetchone()
|
|
total = int(row["total"] or 0)
|
|
healthy = int(row["healthy"] or 0)
|
|
degraded = int(row["degraded"] or 0)
|
|
down = int(row["down_count"] or 0)
|
|
uptime = round(((healthy + degraded) / total) * 100, 2) if total else None
|
|
target = service_row["availability_target"] if service_row else None
|
|
return AvailabilitySummary(
|
|
hours=hours,
|
|
total_checks=total,
|
|
healthy_checks=healthy,
|
|
degraded_checks=degraded,
|
|
down_checks=down,
|
|
uptime_percent=uptime,
|
|
average_latency_ms=round(float(row["avg_latency"]), 2) if row["avg_latency"] is not None else None,
|
|
last_checked_at=datetime.fromisoformat(row["last_checked"]) if row["last_checked"] else None,
|
|
target_percent=target,
|
|
meets_target=(uptime >= target) if uptime is not None and target is not None else None,
|
|
)
|
|
|
|
def availability_summaries(self, service_ids: list[str], hours: int = 24) -> dict[str, AvailabilitySummary]:
|
|
"""Fetch table-ready availability in one grouped query, not N requests."""
|
|
if not service_ids:
|
|
return {}
|
|
hours = max(1, min(hours, 24 * 90))
|
|
since = _now() - timedelta(hours=hours)
|
|
placeholders = ",".join("?" for _ in service_ids)
|
|
with self._connect() as db:
|
|
rows = db.execute(
|
|
f"""
|
|
SELECT s.id, s.availability_target, COUNT(p.id) AS total,
|
|
SUM(CASE WHEN p.status = 'healthy' THEN 1 ELSE 0 END) AS healthy,
|
|
SUM(CASE WHEN p.status = 'degraded' THEN 1 ELSE 0 END) AS degraded,
|
|
SUM(CASE WHEN p.status = 'down' THEN 1 ELSE 0 END) AS down_count,
|
|
AVG(p.latency_ms) AS avg_latency, MAX(p.checked_at) AS last_checked
|
|
FROM services s
|
|
LEFT JOIN probe_results p ON p.service_id=s.id AND p.checked_at >= ?
|
|
WHERE s.id IN ({placeholders})
|
|
GROUP BY s.id, s.availability_target
|
|
""",
|
|
[since.isoformat(), *service_ids],
|
|
).fetchall()
|
|
result: dict[str, AvailabilitySummary] = {}
|
|
for row in rows:
|
|
total = int(row["total"] or 0)
|
|
healthy = int(row["healthy"] or 0)
|
|
degraded = int(row["degraded"] or 0)
|
|
target = row["availability_target"]
|
|
uptime = round(((healthy + degraded) / total) * 100, 2) if total else None
|
|
result[row["id"]] = AvailabilitySummary(
|
|
hours=hours,
|
|
total_checks=total,
|
|
healthy_checks=healthy,
|
|
degraded_checks=degraded,
|
|
down_checks=int(row["down_count"] or 0),
|
|
uptime_percent=uptime,
|
|
average_latency_ms=round(float(row["avg_latency"]), 2) if row["avg_latency"] is not None else None,
|
|
last_checked_at=datetime.fromisoformat(row["last_checked"]) if row["last_checked"] else None,
|
|
target_percent=target,
|
|
meets_target=(uptime >= target) if uptime is not None and target is not None else None,
|
|
)
|
|
return result
|
|
|
|
def next_probe_at(self, service: Service, global_interval: int) -> datetime | None:
|
|
if not service.enabled or not service.monitoring_enabled:
|
|
return None
|
|
last = self.latest_probe(service.id)
|
|
if last is None:
|
|
return _now()
|
|
return last.checked_at + timedelta(seconds=service.interval_seconds or global_interval)
|
|
|
|
def prune_probes(self, retention_days: int, event_retention_days: int | None = None) -> int:
|
|
cutoff = _now() - timedelta(days=retention_days)
|
|
event_cutoff = _now() - timedelta(days=event_retention_days or retention_days)
|
|
with self._connect() as db:
|
|
cursor = db.execute("DELETE FROM probe_results WHERE checked_at < ?", (cutoff.isoformat(),))
|
|
db.execute("DELETE FROM events WHERE created_at < ? AND event_type NOT LIKE 'incident.%'", (event_cutoff.isoformat(),))
|
|
return cursor.rowcount
|
|
|
|
def _insert_event_db(
|
|
self,
|
|
db: sqlite3.Connection,
|
|
event_type: str,
|
|
service_id: str | None,
|
|
incident_id: str | None,
|
|
from_status: str | None,
|
|
to_status: str | None,
|
|
message: str,
|
|
metadata: dict[str, object],
|
|
created_at: datetime | None = None,
|
|
) -> str:
|
|
event_id = str(uuid4())
|
|
db.execute(
|
|
"""
|
|
INSERT INTO events (id, event_type, service_id, incident_id, from_status, to_status, message, metadata, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(event_id, event_type, service_id, incident_id, from_status, to_status, message, json.dumps(metadata), (created_at or _now()).isoformat()),
|
|
)
|
|
return event_id
|
|
|
|
def add_event(
|
|
self,
|
|
event_type: str,
|
|
message: str,
|
|
*,
|
|
service_id: str | None = None,
|
|
incident_id: str | None = None,
|
|
from_status: HealthStatus | None = None,
|
|
to_status: HealthStatus | None = None,
|
|
metadata: dict[str, object] | None = None,
|
|
) -> EventRecord:
|
|
created_at = _now()
|
|
with self._connect() as db:
|
|
event_id = self._insert_event_db(
|
|
db,
|
|
event_type,
|
|
service_id,
|
|
incident_id,
|
|
from_status.value if from_status else None,
|
|
to_status.value if to_status else None,
|
|
message,
|
|
metadata or {},
|
|
created_at,
|
|
)
|
|
row = db.execute("SELECT * FROM events WHERE id = ?", (event_id,)).fetchone()
|
|
result = self._row_to_event(row)
|
|
event_bus.publish(result.model_dump(mode="json"))
|
|
return result
|
|
|
|
def list_events(self, limit: int = 50, service_id: str | None = None) -> list[EventRecord]:
|
|
with self._connect() as db:
|
|
if service_id:
|
|
rows = db.execute("SELECT * FROM events WHERE service_id = ? ORDER BY created_at DESC LIMIT ?", (service_id, limit)).fetchall()
|
|
else:
|
|
rows = db.execute("SELECT * FROM events ORDER BY created_at DESC LIMIT ?", (limit,)).fetchall()
|
|
return [self._row_to_event(row) for row in rows]
|
|
|
|
def demo_active(self) -> bool:
|
|
with self._connect() as db:
|
|
return db.execute("SELECT 1 FROM demo_sessions WHERE id = 1").fetchone() is not None
|
|
|
|
@staticmethod
|
|
def _snapshot_rows(db: sqlite3.Connection) -> tuple[str, str, str, str]:
|
|
services = [dict(row) for row in db.execute("SELECT * FROM services ORDER BY id").fetchall()]
|
|
probes = [dict(row) for row in db.execute("SELECT * FROM probe_results ORDER BY id").fetchall()]
|
|
events = [dict(row) for row in db.execute("SELECT * FROM events ORDER BY id").fetchall()]
|
|
incidents = [dict(row) for row in db.execute("SELECT * FROM incidents ORDER BY id").fetchall()]
|
|
return tuple(json.dumps(item, separators=(",", ":")) for item in (services, probes, events, incidents)) # type: ignore[return-value]
|
|
|
|
@staticmethod
|
|
def _snapshot_discovery_rows(db: sqlite3.Connection) -> str:
|
|
tables = (
|
|
"discovery_runs", "discovery_sources", "service_sources", "discovery_observations",
|
|
"relation_evidence", "topology_suggestions", "topology_overrides", "topology_change_events",
|
|
)
|
|
existing = {row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
|
|
snapshot: dict[str, list[dict[str, object]]] = {}
|
|
for table in tables:
|
|
if table in existing:
|
|
snapshot[table] = [dict(row) for row in db.execute(f"SELECT * FROM {table}").fetchall()]
|
|
return json.dumps(snapshot, separators=(",", ":"))
|
|
|
|
@staticmethod
|
|
def _snapshot_health_evidence_rows(db: sqlite3.Connection) -> str:
|
|
return json.dumps(
|
|
{
|
|
"health_evidence": [dict(row) for row in db.execute("SELECT * FROM health_evidence ORDER BY id").fetchall()],
|
|
"health_evidence_service_links": [
|
|
dict(row) for row in db.execute("SELECT * FROM health_evidence_service_links ORDER BY evidence_id, service_id").fetchall()
|
|
],
|
|
"provider_status": [dict(row) for row in db.execute("SELECT * FROM provider_status ORDER BY provider").fetchall()],
|
|
},
|
|
separators=(",", ":"),
|
|
)
|
|
|
|
@classmethod
|
|
def _raw_service_params(cls, raw: dict[str, Any]) -> dict[str, Any]:
|
|
defaults: dict[str, Any] = {
|
|
"description": None, "tags": "[]", "group": None, "environment": None, "owner": None,
|
|
"enabled": 1, "monitoring_enabled": 1, "monitoring_state": "active_probe", "probe_source": "manual",
|
|
"probe_reachability": "verified", "probe_confidence": "confirmed", "runtime_state": None, "runtime_health": None,
|
|
"application": None, "criticality": "medium", "category": "unknown",
|
|
"visibility": "visible", "lifecycle": "active", "source_type": None, "source_ref": None,
|
|
"first_seen_at": None, "last_seen_at": None, "last_discovery_status": None,
|
|
"container_image": None, "compose_project": None, "layout_x": None, "layout_y": None,
|
|
"layout_pinned": 0, "interval_seconds": None, "timeout_seconds": None, "retries": None,
|
|
"failure_threshold": 1, "recovery_threshold": 1, "expected_status_min": None,
|
|
"expected_status_max": None, "availability_target": None, "maintenance_enabled": 0,
|
|
"maintenance_reason": None, "state": "unknown", "failure_streak": 0, "recovery_streak": 0,
|
|
"flapping": 0, "flap_count": 0, "last_state_change_at": None, "updated_at": raw.get("created_at"),
|
|
"depends_on": "[]",
|
|
}
|
|
params = {**defaults, **raw}
|
|
params["depends_on"] = json.dumps(params["depends_on"]) if isinstance(params["depends_on"], list) else params["depends_on"]
|
|
params["tags"] = json.dumps(params["tags"]) if isinstance(params["tags"], list) else params["tags"]
|
|
params["updated_at"] = params["updated_at"] or params["created_at"]
|
|
params["metadata"] = params.get("metadata", "{}")
|
|
params["trigger"] = params.get("trigger", "manual")
|
|
return params
|
|
|
|
@classmethod
|
|
def _insert_raw_service(cls, db: sqlite3.Connection, raw: dict[str, Any]) -> None:
|
|
params = cls._raw_service_params(raw)
|
|
cls._insert_service_row(db, params)
|
|
|
|
@staticmethod
|
|
def _insert_service_row(db: sqlite3.Connection, params: dict[str, Any]) -> None:
|
|
db.execute(
|
|
f"INSERT INTO services ({SERVICE_COLUMNS}) VALUES ("
|
|
":id, :name, :kind, :url, :host, :port, :depends_on, :description, :tags, :group, "
|
|
":environment, :owner, :enabled, :monitoring_enabled, :monitoring_state, :probe_source, :probe_reachability, :probe_confidence, :runtime_state, :runtime_health, :application, :domain, :component_role, :application_locked, :domain_locked, :component_role_locked, :criticality, :category, :visibility, :lifecycle, "
|
|
":source_type, :source_ref, :first_seen_at, :last_seen_at, :last_discovery_status, :container_image, "
|
|
":compose_project, :layout_x, :layout_y, :layout_pinned, "
|
|
":interval_seconds, :timeout_seconds, :retries, :failure_threshold, :recovery_threshold, "
|
|
":expected_status_min, :expected_status_max, :availability_target, :maintenance_enabled, "
|
|
":maintenance_reason, :state, :failure_streak, :recovery_streak, :flapping, :flap_count, "
|
|
":last_state_change_at, :created_at, :updated_at)",
|
|
params,
|
|
)
|
|
|
|
@classmethod
|
|
def _restore_rows(cls, db: sqlite3.Connection, snapshot_services: str, snapshot_probes: str, snapshot_events: str, snapshot_incidents: str, snapshot_discovery: str = "{}", snapshot_health_evidence: str = "{}") -> None:
|
|
services = json.loads(snapshot_services)
|
|
probes = json.loads(snapshot_probes)
|
|
events = json.loads(snapshot_events or "[]")
|
|
incidents = json.loads(snapshot_incidents or "[]")
|
|
db.execute("DELETE FROM incidents")
|
|
db.execute("DELETE FROM events")
|
|
db.execute("DELETE FROM services")
|
|
for raw in services:
|
|
cls._insert_raw_service(db, raw)
|
|
for raw in probes:
|
|
params = {
|
|
"id": raw["id"],
|
|
"service_id": raw["service_id"],
|
|
"status": raw["status"],
|
|
"latency_ms": raw.get("latency_ms"),
|
|
"detail": raw["detail"],
|
|
"checked_at": raw["checked_at"],
|
|
"trigger": raw.get("trigger", "manual"),
|
|
"metadata": raw.get("metadata", "{}"),
|
|
}
|
|
db.execute(
|
|
"INSERT INTO probe_results (id, service_id, status, latency_ms, detail, checked_at, trigger, metadata) VALUES (:id, :service_id, :status, :latency_ms, :detail, :checked_at, :trigger, :metadata)",
|
|
params,
|
|
)
|
|
for raw in incidents:
|
|
db.execute(
|
|
"INSERT INTO incidents (id, status, root_service_id, started_at, detected_at, acknowledged_at, resolved_at, title, root_cause_reason, affected_service_ids, severity, latest_update_at) VALUES (:id, :status, :root_service_id, :started_at, :detected_at, :acknowledged_at, :resolved_at, :title, :root_cause_reason, :affected_service_ids, :severity, :latest_update_at)",
|
|
raw,
|
|
)
|
|
for raw in events:
|
|
db.execute(
|
|
"INSERT INTO events (id, event_type, service_id, incident_id, from_status, to_status, message, metadata, created_at) VALUES (:id, :event_type, :service_id, :incident_id, :from_status, :to_status, :message, :metadata, :created_at)",
|
|
{**raw, "metadata": raw.get("metadata", "{}")},
|
|
)
|
|
|
|
cls._restore_discovery_rows(db, snapshot_discovery)
|
|
cls._restore_health_evidence_rows(db, snapshot_health_evidence)
|
|
|
|
@staticmethod
|
|
def _restore_health_evidence_rows(db: sqlite3.Connection, snapshot: str) -> None:
|
|
try:
|
|
items = json.loads(snapshot or "{}")
|
|
except json.JSONDecodeError:
|
|
items = {}
|
|
if not isinstance(items, dict):
|
|
return
|
|
db.execute("DELETE FROM health_evidence")
|
|
db.execute("DELETE FROM provider_status")
|
|
for table in ("health_evidence", "health_evidence_service_links", "provider_status"):
|
|
rows = items.get(table, [])
|
|
if not isinstance(rows, list):
|
|
continue
|
|
for raw in rows:
|
|
if not isinstance(raw, dict) or not raw:
|
|
continue
|
|
columns = list(raw)
|
|
quoted = ", ".join(f'"{column}"' for column in columns)
|
|
placeholders = ", ".join("?" for _ in columns)
|
|
db.execute(f"INSERT INTO {table} ({quoted}) VALUES ({placeholders})", tuple(raw[column] for column in columns))
|
|
|
|
@staticmethod
|
|
def _restore_discovery_rows(db: sqlite3.Connection, snapshot: str) -> None:
|
|
try:
|
|
tables = json.loads(snapshot or "{}")
|
|
except json.JSONDecodeError:
|
|
tables = {}
|
|
if not isinstance(tables, dict) or not tables:
|
|
return
|
|
order = (
|
|
"topology_change_events", "topology_overrides", "relation_evidence", "service_sources",
|
|
"discovery_observations", "topology_suggestions", "discovery_sources", "discovery_runs",
|
|
)
|
|
existing = {row["name"] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
|
|
for table in order:
|
|
if table in existing:
|
|
db.execute(f"DELETE FROM {table}")
|
|
for table in reversed(order):
|
|
rows = tables.get(table, [])
|
|
if table not in existing or not isinstance(rows, list):
|
|
continue
|
|
for raw in rows:
|
|
if not isinstance(raw, dict) or not raw:
|
|
continue
|
|
columns = list(raw)
|
|
quoted = ", ".join(f'"{column}"' for column in columns)
|
|
placeholders = ", ".join("?" for _ in columns)
|
|
db.execute(f"INSERT INTO {table} ({quoted}) VALUES ({placeholders})", tuple(raw[column] for column in columns))
|
|
|
|
@classmethod
|
|
def _insert_demo_rows(cls, db: sqlite3.Connection, services: list[Service], probes: list[ProbeResult]) -> None:
|
|
db.execute("DELETE FROM incidents")
|
|
db.execute("DELETE FROM events")
|
|
db.execute("DELETE FROM health_evidence")
|
|
db.execute("DELETE FROM provider_status")
|
|
db.execute("DELETE FROM services")
|
|
latest: dict[str, ProbeResult] = {}
|
|
for probe in probes:
|
|
if probe.service_id not in latest or probe.checked_at > latest[probe.service_id].checked_at:
|
|
latest[probe.service_id] = probe
|
|
for service in services:
|
|
seeded = service.model_copy(update={"state": latest.get(service.id, ProbeResult(id="", service_id=service.id, status=HealthStatus.UNKNOWN, latency_ms=None, detail="", checked_at=_now())).status})
|
|
cls._insert_service_row(db, cls._service_params(seeded))
|
|
for probe in probes:
|
|
db.execute(
|
|
"INSERT INTO probe_results (id, service_id, status, latency_ms, detail, checked_at, trigger, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
(probe.id, probe.service_id, probe.status.value, probe.latency_ms, probe.detail, probe.checked_at.isoformat(), probe.trigger, json.dumps(probe.metadata)),
|
|
)
|
|
for probe in probes:
|
|
service = next((item for item in services if item.id == probe.service_id), None)
|
|
interval = service.interval_seconds if service and service.interval_seconds else 60
|
|
evidence_id = f"demo-probe-{probe.id}"
|
|
db.execute(
|
|
"""
|
|
INSERT INTO health_evidence (
|
|
id, subject_type, subject_id, service_id, provider, provider_ref, status, scope,
|
|
confidence, observed_at, valid_until, received_at, summary, metadata, active
|
|
) VALUES (?, 'service', ?, ?, 'active_probe', 'active-probe', ?, 'component', 1.0, ?, ?, ?, ?, ?, 1)
|
|
ON CONFLICT(id) DO NOTHING
|
|
""",
|
|
(
|
|
evidence_id,
|
|
probe.service_id,
|
|
probe.service_id,
|
|
probe.status.value,
|
|
probe.checked_at.isoformat(),
|
|
(probe.checked_at + timedelta(seconds=max(60, interval * 2))).isoformat(),
|
|
probe.checked_at.isoformat(),
|
|
probe.detail,
|
|
json.dumps({"probe_id": probe.id, "trigger": probe.trigger, **probe.metadata}, sort_keys=True),
|
|
),
|
|
)
|
|
db.execute(
|
|
"INSERT OR IGNORE INTO health_evidence_service_links (evidence_id, service_id) VALUES (?, ?)",
|
|
(evidence_id, probe.service_id),
|
|
)
|
|
|
|
def activate_demo(self, services: list[Service], probes: list[ProbeResult]) -> ConfigImportResult:
|
|
with self._connect() as db:
|
|
db.execute("BEGIN IMMEDIATE")
|
|
session = db.execute("SELECT * FROM demo_sessions WHERE id = 1").fetchone()
|
|
if session is None:
|
|
snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents = self._snapshot_rows(db)
|
|
snapshot_discovery = self._snapshot_discovery_rows(db)
|
|
snapshot_health_evidence = self._snapshot_health_evidence_rows(db)
|
|
started_at = _now().isoformat()
|
|
else:
|
|
snapshot_services = session["snapshot_services"]
|
|
snapshot_probes = session["snapshot_probes"]
|
|
snapshot_events = session["snapshot_events"] or "[]"
|
|
snapshot_incidents = session["snapshot_incidents"] or "[]"
|
|
snapshot_discovery = session["snapshot_discovery"] or "{}"
|
|
snapshot_health_evidence = session["snapshot_health_evidence"] or "{}"
|
|
started_at = session["started_at"]
|
|
self._insert_demo_rows(db, services, probes)
|
|
db.execute(
|
|
"""
|
|
INSERT INTO demo_sessions (id, snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents, snapshot_discovery, snapshot_health_evidence, demo_service_ids, started_at)
|
|
VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(id) DO UPDATE SET snapshot_services=excluded.snapshot_services, snapshot_probes=excluded.snapshot_probes,
|
|
snapshot_events=excluded.snapshot_events, snapshot_incidents=excluded.snapshot_incidents,
|
|
snapshot_discovery=excluded.snapshot_discovery,
|
|
snapshot_health_evidence=excluded.snapshot_health_evidence,
|
|
demo_service_ids=excluded.demo_service_ids, started_at=excluded.started_at
|
|
""",
|
|
(snapshot_services, snapshot_probes, snapshot_events, snapshot_incidents, snapshot_discovery, snapshot_health_evidence, json.dumps([service.id for service in services]), started_at),
|
|
)
|
|
event_bus.publish({"event_type": "topology.changed", "message": "Synthetic demo loaded"})
|
|
return ConfigImportResult(created=len(services), updated=0, total=len(services))
|
|
|
|
def exit_demo(self) -> bool:
|
|
with self._connect() as db:
|
|
db.execute("BEGIN IMMEDIATE")
|
|
session = db.execute("SELECT * FROM demo_sessions WHERE id = 1").fetchone()
|
|
if session is None:
|
|
return False
|
|
self._restore_rows(
|
|
db,
|
|
session["snapshot_services"],
|
|
session["snapshot_probes"],
|
|
session["snapshot_events"],
|
|
session["snapshot_incidents"],
|
|
session["snapshot_discovery"] or "{}",
|
|
session["snapshot_health_evidence"] or "{}",
|
|
)
|
|
db.execute("DELETE FROM demo_sessions WHERE id = 1")
|
|
event_bus.publish({"event_type": "topology.changed", "message": "Synthetic demo exited"})
|
|
return True
|
|
|
|
def list_incidents(self, active_only: bool = False, limit: int = 100) -> list[IncidentRecord]:
|
|
services = {service.id: service for service in self.list_services()}
|
|
with self._connect() as db:
|
|
query = "SELECT * FROM incidents"
|
|
params: tuple[Any, ...] = ()
|
|
if active_only:
|
|
query += " WHERE status != 'resolved'"
|
|
query += " ORDER BY detected_at DESC LIMIT ?"
|
|
params += (limit,)
|
|
rows = db.execute(query, params).fetchall()
|
|
return [self._row_to_incident(row, services) for row in rows]
|
|
|
|
def get_incident(self, incident_id: str) -> IncidentRecord | None:
|
|
services = {service.id: service for service in self.list_services()}
|
|
with self._connect() as db:
|
|
row = db.execute("SELECT * FROM incidents WHERE id = ?", (incident_id,)).fetchone()
|
|
return self._row_to_incident(row, services) if row else None
|
|
|
|
def incident_timeline(self, incident_id: str, limit: int = 200) -> list[IncidentEvent]:
|
|
with self._connect() as db:
|
|
rows = db.execute(
|
|
"SELECT id, incident_id, event_type, message, created_at, service_id FROM events WHERE incident_id = ? ORDER BY created_at ASC LIMIT ?",
|
|
(incident_id, limit),
|
|
).fetchall()
|
|
return [IncidentEvent(**dict(row)) for row in rows]
|
|
|
|
def upsert_incident(self, incident: IncidentRecord) -> IncidentRecord:
|
|
with self._connect() as db:
|
|
db.execute(
|
|
"""
|
|
INSERT INTO incidents (id, status, root_service_id, started_at, detected_at, acknowledged_at, resolved_at,
|
|
title, root_cause_reason, affected_service_ids, severity, latest_update_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(id) DO UPDATE SET status=excluded.status, acknowledged_at=excluded.acknowledged_at,
|
|
resolved_at=excluded.resolved_at, title=excluded.title, root_cause_reason=excluded.root_cause_reason,
|
|
affected_service_ids=excluded.affected_service_ids, severity=excluded.severity, latest_update_at=excluded.latest_update_at
|
|
""",
|
|
(incident.id, incident.status.value, incident.root_service_id, incident.started_at.isoformat(), incident.detected_at.isoformat(), _iso(incident.acknowledged_at), _iso(incident.resolved_at), incident.title, incident.root_cause_reason, json.dumps(incident.affected_service_ids), incident.severity.value, incident.latest_update_at.isoformat()),
|
|
)
|
|
return incident
|
|
|
|
def append_incident_event(self, incident_id: str, event_type: str, message: str, service_id: str | None = None) -> IncidentEvent:
|
|
created_at = _now()
|
|
with self._connect() as db:
|
|
event_id = self._insert_event_db(db, event_type, service_id, incident_id, None, None, message, {}, created_at)
|
|
payload = {"event_type": event_type, "incident_id": incident_id, "service_id": service_id, "message": message, "created_at": created_at.isoformat()}
|
|
event_bus.publish(payload)
|
|
return IncidentEvent(id=event_id, incident_id=incident_id, event_type=event_type, message=message, created_at=created_at, service_id=service_id)
|
|
|
|
def acknowledge_incident(self, incident_id: str) -> IncidentRecord | None:
|
|
incident = self.get_incident(incident_id)
|
|
if incident is None:
|
|
return None
|
|
if incident.status == IncidentStatus.RESOLVED:
|
|
raise ValueError("Resolved incidents cannot be acknowledged")
|
|
now = _now()
|
|
updated = incident.model_copy(update={"status": IncidentStatus.ACKNOWLEDGED, "acknowledged_at": now, "latest_update_at": now})
|
|
self.upsert_incident(updated)
|
|
self.append_incident_event(incident_id, "incident.acknowledged", "Incident acknowledged", incident.root_service_id)
|
|
return updated
|