Initial curated ITWorx_MCP_Hub source from 368fba691ea8d23403b4d56b53b5ec670d7c5020

packages/hub_observability/__init__.py

@@ -0,0 +1,25 @@
+"""Small, dependency-light observability primitives for every Hub service."""
+
+from .telemetry import (
+    MetricsRegistry,
+    StructuredLogger,
+    Telemetry,
+    TraceContext,
+    build_telemetry,
+    new_trace_id,
+    sanitize_fields,
+    trace_from_headers,
+    valid_trace_id,
+)
+
+__all__ = [
+    "MetricsRegistry",
+    "StructuredLogger",
+    "Telemetry",
+    "TraceContext",
+    "build_telemetry",
+    "new_trace_id",
+    "sanitize_fields",
+    "trace_from_headers",
+    "valid_trace_id",
+]

packages/hub_observability/telemetry.py

@@ -0,0 +1,265 @@
+"""Structured telemetry with W3C trace propagation and OTLP/HTTP export.
+
+The module intentionally uses the standard library so the health process and
+connectors can share the same contract without adding a runtime dependency.
+Export is best-effort and never puts authorization, cookies, request bodies,
+or secret values into logs, metrics, or spans.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import secrets
+import sys
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any, TextIO
+
+_TRACE_ID = re.compile(r"^[0-9a-f]{32}$")
+_SPAN_ID = re.compile(r"^[0-9a-f]{16}$")
+_SENSITIVE = re.compile(
+    r"(?:authorization|cookie|password|passwd|secret|token|credential|private.?key|api.?key)",
+    re.IGNORECASE,
+)
+_METRIC = re.compile(r"[^a-zA-Z0-9_:]")
+
+
+def new_trace_id() -> str:
+    return secrets.token_hex(16)
+
+
+def _new_span_id() -> str:
+    return secrets.token_hex(8)
+
+
+def valid_trace_id(value: str | None) -> str | None:
+    if value is None:
+        return None
+    candidate = value.strip().lower()
+    return candidate if _TRACE_ID.fullmatch(candidate) else None
+
+
+def trace_from_headers(headers: Mapping[str, str]) -> TraceContext:
+    """Accept a W3C traceparent trace-id, otherwise create a new root trace."""
+
+    traceparent = headers.get("traceparent", headers.get("Traceparent", ""))
+    pieces = traceparent.split("-")
+    trace_id = valid_trace_id(pieces[1] if len(pieces) == 4 else None)
+    parent = pieces[2].lower() if len(pieces) == 4 and _SPAN_ID.fullmatch(pieces[2]) else None
+    return TraceContext(trace_id=trace_id or new_trace_id(), parent_span_id=parent)
+
+
+@dataclass(frozen=True)
+class TraceContext:
+    trace_id: str
+    parent_span_id: str | None = None
+
+    def __post_init__(self) -> None:
+        if not _TRACE_ID.fullmatch(self.trace_id):
+            raise ValueError("trace_id must be 32 lowercase hexadecimal characters")
+
+    def child(self) -> TraceContext:
+        return TraceContext(self.trace_id, _new_span_id())
+
+    def response_headers(self) -> dict[str, str]:
+        return {"X-Trace-Id": self.trace_id}
+
+
+def sanitize_fields(value: Any, *, _key: str = "") -> Any:
+    """Return JSON-safe fields with sensitive keys replaced and bounded values."""
+
+    if _key and _SENSITIVE.search(_key):
+        return "[REDACTED]"
+    if isinstance(value, Mapping):
+        return {str(key): sanitize_fields(item, _key=str(key)) for key, item in value.items()}
+    if isinstance(value, list | tuple | set):
+        return [sanitize_fields(item) for item in value][:100]
+    if isinstance(value, str | int | float | bool) or value is None:
+        return str(value)[:512] if isinstance(value, str) else value
+    return str(value)[:512]
+
+
+class StructuredLogger:
+    def __init__(self, service: str, *, stream: TextIO | None = None) -> None:
+        self.service = service[:64]
+        self.stream = stream or sys.stdout
+
+    def emit(
+        self,
+        event_code: str,
+        trace_id: str,
+        *,
+        level: str = "info",
+        fields: Mapping[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        safe_trace_id = valid_trace_id(trace_id) or new_trace_id()
+        record = {
+            "timestamp": datetime.now(UTC).isoformat(),
+            "service": self.service,
+            "version": os.environ.get("APP_VERSION", "dev")[:64],
+            "environment": os.environ.get("APP_ENV", "unknown")[:32],
+            "traceId": safe_trace_id,
+            "eventCode": event_code[:96],
+            "level": level[:16],
+            "fields": sanitize_fields(fields or {}),
+        }
+        self.stream.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n")
+        self.stream.flush()
+        return record
+
+
+@dataclass
+class _Metric:
+    kind: str
+    value: float = 0.0
+    observations: list[float] = field(default_factory=list)
+
+
+class MetricsRegistry:
+    """In-process metrics with a stable Prometheus text representation."""
+
+    def __init__(self) -> None:
+        self._metrics: dict[tuple[str, tuple[tuple[str, str], ...]], _Metric] = {}
+
+    @staticmethod
+    def _name(name: str) -> str:
+        normalized = _METRIC.sub("_", name).strip("_").lower()
+        if not normalized or normalized[0].isdigit():
+            raise ValueError("metric name must start with a letter or underscore")
+        return normalized
+
+    @staticmethod
+    def _labels(labels: Mapping[str, str] | None) -> tuple[tuple[str, str], ...]:
+        if not labels:
+            return ()
+        safe = []
+        for key, value in sorted(labels.items()):
+            if _SENSITIVE.search(key):
+                continue
+            safe.append((_METRIC.sub("_", key)[:32], str(value)[:64]))
+        return tuple(safe)
+
+    def _get(self, name: str, labels: Mapping[str, str] | None, kind: str) -> _Metric:
+        key = (self._name(name), self._labels(labels))
+        metric = self._metrics.setdefault(key, _Metric(kind=kind))
+        if metric.kind != kind:
+            raise ValueError(f"metric {name!r} changed type")
+        return metric
+
+    def inc(
+        self, name: str, amount: float = 1.0, *, labels: Mapping[str, str] | None = None
+    ) -> None:
+        if amount < 0:
+            raise ValueError("counter increments cannot be negative")
+        self._get(name, labels, "counter").value += amount
+
+    def set_gauge(
+        self, name: str, value: float, *, labels: Mapping[str, str] | None = None
+    ) -> None:
+        self._get(name, labels, "gauge").value = value
+
+    def observe(self, name: str, value: float, *, labels: Mapping[str, str] | None = None) -> None:
+        metric = self._get(name, labels, "histogram")
+        metric.observations.append(max(0.0, value))
+
+    def render_prometheus(self) -> str:
+        lines: list[str] = []
+        for (name, labels), metric in sorted(self._metrics.items()):
+            label_text = ""
+            if labels:
+                parts = []
+                for key, value in labels:
+                    escaped = value.replace(chr(92), chr(92) * 2).replace(
+                        chr(34), chr(92) + chr(34)
+                    )
+                    parts.append(f'{key}="{escaped}"')
+                label_text = "{" + ",".join(parts) + "}"
+            if metric.kind == "histogram":
+                total = sum(metric.observations)
+                count = len(metric.observations)
+                lines.extend(
+                    [f"{name}_count{label_text} {count}", f"{name}_sum{label_text} {total:g}"]
+                )
+            else:
+                lines.append(f"{name}{label_text} {metric.value:g}")
+        return "\n".join(lines) + ("\n" if lines else "")
+
+
+class OTLPHttpExporter:
+    def __init__(self, endpoint: str, *, timeout: float = 1.0) -> None:
+        if not endpoint.startswith(("http://", "https://")):
+            raise ValueError("OTLP endpoint must use http or https")
+        self.endpoint = endpoint.rstrip("/") + "/v1/traces"
+        self.timeout = max(0.1, min(timeout, 5.0))
+
+    def export(self, span: Mapping[str, Any]) -> bool:
+        payload = json.dumps(
+            {"resourceSpans": [{"scopeSpans": [{"spans": [sanitize_fields(span)]}]}]}
+        ).encode()
+        request = urllib.request.Request(  # noqa: S310
+            self.endpoint,
+            data=payload,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout) as response:  # noqa: S310
+                return 200 <= int(response.status) < 300
+        except (OSError, urllib.error.URLError):
+            return False
+
+
+class Telemetry:
+    def __init__(
+        self,
+        service: str,
+        *,
+        stream: TextIO | None = None,
+        exporter: OTLPHttpExporter | None = None,
+    ) -> None:
+        self.logger = StructuredLogger(service, stream=stream)
+        self.metrics = MetricsRegistry()
+        self.exporter = exporter
+
+    @contextmanager
+    def span(
+        self, name: str, context: TraceContext, *, attributes: Mapping[str, Any] | None = None
+    ) -> Iterator[TraceContext]:
+        child = context.child()
+        start = time.perf_counter()
+        status = "ok"
+        try:
+            yield child
+        except Exception:
+            status = "error"
+            raise
+        finally:
+            duration = time.perf_counter() - start
+            self.metrics.inc("itworx_requests_total", labels={"operation": name, "status": status})
+            self.metrics.observe(
+                "itworx_request_duration_seconds", duration, labels={"operation": name}
+            )
+            span = {
+                "traceId": child.trace_id,
+                "spanId": child.parent_span_id or _new_span_id(),
+                "name": name[:128],
+                "startTimeUnixNano": int((time.time() - duration) * 1_000_000_000),
+                "endTimeUnixNano": int(time.time() * 1_000_000_000),
+                "status": {"code": status},
+                "attributes": attributes or {},
+            }
+            if self.exporter:
+                self.exporter.export(span)
+
+
+def build_telemetry(service: str, *, stream: TextIO | None = None) -> Telemetry:
+    endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip()
+    exporter = OTLPHttpExporter(endpoint) if endpoint else None
+    return Telemetry(service, stream=stream, exporter=exporter)