266 lines
9.3 KiB
Python
266 lines
9.3 KiB
Python
"""Structured telemetry with W3C trace propagation and OTLP/HTTP export.
|
|
|
|
The module intentionally uses the standard library so the health process and
|
|
connectors can share the same contract without adding a runtime dependency.
|
|
Export is best-effort and never puts authorization, cookies, request bodies,
|
|
or secret values into logs, metrics, or spans.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import secrets
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from collections.abc import Iterator, Mapping
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass, field
|
|
from datetime import UTC, datetime
|
|
from typing import Any, TextIO
|
|
|
|
_TRACE_ID = re.compile(r"^[0-9a-f]{32}$")
|
|
_SPAN_ID = re.compile(r"^[0-9a-f]{16}$")
|
|
_SENSITIVE = re.compile(
|
|
r"(?:authorization|cookie|password|passwd|secret|token|credential|private.?key|api.?key)",
|
|
re.IGNORECASE,
|
|
)
|
|
_METRIC = re.compile(r"[^a-zA-Z0-9_:]")
|
|
|
|
|
|
def new_trace_id() -> str:
|
|
return secrets.token_hex(16)
|
|
|
|
|
|
def _new_span_id() -> str:
|
|
return secrets.token_hex(8)
|
|
|
|
|
|
def valid_trace_id(value: str | None) -> str | None:
|
|
if value is None:
|
|
return None
|
|
candidate = value.strip().lower()
|
|
return candidate if _TRACE_ID.fullmatch(candidate) else None
|
|
|
|
|
|
def trace_from_headers(headers: Mapping[str, str]) -> TraceContext:
|
|
"""Accept a W3C traceparent trace-id, otherwise create a new root trace."""
|
|
|
|
traceparent = headers.get("traceparent", headers.get("Traceparent", ""))
|
|
pieces = traceparent.split("-")
|
|
trace_id = valid_trace_id(pieces[1] if len(pieces) == 4 else None)
|
|
parent = pieces[2].lower() if len(pieces) == 4 and _SPAN_ID.fullmatch(pieces[2]) else None
|
|
return TraceContext(trace_id=trace_id or new_trace_id(), parent_span_id=parent)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TraceContext:
|
|
trace_id: str
|
|
parent_span_id: str | None = None
|
|
|
|
def __post_init__(self) -> None:
|
|
if not _TRACE_ID.fullmatch(self.trace_id):
|
|
raise ValueError("trace_id must be 32 lowercase hexadecimal characters")
|
|
|
|
def child(self) -> TraceContext:
|
|
return TraceContext(self.trace_id, _new_span_id())
|
|
|
|
def response_headers(self) -> dict[str, str]:
|
|
return {"X-Trace-Id": self.trace_id}
|
|
|
|
|
|
def sanitize_fields(value: Any, *, _key: str = "") -> Any:
|
|
"""Return JSON-safe fields with sensitive keys replaced and bounded values."""
|
|
|
|
if _key and _SENSITIVE.search(_key):
|
|
return "[REDACTED]"
|
|
if isinstance(value, Mapping):
|
|
return {str(key): sanitize_fields(item, _key=str(key)) for key, item in value.items()}
|
|
if isinstance(value, list | tuple | set):
|
|
return [sanitize_fields(item) for item in value][:100]
|
|
if isinstance(value, str | int | float | bool) or value is None:
|
|
return str(value)[:512] if isinstance(value, str) else value
|
|
return str(value)[:512]
|
|
|
|
|
|
class StructuredLogger:
|
|
def __init__(self, service: str, *, stream: TextIO | None = None) -> None:
|
|
self.service = service[:64]
|
|
self.stream = stream or sys.stdout
|
|
|
|
def emit(
|
|
self,
|
|
event_code: str,
|
|
trace_id: str,
|
|
*,
|
|
level: str = "info",
|
|
fields: Mapping[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
safe_trace_id = valid_trace_id(trace_id) or new_trace_id()
|
|
record = {
|
|
"timestamp": datetime.now(UTC).isoformat(),
|
|
"service": self.service,
|
|
"version": os.environ.get("APP_VERSION", "dev")[:64],
|
|
"environment": os.environ.get("APP_ENV", "unknown")[:32],
|
|
"traceId": safe_trace_id,
|
|
"eventCode": event_code[:96],
|
|
"level": level[:16],
|
|
"fields": sanitize_fields(fields or {}),
|
|
}
|
|
self.stream.write(json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n")
|
|
self.stream.flush()
|
|
return record
|
|
|
|
|
|
@dataclass
|
|
class _Metric:
|
|
kind: str
|
|
value: float = 0.0
|
|
observations: list[float] = field(default_factory=list)
|
|
|
|
|
|
class MetricsRegistry:
|
|
"""In-process metrics with a stable Prometheus text representation."""
|
|
|
|
def __init__(self) -> None:
|
|
self._metrics: dict[tuple[str, tuple[tuple[str, str], ...]], _Metric] = {}
|
|
|
|
@staticmethod
|
|
def _name(name: str) -> str:
|
|
normalized = _METRIC.sub("_", name).strip("_").lower()
|
|
if not normalized or normalized[0].isdigit():
|
|
raise ValueError("metric name must start with a letter or underscore")
|
|
return normalized
|
|
|
|
@staticmethod
|
|
def _labels(labels: Mapping[str, str] | None) -> tuple[tuple[str, str], ...]:
|
|
if not labels:
|
|
return ()
|
|
safe = []
|
|
for key, value in sorted(labels.items()):
|
|
if _SENSITIVE.search(key):
|
|
continue
|
|
safe.append((_METRIC.sub("_", key)[:32], str(value)[:64]))
|
|
return tuple(safe)
|
|
|
|
def _get(self, name: str, labels: Mapping[str, str] | None, kind: str) -> _Metric:
|
|
key = (self._name(name), self._labels(labels))
|
|
metric = self._metrics.setdefault(key, _Metric(kind=kind))
|
|
if metric.kind != kind:
|
|
raise ValueError(f"metric {name!r} changed type")
|
|
return metric
|
|
|
|
def inc(
|
|
self, name: str, amount: float = 1.0, *, labels: Mapping[str, str] | None = None
|
|
) -> None:
|
|
if amount < 0:
|
|
raise ValueError("counter increments cannot be negative")
|
|
self._get(name, labels, "counter").value += amount
|
|
|
|
def set_gauge(
|
|
self, name: str, value: float, *, labels: Mapping[str, str] | None = None
|
|
) -> None:
|
|
self._get(name, labels, "gauge").value = value
|
|
|
|
def observe(self, name: str, value: float, *, labels: Mapping[str, str] | None = None) -> None:
|
|
metric = self._get(name, labels, "histogram")
|
|
metric.observations.append(max(0.0, value))
|
|
|
|
def render_prometheus(self) -> str:
|
|
lines: list[str] = []
|
|
for (name, labels), metric in sorted(self._metrics.items()):
|
|
label_text = ""
|
|
if labels:
|
|
parts = []
|
|
for key, value in labels:
|
|
escaped = value.replace(chr(92), chr(92) * 2).replace(
|
|
chr(34), chr(92) + chr(34)
|
|
)
|
|
parts.append(f'{key}="{escaped}"')
|
|
label_text = "{" + ",".join(parts) + "}"
|
|
if metric.kind == "histogram":
|
|
total = sum(metric.observations)
|
|
count = len(metric.observations)
|
|
lines.extend(
|
|
[f"{name}_count{label_text} {count}", f"{name}_sum{label_text} {total:g}"]
|
|
)
|
|
else:
|
|
lines.append(f"{name}{label_text} {metric.value:g}")
|
|
return "\n".join(lines) + ("\n" if lines else "")
|
|
|
|
|
|
class OTLPHttpExporter:
|
|
def __init__(self, endpoint: str, *, timeout: float = 1.0) -> None:
|
|
if not endpoint.startswith(("http://", "https://")):
|
|
raise ValueError("OTLP endpoint must use http or https")
|
|
self.endpoint = endpoint.rstrip("/") + "/v1/traces"
|
|
self.timeout = max(0.1, min(timeout, 5.0))
|
|
|
|
def export(self, span: Mapping[str, Any]) -> bool:
|
|
payload = json.dumps(
|
|
{"resourceSpans": [{"scopeSpans": [{"spans": [sanitize_fields(span)]}]}]}
|
|
).encode()
|
|
request = urllib.request.Request( # noqa: S310
|
|
self.endpoint,
|
|
data=payload,
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST",
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=self.timeout) as response: # noqa: S310
|
|
return 200 <= int(response.status) < 300
|
|
except (OSError, urllib.error.URLError):
|
|
return False
|
|
|
|
|
|
class Telemetry:
|
|
def __init__(
|
|
self,
|
|
service: str,
|
|
*,
|
|
stream: TextIO | None = None,
|
|
exporter: OTLPHttpExporter | None = None,
|
|
) -> None:
|
|
self.logger = StructuredLogger(service, stream=stream)
|
|
self.metrics = MetricsRegistry()
|
|
self.exporter = exporter
|
|
|
|
@contextmanager
|
|
def span(
|
|
self, name: str, context: TraceContext, *, attributes: Mapping[str, Any] | None = None
|
|
) -> Iterator[TraceContext]:
|
|
child = context.child()
|
|
start = time.perf_counter()
|
|
status = "ok"
|
|
try:
|
|
yield child
|
|
except Exception:
|
|
status = "error"
|
|
raise
|
|
finally:
|
|
duration = time.perf_counter() - start
|
|
self.metrics.inc("itworx_requests_total", labels={"operation": name, "status": status})
|
|
self.metrics.observe(
|
|
"itworx_request_duration_seconds", duration, labels={"operation": name}
|
|
)
|
|
span = {
|
|
"traceId": child.trace_id,
|
|
"spanId": child.parent_span_id or _new_span_id(),
|
|
"name": name[:128],
|
|
"startTimeUnixNano": int((time.time() - duration) * 1_000_000_000),
|
|
"endTimeUnixNano": int(time.time() * 1_000_000_000),
|
|
"status": {"code": status},
|
|
"attributes": attributes or {},
|
|
}
|
|
if self.exporter:
|
|
self.exporter.export(span)
|
|
|
|
|
|
def build_telemetry(service: str, *, stream: TextIO | None = None) -> Telemetry:
|
|
endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip()
|
|
exporter = OTLPHttpExporter(endpoint) if endpoint else None
|
|
return Telemetry(service, stream=stream, exporter=exporter)
|