Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/unit/platform/telemetry/test_attributes.py

@@ -0,0 +1,40 @@
+"""safe_attributes: sensitive query/document text is absent by default."""
+
+from __future__ import annotations
+
+from ragcore.platform.telemetry import safe_attributes
+
+
+def test_sensitive_keys_are_dropped_entirely() -> None:
+    result = safe_attributes(
+        {
+            "query": "what is the secret plan",
+            "document": "confidential contents",
+            "prompt": "system prompt text",
+            "answer": "the generated answer",
+            "excerpt": "a citation excerpt",
+            "duration_ms": 12.5,
+        }
+    )
+    assert result == {"duration_ms": 12.5}
+
+
+def test_matching_is_case_insensitive_and_substring_based() -> None:
+    result = safe_attributes(
+        {"UserQuery": "text", "Document_Id_Content": "text", "status_code": 200}
+    )
+    assert result == {"status_code": 200}
+
+
+def test_unsupported_value_types_are_dropped() -> None:
+    result = safe_attributes({"safe_number": 1, "unsafe_dict": {"a": 1}, "unsafe_list": [1, 2]})
+    assert result == {"safe_number": 1}
+
+
+def test_boolean_and_string_values_pass_through() -> None:
+    result = safe_attributes({"degraded": True, "operation": "search"})
+    assert result == {"degraded": True, "operation": "search"}
+
+
+def test_empty_attributes_produce_an_empty_result() -> None:
+    assert safe_attributes({}) == {}

tests/unit/platform/telemetry/test_dashboards_and_alerts.py

@@ -0,0 +1,65 @@
+"""ops/dashboards and ops/alerts stay in sync with the real metric/attribute names."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from ragcore.platform.telemetry import TelemetryMetrics
+
+_REPO_ROOT = Path(__file__).parents[4]
+
+
+def _known_metric_names() -> set[str]:
+    """The stable metric names TelemetryMetrics actually creates."""
+
+    return {
+        "ragcore_request_duration_seconds",
+        "ragcore_request_errors_total",
+        "ragcore_job_queue_depth",
+        "ragcore_job_age_seconds",
+        "ragcore_qdrant_errors_total",
+        "ragcore_provider_errors_total",
+    }
+
+
+def test_operations_dashboard_only_references_real_metric_names() -> None:
+    document = yaml.safe_load(
+        (_REPO_ROOT / "ops" / "dashboards" / "operations_overview.yaml").read_text()
+    )
+    known = _known_metric_names()
+    for panel in document["panels"]:
+        assert panel["metric"] in known, f"{panel['metric']} is not a real TelemetryMetrics metric"
+
+
+def test_operations_dashboard_correlation_attribute_matches_the_real_span_attribute() -> None:
+    document = yaml.safe_load(
+        (_REPO_ROOT / "ops" / "dashboards" / "operations_overview.yaml").read_text()
+    )
+    assert document["correlation"]["attribute"] == "correlation_id"
+
+
+def test_queue_age_alert_references_a_real_metric_name() -> None:
+    document = yaml.safe_load((_REPO_ROOT / "ops" / "alerts" / "queue_age_slo.yaml").read_text())
+    assert document["alert"] == "JobQueueAgeSloBreached"
+    assert document["signal"]["metric"] in _known_metric_names()
+    assert document["signal"]["metric"] == "ragcore_job_age_seconds"
+
+
+def test_declared_metric_names_are_exhaustive_against_the_real_instruments() -> None:
+    """Guards the other direction: if a new metric is added to
+    TelemetryMetrics without updating the dashboard, this fails instead of
+    the dashboard silently going stale."""
+
+    telemetry = TelemetryMetrics(meter_name="dashboard-completeness-check")
+    instrument_attrs = (
+        "request_duration_seconds",
+        "request_errors_total",
+        "job_queue_depth",
+        "job_age_seconds",
+        "qdrant_errors_total",
+        "provider_errors_total",
+    )
+    for attr in instrument_attrs:
+        assert hasattr(telemetry, attr)

tests/unit/platform/telemetry/test_metrics.py

@@ -0,0 +1,164 @@
+"""TelemetryMetrics: the representative required-metrics subset, recorded correctly."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+from opentelemetry.sdk.resources import Resource
+
+from ragcore.platform.telemetry import TelemetryMetrics
+
+
+def _telemetry() -> tuple[TelemetryMetrics, InMemoryMetricReader]:
+    """A fresh, isolated MeterProvider per call.
+
+    OpenTelemetry's global provider may only be set once per process, so
+    each test builds its own explicit provider and passes it directly to
+    ``TelemetryMetrics`` rather than fighting over global state.
+    """
+
+    reader = InMemoryMetricReader()
+    provider = MeterProvider(
+        resource=Resource.create({"service.name": "test"}), metric_readers=[reader]
+    )
+    return TelemetryMetrics(meter_provider=provider), reader
+
+
+def _metric_named(reader: InMemoryMetricReader, name: str) -> Any | None:
+    data = reader.get_metrics_data()
+    if data is None:
+        return None
+    for resource_metric in data.resource_metrics:
+        for scope_metric in resource_metric.scope_metrics:
+            for metric in scope_metric.metrics:
+                if metric.name == name:
+                    return metric
+    return None
+
+
+def _data_points(metric: Any) -> list[Any]:
+    return list(metric.data.data_points)
+
+
+def test_successful_requests_record_duration_but_no_error() -> None:
+    telemetry, reader = _telemetry()
+    telemetry.request_duration_seconds.record(0.042, attributes={"status_code": 200})
+
+    duration_metric = _metric_named(reader, "ragcore_request_duration_seconds")
+    assert duration_metric is not None
+    points = _data_points(duration_metric)
+    assert len(points) == 1
+    assert points[0].sum == pytest.approx(0.042)
+
+
+def test_server_errors_and_rate_limits_increment_the_error_counter() -> None:
+    telemetry, reader = _telemetry()
+    telemetry.record_request(duration_seconds=0.1, status_code=200)
+    telemetry.record_request(duration_seconds=0.2, status_code=503)
+    telemetry.record_request(duration_seconds=0.3, status_code=429)
+
+    error_metric = _metric_named(reader, "ragcore_request_errors_total")
+    assert error_metric is not None
+    total = sum(point.value for point in _data_points(error_metric))
+    assert total == 2
+
+
+def test_job_queue_depth_rises_on_enqueue_and_falls_on_claim() -> None:
+    telemetry, reader = _telemetry()
+    telemetry.record_job_enqueued()
+    telemetry.record_job_enqueued()
+    telemetry.record_job_claimed(age_seconds=5.0)
+
+    depth_metric = _metric_named(reader, "ragcore_job_queue_depth")
+    assert depth_metric is not None
+    net_depth = sum(point.value for point in _data_points(depth_metric))
+    assert net_depth == 1
+
+    age_metric = _metric_named(reader, "ragcore_job_age_seconds")
+    assert age_metric is not None
+    assert _data_points(age_metric)[0].sum == pytest.approx(5.0)
+
+
+def test_qdrant_and_provider_errors_are_recorded_by_operation_and_provider() -> None:
+    telemetry, reader = _telemetry()
+    telemetry.record_qdrant_error(operation="upsert")
+    telemetry.record_provider_error(provider="ollama-generation")
+
+    qdrant_metric = _metric_named(reader, "ragcore_qdrant_errors_total")
+    provider_metric = _metric_named(reader, "ragcore_provider_errors_total")
+    assert qdrant_metric is not None
+    assert provider_metric is not None
+    (qdrant_point,) = _data_points(qdrant_metric)
+    (provider_point,) = _data_points(provider_metric)
+    assert dict(qdrant_point.attributes)["operation"] == "upsert"
+    assert dict(provider_point.attributes)["provider"] == "ollama-generation"
+
+
+def test_database_errors_are_recorded_by_operation() -> None:
+    telemetry, reader = _telemetry()
+    telemetry.record_database_error(operation="publish_validated")
+
+    metric = _metric_named(reader, "ragcore_database_errors_total")
+    assert metric is not None
+    (point,) = _data_points(metric)
+    assert dict(point.attributes)["operation"] == "publish_validated"
+
+
+def test_provider_duration_is_recorded_by_provider() -> None:
+    telemetry, reader = _telemetry()
+    telemetry.record_provider_duration(provider="ollama-embedding", duration_seconds=0.25)
+
+    metric = _metric_named(reader, "ragcore_provider_duration_seconds")
+    assert metric is not None
+    (point,) = _data_points(metric)
+    assert dict(point.attributes)["provider"] == "ollama-embedding"
+    assert point.sum == pytest.approx(0.25)
+
+
+def test_retrieval_duration_and_zero_result_are_recorded() -> None:
+    telemetry, reader = _telemetry()
+    telemetry.record_retrieval(duration_seconds=0.05, zero_result=False)
+    telemetry.record_retrieval(duration_seconds=0.03, zero_result=True)
+
+    duration_metric = _metric_named(reader, "ragcore_retrieval_duration_seconds")
+    zero_result_metric = _metric_named(reader, "ragcore_retrieval_zero_result_total")
+    assert duration_metric is not None
+    assert zero_result_metric is not None
+    points = _data_points(duration_metric)
+    assert len(points) == 1
+    assert points[0].count == 2
+    assert sum(point.value for point in _data_points(zero_result_metric)) == 1
+
+
+def test_answerability_result_is_recorded() -> None:
+    telemetry, reader = _telemetry()
+    telemetry.record_answerability(result="answerable")
+    telemetry.record_answerability(result="unanswerable")
+
+    metric = _metric_named(reader, "ragcore_answerability_result_total")
+    assert metric is not None
+    results = {dict(point.attributes)["result"] for point in _data_points(metric)}
+    assert results == {"answerable", "unanswerable"}
+
+
+def test_ingestion_job_result_is_recorded_by_operation_and_result() -> None:
+    telemetry, reader = _telemetry()
+    telemetry.record_ingestion_job_result(operation="ingest_upload", result="succeeded")
+
+    metric = _metric_named(reader, "ragcore_ingestion_job_result_total")
+    assert metric is not None
+    (point,) = _data_points(metric)
+    assert dict(point.attributes) == {"operation": "ingest_upload", "result": "succeeded"}
+
+
+def test_rate_limit_refusal_is_recorded_by_operation() -> None:
+    telemetry, reader = _telemetry()
+    telemetry.record_rate_limit_refusal(operation="search")
+
+    metric = _metric_named(reader, "ragcore_rate_limit_refusals_total")
+    assert metric is not None
+    (point,) = _data_points(metric)
+    assert dict(point.attributes)["operation"] == "search"

tests/unit/platform/telemetry/test_tracing.py

@@ -0,0 +1,129 @@
+"""correlated_span: one real OpenTelemetry trace across separate components."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import pytest
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+from ragcore.platform.telemetry import correlated_span, trace_id_from_correlation_id
+
+
+@pytest.fixture
+def tracing() -> Iterator[tuple[TracerProvider, InMemorySpanExporter]]:
+    """A fresh, isolated TracerProvider per test.
+
+    OpenTelemetry's global provider may only be set once per process, so
+    each test gets its own explicit provider passed to ``correlated_span``
+    rather than fighting over global state.
+    """
+
+    span_exporter = InMemorySpanExporter()
+    provider = TracerProvider()
+    provider.add_span_processor(SimpleSpanProcessor(span_exporter))
+    yield provider, span_exporter
+
+
+def test_trace_id_is_deterministic_for_the_same_correlation_id() -> None:
+    assert trace_id_from_correlation_id("req-123") == trace_id_from_correlation_id("req-123")
+
+
+def test_trace_id_differs_for_different_correlation_ids() -> None:
+    assert trace_id_from_correlation_id("req-123") != trace_id_from_correlation_id("req-456")
+
+
+def test_trace_id_is_never_zero() -> None:
+    assert trace_id_from_correlation_id("x") != 0
+
+
+def test_empty_correlation_id_is_rejected() -> None:
+    with pytest.raises(ValueError, match="correlation_id"):
+        trace_id_from_correlation_id("")
+
+
+def test_two_spans_from_different_components_share_one_trace(
+    tracing: tuple[TracerProvider, InMemorySpanExporter],
+) -> None:
+    """The core acceptance proof: an "API request" span and a later "job
+    processing" span, opened independently with the same correlation id,
+    land in the literal same OpenTelemetry trace -- not just a matching
+    attribute a human has to notice.
+    """
+
+    provider, exporter = tracing
+    correlation_id = "req-shared-across-request-and-job"
+
+    with correlated_span(
+        "api.handle_request", correlation_id=correlation_id, tracer_provider=provider
+    ):
+        pass
+
+    with correlated_span("job.process", correlation_id=correlation_id, tracer_provider=provider):
+        pass
+
+    spans = exporter.get_finished_spans()
+    assert len(spans) == 2
+    trace_ids = {span.context.trace_id for span in spans}
+    assert len(trace_ids) == 1
+    assert trace_ids.pop() == trace_id_from_correlation_id(correlation_id)
+
+
+def test_spans_from_different_correlation_ids_are_in_different_traces(
+    tracing: tuple[TracerProvider, InMemorySpanExporter],
+) -> None:
+    provider, exporter = tracing
+    with correlated_span("api.handle_request", correlation_id="req-1", tracer_provider=provider):
+        pass
+    with correlated_span("api.handle_request", correlation_id="req-2", tracer_provider=provider):
+        pass
+
+    spans = exporter.get_finished_spans()
+    trace_ids = {span.context.trace_id for span in spans}
+    assert len(trace_ids) == 2
+
+
+def test_correlation_id_is_always_set_as_a_safe_attribute(
+    tracing: tuple[TracerProvider, InMemorySpanExporter],
+) -> None:
+    provider, exporter = tracing
+    with correlated_span("api.handle_request", correlation_id="req-999", tracer_provider=provider):
+        pass
+
+    (span,) = exporter.get_finished_spans()
+    assert span.attributes["correlation_id"] == "req-999"
+
+
+def test_sensitive_attributes_are_dropped_before_reaching_the_span(
+    tracing: tuple[TracerProvider, InMemorySpanExporter],
+) -> None:
+    provider, exporter = tracing
+    with correlated_span(
+        "api.handle_request",
+        correlation_id="req-1",
+        tracer_provider=provider,
+        attributes={"query": "sensitive text", "duration_ms": 42},
+    ):
+        pass
+
+    (span,) = exporter.get_finished_spans()
+    assert "query" not in span.attributes
+    assert span.attributes["duration_ms"] == 42
+
+
+def test_an_exception_inside_the_span_is_recorded_and_reraised(
+    tracing: tuple[TracerProvider, InMemorySpanExporter],
+) -> None:
+    provider, exporter = tracing
+    with (
+        pytest.raises(RuntimeError, match="boom"),
+        correlated_span("api.handle_request", correlation_id="req-1", tracer_provider=provider),
+    ):
+        raise RuntimeError("boom")
+
+    (span,) = exporter.get_finished_spans()
+    assert span.status.status_code == trace.StatusCode.ERROR
+    assert any(event.name == "exception" for event in span.events)