Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/integration/resilience/test_alert_matches_circuit_event.py

@@ -0,0 +1,36 @@
+"""ops/alerts/provider_circuit_open.yaml stays in sync with the real code."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from ragcore.infrastructure.models.resilience import CIRCUIT_OPEN_EVENT
+
+
+def _alert_document() -> dict[str, object]:
+    path = Path(__file__).parents[3] / "ops" / "alerts" / "provider_circuit_open.yaml"
+    return yaml.safe_load(path.read_text())
+
+
+def test_alert_file_exists_and_parses() -> None:
+    document = _alert_document()
+    assert document["alert"] == "ProviderCircuitOpen"
+    assert document["severity"] == "warning"
+
+
+def test_alert_signal_event_matches_the_literal_logged_event_name() -> None:
+    document = _alert_document()
+    signal = document["signal"]
+    assert signal["type"] == "log_event"
+    # This is the actual regression guard: if CIRCUIT_OPEN_EVENT is ever
+    # renamed in code without updating the alert, this test fails instead
+    # of the alert silently going stale and never firing again.
+    assert signal["event"] == CIRCUIT_OPEN_EVENT
+
+
+def test_alert_declares_the_expected_fields() -> None:
+    document = _alert_document()
+    fields = document["signal"]["fields"]
+    assert set(fields) == {"provider", "from_state", "to_state"}

tests/integration/resilience/test_circuit_breaker.py

@@ -0,0 +1,115 @@
+"""CircuitBreaker: observable state transitions, cooldown and recovery."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+
+import pytest
+
+from ragcore.infrastructure.models.resilience import (
+    CIRCUIT_CLOSED_EVENT,
+    CIRCUIT_HALF_OPEN_EVENT,
+    CIRCUIT_OPEN_EVENT,
+    CircuitBreaker,
+    CircuitOpenError,
+    CircuitState,
+)
+
+
+def _clock(value: dict[str, float]) -> Callable[[], float]:
+    return lambda: value["now"]
+
+
+def test_circuit_starts_closed_and_allows_calls() -> None:
+    breaker = CircuitBreaker(provider="test-provider")
+    assert breaker.state is CircuitState.CLOSED
+    breaker.check()  # must not raise
+
+
+def test_circuit_opens_after_consecutive_failures_and_blocks_calls() -> None:
+    breaker = CircuitBreaker(provider="test-provider", failure_threshold=3)
+    for _ in range(2):
+        breaker.record_failure()
+        assert breaker.state is CircuitState.CLOSED
+    breaker.record_failure()
+    assert breaker.state is CircuitState.OPEN
+    with pytest.raises(CircuitOpenError):
+        breaker.check()
+
+
+def test_circuit_transitions_are_observable_via_state_and_logs(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    breaker = CircuitBreaker(provider="test-provider", failure_threshold=1)
+    with caplog.at_level(logging.WARNING, logger="ragcore.infrastructure.models.resilience"):
+        breaker.record_failure()
+    assert breaker.state is CircuitState.OPEN
+    open_records = [r for r in caplog.records if r.getMessage() == CIRCUIT_OPEN_EVENT]
+    assert len(open_records) == 1
+    assert open_records[0].provider == "test-provider"
+    assert open_records[0].from_state == "closed"
+    assert open_records[0].to_state == "open"
+
+
+def test_circuit_half_opens_after_cooldown_and_closes_on_success_recovery_probe() -> None:
+    time_value = {"now": 0.0}
+    breaker = CircuitBreaker(
+        provider="test-provider",
+        failure_threshold=1,
+        cooldown_seconds=10.0,
+        clock=_clock(time_value),
+    )
+    breaker.record_failure()
+    assert breaker.state is CircuitState.OPEN
+
+    # Still within cooldown: blocked without a real recovery attempt.
+    with pytest.raises(CircuitOpenError):
+        breaker.check()
+    assert breaker.state is CircuitState.OPEN
+
+    # Cooldown elapsed: exactly one recovery probe is allowed through.
+    time_value["now"] = 10.1
+    breaker.check()
+    assert breaker.state is CircuitState.HALF_OPEN
+
+    breaker.record_success()
+    assert breaker.state is CircuitState.CLOSED
+    breaker.check()  # closed again: no longer raises
+
+
+def test_circuit_half_open_failure_reopens_immediately_without_a_fresh_threshold() -> None:
+    time_value = {"now": 0.0}
+    breaker = CircuitBreaker(
+        provider="test-provider",
+        failure_threshold=5,
+        cooldown_seconds=10.0,
+        clock=_clock(time_value),
+    )
+    for _ in range(5):
+        breaker.record_failure()
+    assert breaker.state is CircuitState.OPEN
+
+    time_value["now"] = 10.1
+    breaker.check()
+    assert breaker.state is CircuitState.HALF_OPEN
+
+    # A single half-open failure reopens immediately -- it does not require
+    # accumulating the full failure_threshold all over again.
+    breaker.record_failure()
+    assert breaker.state is CircuitState.OPEN
+
+
+def test_transition_events_are_named_consistently() -> None:
+    assert CIRCUIT_OPEN_EVENT == "provider_circuit_open"
+    assert CIRCUIT_HALF_OPEN_EVENT == "provider_circuit_half_open"
+    assert CIRCUIT_CLOSED_EVENT == "provider_circuit_closed"
+
+
+def test_invalid_construction_arguments_are_rejected() -> None:
+    with pytest.raises(ValueError, match="provider"):
+        CircuitBreaker(provider="")
+    with pytest.raises(ValueError, match="failure_threshold"):
+        CircuitBreaker(provider="p", failure_threshold=0)
+    with pytest.raises(ValueError, match="cooldown_seconds"):
+        CircuitBreaker(provider="p", cooldown_seconds=0)

tests/integration/resilience/test_provider_resilience_guard.py

@@ -0,0 +1,109 @@
+"""ProviderResilienceGuard: bounded overload, and circuit integration."""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+from ragcore.infrastructure.models.resilience import (
+    CircuitOpenError,
+    CircuitState,
+    ProviderOverloadedError,
+    ProviderResilienceGuard,
+)
+
+
+async def test_provider_overload_is_bounded_not_queued_without_limit() -> None:
+    guard = ProviderResilienceGuard(provider="test-provider", max_concurrent=1, max_queued=1)
+    release = asyncio.Event()
+
+    async def occupy() -> None:
+        async with guard.guard():
+            await release.wait()
+
+    first = asyncio.create_task(occupy())
+    await asyncio.sleep(0)  # let `first` acquire the single concurrency slot
+
+    second = asyncio.create_task(occupy())
+    await asyncio.sleep(0)  # `second` should be admitted into the one queue slot
+    assert guard.waiting == 1
+
+    # A third call arrives while the budget and its queue are both full.
+    with pytest.raises(ProviderOverloadedError):
+        async with guard.guard():
+            pass  # pragma: no cover -- must never be entered
+
+    release.set()
+    await first
+    await second
+
+
+async def test_guard_records_success_and_failure_against_the_circuit() -> None:
+    guard = ProviderResilienceGuard(
+        provider="test-provider", max_concurrent=2, max_queued=2, failure_threshold=2
+    )
+
+    async with guard.guard():
+        pass
+    assert guard.circuit.state is CircuitState.CLOSED
+
+    with pytest.raises(RuntimeError, match="boom"):
+        async with guard.guard():
+            raise RuntimeError("boom")
+    assert guard.circuit.state is CircuitState.CLOSED  # one failure, below threshold
+
+    with pytest.raises(RuntimeError, match="boom"):
+        async with guard.guard():
+            raise RuntimeError("boom")
+    assert guard.circuit.state is CircuitState.OPEN  # threshold reached
+
+
+async def test_open_circuit_rejects_before_touching_the_concurrency_budget() -> None:
+    guard = ProviderResilienceGuard(
+        provider="test-provider", max_concurrent=1, max_queued=0, failure_threshold=1
+    )
+    with pytest.raises(RuntimeError, match="boom"):
+        async with guard.guard():
+            raise RuntimeError("boom")
+    assert guard.circuit.state is CircuitState.OPEN
+
+    with pytest.raises(CircuitOpenError):
+        async with guard.guard():
+            pass  # pragma: no cover -- must never be entered
+    assert guard.waiting == 0
+
+
+async def test_cancellation_while_waiting_releases_the_queue_slot() -> None:
+    guard = ProviderResilienceGuard(provider="test-provider", max_concurrent=1, max_queued=1)
+    release = asyncio.Event()
+
+    async def occupy() -> None:
+        async with guard.guard():
+            await release.wait()
+
+    holder = asyncio.create_task(occupy())
+    await asyncio.sleep(0)
+
+    async def waiter() -> None:
+        async with guard.guard():
+            pass  # pragma: no cover -- cancelled before ever acquiring
+
+    waiting_task = asyncio.create_task(waiter())
+    await asyncio.sleep(0)
+    assert guard.waiting == 1
+
+    waiting_task.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await waiting_task
+    assert guard.waiting == 0
+
+    release.set()
+    await holder
+
+
+def test_invalid_construction_arguments_are_rejected() -> None:
+    with pytest.raises(ValueError, match="max_concurrent"):
+        ProviderResilienceGuard(provider="p", max_concurrent=0, max_queued=1)
+    with pytest.raises(ValueError, match="max_queued"):
+        ProviderResilienceGuard(provider="p", max_concurrent=1, max_queued=-1)

tests/integration/resilience/test_search_context_independence.py

@@ -0,0 +1,236 @@
+"""Search/context succeed even while every generation provider is down.
+
+Search and Context never call a generation provider at all (confirmed by
+grep: neither module imports anything from ``ragcore.generation``,
+``ragcore.infrastructure.models``, or ``ragcore.application.models``).
+This test proves that architectural independence behaviorally: a
+:class:`ProviderResilienceGuard` standing in for "the generation provider"
+is driven fully open (simulating every generation provider being down),
+and real ``/v1/search``/``/v1/context`` requests -- built the same way
+their own contract tests build them -- still return 200.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import replace
+from datetime import UTC, datetime
+from uuid import UUID, uuid4
+
+import pytest
+from fastapi.testclient import TestClient
+
+from ragcore.api.v1.context.dependencies import get_context_application, get_context_principal
+from ragcore.api.v1.context.service import (
+    ContextExecutionRequest,
+    ContextExecutionResult,
+    ContextService,
+)
+from ragcore.api.v1.search.dependencies import get_search_application, get_search_principal
+from ragcore.api.v1.search.service import (
+    AuthorizationInputs,
+    SearchExecutionRequest,
+    SearchExecutionResult,
+    SearchHit,
+    SearchService,
+)
+from ragcore.application.retrieval.authorization import RetrievalAuthorizationService
+from ragcore.domain.authorization import Principal, PrincipalType
+from ragcore.domain.citations import Citation
+from ragcore.domain.control import Role, Workspace
+from ragcore.infrastructure.models.resilience import (
+    CircuitOpenError,
+    CircuitState,
+    ProviderResilienceGuard,
+)
+from ragcore.main import create_app
+from ragcore.retrieval.context import ContextCandidate
+from ragcore.retrieval.ranking import FusedCandidate
+from ragcore.retrieval.search.models import Candidate, CandidateBackendKind, CandidateProvenance
+
+
+def _fully_down_generation_guard() -> ProviderResilienceGuard:
+    """Simulate every generation provider being down: an open circuit."""
+
+    guard = ProviderResilienceGuard(
+        provider="generation-ollama", max_concurrent=1, max_queued=0, failure_threshold=1
+    )
+
+    # A single synthetic failure trips the circuit open, matching what a
+    # real generation adapter would do after its own connection attempts
+    # fail (see OllamaGenerationAdapter's own circuit breaker).
+    async def _fail() -> None:
+        async with guard.guard():
+            raise RuntimeError("every generation provider is unreachable")
+
+    with pytest.raises(RuntimeError):
+        asyncio.run(_fail())
+    assert guard.circuit.state is CircuitState.OPEN
+    return guard
+
+
+class _SearchInputs:
+    def __init__(self, workspace: Workspace, space_id: UUID) -> None:
+        self.workspace = workspace
+        self.space_id = space_id
+
+    async def load(self, principal: Principal) -> AuthorizationInputs:
+        return AuthorizationInputs(
+            spaces=(
+                self.workspace.create_knowledge_space(
+                    id=self.space_id, name="Space", slug="space"
+                ),
+            ),
+            user_grants=(
+                self.workspace.grant_user(
+                    id=uuid4(),
+                    user_id=principal.id,
+                    knowledge_space=self.workspace.create_knowledge_space(
+                        id=self.space_id, name="Space", slug="space"
+                    ),
+                    role=Role.EDITOR,
+                ),
+            ),
+        )
+
+
+class _SearchExecutor:
+    def __init__(self, hit: SearchHit) -> None:
+        self.hit = hit
+
+    async def execute(self, request: SearchExecutionRequest) -> SearchExecutionResult:
+        return SearchExecutionResult(hits=(_refresh_hit(self.hit, request.retrieval_run_id),))
+
+
+class _ContextExecutor:
+    def __init__(self, candidate: ContextCandidate) -> None:
+        self.candidate = candidate
+
+    async def execute(self, request: ContextExecutionRequest) -> ContextExecutionResult:
+        return ContextExecutionResult(
+            candidates=(_refresh_candidate(self.candidate, request.retrieval_run_id),)
+        )
+
+
+def _refresh_citation(citation: Citation, retrieval_run_id: UUID) -> Citation:
+    return Citation.create(
+        workspace_id=citation.workspace_id,
+        space_id=citation.space_id,
+        document_id=citation.document_id,
+        document_version_id=citation.document_version_id,
+        chunk_id=citation.chunk_id,
+        parent_chunk_id=citation.parent_chunk_id,
+        location=citation.location,
+        excerpt=citation.excerpt,
+        source_uri=citation.source_uri,
+        acquired_at=citation.acquired_at,
+        retrieval_run_id=retrieval_run_id,
+        index_alias=citation.index_alias,
+        profile_version=citation.profile_version,
+    )
+
+
+def _refresh_hit(hit: SearchHit, retrieval_run_id: UUID) -> SearchHit:
+    return replace(hit, citation=_refresh_citation(hit.citation, retrieval_run_id))
+
+
+def _refresh_candidate(candidate: ContextCandidate, retrieval_run_id: UUID) -> ContextCandidate:
+    return replace(candidate, citation=_refresh_citation(candidate.citation, retrieval_run_id))
+
+
+def _fixture() -> tuple[object, UUID]:
+    workspace = Workspace(uuid4(), "Resilience workspace", "resilience-workspace")
+    space_id = uuid4()
+    principal = Principal(uuid4(), workspace.id, PrincipalType.HUMAN, frozenset({Role.EDITOR}))
+    document_id, version_id, chunk_id = uuid4(), uuid4(), uuid4()
+    citation = Citation.create(
+        workspace_id=workspace.id,
+        space_id=space_id,
+        document_id=document_id,
+        document_version_id=version_id,
+        chunk_id=chunk_id,
+        parent_chunk_id=None,
+        location={"page_start": 1},
+        excerpt="Independent evidence",
+        source_uri="https://example.test/doc.md",
+        acquired_at=datetime(2026, 8, 1, 12, tzinfo=UTC),
+        retrieval_run_id=uuid4(),
+        index_alias="rag_dense_hybrid_active",
+        profile_version="hybrid-v1",
+    )
+    candidate = Candidate(
+        point_id=uuid4(),
+        score=0.9,
+        rank=1,
+        backend=CandidateBackendKind.DENSE,
+        provenance=CandidateProvenance(
+            workspace_id=workspace.id,
+            space_id=space_id,
+            document_id=document_id,
+            document_version_id=version_id,
+            chunk_id=chunk_id,
+            source_id=uuid4(),
+            parent_chunk_id=None,
+            status="published",
+            content_type="text/markdown",
+            language="en",
+            trust_level="trusted_internal",
+        ),
+    )
+    hit = SearchHit(
+        fused=FusedCandidate(candidate, 1 / 61, 1, None),
+        text="Independent evidence",
+        title="Doc",
+        citation=citation,
+    )
+    context_candidate = ContextCandidate(citation, "Independent evidence", 1, 1.0)
+
+    search_service = SearchService(
+        authorization=RetrievalAuthorizationService(),
+        inputs=_SearchInputs(workspace, space_id),
+        executor=_SearchExecutor(hit),
+    )
+    context_service = ContextService(
+        authorization=RetrievalAuthorizationService(),
+        inputs=_SearchInputs(workspace, space_id),
+        executor=_ContextExecutor(context_candidate),
+    )
+
+    api = create_app()
+    api.dependency_overrides[get_search_principal] = lambda: principal
+    api.dependency_overrides[get_context_principal] = lambda: principal
+    api.dependency_overrides[get_search_application] = lambda: search_service
+    api.dependency_overrides[get_context_application] = lambda: context_service
+    return api, space_id
+
+
+def test_search_succeeds_while_the_generation_provider_circuit_is_open() -> None:
+    generation_guard = _fully_down_generation_guard()
+    api, space_id = _fixture()
+
+    response = TestClient(api).post(
+        "/v1/search",
+        json={"query": "anything", "requested_space_ids": [str(space_id)]},
+    )
+
+    assert response.status_code == 200
+    assert response.json()["results"]
+    # The generation provider's outage is still in effect throughout --
+    # search never touched it, so it never has a chance to recover either.
+    assert generation_guard.circuit.state is CircuitState.OPEN
+    with pytest.raises(CircuitOpenError):
+        generation_guard.circuit.check()
+
+
+def test_context_succeeds_while_the_generation_provider_circuit_is_open() -> None:
+    generation_guard = _fully_down_generation_guard()
+    api, space_id = _fixture()
+
+    response = TestClient(api).post(
+        "/v1/context",
+        json={"query": "anything", "requested_space_ids": [str(space_id)]},
+    )
+
+    assert response.status_code == 200
+    assert response.json()["blocks"]
+    assert generation_guard.circuit.state is CircuitState.OPEN