Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e
This commit is contained in:
Public Source Publisher committed
commit
7ca726caf2
659 files changed
+98171
No files matched your search
@@ -0,0 +1,184 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from ragcore.application.profiles import (
|
||||
ActivationDecision,
|
||||
AlreadyRolledBackError,
|
||||
CandidateAliasTarget,
|
||||
IncompleteEvaluationError,
|
||||
NotActivatedError,
|
||||
ProfileActivationError,
|
||||
require_complete_run,
|
||||
)
|
||||
from ragcore.evaluation import EvaluationCase, EvaluationDataset, EvaluationRun, RetrievalRunProfile
|
||||
from ragcore.evaluation.metrics.models import GateResult, RegressionFinding, RegressionMetric
|
||||
from ragcore.evaluation.models import AnswerabilityStatus, CaseResult
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _run_profile(index_version: str = "v1") -> RetrievalRunProfile:
|
||||
return RetrievalRunProfile(
|
||||
embedding_profile_digest="a" * 64,
|
||||
sparse_profile_version="technical-v1",
|
||||
index_alias="rag_dense_hybrid_active",
|
||||
index_collection_version=index_version,
|
||||
)
|
||||
|
||||
|
||||
def _passing_gate() -> GateResult:
|
||||
return GateResult(thresholds_version="v1", passed=True, regressions=())
|
||||
|
||||
|
||||
def _target() -> CandidateAliasTarget:
|
||||
return CandidateAliasTarget(
|
||||
alias_name="rag_dense_hybrid_active",
|
||||
collection_name="rag_dense_hybrid_v2",
|
||||
version="v2",
|
||||
model_digest="b" * 64,
|
||||
)
|
||||
|
||||
|
||||
def _decision(**overrides: object) -> ActivationDecision:
|
||||
values: dict[str, object] = {
|
||||
"id": uuid4(),
|
||||
"created_at": _now(),
|
||||
"dataset_id": "sample",
|
||||
"dataset_version": "v1",
|
||||
"baseline_run_id": uuid4(),
|
||||
"candidate_run_id": uuid4(),
|
||||
"baseline_profile": _run_profile("v1"),
|
||||
"candidate_profile": _run_profile("v2"),
|
||||
"gate": _passing_gate(),
|
||||
"activated": True,
|
||||
"candidate_target": _target(),
|
||||
"index_operation_id": uuid4(),
|
||||
"previous_collection": "rag_dense_hybrid_v1",
|
||||
}
|
||||
values.update(overrides)
|
||||
return ActivationDecision(**values)
|
||||
|
||||
|
||||
def test_candidate_alias_target_rejects_unsafe_identifiers() -> None:
|
||||
with pytest.raises(ProfileActivationError):
|
||||
CandidateAliasTarget(
|
||||
alias_name="not a safe id!", collection_name="c", version="v1", model_digest="d"
|
||||
)
|
||||
|
||||
|
||||
def test_activated_decision_requires_candidate_target_and_operation_id() -> None:
|
||||
with pytest.raises(ProfileActivationError):
|
||||
_decision(candidate_target=None)
|
||||
with pytest.raises(ProfileActivationError):
|
||||
_decision(index_operation_id=None)
|
||||
|
||||
|
||||
def test_rejected_decision_must_not_carry_activation_fields() -> None:
|
||||
with pytest.raises(ProfileActivationError):
|
||||
_decision(activated=False, gate=_failing_gate())
|
||||
|
||||
|
||||
def _failing_gate() -> GateResult:
|
||||
return GateResult(
|
||||
thresholds_version="v1",
|
||||
passed=False,
|
||||
regressions=(
|
||||
RegressionFinding(
|
||||
metric=RegressionMetric.RECALL_AT_5,
|
||||
tag=None,
|
||||
baseline_value=0.9,
|
||||
candidate_value=0.5,
|
||||
delta=-0.4,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_failed_gate_can_never_be_recorded_as_activated() -> None:
|
||||
with pytest.raises(ProfileActivationError):
|
||||
_decision(gate=_failing_gate(), activated=True)
|
||||
|
||||
|
||||
def test_rollback_requires_an_activated_decision() -> None:
|
||||
rejected = _decision(
|
||||
activated=False, gate=_failing_gate(), candidate_target=None, index_operation_id=None
|
||||
)
|
||||
with pytest.raises(NotActivatedError):
|
||||
rejected.with_rollback(rolled_back_at=_now())
|
||||
|
||||
|
||||
def test_rollback_cannot_happen_twice() -> None:
|
||||
decision = _decision()
|
||||
rolled_back = decision.with_rollback(rolled_back_at=_now())
|
||||
assert rolled_back.rolled_back is True
|
||||
with pytest.raises(AlreadyRolledBackError):
|
||||
rolled_back.with_rollback(rolled_back_at=_now())
|
||||
|
||||
|
||||
def _dataset() -> EvaluationDataset:
|
||||
case = EvaluationCase(
|
||||
id="case-1",
|
||||
question="What restarts the payment service?",
|
||||
language="en",
|
||||
expected_answerability=AnswerabilityStatus.ANSWERABLE,
|
||||
expected_document_ids=(uuid4(),),
|
||||
expected_chunk_ids=(uuid4(),),
|
||||
required_facts=("restart via kubectl rollout restart",),
|
||||
)
|
||||
return EvaluationDataset(id="sample", version="v1", cases=(case,))
|
||||
|
||||
|
||||
def _run(dataset: EvaluationDataset, *, cover_all: bool = True) -> EvaluationRun:
|
||||
case_results = tuple(
|
||||
CaseResult(
|
||||
case_id=case.id,
|
||||
retrieved_chunk_ids=case.expected_chunk_ids,
|
||||
matched_expected_ids=case.expected_chunk_ids,
|
||||
missing_expected_ids=(),
|
||||
unexpected_ids=(),
|
||||
)
|
||||
for case in dataset.cases
|
||||
)
|
||||
if not cover_all:
|
||||
case_results = case_results[:0]
|
||||
return EvaluationRun(
|
||||
id=uuid4(),
|
||||
dataset_id=dataset.id,
|
||||
dataset_version=dataset.version,
|
||||
run_profile=_run_profile(),
|
||||
started_at=_now(),
|
||||
case_results=case_results or (
|
||||
CaseResult(
|
||||
case_id="unrelated-case",
|
||||
retrieved_chunk_ids=(),
|
||||
matched_expected_ids=(),
|
||||
missing_expected_ids=(),
|
||||
unexpected_ids=(),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_require_complete_run_accepts_full_coverage() -> None:
|
||||
dataset = _dataset()
|
||||
require_complete_run(dataset, _run(dataset))
|
||||
|
||||
|
||||
def test_require_complete_run_rejects_partial_coverage() -> None:
|
||||
dataset = _dataset()
|
||||
with pytest.raises(IncompleteEvaluationError):
|
||||
require_complete_run(dataset, _run(dataset, cover_all=False))
|
||||
|
||||
|
||||
def test_require_complete_run_rejects_wrong_dataset_version() -> None:
|
||||
dataset = _dataset()
|
||||
run = _run(dataset)
|
||||
other_dataset = EvaluationDataset(id="sample", version="v2", cases=dataset.cases)
|
||||
with pytest.raises(IncompleteEvaluationError):
|
||||
require_complete_run(other_dataset, run)
|
||||
@@ -0,0 +1,371 @@
|
||||
"""Fast unit coverage for ProfileActivationService using in-memory index stores.
|
||||
|
||||
Reuses the real ``IndexBuildService`` (never reimplemented), backed by
|
||||
in-memory fakes for ``DerivedIndexStore``/``IndexOperationStore`` instead of
|
||||
real Qdrant/Postgres -- this proves ProfileActivationService's own gating,
|
||||
decision-persistence and rollback-reconstruction logic quickly and
|
||||
deterministically. Slower, real-infrastructure end-to-end coverage lives in
|
||||
``tests/integration/profiles``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from ragcore.application.indexes import (
|
||||
IndexBuildService,
|
||||
IndexOperationStatus,
|
||||
MalformedIndexPoint,
|
||||
StoredIndexPoint,
|
||||
)
|
||||
from ragcore.application.profiles import (
|
||||
ActivationDecisionNotFoundError,
|
||||
ActivationRejectedError,
|
||||
AlreadyRolledBackError,
|
||||
CandidateAliasTarget,
|
||||
IncompleteEvaluationError,
|
||||
ProfileActivationService,
|
||||
)
|
||||
from ragcore.evaluation import EvaluationCase, EvaluationDataset, EvaluationRun, RetrievalRunProfile
|
||||
from ragcore.evaluation.metrics.models import RegressionThresholds
|
||||
from ragcore.evaluation.models import AnswerabilityStatus, CaseResult
|
||||
|
||||
|
||||
class InMemoryDerivedIndexStore:
|
||||
def __init__(self) -> None:
|
||||
self.aliases: dict[str, str] = {}
|
||||
self.shadows: set[str] = set()
|
||||
self.build_calls = 0
|
||||
self.rollback_calls: list[tuple[str, str | None]] = []
|
||||
|
||||
async def create_shadow(self, profile: object) -> None:
|
||||
self.shadows.add(profile.collection_name) # type: ignore[attr-defined]
|
||||
self.build_calls += 1
|
||||
|
||||
async def delete_shadow(self, profile: object) -> None:
|
||||
self.shadows.discard(profile.collection_name) # type: ignore[attr-defined]
|
||||
|
||||
async def upsert_points(self, profile: object, points: Sequence[object]) -> None:
|
||||
return None
|
||||
|
||||
async def delete_points(self, profile: object, point_ids: Sequence[UUID]) -> None:
|
||||
return None
|
||||
|
||||
async def delete_raw_points(self, profile: object, raw_ids: Sequence[str]) -> None:
|
||||
return None
|
||||
|
||||
async def count_points(self, profile: object) -> int:
|
||||
return 0
|
||||
|
||||
async def list_points(
|
||||
self, profile: object
|
||||
) -> tuple[tuple[StoredIndexPoint, ...], tuple[MalformedIndexPoint, ...]]:
|
||||
return (), ()
|
||||
|
||||
async def current_alias_target(self, alias_name: str) -> str | None:
|
||||
return self.aliases.get(alias_name)
|
||||
|
||||
async def activate(self, profile: object) -> None:
|
||||
self.aliases[profile.alias_name] = profile.collection_name # type: ignore[attr-defined]
|
||||
|
||||
async def rollback(self, alias_name: str, previous_collection: str | None) -> None:
|
||||
self.rollback_calls.append((alias_name, previous_collection))
|
||||
if previous_collection is None:
|
||||
self.aliases.pop(alias_name, None)
|
||||
else:
|
||||
self.aliases[alias_name] = previous_collection
|
||||
|
||||
|
||||
class InMemoryIndexOperationStore:
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[UUID, dict[str, object]] = {}
|
||||
|
||||
async def begin(
|
||||
self,
|
||||
*,
|
||||
alias_name: str,
|
||||
operation: str,
|
||||
collection_name: str,
|
||||
owner_token: str,
|
||||
now: datetime,
|
||||
lease_duration: timedelta,
|
||||
) -> UUID | None:
|
||||
for row in self._rows.values():
|
||||
in_progress = row["status"] == IndexOperationStatus.IN_PROGRESS
|
||||
if row["alias_name"] == alias_name and in_progress:
|
||||
return None
|
||||
operation_id = uuid4()
|
||||
self._rows[operation_id] = {
|
||||
"alias_name": alias_name,
|
||||
"operation": operation,
|
||||
"collection_name": collection_name,
|
||||
"owner_token": owner_token,
|
||||
"status": IndexOperationStatus.IN_PROGRESS,
|
||||
"previous_collection": None,
|
||||
}
|
||||
return operation_id
|
||||
|
||||
async def set_previous_collection(
|
||||
self,
|
||||
operation_id: UUID,
|
||||
owner_token: str,
|
||||
previous_collection: str | None,
|
||||
*,
|
||||
now: datetime,
|
||||
) -> None:
|
||||
self._rows[operation_id]["previous_collection"] = previous_collection
|
||||
|
||||
async def record_step(
|
||||
self,
|
||||
operation_id: UUID,
|
||||
owner_token: str,
|
||||
step: str,
|
||||
*,
|
||||
now: datetime,
|
||||
details: Mapping[str, object] | None = None,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
async def finish(
|
||||
self,
|
||||
operation_id: UUID,
|
||||
owner_token: str,
|
||||
status: IndexOperationStatus,
|
||||
*,
|
||||
now: datetime,
|
||||
error_code: str | None = None,
|
||||
error_details: Mapping[str, object] | None = None,
|
||||
) -> bool:
|
||||
self._rows[operation_id]["status"] = status
|
||||
return True
|
||||
|
||||
|
||||
class FixtureSource:
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
async def iter_points(self, profile: object) -> AsyncIterator[object]:
|
||||
return
|
||||
yield # pragma: no cover - makes this an async generator with zero points
|
||||
|
||||
|
||||
class PassingSmoke:
|
||||
async def run(self, profile: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class InMemoryActivationDecisionStore:
|
||||
def __init__(self) -> None:
|
||||
self._decisions: dict[UUID, object] = {}
|
||||
|
||||
async def save(self, decision: object) -> None:
|
||||
self._decisions[decision.id] = decision # type: ignore[attr-defined]
|
||||
|
||||
async def get(self, decision_id: UUID) -> object | None:
|
||||
return self._decisions.get(decision_id)
|
||||
|
||||
async def list_recent(self, limit: int = 20) -> tuple[object, ...]:
|
||||
return tuple(self._decisions.values())[:limit]
|
||||
|
||||
async def mark_rolled_back(self, decision_id: UUID, *, rolled_back_at: datetime) -> None:
|
||||
decision = self._decisions[decision_id]
|
||||
self._decisions[decision_id] = decision.with_rollback(rolled_back_at=rolled_back_at) # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def _dataset() -> EvaluationDataset:
|
||||
case = EvaluationCase(
|
||||
id="case-1",
|
||||
question="How do I restart the payment service?",
|
||||
language="en",
|
||||
expected_answerability=AnswerabilityStatus.ANSWERABLE,
|
||||
expected_document_ids=(uuid4(),),
|
||||
expected_chunk_ids=(uuid4(),),
|
||||
required_facts=("restart via kubectl rollout restart",),
|
||||
)
|
||||
return EvaluationDataset(id="sample", version="v1", cases=(case,))
|
||||
|
||||
|
||||
def _run_profile(version: str) -> RetrievalRunProfile:
|
||||
return RetrievalRunProfile(
|
||||
embedding_profile_digest="a" * 64,
|
||||
sparse_profile_version="technical-v1",
|
||||
index_alias="rag_dense_hybrid_active",
|
||||
index_collection_version=version,
|
||||
)
|
||||
|
||||
|
||||
def _complete_run(dataset: EvaluationDataset, *, version: str, hit: bool = True) -> EvaluationRun:
|
||||
case = dataset.cases[0]
|
||||
retrieved = case.expected_chunk_ids if hit else ()
|
||||
return EvaluationRun(
|
||||
id=uuid4(),
|
||||
dataset_id=dataset.id,
|
||||
dataset_version=dataset.version,
|
||||
run_profile=_run_profile(version),
|
||||
started_at=datetime.now(UTC),
|
||||
case_results=(
|
||||
CaseResult(
|
||||
case_id=case.id,
|
||||
retrieved_chunk_ids=retrieved,
|
||||
matched_expected_ids=retrieved,
|
||||
missing_expected_ids=() if hit else case.expected_chunk_ids,
|
||||
unexpected_ids=(),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _incomplete_run(dataset: EvaluationDataset) -> EvaluationRun:
|
||||
return EvaluationRun(
|
||||
id=uuid4(),
|
||||
dataset_id=dataset.id,
|
||||
dataset_version=dataset.version,
|
||||
run_profile=_run_profile("v2"),
|
||||
started_at=datetime.now(UTC),
|
||||
case_results=(
|
||||
CaseResult(
|
||||
case_id="a-different-case",
|
||||
retrieved_chunk_ids=(),
|
||||
matched_expected_ids=(),
|
||||
missing_expected_ids=(),
|
||||
unexpected_ids=(),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _target() -> CandidateAliasTarget:
|
||||
return CandidateAliasTarget(
|
||||
alias_name="rag_dense_hybrid_active",
|
||||
collection_name="rag_dense_hybrid_v2",
|
||||
version="v2",
|
||||
model_digest="b" * 64,
|
||||
)
|
||||
|
||||
|
||||
_ServiceFixture = tuple[
|
||||
ProfileActivationService, InMemoryDerivedIndexStore, InMemoryActivationDecisionStore
|
||||
]
|
||||
|
||||
|
||||
def _service() -> _ServiceFixture:
|
||||
store = InMemoryDerivedIndexStore()
|
||||
builder = IndexBuildService(store, InMemoryIndexOperationStore())
|
||||
decisions = InMemoryActivationDecisionStore()
|
||||
return ProfileActivationService(index_builder=builder, decisions=decisions), store, decisions
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_incomplete_candidate_run_is_rejected_before_any_qdrant_call() -> None:
|
||||
service, store, decisions = _service()
|
||||
dataset = _dataset()
|
||||
baseline = _complete_run(dataset, version="v1")
|
||||
with pytest.raises(IncompleteEvaluationError):
|
||||
await service.activate(
|
||||
dataset=dataset,
|
||||
baseline_run=baseline,
|
||||
candidate_run=_incomplete_run(dataset),
|
||||
thresholds=RegressionThresholds(version="v1"),
|
||||
candidate_profile=_target(),
|
||||
source=FixtureSource(),
|
||||
smoke=PassingSmoke(),
|
||||
)
|
||||
assert store.build_calls == 0
|
||||
assert await decisions.list_recent() == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_regressed_candidate_is_rejected_and_recorded_without_activating() -> None:
|
||||
service, store, decisions = _service()
|
||||
dataset = _dataset()
|
||||
baseline = _complete_run(dataset, version="v1", hit=True)
|
||||
candidate = _complete_run(dataset, version="v2", hit=False)
|
||||
with pytest.raises(ActivationRejectedError) as excinfo:
|
||||
await service.activate(
|
||||
dataset=dataset,
|
||||
baseline_run=baseline,
|
||||
candidate_run=candidate,
|
||||
thresholds=RegressionThresholds(version="v1"),
|
||||
candidate_profile=_target(),
|
||||
source=FixtureSource(),
|
||||
smoke=PassingSmoke(),
|
||||
)
|
||||
assert excinfo.value.gate.passed is False
|
||||
assert store.build_calls == 0
|
||||
recorded = (await decisions.list_recent())[0]
|
||||
assert recorded.activated is False
|
||||
assert recorded.candidate_target is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_passing_candidate_is_activated_and_recorded() -> None:
|
||||
service, store, decisions = _service()
|
||||
dataset = _dataset()
|
||||
baseline = _complete_run(dataset, version="v1", hit=True)
|
||||
candidate = _complete_run(dataset, version="v2", hit=True)
|
||||
decision = await service.activate(
|
||||
dataset=dataset,
|
||||
baseline_run=baseline,
|
||||
candidate_run=candidate,
|
||||
thresholds=RegressionThresholds(version="v1"),
|
||||
candidate_profile=_target(),
|
||||
source=FixtureSource(),
|
||||
smoke=PassingSmoke(),
|
||||
)
|
||||
assert decision.activated is True
|
||||
assert store.aliases["rag_dense_hybrid_active"] == "rag_dense_hybrid_v2"
|
||||
stored = await decisions.get(decision.id)
|
||||
assert stored is not None
|
||||
assert stored.index_operation_id == decision.index_operation_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rollback_restores_the_previous_alias_and_marks_the_decision() -> None:
|
||||
service, store, _decisions = _service()
|
||||
dataset = _dataset()
|
||||
baseline = _complete_run(dataset, version="v1", hit=True)
|
||||
candidate = _complete_run(dataset, version="v2", hit=True)
|
||||
decision = await service.activate(
|
||||
dataset=dataset,
|
||||
baseline_run=baseline,
|
||||
candidate_run=candidate,
|
||||
thresholds=RegressionThresholds(version="v1"),
|
||||
candidate_profile=_target(),
|
||||
source=FixtureSource(),
|
||||
smoke=PassingSmoke(),
|
||||
)
|
||||
assert decision.previous_collection is None # nothing was active before this test
|
||||
|
||||
rolled_back = await service.rollback(decision.id)
|
||||
assert rolled_back.rolled_back is True
|
||||
assert store.rollback_calls == [("rag_dense_hybrid_active", None)]
|
||||
assert "rag_dense_hybrid_active" not in store.aliases
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rollback_twice_is_rejected() -> None:
|
||||
service, _, _ = _service()
|
||||
dataset = _dataset()
|
||||
decision = await service.activate(
|
||||
dataset=dataset,
|
||||
baseline_run=_complete_run(dataset, version="v1", hit=True),
|
||||
candidate_run=_complete_run(dataset, version="v2", hit=True),
|
||||
thresholds=RegressionThresholds(version="v1"),
|
||||
candidate_profile=_target(),
|
||||
source=FixtureSource(),
|
||||
smoke=PassingSmoke(),
|
||||
)
|
||||
await service.rollback(decision.id)
|
||||
with pytest.raises(AlreadyRolledBackError):
|
||||
await service.rollback(decision.id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rollback_of_unknown_decision_is_rejected() -> None:
|
||||
service, _, _ = _service()
|
||||
with pytest.raises(ActivationDecisionNotFoundError):
|
||||
await service.rollback(uuid4())
|
||||
Reference in new issue
Block a user