Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e
This commit is contained in:
Public Source Publisher committed
commit
7ca726caf2
659 files changed
+98171
No files matched your search
@@ -0,0 +1,335 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from ragcore.application.models.selection import (
|
||||
EmbeddingCutoverRejected,
|
||||
EmbeddingSelectionConflict,
|
||||
EmbeddingSelectionNotInitialized,
|
||||
EmbeddingSelectionOperationResult,
|
||||
EmbeddingSelectionState,
|
||||
EmbeddingSelectionVerificationResult,
|
||||
)
|
||||
from ragcore.config import Settings
|
||||
from ragcore.domain.control import DataPolicy
|
||||
from ragcore.domain.models import (
|
||||
EmbeddingProfile,
|
||||
EmbeddingProvider,
|
||||
EmbeddingSelection,
|
||||
NormalizationMode,
|
||||
)
|
||||
from ragcore.infrastructure.models.persistence import JsonEmbeddingProfileStore
|
||||
from ragcore.ops import modelforge_embedding_bootstrap as bootstrap
|
||||
|
||||
_OPERATOR_TOKEN = "operator-token-with-at-least-32-random-characters"
|
||||
|
||||
|
||||
class MemorySelectionStore:
|
||||
def __init__(self) -> None:
|
||||
self.state: EmbeddingSelectionState | None = None
|
||||
self.operations: dict[UUID, tuple[str, str, EmbeddingSelectionOperationResult]] = {}
|
||||
|
||||
async def get(self) -> EmbeddingSelectionState:
|
||||
if self.state is None:
|
||||
raise EmbeddingSelectionNotInitialized("not initialized")
|
||||
return self.state
|
||||
|
||||
async def find_operation(
|
||||
self,
|
||||
operation_id: UUID,
|
||||
*,
|
||||
request_fingerprint: str,
|
||||
) -> EmbeddingSelectionOperationResult | None:
|
||||
entry = self.operations.get(operation_id)
|
||||
if entry is None:
|
||||
return None
|
||||
_action, old_fingerprint, result = entry
|
||||
if old_fingerprint != request_fingerprint:
|
||||
raise EmbeddingSelectionConflict("operation id conflict")
|
||||
return EmbeddingSelectionOperationResult(
|
||||
operation_id=result.operation_id,
|
||||
generation=result.generation,
|
||||
selection=result.selection,
|
||||
replayed=True,
|
||||
evidence=result.evidence,
|
||||
)
|
||||
|
||||
async def compare_and_swap(
|
||||
self,
|
||||
*,
|
||||
expected_generation: int,
|
||||
target: EmbeddingSelection,
|
||||
operation_id: UUID,
|
||||
action: str,
|
||||
request_fingerprint: str,
|
||||
operation_evidence: Mapping[str, object] | None = None,
|
||||
) -> EmbeddingSelectionOperationResult:
|
||||
if operation_id in self.operations:
|
||||
replay = await self.find_operation(
|
||||
operation_id,
|
||||
request_fingerprint=request_fingerprint,
|
||||
)
|
||||
assert replay is not None
|
||||
return replay
|
||||
if action != "initialize" or expected_generation != 0 or self.state is not None:
|
||||
raise EmbeddingSelectionConflict("already initialized or stale")
|
||||
self.state = EmbeddingSelectionState(0, target, None)
|
||||
result = EmbeddingSelectionOperationResult(
|
||||
operation_id=operation_id,
|
||||
generation=0,
|
||||
selection=target,
|
||||
evidence=operation_evidence or {},
|
||||
)
|
||||
self.operations[operation_id] = (action, request_fingerprint, result)
|
||||
return result
|
||||
|
||||
|
||||
class RecordingVerifier:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[EmbeddingSelection, EmbeddingProfile]] = []
|
||||
self.gateway_request_id = uuid4()
|
||||
|
||||
async def verify(
|
||||
self, selection: EmbeddingSelection, profile: EmbeddingProfile
|
||||
) -> EmbeddingSelectionVerificationResult:
|
||||
self.calls.append((selection, profile))
|
||||
return EmbeddingSelectionVerificationResult(
|
||||
gateway_request_ids=(self.gateway_request_id,), verified_point_count=12
|
||||
)
|
||||
|
||||
|
||||
class ClosingResource:
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
async def aclose(self) -> None:
|
||||
return None
|
||||
|
||||
async def dispose(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _rc_settings(tmp_path: Path) -> Settings:
|
||||
return Settings(
|
||||
env="rc",
|
||||
secret_key="s" * 48,
|
||||
api_token_pepper="p" * 48,
|
||||
qdrant_api_key="q" * 48,
|
||||
database_url="postgresql+psycopg://rc:rc-very-long-random-password@localhost:5432/ragcore_rc",
|
||||
public_base_url="https://ragcore-rc.example.test",
|
||||
trusted_proxy_ips=("127.0.0.1",),
|
||||
blob_root=tmp_path / "blobs",
|
||||
parsed_root=tmp_path / "parsed",
|
||||
temp_root=tmp_path / "tmp",
|
||||
model_state_root=tmp_path / "models",
|
||||
embedding_cutover_operator_id=uuid4(),
|
||||
embedding_cutover_operator_token_sha256=hashlib.sha256(
|
||||
_OPERATOR_TOKEN.encode("utf-8")
|
||||
).hexdigest(),
|
||||
)
|
||||
|
||||
|
||||
def _manifest(
|
||||
path: Path, *, profile: str = "modelforge-rc-con01", version: str = "rc1"
|
||||
) -> dict[str, object]:
|
||||
value: dict[str, object] = {
|
||||
"profile": profile,
|
||||
"version": version,
|
||||
"embedding_space_id": str(uuid4()),
|
||||
"embedding_space_digest": "a" * 64,
|
||||
"dimensions": 1024,
|
||||
"experiment_route_key": "rc-canary-2026-09",
|
||||
}
|
||||
path.write_text(json.dumps(value), encoding="utf-8")
|
||||
return value
|
||||
|
||||
|
||||
def _arguments(path: Path, *, operation_id: UUID | None = None) -> argparse.Namespace:
|
||||
return bootstrap.parser().parse_args(
|
||||
[
|
||||
"initialize",
|
||||
"--expected-generation",
|
||||
"0",
|
||||
"--operation-id",
|
||||
str(operation_id or uuid4()),
|
||||
"--manifest-file",
|
||||
str(path),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_parser_requires_generation_zero_operation_id_and_manifest() -> None:
|
||||
manifest = Path("manifest.json")
|
||||
args = bootstrap.parser().parse_args(
|
||||
[
|
||||
"initialize",
|
||||
"--expected-generation",
|
||||
"0",
|
||||
"--operation-id",
|
||||
str(uuid4()),
|
||||
"--manifest-file",
|
||||
str(manifest),
|
||||
]
|
||||
)
|
||||
|
||||
assert args.operation == "initialize"
|
||||
assert args.expected_generation == 0
|
||||
assert args.manifest_file == manifest
|
||||
with pytest.raises(SystemExit):
|
||||
bootstrap.parser().parse_args(
|
||||
["initialize", "--expected-generation", "-1", "--operation-id", str(uuid4())]
|
||||
)
|
||||
|
||||
|
||||
def test_manifest_rejects_duplicate_or_unknown_fields(tmp_path: Path) -> None:
|
||||
path = tmp_path / "manifest.json"
|
||||
path.write_text('{"profile":"a","profile":"b"}', encoding="utf-8")
|
||||
|
||||
with pytest.raises(bootstrap.EmbeddingBootstrapOperatorError, match="valid JSON"):
|
||||
bootstrap._read_manifest(path)
|
||||
|
||||
path.write_text(json.dumps({"unexpected": True}), encoding="utf-8")
|
||||
with pytest.raises(bootstrap.EmbeddingBootstrapOperatorError, match="field set"):
|
||||
bootstrap._read_manifest(path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bootstrap_rejects_collection_already_referenced_by_qdrant_alias() -> None:
|
||||
space_id = uuid4()
|
||||
profile = EmbeddingProfile(
|
||||
id=space_id,
|
||||
profile="modelforge-rc-con01",
|
||||
version="rc1",
|
||||
provider=EmbeddingProvider.MODELFORGE,
|
||||
model_identifier="rag.embedding@1",
|
||||
model_digest="a" * 64,
|
||||
dimensions=4,
|
||||
context_limit=4096,
|
||||
tokenizer="capability-managed",
|
||||
data_policy=DataPolicy.LOCAL_ONLY,
|
||||
normalization=NormalizationMode.L2,
|
||||
embedding_space_id=space_id,
|
||||
)
|
||||
selection = EmbeddingSelection.from_profile(profile, experiment_route_key="rc-canary-route")
|
||||
|
||||
class AliasedClient:
|
||||
async def get_aliases(self) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
aliases=[SimpleNamespace(collection_name=selection.collection_name)]
|
||||
)
|
||||
|
||||
verifier = bootstrap._UnaliasedRCCollectionVerifier(AliasedClient()) # type: ignore[arg-type]
|
||||
with pytest.raises(EmbeddingCutoverRejected, match="already a Qdrant alias target"):
|
||||
await verifier.verify(selection, profile)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_registers_and_initializes_exact_verified_rc_selection_idempotently(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
settings = _rc_settings(tmp_path)
|
||||
manifest_path = tmp_path / "bootstrap.json"
|
||||
manifest = _manifest(manifest_path)
|
||||
operation_id = uuid4()
|
||||
profile_store = JsonEmbeddingProfileStore(settings.embedding_profiles_path)
|
||||
selection_store = MemorySelectionStore()
|
||||
verifier = RecordingVerifier()
|
||||
from ragcore.application.models.selection import EmbeddingSelectionService
|
||||
|
||||
service = EmbeddingSelectionService(
|
||||
profiles=profile_store,
|
||||
selections=selection_store,
|
||||
verifier=verifier,
|
||||
)
|
||||
|
||||
async def build_service(
|
||||
_settings: Settings,
|
||||
) -> tuple[object, ClosingResource, ClosingResource, ClosingResource]:
|
||||
return _service_tuple(service)
|
||||
|
||||
monkeypatch.setattr(bootstrap, "_build_service", build_service)
|
||||
arguments = _arguments(manifest_path, operation_id=operation_id)
|
||||
|
||||
first = await bootstrap.execute(
|
||||
arguments,
|
||||
operator_token=_OPERATOR_TOKEN,
|
||||
settings=settings,
|
||||
)
|
||||
replay = await bootstrap.execute(
|
||||
arguments,
|
||||
operator_token=_OPERATOR_TOKEN,
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
space_id = UUID(str(manifest["embedding_space_id"]))
|
||||
profile = await profile_store.get(space_id)
|
||||
assert profile is not None
|
||||
assert profile.model_digest == manifest["embedding_space_digest"]
|
||||
assert profile.dimensions == manifest["dimensions"]
|
||||
assert profile.embedding_space_id == space_id
|
||||
assert await profile_store.get_active() is None
|
||||
assert selection_store.state is not None
|
||||
selection = selection_store.state.active
|
||||
assert selection.profile_id == space_id
|
||||
assert selection.experiment_route_key == manifest["experiment_route_key"]
|
||||
assert selection.collection_name == "rag_dense_modelforge-rc-con01_rc1"
|
||||
assert verifier.calls == [(selection, profile)]
|
||||
assert first["status"] == "applied"
|
||||
assert replay["status"] == "replayed"
|
||||
assert first["generation"] == replay["generation"] == 0
|
||||
assert first["verification"]["verified_point_count"] == 12 # type: ignore[index]
|
||||
assert _OPERATOR_TOKEN not in json.dumps(first)
|
||||
|
||||
|
||||
def _service_tuple(
|
||||
service: object,
|
||||
) -> tuple[object, ClosingResource, ClosingResource, ClosingResource]:
|
||||
resource = ClosingResource()
|
||||
return service, resource, resource, resource
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_rejects_non_rc_collection_namespace_before_connecting(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
settings = _rc_settings(tmp_path)
|
||||
manifest_path = tmp_path / "bootstrap.json"
|
||||
_manifest(manifest_path, profile="modelforge-rag", version="m6")
|
||||
|
||||
async def no_connect(_settings: Settings) -> object:
|
||||
raise AssertionError("service connections must follow manifest validation")
|
||||
|
||||
monkeypatch.setattr(bootstrap, "_build_service", no_connect)
|
||||
with pytest.raises(bootstrap.EmbeddingBootstrapOperatorError, match="modelforge-rc-"):
|
||||
await bootstrap.execute(
|
||||
_arguments(manifest_path),
|
||||
operator_token=_OPERATOR_TOKEN,
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_rejects_non_rc_or_wrong_operator_token_before_manifest_read(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
missing_manifest = tmp_path / "does-not-exist.json"
|
||||
arguments = _arguments(missing_manifest)
|
||||
|
||||
with pytest.raises(bootstrap.EmbeddingBootstrapOperatorError, match="restricted"):
|
||||
await bootstrap.execute(arguments, operator_token=_OPERATOR_TOKEN, settings=Settings())
|
||||
|
||||
settings = _rc_settings(tmp_path)
|
||||
with pytest.raises(bootstrap.EmbeddingBootstrapOperatorError, match="authentication failed"):
|
||||
await bootstrap.execute(
|
||||
arguments,
|
||||
operator_token="wrong-token-value-that-is-long-enough",
|
||||
settings=settings,
|
||||
)
|
||||
@@ -0,0 +1,255 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from ragcore.application.models.selection import EmbeddingSelectionOperationResult
|
||||
from ragcore.config import Settings
|
||||
from ragcore.domain.models import EmbeddingProvider, EmbeddingSelection
|
||||
from ragcore.ops.modelforge_embedding_cutover import (
|
||||
_PROMOTION_EVIDENCE_FIELDS,
|
||||
EmbeddingCutoverOperatorError,
|
||||
_authorize_operator,
|
||||
_read_evidence,
|
||||
_redacted_receipt,
|
||||
execute,
|
||||
parser,
|
||||
)
|
||||
|
||||
_OPERATOR_TOKEN = "operator-token-with-at-least-32-random-characters"
|
||||
|
||||
|
||||
def _rc_settings(tmp_path: Path, *, token_hash: str | None = None) -> Settings:
|
||||
return Settings(
|
||||
env="rc",
|
||||
secret_key="s" * 48,
|
||||
api_token_pepper="p" * 48,
|
||||
qdrant_api_key="q" * 48,
|
||||
database_url="postgresql+psycopg://rc:rc-very-long-random-password@localhost:5432/ragcore_rc",
|
||||
public_base_url="https://ragcore-rc.example.test",
|
||||
trusted_proxy_ips=("127.0.0.1",),
|
||||
blob_root=tmp_path / "blobs",
|
||||
parsed_root=tmp_path / "parsed",
|
||||
temp_root=tmp_path / "tmp",
|
||||
model_state_root=tmp_path / "models",
|
||||
embedding_cutover_operator_id=uuid4() if token_hash is not None else None,
|
||||
embedding_cutover_operator_token_sha256=token_hash,
|
||||
)
|
||||
|
||||
|
||||
def test_parser_requires_bounded_operation_identity_and_evidence_file(tmp_path: Path) -> None:
|
||||
evidence = tmp_path / "evidence.json"
|
||||
args = parser().parse_args(
|
||||
[
|
||||
"promote",
|
||||
"--expected-generation",
|
||||
"7",
|
||||
"--operation-id",
|
||||
str(uuid4()),
|
||||
"--evidence-file",
|
||||
str(evidence),
|
||||
]
|
||||
)
|
||||
|
||||
assert args.operation == "promote"
|
||||
assert args.expected_generation == 7
|
||||
assert args.evidence_file == evidence
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
parser().parse_args(
|
||||
["rollback", "--expected-generation", "-1", "--operation-id", str(uuid4())]
|
||||
)
|
||||
|
||||
|
||||
def test_parser_rejects_nil_operation_ids() -> None:
|
||||
with pytest.raises(SystemExit):
|
||||
parser().parse_args(
|
||||
[
|
||||
"rollback",
|
||||
"--expected-generation",
|
||||
"0",
|
||||
"--operation-id",
|
||||
"00000000-0000-0000-0000-000000000000",
|
||||
"--evidence-file",
|
||||
"evidence.json",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_operator_authentication_requires_rc_configuration_and_exact_secret(tmp_path: Path) -> None:
|
||||
expected = hashlib.sha256(_OPERATOR_TOKEN.encode()).hexdigest()
|
||||
settings = _rc_settings(tmp_path, token_hash=expected)
|
||||
|
||||
_authorize_operator(settings, _OPERATOR_TOKEN)
|
||||
with pytest.raises(EmbeddingCutoverOperatorError, match="authentication failed"):
|
||||
_authorize_operator(settings, "wrong-token-value-that-is-long-enough")
|
||||
|
||||
with pytest.raises(EmbeddingCutoverOperatorError, match="identity is not configured"):
|
||||
_authorize_operator(_rc_settings(tmp_path), _OPERATOR_TOKEN)
|
||||
|
||||
with pytest.raises(EmbeddingCutoverOperatorError, match="restricted"):
|
||||
_authorize_operator(Settings(env="development"), _OPERATOR_TOKEN)
|
||||
|
||||
|
||||
def test_rc_settings_reject_malformed_operator_token_hash() -> None:
|
||||
with pytest.raises(ValidationError, match="lowercase SHA-256"):
|
||||
Settings(embedding_cutover_operator_token_sha256="not-a-hash")
|
||||
|
||||
|
||||
def test_rc_requires_production_strength_settings() -> None:
|
||||
with pytest.raises(ValidationError, match="secret_key"):
|
||||
Settings(env="rc", secret_key="development-only-change-me")
|
||||
|
||||
|
||||
def test_rc_operator_identity_and_token_hash_must_be_paired() -> None:
|
||||
with pytest.raises(ValidationError, match="must be set together"):
|
||||
Settings(embedding_cutover_operator_id=uuid4())
|
||||
with pytest.raises(ValidationError, match="must not be the nil UUID"):
|
||||
Settings(
|
||||
embedding_cutover_operator_id="00000000-0000-0000-0000-000000000000",
|
||||
embedding_cutover_operator_token_sha256="a" * 64,
|
||||
)
|
||||
|
||||
|
||||
def test_evidence_parser_rejects_unexpected_fields(tmp_path: Path) -> None:
|
||||
path = tmp_path / "evidence.json"
|
||||
path.write_text(
|
||||
json.dumps({"comparison_id": str(uuid4()), "secret": "extra"}), encoding="utf-8"
|
||||
)
|
||||
|
||||
with pytest.raises(EmbeddingCutoverOperatorError, match="unexpected field set"):
|
||||
_read_evidence(path, expected_fields=_PROMOTION_EVIDENCE_FIELDS)
|
||||
|
||||
|
||||
def test_evidence_parser_rejects_duplicate_keys(tmp_path: Path) -> None:
|
||||
path = tmp_path / "evidence.json"
|
||||
path.write_text('{"operator_id":"a","operator_id":"b"}', encoding="utf-8")
|
||||
|
||||
with pytest.raises(EmbeddingCutoverOperatorError, match="valid JSON"):
|
||||
_read_evidence(path, expected_fields=frozenset({"operator_id"}))
|
||||
|
||||
|
||||
def test_receipt_omits_submitted_evidence_and_retains_only_bounded_verification() -> None:
|
||||
operation_id = uuid4()
|
||||
selection = EmbeddingSelection(
|
||||
profile_id=uuid4(),
|
||||
profile="rc-candidate",
|
||||
version="v2",
|
||||
provider=EmbeddingProvider.MODELFORGE,
|
||||
model_identifier="rag.embedding@1",
|
||||
model_digest="a" * 64,
|
||||
dimensions=4,
|
||||
embedding_space_id=uuid4(),
|
||||
experiment_route_key="lab-candidate-route",
|
||||
collection_name="rag_dense_rc-candidate_v2",
|
||||
)
|
||||
result = EmbeddingSelectionOperationResult(
|
||||
operation_id=operation_id,
|
||||
generation=9,
|
||||
selection=selection,
|
||||
evidence={
|
||||
"operator_id": str(uuid4()),
|
||||
"approval_id": str(uuid4()),
|
||||
"reason": "private change note",
|
||||
"verification": {
|
||||
"gateway_request_ids": [str(uuid4())],
|
||||
"verified_point_count": 25,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
receipt = _redacted_receipt("promote", result)
|
||||
|
||||
assert receipt["operation_id"] == str(operation_id)
|
||||
assert "operator_id" not in receipt and "approval_id" not in receipt
|
||||
assert "private change note" not in json.dumps(receipt)
|
||||
assert receipt["verification"]["verified_point_count"] == 25 # type: ignore[index]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_rejects_non_rc_before_reading_evidence_or_connecting(tmp_path: Path) -> None:
|
||||
evidence = tmp_path / "missing-evidence.json"
|
||||
arguments = parser().parse_args(
|
||||
[
|
||||
"promote",
|
||||
"--expected-generation",
|
||||
"0",
|
||||
"--operation-id",
|
||||
str(uuid4()),
|
||||
"--evidence-file",
|
||||
str(evidence),
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(EmbeddingCutoverOperatorError, match="restricted"):
|
||||
await execute(arguments, settings=Settings(env="development"), operator_token="x" * 40)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_rejects_evidence_claimed_by_another_operator(tmp_path: Path) -> None:
|
||||
token_hash = hashlib.sha256(_OPERATOR_TOKEN.encode()).hexdigest()
|
||||
settings = _rc_settings(tmp_path, token_hash=token_hash)
|
||||
evidence_path = tmp_path / "promotion.json"
|
||||
evidence_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"comparison_id": str(uuid4()),
|
||||
"comparison_reviewed": True,
|
||||
"comparison_passed": True,
|
||||
"operator_id": str(uuid4()),
|
||||
"approval_id": str(uuid4()),
|
||||
"operator_approved_impact_and_reindex": True,
|
||||
"index_operation_id": str(uuid4()),
|
||||
"authoritative_reindex_completed": True,
|
||||
"collection_identity_pins_verified": True,
|
||||
"consumer_smoke_passed": True,
|
||||
"candidate_profile_id": str(uuid4()),
|
||||
"candidate_experiment_route_key": "lab-candidate-route",
|
||||
"gateway_route_key": "lab-candidate-route",
|
||||
"gateway_embedding_space_id": str(uuid4()),
|
||||
"gateway_dimensions": 4,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
arguments = parser().parse_args(
|
||||
[
|
||||
"promote",
|
||||
"--expected-generation",
|
||||
"0",
|
||||
"--operation-id",
|
||||
str(uuid4()),
|
||||
"--evidence-file",
|
||||
str(evidence_path),
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(EmbeddingCutoverOperatorError, match="authenticated identity"):
|
||||
await execute(arguments, settings=settings, operator_token=_OPERATOR_TOKEN)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_rejects_invalid_evidence_before_connecting(tmp_path: Path) -> None:
|
||||
token_hash = hashlib.sha256(_OPERATOR_TOKEN.encode()).hexdigest()
|
||||
settings = _rc_settings(tmp_path, token_hash=token_hash)
|
||||
evidence_path = tmp_path / "malformed.json"
|
||||
evidence_path.write_text("not-json", encoding="utf-8")
|
||||
arguments = parser().parse_args(
|
||||
[
|
||||
"rollback",
|
||||
"--expected-generation",
|
||||
"3",
|
||||
"--operation-id",
|
||||
str(uuid4()),
|
||||
"--evidence-file",
|
||||
str(evidence_path),
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(EmbeddingCutoverOperatorError, match="not valid JSON"):
|
||||
await execute(arguments, settings=settings, operator_token=_OPERATOR_TOKEN)
|
||||
@@ -0,0 +1,720 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from qdrant_client.http import models
|
||||
|
||||
from ragcore.ops.modelforge_selected_comparison import (
|
||||
ComparisonCase,
|
||||
ComparisonClaim,
|
||||
ComparisonManifest,
|
||||
ComparisonPins,
|
||||
ComparisonStatus,
|
||||
ComparisonTargetPin,
|
||||
ModelForgeComparisonClient,
|
||||
PinnedIndex,
|
||||
SelectedComparisonError,
|
||||
SelectedComparisonRunner,
|
||||
_parse_pins,
|
||||
)
|
||||
from ragcore.retrieval.ranking import FusionConfig
|
||||
from ragcore.retrieval.sparse import SparseEncoderProfile
|
||||
|
||||
COMPARISON_ID = uuid4()
|
||||
PROJECT_ID = uuid4()
|
||||
SUITE_REVISION_ID = uuid4()
|
||||
WORKSPACE_ID = uuid4()
|
||||
CASE_ID = uuid4()
|
||||
CHUNK_ID = uuid4()
|
||||
SPACE_ID = uuid4()
|
||||
DOCUMENT_ID = uuid4()
|
||||
CURRENT_DEPLOYMENT_ID = uuid4()
|
||||
CANDIDATE_DEPLOYMENT_ID = uuid4()
|
||||
CURRENT_SPACE_ID = uuid4()
|
||||
CANDIDATE_SPACE_ID = uuid4()
|
||||
|
||||
|
||||
def _index(ref: str, profile: str, digest: str, space_id: UUID) -> PinnedIndex:
|
||||
return PinnedIndex(
|
||||
ref=ref,
|
||||
profile=profile,
|
||||
version="v1",
|
||||
dimensions=2,
|
||||
model_digest=digest,
|
||||
embedding_space_id=space_id,
|
||||
embedding_space_digest=("e" if profile == "current" else "f") * 64,
|
||||
)
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path) -> ComparisonManifest:
|
||||
retrieval = {
|
||||
"candidate_k": 40,
|
||||
"dense_limit": 40,
|
||||
"sparse_limit": 40,
|
||||
"output_k": 10,
|
||||
"timeout_seconds": 2.0,
|
||||
"sparse_profile": {
|
||||
"name": "technical",
|
||||
"version": "v1",
|
||||
"hash_namespace": "ragcore-sparse-v1",
|
||||
"max_tokens": 4096,
|
||||
"term_weight": 1.0,
|
||||
"identifier_weight": 3.0,
|
||||
"identifier_component_weight": 0.5,
|
||||
"normalization": "none",
|
||||
},
|
||||
"fusion": {"rrf_k": 60, "max_fused_candidates": 40, "max_per_document": 3},
|
||||
}
|
||||
return ComparisonManifest(
|
||||
comparison_id=COMPARISON_ID,
|
||||
project_id=PROJECT_ID,
|
||||
suite_revision_id=SUITE_REVISION_ID,
|
||||
workspace_id=WORKSPACE_ID,
|
||||
corpus_sha256="a" * 64,
|
||||
consumer_revision_sha256="b" * 64,
|
||||
modelforge_base_url="http://modelforge:8000",
|
||||
credential_file=tmp_path / "credential",
|
||||
qdrant_url="http://qdrant:6333",
|
||||
current_index=_index("current-index", "current", "c" * 64, CURRENT_SPACE_ID),
|
||||
candidate_index=_index("candidate-index", "candidate", "d" * 64, CANDIDATE_SPACE_ID),
|
||||
retrieval_config=retrieval,
|
||||
sparse_profile=SparseEncoderProfile(),
|
||||
fusion_config=FusionConfig(),
|
||||
output_k=10,
|
||||
timeout_seconds=2.0,
|
||||
)
|
||||
|
||||
|
||||
def _write_credential(path: Path) -> None:
|
||||
path.write_text("mf_" + "x" * 48, encoding="utf-8")
|
||||
path.chmod(0o600)
|
||||
|
||||
|
||||
def _manifest_payload(manifest: ComparisonManifest) -> dict[str, object]:
|
||||
def index_payload(index: PinnedIndex) -> dict[str, object]:
|
||||
return {
|
||||
"ref": index.ref,
|
||||
"profile": index.profile,
|
||||
"version": index.version,
|
||||
"dimensions": index.dimensions,
|
||||
"model_digest": index.model_digest,
|
||||
"embedding_space_id": str(index.embedding_space_id),
|
||||
"embedding_space_digest": index.embedding_space_digest,
|
||||
}
|
||||
|
||||
return {
|
||||
"comparison_id": str(manifest.comparison_id),
|
||||
"project_id": str(manifest.project_id),
|
||||
"suite_revision_id": str(manifest.suite_revision_id),
|
||||
"workspace_id": str(manifest.workspace_id),
|
||||
"corpus_sha256": manifest.corpus_sha256,
|
||||
"consumer_revision_sha256": manifest.consumer_revision_sha256,
|
||||
"modelforge_base_url": manifest.modelforge_base_url,
|
||||
"credential_file": str(manifest.credential_file),
|
||||
"qdrant_url": manifest.qdrant_url,
|
||||
"current_index": index_payload(manifest.current_index),
|
||||
"candidate_index": index_payload(manifest.candidate_index),
|
||||
"retrieval_config": dict(manifest.retrieval_config),
|
||||
}
|
||||
|
||||
|
||||
def _claim(
|
||||
manifest: ComparisonManifest,
|
||||
*,
|
||||
current_space: UUID = CURRENT_SPACE_ID,
|
||||
current_route_key: str | None = None,
|
||||
) -> ComparisonClaim:
|
||||
now = datetime.now(UTC)
|
||||
current = ComparisonTargetPin(
|
||||
deployment_id=CURRENT_DEPLOYMENT_ID,
|
||||
embedding_space_id=current_space,
|
||||
embedding_space_digest="e" * 64,
|
||||
dimension=2,
|
||||
artifact_sha256="1" * 64,
|
||||
runtime_fingerprint="runtime-current",
|
||||
)
|
||||
candidate = ComparisonTargetPin(
|
||||
deployment_id=CANDIDATE_DEPLOYMENT_ID,
|
||||
embedding_space_id=CANDIDATE_SPACE_ID,
|
||||
embedding_space_digest="f" * 64,
|
||||
dimension=2,
|
||||
artifact_sha256="2" * 64,
|
||||
runtime_fingerprint="runtime-candidate",
|
||||
)
|
||||
raw_pins = {
|
||||
"project_id": str(PROJECT_ID),
|
||||
"current_index_ref": manifest.current_index.ref,
|
||||
"candidate_index_ref": manifest.candidate_index.ref,
|
||||
"corpus_sha256": manifest.corpus_sha256,
|
||||
"retrieval_config_sha256": manifest.retrieval_config_sha256,
|
||||
"candidate_route_key": "candidate-route",
|
||||
"current": {
|
||||
"deployment_id": str(CURRENT_DEPLOYMENT_ID),
|
||||
"embedding_space_id": str(current_space),
|
||||
"embedding_space_digest": "e" * 64,
|
||||
"dimension": 2,
|
||||
"artifact_sha256": "1" * 64,
|
||||
"runtime_fingerprint": "runtime-current",
|
||||
},
|
||||
"candidate": {
|
||||
"deployment_id": str(CANDIDATE_DEPLOYMENT_ID),
|
||||
"embedding_space_id": str(CANDIDATE_SPACE_ID),
|
||||
"embedding_space_digest": "f" * 64,
|
||||
"dimension": 2,
|
||||
"artifact_sha256": "2" * 64,
|
||||
"runtime_fingerprint": "runtime-candidate",
|
||||
},
|
||||
}
|
||||
if current_route_key is not None:
|
||||
raw_pins["current_route_key"] = current_route_key
|
||||
return ComparisonClaim(
|
||||
comparison_id=COMPARISON_ID,
|
||||
suite_revision_id=SUITE_REVISION_ID,
|
||||
current_deployment_id=CURRENT_DEPLOYMENT_ID,
|
||||
candidate_deployment_id=CANDIDATE_DEPLOYMENT_ID,
|
||||
expected_cases=1,
|
||||
claim_token="claim-token-" + "x" * 32,
|
||||
claim_expires_at=now + timedelta(seconds=120),
|
||||
deadline_at=now + timedelta(minutes=10),
|
||||
pins_digest=hashlib.sha256(
|
||||
json.dumps(raw_pins, sort_keys=True, separators=(",", ":"), default=str).encode()
|
||||
).hexdigest(),
|
||||
pins=ComparisonPins(
|
||||
raw=raw_pins,
|
||||
project_id=PROJECT_ID,
|
||||
corpus_sha256=manifest.corpus_sha256,
|
||||
retrieval_config_sha256=manifest.retrieval_config_sha256,
|
||||
current_index_ref=manifest.current_index.ref,
|
||||
candidate_index_ref=manifest.candidate_index.ref,
|
||||
current_route_key=current_route_key,
|
||||
candidate_route_key="candidate-route",
|
||||
current=current,
|
||||
candidate=candidate,
|
||||
),
|
||||
cases=(
|
||||
ComparisonCase(
|
||||
id=CASE_ID,
|
||||
suite_revision_id=SUITE_REVISION_ID,
|
||||
case_key="one",
|
||||
query="where is the answer",
|
||||
relevant_chunk_ids=(CHUNK_ID,),
|
||||
critical=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class FakeControl:
|
||||
def __init__(self, claim: ComparisonClaim) -> None:
|
||||
self.claim_value = claim
|
||||
self.progress_calls: list[tuple[int, int]] = []
|
||||
self.finish_calls: list[tuple[str, str]] = []
|
||||
self.submitted: tuple[object, ...] | None = None
|
||||
self.cancel = False
|
||||
|
||||
async def claim(self, comparison_id: UUID) -> ComparisonClaim:
|
||||
assert comparison_id == self.claim_value.comparison_id
|
||||
return self.claim_value
|
||||
|
||||
async def progress(
|
||||
self,
|
||||
comparison_id: UUID,
|
||||
claim_token: str,
|
||||
current_completed: int,
|
||||
candidate_completed: int,
|
||||
) -> ComparisonStatus:
|
||||
assert comparison_id == self.claim_value.comparison_id
|
||||
assert claim_token == self.claim_value.claim_token
|
||||
self.progress_calls.append((current_completed, candidate_completed))
|
||||
return ComparisonStatus(
|
||||
status="cancel_requested" if self.cancel else "running",
|
||||
current_completed_cases=current_completed,
|
||||
candidate_completed_cases=candidate_completed,
|
||||
)
|
||||
|
||||
async def submit_result(
|
||||
self,
|
||||
comparison_id: UUID,
|
||||
claim_token: str,
|
||||
pins_digest: str,
|
||||
consumer_revision_sha256: str,
|
||||
current_results: object,
|
||||
candidate_results: object,
|
||||
current_gateway_requests: object,
|
||||
candidate_gateway_requests: object,
|
||||
) -> ComparisonStatus:
|
||||
self.submitted = (
|
||||
pins_digest,
|
||||
consumer_revision_sha256,
|
||||
current_results,
|
||||
candidate_results,
|
||||
current_gateway_requests,
|
||||
candidate_gateway_requests,
|
||||
)
|
||||
return ComparisonStatus(
|
||||
status="completed",
|
||||
current_completed_cases=1,
|
||||
candidate_completed_cases=1,
|
||||
baseline_run_id=uuid4(),
|
||||
candidate_run_id=uuid4(),
|
||||
evaluation_comparison_id=uuid4(),
|
||||
)
|
||||
|
||||
async def finish(
|
||||
self, comparison_id: UUID, claim_token: str, outcome: str, error_code: str
|
||||
) -> ComparisonStatus:
|
||||
self.finish_calls.append((outcome, error_code))
|
||||
return ComparisonStatus(
|
||||
status=outcome,
|
||||
current_completed_cases=0,
|
||||
candidate_completed_cases=0,
|
||||
error_code=error_code,
|
||||
)
|
||||
|
||||
|
||||
class FakeQdrant:
|
||||
def __init__(self, manifest: ComparisonManifest) -> None:
|
||||
self.manifest = manifest
|
||||
self.request_ids: list[str] = []
|
||||
|
||||
async def retrieve(self, *, collection_name: str, ids: list[str], **_: object) -> list[object]:
|
||||
assert collection_name == self.manifest.current_index.ref
|
||||
assert ids == [str(CHUNK_ID)]
|
||||
return [
|
||||
SimpleNamespace(
|
||||
id=str(CHUNK_ID),
|
||||
payload={"workspace_id": str(WORKSPACE_ID), "space_id": str(SPACE_ID)},
|
||||
)
|
||||
]
|
||||
|
||||
async def get_aliases(self) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
aliases=[
|
||||
SimpleNamespace(
|
||||
alias_name="rag_dense_current_active",
|
||||
collection_name=self.manifest.current_index.ref,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
async def get_collection(self, collection_name: str) -> SimpleNamespace:
|
||||
assert collection_name in {
|
||||
self.manifest.current_index.ref,
|
||||
self.manifest.candidate_index.ref,
|
||||
}
|
||||
return SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
params=SimpleNamespace(
|
||||
vectors={"dense": models.VectorParams(size=2, distance=models.Distance.COSINE)},
|
||||
sparse_vectors={"sparse": models.SparseVectorParams()},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
async def query_points(self, *, collection_name: str, **_: object) -> models.QueryResponse:
|
||||
index = (
|
||||
self.manifest.current_index
|
||||
if collection_name == self.manifest.current_index.ref
|
||||
else self.manifest.candidate_index
|
||||
)
|
||||
point_id = CHUNK_ID
|
||||
self.request_ids.append(collection_name)
|
||||
return models.QueryResponse(
|
||||
points=[
|
||||
models.ScoredPoint(
|
||||
id=str(point_id),
|
||||
version=0,
|
||||
score=0.9,
|
||||
payload={
|
||||
"workspace_id": str(WORKSPACE_ID),
|
||||
"space_id": str(SPACE_ID),
|
||||
"document_id": str(DOCUMENT_ID),
|
||||
"document_version_id": str(UUID("10000000-0000-0000-0000-000000000001")),
|
||||
"chunk_id": str(CHUNK_ID),
|
||||
"source_id": str(UUID("10000000-0000-0000-0000-000000000002")),
|
||||
"parent_chunk_id": None,
|
||||
"status": "published",
|
||||
"content_type": "text/plain",
|
||||
"language": "en",
|
||||
"trust_level": "source",
|
||||
"embedding_model_digest": index.model_digest,
|
||||
"chunk_profile_version": index.version,
|
||||
"embedding_space_id": str(index.embedding_space_id),
|
||||
},
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _gateway_transport(
|
||||
manifest: ComparisonManifest,
|
||||
*,
|
||||
duplicate: bool = False,
|
||||
current_route_key: str | None = None,
|
||||
fail_route_key: str | None = None,
|
||||
seen: list[str] | None = None,
|
||||
) -> httpx.MockTransport:
|
||||
request_id = uuid4()
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path.endswith("/invoke"):
|
||||
assert json.loads(request.content)["input"] == ["where is the answer"]
|
||||
if seen is not None:
|
||||
seen.append(request.url.path)
|
||||
if fail_route_key is not None and request.url.path.endswith(
|
||||
f"/{fail_route_key}/invoke"
|
||||
):
|
||||
return httpx.Response(503)
|
||||
if current_route_key is not None and request.url.path.endswith(
|
||||
f"/{current_route_key}/invoke"
|
||||
):
|
||||
space_id = manifest.current_index.embedding_space_id
|
||||
response_id = uuid4()
|
||||
elif request.url.path.endswith("/candidate-route/invoke"):
|
||||
space_id = manifest.candidate_index.embedding_space_id
|
||||
response_id = request_id if duplicate else uuid4()
|
||||
else:
|
||||
space_id = manifest.current_index.embedding_space_id
|
||||
response_id = request_id
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"capability": "rag.embedding@1",
|
||||
"dimension": 2,
|
||||
"normalized": True,
|
||||
"embedding_space_id": str(space_id),
|
||||
"data": [[1.0, 0.0]],
|
||||
"request_id": str(response_id),
|
||||
},
|
||||
)
|
||||
raise AssertionError(request.url.path)
|
||||
|
||||
return httpx.MockTransport(handler)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_selected_runner_executes_both_pinned_legs_and_submits_gateway_evidence(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manifest = _manifest(tmp_path)
|
||||
_write_credential(manifest.credential_file)
|
||||
control = FakeControl(_claim(manifest))
|
||||
qdrant = FakeQdrant(manifest)
|
||||
seen: list[str] = []
|
||||
async with httpx.AsyncClient(
|
||||
transport=_gateway_transport(manifest, seen=seen), follow_redirects=False
|
||||
) as client:
|
||||
result = await SelectedComparisonRunner(manifest, control, qdrant, client).run()
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert control.progress_calls == [(1, 0), (1, 1)]
|
||||
assert control.finish_calls == []
|
||||
assert control.submitted is not None
|
||||
assert qdrant.request_ids == [
|
||||
"current-index",
|
||||
"current-index",
|
||||
"candidate-index",
|
||||
"candidate-index",
|
||||
]
|
||||
assert result["current_gateway_requests"] != result["candidate_gateway_requests"]
|
||||
assert seen == [
|
||||
"/api/v1/capabilities/rag.embedding@1/invoke",
|
||||
"/api/v1/capability-experiments/candidate-route/invoke",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_selected_runner_uses_optional_current_lab_route_pin(tmp_path: Path) -> None:
|
||||
manifest = _manifest(tmp_path)
|
||||
_write_credential(manifest.credential_file)
|
||||
current_route_key = "current-lab-route"
|
||||
control = FakeControl(_claim(manifest, current_route_key=current_route_key))
|
||||
seen: list[str] = []
|
||||
async with httpx.AsyncClient(
|
||||
transport=_gateway_transport(manifest, current_route_key=current_route_key, seen=seen),
|
||||
follow_redirects=False,
|
||||
) as client:
|
||||
result = await SelectedComparisonRunner(
|
||||
manifest, control, FakeQdrant(manifest), client
|
||||
).run()
|
||||
|
||||
assert result["status"] == "completed"
|
||||
assert seen == [
|
||||
"/api/v1/capability-experiments/current-lab-route/invoke",
|
||||
"/api/v1/capability-experiments/candidate-route/invoke",
|
||||
]
|
||||
|
||||
|
||||
def test_claim_pins_reject_malformed_optional_current_route_key(tmp_path: Path) -> None:
|
||||
manifest = _manifest(tmp_path)
|
||||
malformed = {**_claim(manifest).pins.raw, "current_route_key": "../stable"}
|
||||
|
||||
with pytest.raises(SelectedComparisonError, match="CURRENT_ROUTE_KEY_INVALID"):
|
||||
_parse_pins(malformed)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_current_lab_route_failure_does_not_fall_back_to_stable_or_ollama(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manifest = _manifest(tmp_path)
|
||||
_write_credential(manifest.credential_file)
|
||||
current_route_key = "current-lab-route"
|
||||
control = FakeControl(_claim(manifest, current_route_key=current_route_key))
|
||||
seen: list[str] = []
|
||||
async with httpx.AsyncClient(
|
||||
transport=_gateway_transport(
|
||||
manifest,
|
||||
current_route_key=current_route_key,
|
||||
fail_route_key=current_route_key,
|
||||
seen=seen,
|
||||
),
|
||||
follow_redirects=False,
|
||||
) as client:
|
||||
result = await SelectedComparisonRunner(
|
||||
manifest, control, FakeQdrant(manifest), client
|
||||
).run()
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert seen == ["/api/v1/capability-experiments/current-lab-route/invoke"]
|
||||
assert control.submitted is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_selected_runner_fails_closed_on_duplicate_gateway_identity(tmp_path: Path) -> None:
|
||||
manifest = _manifest(tmp_path)
|
||||
_write_credential(manifest.credential_file)
|
||||
control = FakeControl(_claim(manifest))
|
||||
async with httpx.AsyncClient(
|
||||
transport=_gateway_transport(manifest, duplicate=True), follow_redirects=False
|
||||
) as client:
|
||||
result = await SelectedComparisonRunner(
|
||||
manifest, control, FakeQdrant(manifest), client
|
||||
).run()
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["error_code"] == "DUPLICATE_GATEWAY_REQUEST_ID"
|
||||
assert control.submitted is None
|
||||
assert control.finish_calls == [("failed", "DUPLICATE_GATEWAY_REQUEST_ID")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_selected_runner_honours_cancellation_before_candidate_leg(tmp_path: Path) -> None:
|
||||
manifest = _manifest(tmp_path)
|
||||
_write_credential(manifest.credential_file)
|
||||
control = FakeControl(_claim(manifest))
|
||||
control.cancel = True
|
||||
qdrant = FakeQdrant(manifest)
|
||||
async with httpx.AsyncClient(
|
||||
transport=_gateway_transport(manifest), follow_redirects=False
|
||||
) as client:
|
||||
result = await SelectedComparisonRunner(manifest, control, qdrant, client).run()
|
||||
|
||||
assert result["status"] == "cancelled"
|
||||
assert result["error_code"] == "COMPARISON_CANCEL_REQUESTED"
|
||||
assert result["completed_cases"] == 0
|
||||
assert control.submitted is None
|
||||
assert control.finish_calls == [("cancelled", "COMPARISON_CANCEL_REQUESTED")]
|
||||
assert qdrant.request_ids == ["current-index", "current-index"]
|
||||
|
||||
|
||||
class FailingProgressControl(FakeControl):
|
||||
async def progress(
|
||||
self,
|
||||
comparison_id: UUID,
|
||||
claim_token: str,
|
||||
current_completed: int,
|
||||
candidate_completed: int,
|
||||
) -> ComparisonStatus:
|
||||
del comparison_id, claim_token, current_completed, candidate_completed
|
||||
raise SelectedComparisonError("LEASE_RENEWAL_FAILED")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_selected_runner_reports_lease_failure_without_submitting_partial_results(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
manifest = _manifest(tmp_path)
|
||||
_write_credential(manifest.credential_file)
|
||||
control = FailingProgressControl(_claim(manifest))
|
||||
async with httpx.AsyncClient(
|
||||
transport=_gateway_transport(manifest), follow_redirects=False
|
||||
) as client:
|
||||
result = await SelectedComparisonRunner(
|
||||
manifest, control, FakeQdrant(manifest), client
|
||||
).run()
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["error_code"] == "LEASE_RENEWAL_FAILED"
|
||||
assert result["completed_cases"] == 0
|
||||
assert control.submitted is None
|
||||
assert control.finish_calls == [("failed", "LEASE_RENEWAL_FAILED")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_selected_runner_rejects_expired_claim_before_inference(tmp_path: Path) -> None:
|
||||
manifest = _manifest(tmp_path)
|
||||
claim = replace(
|
||||
_claim(manifest),
|
||||
claim_expires_at=datetime.now(UTC) - timedelta(seconds=1),
|
||||
deadline_at=datetime.now(UTC) + timedelta(minutes=5),
|
||||
)
|
||||
control = FakeControl(claim)
|
||||
async with httpx.AsyncClient(transport=_gateway_transport(manifest)) as client:
|
||||
result = await SelectedComparisonRunner(
|
||||
manifest, control, FakeQdrant(manifest), client
|
||||
).run()
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert result["error_code"] == "COMPARISON_CLAIM_EXPIRED"
|
||||
assert control.finish_calls == [("failed", "COMPARISON_CLAIM_EXPIRED")]
|
||||
|
||||
|
||||
def test_manifest_requires_exact_runtime_contract(tmp_path: Path) -> None:
|
||||
manifest = {
|
||||
"comparison_id": str(COMPARISON_ID),
|
||||
"project_id": str(PROJECT_ID),
|
||||
"suite_revision_id": str(SUITE_REVISION_ID),
|
||||
"workspace_id": str(WORKSPACE_ID),
|
||||
"corpus_sha256": "a" * 64,
|
||||
"consumer_revision_sha256": "b" * 64,
|
||||
"modelforge_base_url": "https://api.example.com",
|
||||
"credential_file": "credential",
|
||||
"qdrant_url": "http://qdrant:6333",
|
||||
"current_index": {},
|
||||
"candidate_index": {},
|
||||
"retrieval_config": {},
|
||||
}
|
||||
path = tmp_path / "manifest.json"
|
||||
path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
with pytest.raises(SelectedComparisonError, match="MODELFORGE_BASE_URL_MUST_BE_LOCAL_ORIGIN"):
|
||||
ComparisonManifest.from_path(path)
|
||||
|
||||
|
||||
def test_manifest_loads_the_pinned_index_and_retrieval_contract(tmp_path: Path) -> None:
|
||||
manifest = _manifest(tmp_path)
|
||||
path = tmp_path / "manifest.json"
|
||||
path.write_text(json.dumps(_manifest_payload(manifest)), encoding="utf-8")
|
||||
|
||||
loaded = ComparisonManifest.from_path(path)
|
||||
|
||||
assert loaded.current_index == manifest.current_index
|
||||
assert loaded.candidate_index == manifest.candidate_index
|
||||
assert loaded.retrieval_config_sha256 == manifest.retrieval_config_sha256
|
||||
assert loaded.sparse_profile.digest == manifest.sparse_profile.digest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_client_parses_and_verifies_claim_pins(tmp_path: Path) -> None:
|
||||
manifest = _manifest(tmp_path)
|
||||
_write_credential(manifest.credential_file)
|
||||
claim = _claim(manifest, current_route_key="current-lab-route")
|
||||
response_body = {
|
||||
"comparison": {
|
||||
"id": str(claim.comparison_id),
|
||||
"suite_revision_id": str(claim.suite_revision_id),
|
||||
"current_deployment_id": str(claim.current_deployment_id),
|
||||
"candidate_deployment_id": str(claim.candidate_deployment_id),
|
||||
"expected_cases": claim.expected_cases,
|
||||
"deadline_at": claim.deadline_at.isoformat(),
|
||||
"pins": claim.pins.raw,
|
||||
"pins_digest": claim.pins_digest,
|
||||
},
|
||||
"claim_token": claim.claim_token,
|
||||
"claim_expires_at": claim.claim_expires_at.isoformat(),
|
||||
"cases": [
|
||||
{
|
||||
"id": str(CASE_ID),
|
||||
"suite_revision_id": str(SUITE_REVISION_ID),
|
||||
"case_key": "one",
|
||||
"query": "where is the answer",
|
||||
"relevant_chunk_ids": [str(CHUNK_ID)],
|
||||
"relevant_document_ids": [],
|
||||
"relevance_grades": {},
|
||||
"label_provenance": {},
|
||||
"critical": True,
|
||||
"review_status": "reviewed",
|
||||
"created_at": claim.claim_expires_at.isoformat(),
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
def handler(_: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=response_body)
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(handler), follow_redirects=False
|
||||
) as client:
|
||||
parsed = await ModelForgeComparisonClient(
|
||||
client,
|
||||
base_url=manifest.modelforge_base_url,
|
||||
credential_file=manifest.credential_file,
|
||||
).claim(COMPARISON_ID)
|
||||
|
||||
assert parsed.pins.current.dimension == 2
|
||||
assert parsed.pins.current_route_key == "current-lab-route"
|
||||
assert parsed.cases[0].relevant_chunk_ids == (CHUNK_ID,)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_client_serializes_uuid_results_without_secret_echo(tmp_path: Path) -> None:
|
||||
credential = tmp_path / "credential"
|
||||
_write_credential(credential)
|
||||
seen: list[httpx.Request] = []
|
||||
response_body = {
|
||||
"status": "completed",
|
||||
"current_completed_cases": 1,
|
||||
"candidate_completed_cases": 1,
|
||||
"claim_expires_at": None,
|
||||
"deadline_at": None,
|
||||
"error_code": None,
|
||||
"baseline_run_id": None,
|
||||
"candidate_run_id": None,
|
||||
"evaluation_comparison_id": None,
|
||||
}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request)
|
||||
assert "mf_" not in request.url.path
|
||||
return httpx.Response(200, json=response_body)
|
||||
|
||||
case_id, result_id = uuid4(), uuid4()
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(handler), follow_redirects=False
|
||||
) as client:
|
||||
api = ModelForgeComparisonClient(
|
||||
client,
|
||||
base_url="http://modelforge:8000",
|
||||
credential_file=credential,
|
||||
)
|
||||
await api.submit_result(
|
||||
COMPARISON_ID,
|
||||
"claim-token-" + "x" * 32,
|
||||
"a" * 64,
|
||||
"b" * 64,
|
||||
[
|
||||
{
|
||||
"case_id": case_id,
|
||||
"ranked_results": [{"chunk_id": result_id, "score": 0.5}],
|
||||
"latency_ms": 1.0,
|
||||
"error_code": None,
|
||||
}
|
||||
],
|
||||
[{"case_id": case_id, "ranked_results": [], "latency_ms": 1.0, "error_code": None}],
|
||||
{case_id: uuid4()},
|
||||
{case_id: uuid4()},
|
||||
)
|
||||
|
||||
body = seen[0].content.decode("utf-8")
|
||||
assert str(result_id) in body
|
||||
assert "claim-token" in body
|
||||
assert "mf_" not in body
|
||||
Reference in new issue
Block a user