740 lines
28 KiB
Python
740 lines
28 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
from unittest.mock import patch
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, func, select, update
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modelforge_api.domain.evaluation import (
|
|
EvaluationCaseCreate,
|
|
EvaluationCaseResultCreate,
|
|
EvaluationRevisionCreate,
|
|
EvaluationRunComplete,
|
|
EvaluationSuiteCreate,
|
|
RankedResult,
|
|
)
|
|
from modelforge_api.domain.selected_model_comparison import (
|
|
ComparisonCancel,
|
|
ComparisonCreate,
|
|
ComparisonFinish,
|
|
ComparisonProgress,
|
|
ComparisonResult,
|
|
)
|
|
from modelforge_api.persistence.models import (
|
|
ArtifactSet,
|
|
AuditEvent,
|
|
Base,
|
|
Capability,
|
|
CapabilityContract,
|
|
CapabilityDeployment,
|
|
CapabilityExperimentRoute,
|
|
DeploymentCandidate,
|
|
EmbeddingSpace,
|
|
EvaluationCase,
|
|
EvaluationComparison,
|
|
EvaluationRun,
|
|
GatewayRequest,
|
|
Model,
|
|
ModelRevision,
|
|
Project,
|
|
ProjectBinding,
|
|
RuntimeProfile,
|
|
SelectedModelComparison,
|
|
ServiceClient,
|
|
)
|
|
from modelforge_api.services.audit import AuditContext
|
|
from modelforge_api.services.evaluation import EvaluationError, EvaluationService
|
|
from modelforge_api.services.selected_model_comparison import (
|
|
SelectedModelComparisonService,
|
|
_digest,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def setup():
|
|
engine = create_engine("sqlite://")
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
|
|
def add(value):
|
|
session.add(value)
|
|
session.flush()
|
|
return value
|
|
|
|
project = add(Project(key="owner", name="Owner", description="fixture"))
|
|
capability = add(Capability(key="rag.embedding", description="Embeddings"))
|
|
contract = add(
|
|
CapabilityContract(
|
|
capability_id=capability.id,
|
|
version=1,
|
|
input_schema={},
|
|
output_schema={},
|
|
contract={},
|
|
upgrade_class="requires_reindex",
|
|
)
|
|
)
|
|
binding = add(
|
|
ProjectBinding(
|
|
project_id=project.id,
|
|
capability_contract_id=contract.id,
|
|
channel="stable",
|
|
priority="production",
|
|
)
|
|
)
|
|
client = add(
|
|
ServiceClient(
|
|
name="designated",
|
|
allowed_capabilities=["rag.embedding@1"],
|
|
project_binding_id=binding.id,
|
|
)
|
|
)
|
|
model = add(
|
|
Model(
|
|
key="chosen",
|
|
display_name="Chosen",
|
|
upstream_provider="huggingface",
|
|
upstream_source="org/chosen",
|
|
)
|
|
)
|
|
deployments = []
|
|
for index, variant in enumerate(("stable", "experiment", "experiment-current")):
|
|
channel = "stable" if index == 0 else "experiment"
|
|
revision = add(
|
|
ModelRevision(
|
|
model_id=model.id,
|
|
upstream_revision=variant,
|
|
resolved_commit_sha=str(index + 1) * 40,
|
|
)
|
|
)
|
|
artifact = add(
|
|
ArtifactSet(
|
|
revision_id=revision.id,
|
|
snapshot_id=uuid4(),
|
|
variant_key=variant,
|
|
label=variant,
|
|
selection_reason="fixture",
|
|
immutable_at=datetime.now(UTC),
|
|
)
|
|
)
|
|
profile = add(
|
|
RuntimeProfile(
|
|
name=variant,
|
|
artifact_set_id=artifact.id,
|
|
runtime_type="sentence_transformers",
|
|
runtime_version="fixture",
|
|
artifact_sha256=str(index + 1) * 64,
|
|
fingerprint=str(index + 3) * 64,
|
|
immutable_at=datetime.now(UTC),
|
|
)
|
|
)
|
|
candidate = add(
|
|
DeploymentCandidate(
|
|
artifact_set_id=artifact.id,
|
|
runtime_profile_id=profile.id,
|
|
compute_node_id=uuid4(),
|
|
compatibility_assessment_id=uuid4(),
|
|
runtime_probe_id=uuid4(),
|
|
)
|
|
)
|
|
space = add(
|
|
EmbeddingSpace(
|
|
capability_contract_id=contract.id,
|
|
artifact_set_id=artifact.id,
|
|
runtime_profile_id=profile.id,
|
|
identity_digest=str(index + 5) * 64,
|
|
dimension=4,
|
|
normalized=True,
|
|
identity_facts={},
|
|
)
|
|
)
|
|
deployments.append(
|
|
add(
|
|
CapabilityDeployment(
|
|
capability_contract_id=contract.id,
|
|
deployment_candidate_id=candidate.id,
|
|
embedding_space_id=space.id,
|
|
artifact_set_id=artifact.id,
|
|
runtime_profile_id=profile.id,
|
|
compute_node_id=candidate.compute_node_id,
|
|
accelerator_id=uuid4(),
|
|
channel=channel,
|
|
status="stable" if index == 0 else "candidate",
|
|
production=index == 0,
|
|
config_fingerprint=str(index + 7) * 64,
|
|
provenance={},
|
|
)
|
|
)
|
|
)
|
|
route = add(
|
|
CapabilityExperimentRoute(
|
|
route_key="selected-candidate",
|
|
capability_contract_id=contract.id,
|
|
capability_deployment_id=deployments[1].id,
|
|
purpose="fixture",
|
|
evidence={},
|
|
)
|
|
)
|
|
add(
|
|
CapabilityExperimentRoute(
|
|
route_key="selected-current-experiment",
|
|
capability_contract_id=contract.id,
|
|
capability_deployment_id=deployments[2].id,
|
|
purpose="fixture current experiment",
|
|
evidence={},
|
|
)
|
|
)
|
|
session.commit()
|
|
relevant = uuid4()
|
|
suite = EvaluationService(session).create_suite(
|
|
EvaluationSuiteCreate(
|
|
project_id=project.id,
|
|
key="reviewed",
|
|
name="Reviewed",
|
|
description="fixture",
|
|
revision=EvaluationRevisionCreate(
|
|
revision="v1",
|
|
dataset_revision="fixture-v1",
|
|
retrieval_settings={"top_k": 10},
|
|
cases=[
|
|
EvaluationCaseCreate(
|
|
case_key="one",
|
|
query="Where is the answer?",
|
|
relevant_chunk_ids=[relevant],
|
|
label_provenance={
|
|
"source": "owner_review",
|
|
"corpus_revision": "a" * 64,
|
|
},
|
|
critical=True,
|
|
review_status="reviewed",
|
|
)
|
|
],
|
|
),
|
|
)
|
|
)
|
|
service = SelectedModelComparisonService(session, AuditContext("operator", "fixture"))
|
|
request = ComparisonCreate(
|
|
project_binding_id=binding.id,
|
|
consumer_client_id=client.id,
|
|
suite_revision_id=suite.latest_revision_id,
|
|
current_deployment_id=deployments[0].id,
|
|
candidate_deployment_id=deployments[1].id,
|
|
candidate_route_id=route.id,
|
|
corpus_sha256="a" * 64,
|
|
retrieval_config_sha256=_digest({"top_k": 10}),
|
|
current_index_ref="current-exact",
|
|
candidate_index_ref="candidate-exact",
|
|
idempotency_key="one",
|
|
)
|
|
yield session, service, request, client, relevant
|
|
engine.dispose()
|
|
|
|
|
|
def _claimed(setup, selected_request=None):
|
|
session, service, request, client, relevant = setup
|
|
if selected_request is not None:
|
|
request = selected_request
|
|
created = service.create(request)
|
|
claim = service.claim(created.id, client)
|
|
case = session.scalar(
|
|
select(EvaluationCase).where(EvaluationCase.suite_revision_id == request.suite_revision_id)
|
|
)
|
|
gateways = []
|
|
for deployment_id in (request.current_deployment_id, request.candidate_deployment_id):
|
|
gateway = GatewayRequest(
|
|
request_id=uuid4(),
|
|
service_client_id=client.id,
|
|
capability_key="rag.embedding",
|
|
capability_version=1,
|
|
capability_deployment_id=deployment_id,
|
|
status="completed",
|
|
priority="production",
|
|
input_sha256=_digest([case.query]),
|
|
input_count=1,
|
|
created_at=datetime.now(UTC),
|
|
finished_at=datetime.now(UTC),
|
|
decision_evidence={},
|
|
)
|
|
session.add(gateway)
|
|
gateways.append(gateway)
|
|
session.commit()
|
|
leg = EvaluationRunComplete(
|
|
results=[
|
|
EvaluationCaseResultCreate(
|
|
case_id=case.id,
|
|
ranked_results=[RankedResult(chunk_id=relevant, score=1)],
|
|
latency_ms=2,
|
|
)
|
|
]
|
|
)
|
|
result = ComparisonResult(
|
|
claim_token=claim.claim_token,
|
|
pins_digest=claim.comparison.pins_digest,
|
|
consumer_revision_sha256="c" * 64,
|
|
current=leg,
|
|
candidate=leg,
|
|
current_gateway_requests={case.id: gateways[0].request_id},
|
|
candidate_gateway_requests={case.id: gateways[1].request_id},
|
|
)
|
|
return claim, result, gateways
|
|
|
|
|
|
def test_real_pair_recomputes_metrics_and_retries_without_duplicate_evidence(setup):
|
|
session, service, request, client, _ = setup
|
|
claim, result, _ = _claimed(setup)
|
|
assert (
|
|
claim.comparison.pins["current"]["dimension"]
|
|
== claim.comparison.pins["candidate"]["dimension"]
|
|
)
|
|
assert claim.comparison.pins["current"]["dimension"] > 0
|
|
completed = service.result(claim.comparison.id, result, client)
|
|
assert completed.status == "completed"
|
|
comparison = session.get(EvaluationComparison, completed.evaluation_comparison_id)
|
|
assert comparison.comparability == "comparable"
|
|
assert comparison.promotion_eligibility == "eligible"
|
|
assert completed.evaluation_summary.eligibility == "eligible"
|
|
assert completed.evaluation_summary.critical_regressions == 0
|
|
assert comparison.eligibility_evidence["promotion_performed"] is False
|
|
assert service.result(completed.id, result, client).id == completed.id
|
|
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 2
|
|
assert session.scalar(select(func.count()).select_from(EvaluationComparison)) == 1
|
|
assert session.get(CapabilityDeployment, request.candidate_deployment_id).production is False
|
|
history = service.get(completed.id).model_dump_json()
|
|
assert claim.claim_token not in history and "claim_token_hash" not in history
|
|
assert all(
|
|
claim.claim_token not in str(event.details) for event in session.scalars(select(AuditEvent))
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"field,value", [("corpus_sha256", "d" * 64), ("current_index_ref", "candidate-exact")]
|
|
)
|
|
def test_rejects_mismatched_corpus_and_shared_vector_index(setup, field, value):
|
|
_, service, request, _, _ = setup
|
|
with pytest.raises(EvaluationError):
|
|
service.create(request.model_copy(update={field: value}))
|
|
|
|
|
|
def test_idempotency_active_bound_and_exact_consumer(setup):
|
|
session, service, request, client, _ = setup
|
|
created = service.create(request)
|
|
legacy_digest = _digest(request.model_dump(mode="json", exclude={"current_route_id"}))
|
|
assert session.get(SelectedModelComparison, created.id).request_digest == legacy_digest
|
|
assert service.create(request).id == created.id
|
|
with pytest.raises(EvaluationError):
|
|
service.create(request.model_copy(update={"candidate_index_ref": "different"}))
|
|
with pytest.raises(EvaluationError):
|
|
service.create(request.model_copy(update={"idempotency_key": "second"}))
|
|
stranger = ServiceClient(
|
|
name="other",
|
|
project_binding_id=client.project_binding_id,
|
|
allowed_capabilities=["rag.embedding@1"],
|
|
)
|
|
session.add(stranger)
|
|
session.commit()
|
|
assert service.assigned(stranger).items == []
|
|
with pytest.raises(EvaluationError):
|
|
service.claim(created.id, stranger)
|
|
assert service.assigned(client).items[0].id == created.id
|
|
|
|
|
|
def test_selected_experiment_pair_requires_both_exact_routes_and_is_idempotent(setup):
|
|
session, service, request, client, _ = setup
|
|
current_route = session.scalar(
|
|
select(CapabilityExperimentRoute).where(
|
|
CapabilityExperimentRoute.route_key == "selected-current-experiment"
|
|
)
|
|
)
|
|
selected = request.model_copy(
|
|
update={
|
|
"current_deployment_id": current_route.capability_deployment_id,
|
|
"current_route_id": current_route.id,
|
|
"idempotency_key": "two-experiments",
|
|
}
|
|
)
|
|
|
|
created = service.create(selected)
|
|
assert service.create(selected).id == created.id
|
|
with pytest.raises(EvaluationError, match="comparison idempotency conflict"):
|
|
service.create(selected.model_copy(update={"current_route_id": request.candidate_route_id}))
|
|
assert created.pins["current_route_id"] == str(current_route.id)
|
|
assert created.pins["current_route_key"] == current_route.route_key
|
|
assert created.pins["candidate_route_id"] == str(request.candidate_route_id)
|
|
identity_pins = {
|
|
"config_fingerprint",
|
|
"artifact_set_id",
|
|
"revision_id",
|
|
"upstream_commit_sha",
|
|
"runtime_profile_id",
|
|
"runtime_fingerprint",
|
|
"artifact_sha256",
|
|
"embedding_space_id",
|
|
"embedding_space_digest",
|
|
"dimension",
|
|
"compute_node_id",
|
|
"accelerator_id",
|
|
}
|
|
assert identity_pins <= set(created.pins["current"])
|
|
assert identity_pins <= set(created.pins["candidate"])
|
|
assert (
|
|
created.pins["current"]["embedding_space_id"]
|
|
!= created.pins["candidate"]["embedding_space_id"]
|
|
)
|
|
claim, result, _ = _claimed(setup, selected)
|
|
completed = service.result(created.id, result, client)
|
|
assert claim.comparison.id == created.id
|
|
assert completed.status == "completed"
|
|
assert completed.pins["current_route_id"] == str(current_route.id)
|
|
current_route.status = "inactive"
|
|
session.commit()
|
|
assert service.result(created.id, result, client).id == completed.id
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
["production_current", "wrong_current_route", "inactive_current_route", "same_route"],
|
|
)
|
|
def test_selected_experiment_pair_rejects_non_experiment_or_mismatched_current_route(
|
|
setup, mutation
|
|
):
|
|
session, service, request, _, _ = setup
|
|
current_route = session.scalar(
|
|
select(CapabilityExperimentRoute).where(
|
|
CapabilityExperimentRoute.route_key == "selected-current-experiment"
|
|
)
|
|
)
|
|
selected = request.model_copy(
|
|
update={
|
|
"current_deployment_id": current_route.capability_deployment_id,
|
|
"current_route_id": current_route.id,
|
|
"idempotency_key": f"invalid-{mutation}",
|
|
}
|
|
)
|
|
if mutation == "production_current":
|
|
selected = selected.model_copy(
|
|
update={"current_deployment_id": request.current_deployment_id}
|
|
)
|
|
elif mutation == "wrong_current_route":
|
|
selected = selected.model_copy(update={"current_route_id": request.candidate_route_id})
|
|
elif mutation == "inactive_current_route":
|
|
current_route.status = "inactive"
|
|
session.commit()
|
|
else:
|
|
selected = selected.model_copy(update={"candidate_route_id": current_route.id})
|
|
|
|
with pytest.raises(EvaluationError):
|
|
service.create(selected)
|
|
assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
|
|
|
|
|
|
def test_current_experiment_route_key_is_revalidated_before_claim_and_result(setup):
|
|
session, service, request, client, _ = setup
|
|
current_route = session.scalar(
|
|
select(CapabilityExperimentRoute).where(
|
|
CapabilityExperimentRoute.route_key == "selected-current-experiment"
|
|
)
|
|
)
|
|
selected = request.model_copy(
|
|
update={
|
|
"current_deployment_id": current_route.capability_deployment_id,
|
|
"current_route_id": current_route.id,
|
|
"idempotency_key": "route-revalidation",
|
|
}
|
|
)
|
|
created = service.create(selected)
|
|
current_route.route_key = "changed-before-claim"
|
|
session.commit()
|
|
with pytest.raises(EvaluationError, match="comparison pins changed"):
|
|
service.claim(created.id, client)
|
|
|
|
current_route.route_key = "selected-current-experiment"
|
|
session.commit()
|
|
claim, result, _ = _claimed(setup, selected)
|
|
assert claim.comparison.id == created.id
|
|
current_route.route_key = "changed-before-result"
|
|
session.commit()
|
|
with pytest.raises(EvaluationError, match="comparison pins changed"):
|
|
service.result(created.id, result, client)
|
|
|
|
|
|
@pytest.mark.parametrize("phase", ["claim", "result"])
|
|
@pytest.mark.parametrize("mutation", ["inactive", "remapped"])
|
|
def test_current_experiment_route_deactivation_or_remap_fails_closed(setup, phase, mutation):
|
|
session, service, request, client, _ = setup
|
|
current_route = session.scalar(
|
|
select(CapabilityExperimentRoute).where(
|
|
CapabilityExperimentRoute.route_key == "selected-current-experiment"
|
|
)
|
|
)
|
|
selected = request.model_copy(
|
|
update={
|
|
"current_deployment_id": current_route.capability_deployment_id,
|
|
"current_route_id": current_route.id,
|
|
"idempotency_key": f"route-{phase}-{mutation}",
|
|
}
|
|
)
|
|
|
|
if phase == "claim":
|
|
comparison = service.create(selected)
|
|
claim = None
|
|
result = None
|
|
else:
|
|
claim, result, _ = _claimed(setup, selected)
|
|
comparison = claim.comparison
|
|
|
|
if mutation == "inactive":
|
|
current_route.status = "inactive"
|
|
else:
|
|
current_route.capability_deployment_id = selected.candidate_deployment_id
|
|
session.commit()
|
|
|
|
with pytest.raises(EvaluationError):
|
|
if phase == "claim":
|
|
service.claim(comparison.id, client)
|
|
else:
|
|
assert claim is not None and result is not None
|
|
service.result(comparison.id, result, client)
|
|
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
|
|
|
|
|
|
def test_expiry_never_renews_or_reclaims(setup):
|
|
session, service, _, client, _ = setup
|
|
claim, result, _ = _claimed(setup)
|
|
row = session.get(SelectedModelComparison, claim.comparison.id)
|
|
row.claim_expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
session.commit()
|
|
with pytest.raises(EvaluationError):
|
|
service.result(row.id, result, client)
|
|
assert service.get(row.id).status == "interrupted"
|
|
assert row.claim_token_hash is None
|
|
with pytest.raises(EvaluationError):
|
|
service.claim(row.id, client)
|
|
|
|
|
|
def test_cancel_is_request_until_consumer_acknowledges(setup):
|
|
_, service, _, client, _ = setup
|
|
claim, result, _ = _claimed(setup)
|
|
response = service.cancel(
|
|
claim.comparison.id, ComparisonCancel(expected_version=claim.comparison.version)
|
|
)
|
|
assert response.status == "cancel_requested" and response.finished_at is None
|
|
with pytest.raises(EvaluationError):
|
|
service.result(response.id, result, client)
|
|
completed = service.finish(
|
|
response.id, ComparisonFinish(claim_token=claim.claim_token, outcome="cancelled"), client
|
|
)
|
|
assert completed.status == "cancelled"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
["historical", "wrong_target", "wrong_client", "wrong_query", "not_completed", "duplicate"],
|
|
)
|
|
def test_gateway_provenance_rejects_unrelated_or_reused_requests(setup, mutation):
|
|
session, service, request, client, _ = setup
|
|
claim, result, gateways = _claimed(setup)
|
|
if mutation == "historical":
|
|
gateways[0].created_at = datetime.now(UTC) - timedelta(hours=1)
|
|
elif mutation == "wrong_target":
|
|
gateways[0].capability_deployment_id = request.candidate_deployment_id
|
|
elif mutation == "wrong_client":
|
|
gateways[0].service_client_id = uuid4()
|
|
elif mutation == "wrong_query":
|
|
gateways[0].input_sha256 = "d" * 64
|
|
elif mutation == "not_completed":
|
|
gateways[0].status = "failed"
|
|
else:
|
|
result.candidate_gateway_requests = result.current_gateway_requests
|
|
session.commit()
|
|
with pytest.raises(EvaluationError):
|
|
service.result(claim.comparison.id, result, client)
|
|
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
|
|
|
|
|
|
def test_result_failure_rolls_back_both_runs_and_keeps_claim(setup):
|
|
session, service, _, client, _ = setup
|
|
claim, result, _ = _claimed(setup)
|
|
with (
|
|
patch.object(EvaluationService, "compare", side_effect=RuntimeError("test interruption")),
|
|
pytest.raises(RuntimeError),
|
|
):
|
|
service.result(claim.comparison.id, result, client)
|
|
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
|
|
assert service.get(claim.comparison.id).status == "running"
|
|
assert service.result(claim.comparison.id, result, client).status == "completed"
|
|
|
|
|
|
def test_progress_is_monotonic_and_claim_cannot_be_stolen(setup):
|
|
_, service, _, client, _ = setup
|
|
claim, _, _ = _claimed(setup)
|
|
with pytest.raises(EvaluationError):
|
|
service.claim(claim.comparison.id, client)
|
|
progress = ComparisonProgress(
|
|
claim_token=claim.claim_token, current_completed_cases=1, candidate_completed_cases=0
|
|
)
|
|
service.progress(claim.comparison.id, progress, client)
|
|
with pytest.raises(EvaluationError):
|
|
service.progress(
|
|
claim.comparison.id, progress.model_copy(update={"current_completed_cases": 0}), client
|
|
)
|
|
with pytest.raises(EvaluationError):
|
|
service.progress(
|
|
claim.comparison.id,
|
|
progress.model_copy(update={"candidate_completed_cases": 2}),
|
|
client,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
[
|
|
"wrong_project",
|
|
"unreviewed",
|
|
"missing_corpus",
|
|
"wrong_contract",
|
|
"same_target",
|
|
"inactive_route",
|
|
"space_mismatch",
|
|
],
|
|
)
|
|
def test_create_rejects_incoherent_selected_project_provenance(setup, mutation):
|
|
session, service, request, _, _ = setup
|
|
if mutation == "wrong_project":
|
|
from modelforge_api.persistence.models import EvaluationSuite, EvaluationSuiteRevision
|
|
|
|
revision = session.get(EvaluationSuiteRevision, request.suite_revision_id)
|
|
session.get(EvaluationSuite, revision.suite_id).project_id = uuid4()
|
|
elif mutation in {"unreviewed", "missing_corpus"}:
|
|
case = session.scalar(select(EvaluationCase))
|
|
# Simulate imported corrupt provenance, bypassing the normal immutable ORM guard.
|
|
if mutation == "unreviewed":
|
|
session.execute(
|
|
update(EvaluationCase)
|
|
.where(EvaluationCase.id == case.id)
|
|
.values(review_status="draft")
|
|
)
|
|
else:
|
|
session.execute(
|
|
update(EvaluationCase)
|
|
.where(EvaluationCase.id == case.id)
|
|
.values(label_provenance={"source": "no reviewed corpus"})
|
|
)
|
|
elif mutation == "same_target":
|
|
request = request.model_copy(
|
|
update={"candidate_deployment_id": request.current_deployment_id}
|
|
)
|
|
elif mutation == "inactive_route":
|
|
session.get(CapabilityExperimentRoute, request.candidate_route_id).status = "inactive"
|
|
else:
|
|
deployment = session.get(CapabilityDeployment, request.candidate_deployment_id)
|
|
if mutation == "wrong_contract":
|
|
deployment.capability_contract_id = uuid4()
|
|
else:
|
|
session.execute(
|
|
update(EmbeddingSpace)
|
|
.where(EmbeddingSpace.id == deployment.embedding_space_id)
|
|
.values(runtime_profile_id=uuid4())
|
|
)
|
|
session.commit()
|
|
with pytest.raises(EvaluationError):
|
|
service.create(request)
|
|
assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
|
|
|
|
|
|
def test_claim_and_result_revalidate_pins_and_client_revocation(setup):
|
|
session, service, request, client, _ = setup
|
|
created = service.create(request)
|
|
deployment = session.get(CapabilityDeployment, request.candidate_deployment_id)
|
|
original = deployment.config_fingerprint
|
|
deployment.config_fingerprint = "d" * 64
|
|
session.commit()
|
|
with pytest.raises(EvaluationError):
|
|
service.claim(created.id, client)
|
|
deployment.config_fingerprint = original
|
|
session.commit()
|
|
claim, result, _ = _claimed(setup)
|
|
client.disabled_at = datetime.now(UTC)
|
|
session.commit()
|
|
with pytest.raises(EvaluationError):
|
|
service.result(claim.comparison.id, result, client)
|
|
client.disabled_at = None
|
|
deployment.config_fingerprint = "d" * 64
|
|
session.commit()
|
|
with pytest.raises(EvaluationError):
|
|
service.result(claim.comparison.id, result, client)
|
|
|
|
|
|
def test_failed_case_cannot_qualify_and_does_not_require_gateway_success(setup):
|
|
session, service, _, client, _ = setup
|
|
claim, result, _ = _claimed(setup)
|
|
failed = result.current.results[0].model_copy(
|
|
update={"error_code": "UNAVAILABLE", "ranked_results": []}
|
|
)
|
|
result.current = EvaluationRunComplete(results=[failed])
|
|
result.current_gateway_requests = {}
|
|
completed = service.result(claim.comparison.id, result, client)
|
|
assert (
|
|
session.get(EvaluationComparison, completed.evaluation_comparison_id).promotion_eligibility
|
|
== "not_eligible"
|
|
)
|
|
|
|
|
|
def test_missing_case_or_mapping_and_changed_retry_cannot_publish(setup):
|
|
session, service, _, client, _ = setup
|
|
claim, result, _ = _claimed(setup)
|
|
original = result.current_gateway_requests
|
|
result.current_gateway_requests = {}
|
|
with pytest.raises(EvaluationError):
|
|
service.result(claim.comparison.id, result, client)
|
|
result.current_gateway_requests = original
|
|
completed = service.result(claim.comparison.id, result, client)
|
|
result.consumer_revision_sha256 = "d" * 64
|
|
with pytest.raises(EvaluationError):
|
|
service.result(completed.id, result, client)
|
|
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 2
|
|
|
|
|
|
@pytest.mark.parametrize("reader", ["list", "assigned", "create"])
|
|
def test_expiry_is_reconciled_by_history_assignment_and_next_intent_without_early_commit(
|
|
setup, reader
|
|
):
|
|
session, service, request, client, _ = setup
|
|
claim, _, _ = _claimed(setup)
|
|
row = session.get(SelectedModelComparison, claim.comparison.id)
|
|
row.claim_expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
session.commit()
|
|
with patch.object(session, "commit", wraps=session.commit) as commits:
|
|
if reader == "list":
|
|
assert service.list().items[0].status == "interrupted"
|
|
elif reader == "assigned":
|
|
assert service.assigned(client).items[0].status == "interrupted"
|
|
else:
|
|
following = service.create(
|
|
request.model_copy(update={"idempotency_key": "explicit-new-intent"})
|
|
)
|
|
assert following.status == "pending" and following.id != row.id
|
|
assert commits.call_count == 1
|
|
assert session.get(SelectedModelComparison, row.id).status == "interrupted"
|
|
|
|
|
|
def test_candidate_must_use_the_gateway_experiment_channel(setup):
|
|
session, service, request, _, _ = setup
|
|
candidate = session.get(CapabilityDeployment, request.candidate_deployment_id)
|
|
candidate.channel = "lab"
|
|
session.commit()
|
|
with pytest.raises(EvaluationError, match="comparison selection invalid"):
|
|
service.create(request)
|
|
assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
|
|
|
|
|
|
def test_summary_fails_closed_when_linked_comparison_is_missing(setup):
|
|
session, service, _, client, _ = setup
|
|
claim, result, _ = _claimed(setup)
|
|
completed = service.result(claim.comparison.id, result, client)
|
|
session.get(SelectedModelComparison, completed.id).evaluation_comparison_id = uuid4()
|
|
session.commit()
|
|
with pytest.raises(EvaluationError):
|
|
service.get(completed.id)
|