Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0
This commit is contained in:
Public Source Publisher committed
1 parent
7082ab955a
commit
848f0d8f8a
434 files changed
+67298
-4315
No files matched your search
@@ -0,0 +1,739 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from modelforge_api.domain.evaluation import (
|
||||
EvaluationCaseCreate,
|
||||
EvaluationCaseResultCreate,
|
||||
EvaluationRevisionCreate,
|
||||
EvaluationRunComplete,
|
||||
EvaluationSuiteCreate,
|
||||
RankedResult,
|
||||
)
|
||||
from modelforge_api.domain.selected_model_comparison import (
|
||||
ComparisonCancel,
|
||||
ComparisonCreate,
|
||||
ComparisonFinish,
|
||||
ComparisonProgress,
|
||||
ComparisonResult,
|
||||
)
|
||||
from modelforge_api.persistence.models import (
|
||||
ArtifactSet,
|
||||
AuditEvent,
|
||||
Base,
|
||||
Capability,
|
||||
CapabilityContract,
|
||||
CapabilityDeployment,
|
||||
CapabilityExperimentRoute,
|
||||
DeploymentCandidate,
|
||||
EmbeddingSpace,
|
||||
EvaluationCase,
|
||||
EvaluationComparison,
|
||||
EvaluationRun,
|
||||
GatewayRequest,
|
||||
Model,
|
||||
ModelRevision,
|
||||
Project,
|
||||
ProjectBinding,
|
||||
RuntimeProfile,
|
||||
SelectedModelComparison,
|
||||
ServiceClient,
|
||||
)
|
||||
from modelforge_api.services.audit import AuditContext
|
||||
from modelforge_api.services.evaluation import EvaluationError, EvaluationService
|
||||
from modelforge_api.services.selected_model_comparison import (
|
||||
SelectedModelComparisonService,
|
||||
_digest,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def setup():
|
||||
engine = create_engine("sqlite://")
|
||||
Base.metadata.create_all(engine)
|
||||
with Session(engine) as session:
|
||||
|
||||
def add(value):
|
||||
session.add(value)
|
||||
session.flush()
|
||||
return value
|
||||
|
||||
project = add(Project(key="owner", name="Owner", description="fixture"))
|
||||
capability = add(Capability(key="rag.embedding", description="Embeddings"))
|
||||
contract = add(
|
||||
CapabilityContract(
|
||||
capability_id=capability.id,
|
||||
version=1,
|
||||
input_schema={},
|
||||
output_schema={},
|
||||
contract={},
|
||||
upgrade_class="requires_reindex",
|
||||
)
|
||||
)
|
||||
binding = add(
|
||||
ProjectBinding(
|
||||
project_id=project.id,
|
||||
capability_contract_id=contract.id,
|
||||
channel="stable",
|
||||
priority="production",
|
||||
)
|
||||
)
|
||||
client = add(
|
||||
ServiceClient(
|
||||
name="designated",
|
||||
allowed_capabilities=["rag.embedding@1"],
|
||||
project_binding_id=binding.id,
|
||||
)
|
||||
)
|
||||
model = add(
|
||||
Model(
|
||||
key="chosen",
|
||||
display_name="Chosen",
|
||||
upstream_provider="huggingface",
|
||||
upstream_source="org/chosen",
|
||||
)
|
||||
)
|
||||
deployments = []
|
||||
for index, variant in enumerate(("stable", "experiment", "experiment-current")):
|
||||
channel = "stable" if index == 0 else "experiment"
|
||||
revision = add(
|
||||
ModelRevision(
|
||||
model_id=model.id,
|
||||
upstream_revision=variant,
|
||||
resolved_commit_sha=str(index + 1) * 40,
|
||||
)
|
||||
)
|
||||
artifact = add(
|
||||
ArtifactSet(
|
||||
revision_id=revision.id,
|
||||
snapshot_id=uuid4(),
|
||||
variant_key=variant,
|
||||
label=variant,
|
||||
selection_reason="fixture",
|
||||
immutable_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
profile = add(
|
||||
RuntimeProfile(
|
||||
name=variant,
|
||||
artifact_set_id=artifact.id,
|
||||
runtime_type="sentence_transformers",
|
||||
runtime_version="fixture",
|
||||
artifact_sha256=str(index + 1) * 64,
|
||||
fingerprint=str(index + 3) * 64,
|
||||
immutable_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
candidate = add(
|
||||
DeploymentCandidate(
|
||||
artifact_set_id=artifact.id,
|
||||
runtime_profile_id=profile.id,
|
||||
compute_node_id=uuid4(),
|
||||
compatibility_assessment_id=uuid4(),
|
||||
runtime_probe_id=uuid4(),
|
||||
)
|
||||
)
|
||||
space = add(
|
||||
EmbeddingSpace(
|
||||
capability_contract_id=contract.id,
|
||||
artifact_set_id=artifact.id,
|
||||
runtime_profile_id=profile.id,
|
||||
identity_digest=str(index + 5) * 64,
|
||||
dimension=4,
|
||||
normalized=True,
|
||||
identity_facts={},
|
||||
)
|
||||
)
|
||||
deployments.append(
|
||||
add(
|
||||
CapabilityDeployment(
|
||||
capability_contract_id=contract.id,
|
||||
deployment_candidate_id=candidate.id,
|
||||
embedding_space_id=space.id,
|
||||
artifact_set_id=artifact.id,
|
||||
runtime_profile_id=profile.id,
|
||||
compute_node_id=candidate.compute_node_id,
|
||||
accelerator_id=uuid4(),
|
||||
channel=channel,
|
||||
status="stable" if index == 0 else "candidate",
|
||||
production=index == 0,
|
||||
config_fingerprint=str(index + 7) * 64,
|
||||
provenance={},
|
||||
)
|
||||
)
|
||||
)
|
||||
route = add(
|
||||
CapabilityExperimentRoute(
|
||||
route_key="selected-candidate",
|
||||
capability_contract_id=contract.id,
|
||||
capability_deployment_id=deployments[1].id,
|
||||
purpose="fixture",
|
||||
evidence={},
|
||||
)
|
||||
)
|
||||
add(
|
||||
CapabilityExperimentRoute(
|
||||
route_key="selected-current-experiment",
|
||||
capability_contract_id=contract.id,
|
||||
capability_deployment_id=deployments[2].id,
|
||||
purpose="fixture current experiment",
|
||||
evidence={},
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
relevant = uuid4()
|
||||
suite = EvaluationService(session).create_suite(
|
||||
EvaluationSuiteCreate(
|
||||
project_id=project.id,
|
||||
key="reviewed",
|
||||
name="Reviewed",
|
||||
description="fixture",
|
||||
revision=EvaluationRevisionCreate(
|
||||
revision="v1",
|
||||
dataset_revision="fixture-v1",
|
||||
retrieval_settings={"top_k": 10},
|
||||
cases=[
|
||||
EvaluationCaseCreate(
|
||||
case_key="one",
|
||||
query="Where is the answer?",
|
||||
relevant_chunk_ids=[relevant],
|
||||
label_provenance={
|
||||
"source": "owner_review",
|
||||
"corpus_revision": "a" * 64,
|
||||
},
|
||||
critical=True,
|
||||
review_status="reviewed",
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
)
|
||||
service = SelectedModelComparisonService(session, AuditContext("operator", "fixture"))
|
||||
request = ComparisonCreate(
|
||||
project_binding_id=binding.id,
|
||||
consumer_client_id=client.id,
|
||||
suite_revision_id=suite.latest_revision_id,
|
||||
current_deployment_id=deployments[0].id,
|
||||
candidate_deployment_id=deployments[1].id,
|
||||
candidate_route_id=route.id,
|
||||
corpus_sha256="a" * 64,
|
||||
retrieval_config_sha256=_digest({"top_k": 10}),
|
||||
current_index_ref="current-exact",
|
||||
candidate_index_ref="candidate-exact",
|
||||
idempotency_key="one",
|
||||
)
|
||||
yield session, service, request, client, relevant
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _claimed(setup, selected_request=None):
|
||||
session, service, request, client, relevant = setup
|
||||
if selected_request is not None:
|
||||
request = selected_request
|
||||
created = service.create(request)
|
||||
claim = service.claim(created.id, client)
|
||||
case = session.scalar(
|
||||
select(EvaluationCase).where(EvaluationCase.suite_revision_id == request.suite_revision_id)
|
||||
)
|
||||
gateways = []
|
||||
for deployment_id in (request.current_deployment_id, request.candidate_deployment_id):
|
||||
gateway = GatewayRequest(
|
||||
request_id=uuid4(),
|
||||
service_client_id=client.id,
|
||||
capability_key="rag.embedding",
|
||||
capability_version=1,
|
||||
capability_deployment_id=deployment_id,
|
||||
status="completed",
|
||||
priority="production",
|
||||
input_sha256=_digest([case.query]),
|
||||
input_count=1,
|
||||
created_at=datetime.now(UTC),
|
||||
finished_at=datetime.now(UTC),
|
||||
decision_evidence={},
|
||||
)
|
||||
session.add(gateway)
|
||||
gateways.append(gateway)
|
||||
session.commit()
|
||||
leg = EvaluationRunComplete(
|
||||
results=[
|
||||
EvaluationCaseResultCreate(
|
||||
case_id=case.id,
|
||||
ranked_results=[RankedResult(chunk_id=relevant, score=1)],
|
||||
latency_ms=2,
|
||||
)
|
||||
]
|
||||
)
|
||||
result = ComparisonResult(
|
||||
claim_token=claim.claim_token,
|
||||
pins_digest=claim.comparison.pins_digest,
|
||||
consumer_revision_sha256="c" * 64,
|
||||
current=leg,
|
||||
candidate=leg,
|
||||
current_gateway_requests={case.id: gateways[0].request_id},
|
||||
candidate_gateway_requests={case.id: gateways[1].request_id},
|
||||
)
|
||||
return claim, result, gateways
|
||||
|
||||
|
||||
def test_real_pair_recomputes_metrics_and_retries_without_duplicate_evidence(setup):
|
||||
session, service, request, client, _ = setup
|
||||
claim, result, _ = _claimed(setup)
|
||||
assert (
|
||||
claim.comparison.pins["current"]["dimension"]
|
||||
== claim.comparison.pins["candidate"]["dimension"]
|
||||
)
|
||||
assert claim.comparison.pins["current"]["dimension"] > 0
|
||||
completed = service.result(claim.comparison.id, result, client)
|
||||
assert completed.status == "completed"
|
||||
comparison = session.get(EvaluationComparison, completed.evaluation_comparison_id)
|
||||
assert comparison.comparability == "comparable"
|
||||
assert comparison.promotion_eligibility == "eligible"
|
||||
assert completed.evaluation_summary.eligibility == "eligible"
|
||||
assert completed.evaluation_summary.critical_regressions == 0
|
||||
assert comparison.eligibility_evidence["promotion_performed"] is False
|
||||
assert service.result(completed.id, result, client).id == completed.id
|
||||
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 2
|
||||
assert session.scalar(select(func.count()).select_from(EvaluationComparison)) == 1
|
||||
assert session.get(CapabilityDeployment, request.candidate_deployment_id).production is False
|
||||
history = service.get(completed.id).model_dump_json()
|
||||
assert claim.claim_token not in history and "claim_token_hash" not in history
|
||||
assert all(
|
||||
claim.claim_token not in str(event.details) for event in session.scalars(select(AuditEvent))
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value", [("corpus_sha256", "d" * 64), ("current_index_ref", "candidate-exact")]
|
||||
)
|
||||
def test_rejects_mismatched_corpus_and_shared_vector_index(setup, field, value):
|
||||
_, service, request, _, _ = setup
|
||||
with pytest.raises(EvaluationError):
|
||||
service.create(request.model_copy(update={field: value}))
|
||||
|
||||
|
||||
def test_idempotency_active_bound_and_exact_consumer(setup):
|
||||
session, service, request, client, _ = setup
|
||||
created = service.create(request)
|
||||
legacy_digest = _digest(request.model_dump(mode="json", exclude={"current_route_id"}))
|
||||
assert session.get(SelectedModelComparison, created.id).request_digest == legacy_digest
|
||||
assert service.create(request).id == created.id
|
||||
with pytest.raises(EvaluationError):
|
||||
service.create(request.model_copy(update={"candidate_index_ref": "different"}))
|
||||
with pytest.raises(EvaluationError):
|
||||
service.create(request.model_copy(update={"idempotency_key": "second"}))
|
||||
stranger = ServiceClient(
|
||||
name="other",
|
||||
project_binding_id=client.project_binding_id,
|
||||
allowed_capabilities=["rag.embedding@1"],
|
||||
)
|
||||
session.add(stranger)
|
||||
session.commit()
|
||||
assert service.assigned(stranger).items == []
|
||||
with pytest.raises(EvaluationError):
|
||||
service.claim(created.id, stranger)
|
||||
assert service.assigned(client).items[0].id == created.id
|
||||
|
||||
|
||||
def test_selected_experiment_pair_requires_both_exact_routes_and_is_idempotent(setup):
|
||||
session, service, request, client, _ = setup
|
||||
current_route = session.scalar(
|
||||
select(CapabilityExperimentRoute).where(
|
||||
CapabilityExperimentRoute.route_key == "selected-current-experiment"
|
||||
)
|
||||
)
|
||||
selected = request.model_copy(
|
||||
update={
|
||||
"current_deployment_id": current_route.capability_deployment_id,
|
||||
"current_route_id": current_route.id,
|
||||
"idempotency_key": "two-experiments",
|
||||
}
|
||||
)
|
||||
|
||||
created = service.create(selected)
|
||||
assert service.create(selected).id == created.id
|
||||
with pytest.raises(EvaluationError, match="comparison idempotency conflict"):
|
||||
service.create(selected.model_copy(update={"current_route_id": request.candidate_route_id}))
|
||||
assert created.pins["current_route_id"] == str(current_route.id)
|
||||
assert created.pins["current_route_key"] == current_route.route_key
|
||||
assert created.pins["candidate_route_id"] == str(request.candidate_route_id)
|
||||
identity_pins = {
|
||||
"config_fingerprint",
|
||||
"artifact_set_id",
|
||||
"revision_id",
|
||||
"upstream_commit_sha",
|
||||
"runtime_profile_id",
|
||||
"runtime_fingerprint",
|
||||
"artifact_sha256",
|
||||
"embedding_space_id",
|
||||
"embedding_space_digest",
|
||||
"dimension",
|
||||
"compute_node_id",
|
||||
"accelerator_id",
|
||||
}
|
||||
assert identity_pins <= set(created.pins["current"])
|
||||
assert identity_pins <= set(created.pins["candidate"])
|
||||
assert (
|
||||
created.pins["current"]["embedding_space_id"]
|
||||
!= created.pins["candidate"]["embedding_space_id"]
|
||||
)
|
||||
claim, result, _ = _claimed(setup, selected)
|
||||
completed = service.result(created.id, result, client)
|
||||
assert claim.comparison.id == created.id
|
||||
assert completed.status == "completed"
|
||||
assert completed.pins["current_route_id"] == str(current_route.id)
|
||||
current_route.status = "inactive"
|
||||
session.commit()
|
||||
assert service.result(created.id, result, client).id == completed.id
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutation",
|
||||
["production_current", "wrong_current_route", "inactive_current_route", "same_route"],
|
||||
)
|
||||
def test_selected_experiment_pair_rejects_non_experiment_or_mismatched_current_route(
|
||||
setup, mutation
|
||||
):
|
||||
session, service, request, _, _ = setup
|
||||
current_route = session.scalar(
|
||||
select(CapabilityExperimentRoute).where(
|
||||
CapabilityExperimentRoute.route_key == "selected-current-experiment"
|
||||
)
|
||||
)
|
||||
selected = request.model_copy(
|
||||
update={
|
||||
"current_deployment_id": current_route.capability_deployment_id,
|
||||
"current_route_id": current_route.id,
|
||||
"idempotency_key": f"invalid-{mutation}",
|
||||
}
|
||||
)
|
||||
if mutation == "production_current":
|
||||
selected = selected.model_copy(
|
||||
update={"current_deployment_id": request.current_deployment_id}
|
||||
)
|
||||
elif mutation == "wrong_current_route":
|
||||
selected = selected.model_copy(update={"current_route_id": request.candidate_route_id})
|
||||
elif mutation == "inactive_current_route":
|
||||
current_route.status = "inactive"
|
||||
session.commit()
|
||||
else:
|
||||
selected = selected.model_copy(update={"candidate_route_id": current_route.id})
|
||||
|
||||
with pytest.raises(EvaluationError):
|
||||
service.create(selected)
|
||||
assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
|
||||
|
||||
|
||||
def test_current_experiment_route_key_is_revalidated_before_claim_and_result(setup):
|
||||
session, service, request, client, _ = setup
|
||||
current_route = session.scalar(
|
||||
select(CapabilityExperimentRoute).where(
|
||||
CapabilityExperimentRoute.route_key == "selected-current-experiment"
|
||||
)
|
||||
)
|
||||
selected = request.model_copy(
|
||||
update={
|
||||
"current_deployment_id": current_route.capability_deployment_id,
|
||||
"current_route_id": current_route.id,
|
||||
"idempotency_key": "route-revalidation",
|
||||
}
|
||||
)
|
||||
created = service.create(selected)
|
||||
current_route.route_key = "changed-before-claim"
|
||||
session.commit()
|
||||
with pytest.raises(EvaluationError, match="comparison pins changed"):
|
||||
service.claim(created.id, client)
|
||||
|
||||
current_route.route_key = "selected-current-experiment"
|
||||
session.commit()
|
||||
claim, result, _ = _claimed(setup, selected)
|
||||
assert claim.comparison.id == created.id
|
||||
current_route.route_key = "changed-before-result"
|
||||
session.commit()
|
||||
with pytest.raises(EvaluationError, match="comparison pins changed"):
|
||||
service.result(created.id, result, client)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("phase", ["claim", "result"])
|
||||
@pytest.mark.parametrize("mutation", ["inactive", "remapped"])
|
||||
def test_current_experiment_route_deactivation_or_remap_fails_closed(setup, phase, mutation):
|
||||
session, service, request, client, _ = setup
|
||||
current_route = session.scalar(
|
||||
select(CapabilityExperimentRoute).where(
|
||||
CapabilityExperimentRoute.route_key == "selected-current-experiment"
|
||||
)
|
||||
)
|
||||
selected = request.model_copy(
|
||||
update={
|
||||
"current_deployment_id": current_route.capability_deployment_id,
|
||||
"current_route_id": current_route.id,
|
||||
"idempotency_key": f"route-{phase}-{mutation}",
|
||||
}
|
||||
)
|
||||
|
||||
if phase == "claim":
|
||||
comparison = service.create(selected)
|
||||
claim = None
|
||||
result = None
|
||||
else:
|
||||
claim, result, _ = _claimed(setup, selected)
|
||||
comparison = claim.comparison
|
||||
|
||||
if mutation == "inactive":
|
||||
current_route.status = "inactive"
|
||||
else:
|
||||
current_route.capability_deployment_id = selected.candidate_deployment_id
|
||||
session.commit()
|
||||
|
||||
with pytest.raises(EvaluationError):
|
||||
if phase == "claim":
|
||||
service.claim(comparison.id, client)
|
||||
else:
|
||||
assert claim is not None and result is not None
|
||||
service.result(comparison.id, result, client)
|
||||
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
|
||||
|
||||
|
||||
def test_expiry_never_renews_or_reclaims(setup):
|
||||
session, service, _, client, _ = setup
|
||||
claim, result, _ = _claimed(setup)
|
||||
row = session.get(SelectedModelComparison, claim.comparison.id)
|
||||
row.claim_expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
session.commit()
|
||||
with pytest.raises(EvaluationError):
|
||||
service.result(row.id, result, client)
|
||||
assert service.get(row.id).status == "interrupted"
|
||||
assert row.claim_token_hash is None
|
||||
with pytest.raises(EvaluationError):
|
||||
service.claim(row.id, client)
|
||||
|
||||
|
||||
def test_cancel_is_request_until_consumer_acknowledges(setup):
|
||||
_, service, _, client, _ = setup
|
||||
claim, result, _ = _claimed(setup)
|
||||
response = service.cancel(
|
||||
claim.comparison.id, ComparisonCancel(expected_version=claim.comparison.version)
|
||||
)
|
||||
assert response.status == "cancel_requested" and response.finished_at is None
|
||||
with pytest.raises(EvaluationError):
|
||||
service.result(response.id, result, client)
|
||||
completed = service.finish(
|
||||
response.id, ComparisonFinish(claim_token=claim.claim_token, outcome="cancelled"), client
|
||||
)
|
||||
assert completed.status == "cancelled"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutation",
|
||||
["historical", "wrong_target", "wrong_client", "wrong_query", "not_completed", "duplicate"],
|
||||
)
|
||||
def test_gateway_provenance_rejects_unrelated_or_reused_requests(setup, mutation):
|
||||
session, service, request, client, _ = setup
|
||||
claim, result, gateways = _claimed(setup)
|
||||
if mutation == "historical":
|
||||
gateways[0].created_at = datetime.now(UTC) - timedelta(hours=1)
|
||||
elif mutation == "wrong_target":
|
||||
gateways[0].capability_deployment_id = request.candidate_deployment_id
|
||||
elif mutation == "wrong_client":
|
||||
gateways[0].service_client_id = uuid4()
|
||||
elif mutation == "wrong_query":
|
||||
gateways[0].input_sha256 = "d" * 64
|
||||
elif mutation == "not_completed":
|
||||
gateways[0].status = "failed"
|
||||
else:
|
||||
result.candidate_gateway_requests = result.current_gateway_requests
|
||||
session.commit()
|
||||
with pytest.raises(EvaluationError):
|
||||
service.result(claim.comparison.id, result, client)
|
||||
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
|
||||
|
||||
|
||||
def test_result_failure_rolls_back_both_runs_and_keeps_claim(setup):
|
||||
session, service, _, client, _ = setup
|
||||
claim, result, _ = _claimed(setup)
|
||||
with (
|
||||
patch.object(EvaluationService, "compare", side_effect=RuntimeError("test interruption")),
|
||||
pytest.raises(RuntimeError),
|
||||
):
|
||||
service.result(claim.comparison.id, result, client)
|
||||
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
|
||||
assert service.get(claim.comparison.id).status == "running"
|
||||
assert service.result(claim.comparison.id, result, client).status == "completed"
|
||||
|
||||
|
||||
def test_progress_is_monotonic_and_claim_cannot_be_stolen(setup):
|
||||
_, service, _, client, _ = setup
|
||||
claim, _, _ = _claimed(setup)
|
||||
with pytest.raises(EvaluationError):
|
||||
service.claim(claim.comparison.id, client)
|
||||
progress = ComparisonProgress(
|
||||
claim_token=claim.claim_token, current_completed_cases=1, candidate_completed_cases=0
|
||||
)
|
||||
service.progress(claim.comparison.id, progress, client)
|
||||
with pytest.raises(EvaluationError):
|
||||
service.progress(
|
||||
claim.comparison.id, progress.model_copy(update={"current_completed_cases": 0}), client
|
||||
)
|
||||
with pytest.raises(EvaluationError):
|
||||
service.progress(
|
||||
claim.comparison.id,
|
||||
progress.model_copy(update={"candidate_completed_cases": 2}),
|
||||
client,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutation",
|
||||
[
|
||||
"wrong_project",
|
||||
"unreviewed",
|
||||
"missing_corpus",
|
||||
"wrong_contract",
|
||||
"same_target",
|
||||
"inactive_route",
|
||||
"space_mismatch",
|
||||
],
|
||||
)
|
||||
def test_create_rejects_incoherent_selected_project_provenance(setup, mutation):
|
||||
session, service, request, _, _ = setup
|
||||
if mutation == "wrong_project":
|
||||
from modelforge_api.persistence.models import EvaluationSuite, EvaluationSuiteRevision
|
||||
|
||||
revision = session.get(EvaluationSuiteRevision, request.suite_revision_id)
|
||||
session.get(EvaluationSuite, revision.suite_id).project_id = uuid4()
|
||||
elif mutation in {"unreviewed", "missing_corpus"}:
|
||||
case = session.scalar(select(EvaluationCase))
|
||||
# Simulate imported corrupt provenance, bypassing the normal immutable ORM guard.
|
||||
if mutation == "unreviewed":
|
||||
session.execute(
|
||||
update(EvaluationCase)
|
||||
.where(EvaluationCase.id == case.id)
|
||||
.values(review_status="draft")
|
||||
)
|
||||
else:
|
||||
session.execute(
|
||||
update(EvaluationCase)
|
||||
.where(EvaluationCase.id == case.id)
|
||||
.values(label_provenance={"source": "no reviewed corpus"})
|
||||
)
|
||||
elif mutation == "same_target":
|
||||
request = request.model_copy(
|
||||
update={"candidate_deployment_id": request.current_deployment_id}
|
||||
)
|
||||
elif mutation == "inactive_route":
|
||||
session.get(CapabilityExperimentRoute, request.candidate_route_id).status = "inactive"
|
||||
else:
|
||||
deployment = session.get(CapabilityDeployment, request.candidate_deployment_id)
|
||||
if mutation == "wrong_contract":
|
||||
deployment.capability_contract_id = uuid4()
|
||||
else:
|
||||
session.execute(
|
||||
update(EmbeddingSpace)
|
||||
.where(EmbeddingSpace.id == deployment.embedding_space_id)
|
||||
.values(runtime_profile_id=uuid4())
|
||||
)
|
||||
session.commit()
|
||||
with pytest.raises(EvaluationError):
|
||||
service.create(request)
|
||||
assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
|
||||
|
||||
|
||||
def test_claim_and_result_revalidate_pins_and_client_revocation(setup):
|
||||
session, service, request, client, _ = setup
|
||||
created = service.create(request)
|
||||
deployment = session.get(CapabilityDeployment, request.candidate_deployment_id)
|
||||
original = deployment.config_fingerprint
|
||||
deployment.config_fingerprint = "d" * 64
|
||||
session.commit()
|
||||
with pytest.raises(EvaluationError):
|
||||
service.claim(created.id, client)
|
||||
deployment.config_fingerprint = original
|
||||
session.commit()
|
||||
claim, result, _ = _claimed(setup)
|
||||
client.disabled_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
with pytest.raises(EvaluationError):
|
||||
service.result(claim.comparison.id, result, client)
|
||||
client.disabled_at = None
|
||||
deployment.config_fingerprint = "d" * 64
|
||||
session.commit()
|
||||
with pytest.raises(EvaluationError):
|
||||
service.result(claim.comparison.id, result, client)
|
||||
|
||||
|
||||
def test_failed_case_cannot_qualify_and_does_not_require_gateway_success(setup):
|
||||
session, service, _, client, _ = setup
|
||||
claim, result, _ = _claimed(setup)
|
||||
failed = result.current.results[0].model_copy(
|
||||
update={"error_code": "UNAVAILABLE", "ranked_results": []}
|
||||
)
|
||||
result.current = EvaluationRunComplete(results=[failed])
|
||||
result.current_gateway_requests = {}
|
||||
completed = service.result(claim.comparison.id, result, client)
|
||||
assert (
|
||||
session.get(EvaluationComparison, completed.evaluation_comparison_id).promotion_eligibility
|
||||
== "not_eligible"
|
||||
)
|
||||
|
||||
|
||||
def test_missing_case_or_mapping_and_changed_retry_cannot_publish(setup):
|
||||
session, service, _, client, _ = setup
|
||||
claim, result, _ = _claimed(setup)
|
||||
original = result.current_gateway_requests
|
||||
result.current_gateway_requests = {}
|
||||
with pytest.raises(EvaluationError):
|
||||
service.result(claim.comparison.id, result, client)
|
||||
result.current_gateway_requests = original
|
||||
completed = service.result(claim.comparison.id, result, client)
|
||||
result.consumer_revision_sha256 = "d" * 64
|
||||
with pytest.raises(EvaluationError):
|
||||
service.result(completed.id, result, client)
|
||||
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reader", ["list", "assigned", "create"])
|
||||
def test_expiry_is_reconciled_by_history_assignment_and_next_intent_without_early_commit(
|
||||
setup, reader
|
||||
):
|
||||
session, service, request, client, _ = setup
|
||||
claim, _, _ = _claimed(setup)
|
||||
row = session.get(SelectedModelComparison, claim.comparison.id)
|
||||
row.claim_expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
||||
session.commit()
|
||||
with patch.object(session, "commit", wraps=session.commit) as commits:
|
||||
if reader == "list":
|
||||
assert service.list().items[0].status == "interrupted"
|
||||
elif reader == "assigned":
|
||||
assert service.assigned(client).items[0].status == "interrupted"
|
||||
else:
|
||||
following = service.create(
|
||||
request.model_copy(update={"idempotency_key": "explicit-new-intent"})
|
||||
)
|
||||
assert following.status == "pending" and following.id != row.id
|
||||
assert commits.call_count == 1
|
||||
assert session.get(SelectedModelComparison, row.id).status == "interrupted"
|
||||
|
||||
|
||||
def test_candidate_must_use_the_gateway_experiment_channel(setup):
|
||||
session, service, request, _, _ = setup
|
||||
candidate = session.get(CapabilityDeployment, request.candidate_deployment_id)
|
||||
candidate.channel = "lab"
|
||||
session.commit()
|
||||
with pytest.raises(EvaluationError, match="comparison selection invalid"):
|
||||
service.create(request)
|
||||
assert session.scalar(select(func.count()).select_from(SelectedModelComparison)) == 0
|
||||
|
||||
|
||||
def test_summary_fails_closed_when_linked_comparison_is_missing(setup):
|
||||
session, service, _, client, _ = setup
|
||||
claim, result, _ = _claimed(setup)
|
||||
completed = service.result(claim.comparison.id, result, client)
|
||||
session.get(SelectedModelComparison, completed.id).evaluation_comparison_id = uuid4()
|
||||
session.commit()
|
||||
with pytest.raises(EvaluationError):
|
||||
service.get(completed.id)
|
||||
Reference in new issue
Block a user