121 lines
6.1 KiB
Python
121 lines
6.1 KiB
Python
from uuid import uuid4
|
|
|
|
import pytest
|
|
from sqlalchemy import func, select, update
|
|
|
|
from modelforge_api.domain.evaluation import (
|
|
EvaluationCaseResultCreate,
|
|
EvaluationComparisonCreate,
|
|
EvaluationRunComplete,
|
|
EvaluationRunCreate,
|
|
)
|
|
from modelforge_api.persistence.models import (
|
|
EvaluationCase,
|
|
EvaluationCaseResult,
|
|
EvaluationComparison,
|
|
EvaluationRun,
|
|
Project,
|
|
)
|
|
from modelforge_api.services.evaluation import EvaluationError, EvaluationService
|
|
from tests.test_evaluation import _run, _session, _suite
|
|
|
|
|
|
def test_run_creation_rejects_another_projects_suite_without_writing_run() -> None:
|
|
with _session() as session:
|
|
owner = Project(key="owner", name="Owner", description="test")
|
|
other = Project(key="other", name="Other", description="test")
|
|
session.add_all([owner, other])
|
|
session.commit()
|
|
service = EvaluationService(session)
|
|
suite = _suite(service, owner.id, uuid4())
|
|
request = EvaluationRunCreate(project_id=other.id, suite_revision_id=suite.latest_revision_id,
|
|
target_kind="current", target_index_ref="current", embedding_space_ref="space",
|
|
corpus_revision="corpus", retrieval_config_digest="a" * 64,
|
|
environment_fingerprint={"engine": "test"})
|
|
with pytest.raises(EvaluationError) as rejected:
|
|
service.create_run(request)
|
|
assert rejected.value.code == "suite_project_mismatch"
|
|
assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
|
|
|
|
|
|
def test_legacy_wrong_project_run_cannot_complete_or_become_eligible() -> None:
|
|
with _session() as session:
|
|
owner = Project(key="owner", name="Owner", description="test")
|
|
other = Project(key="other", name="Other", description="test")
|
|
session.add_all([owner, other])
|
|
session.commit()
|
|
service = EvaluationService(session)
|
|
relevant = uuid4()
|
|
suite = _suite(service, owner.id, relevant)
|
|
case = session.scalar(select(EvaluationCase))
|
|
assert case is not None
|
|
baseline = _run(service, owner.id, suite.latest_revision_id, case.id, relevant,
|
|
target="current", rank=1)
|
|
candidate = _run(service, owner.id, suite.latest_revision_id, case.id, relevant,
|
|
target="shadow", rank=1)
|
|
# Model historical rows admitted before the ownership check, without changing their suite.
|
|
session.execute(update(EvaluationRun).values(project_id=other.id))
|
|
session.commit()
|
|
comparison = service.compare(EvaluationComparisonCreate(
|
|
baseline_run_id=baseline.id, candidate_run_id=candidate.id))
|
|
assert comparison.comparability == "not_comparable"
|
|
assert comparison.promotion_eligibility == "requires_review"
|
|
assert comparison.comparability_evidence["baseline_suite_project"] is False
|
|
session.execute(update(EvaluationRun).where(EvaluationRun.id == candidate.id)
|
|
.values(status="running"))
|
|
session.commit()
|
|
with pytest.raises(EvaluationError) as rejected:
|
|
service.complete_run(candidate.id, EvaluationRunComplete(results=[
|
|
EvaluationCaseResultCreate(case_id=case.id, ranked_results=[], latency_ms=1)]))
|
|
assert rejected.value.code == "suite_project_mismatch"
|
|
|
|
|
|
def test_failed_baseline_remains_reportable_but_never_eligible_even_with_error_tolerance() -> None:
|
|
with _session() as session:
|
|
project = Project(key="owner", name="Owner", description="test")
|
|
session.add(project)
|
|
session.commit()
|
|
service = EvaluationService(session)
|
|
relevant = uuid4()
|
|
suite = _suite(service, project.id, relevant)
|
|
case = session.scalar(select(EvaluationCase))
|
|
assert case is not None
|
|
baseline = service.create_run(EvaluationRunCreate(project_id=project.id,
|
|
suite_revision_id=suite.latest_revision_id, target_kind="current", target_index_ref="current",
|
|
embedding_space_ref="space", corpus_revision="corpus-v1", retrieval_config_digest="a" * 64,
|
|
environment_fingerprint={"engine": "test-v1"}))
|
|
service.complete_run(baseline.id, EvaluationRunComplete(results=[EvaluationCaseResultCreate(
|
|
case_id=case.id, ranked_results=[], latency_ms=1, error_code="upstream_failed")]))
|
|
candidate = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
|
|
target="shadow", rank=1)
|
|
comparison = service.compare(EvaluationComparisonCreate(baseline_run_id=baseline.id,
|
|
candidate_run_id=candidate.id, maximum_error_rate=1))
|
|
assert comparison.comparability == "comparable"
|
|
assert comparison.metric_deltas["recall_at_10"] == 1
|
|
assert comparison.promotion_eligibility == "not_eligible"
|
|
assert comparison.eligibility_evidence["baseline_error_count"] == 1
|
|
assert comparison.eligibility_evidence["baseline_error_free"] is False
|
|
|
|
|
|
def test_caller_owned_transaction_rolls_back_runs_case_results_and_comparison() -> None:
|
|
with _session() as session:
|
|
project = Project(key="owner", name="Owner", description="test")
|
|
session.add(project)
|
|
session.commit()
|
|
relevant = uuid4()
|
|
suite = _suite(EvaluationService(session), project.id, relevant)
|
|
case = session.scalar(select(EvaluationCase))
|
|
assert case is not None
|
|
service = EvaluationService(session, commit=False)
|
|
baseline = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
|
|
target="current", rank=2)
|
|
candidate = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
|
|
target="shadow", rank=1)
|
|
comparison = service.compare(EvaluationComparisonCreate(
|
|
baseline_run_id=baseline.id, candidate_run_id=candidate.id))
|
|
assert comparison.promotion_eligibility == "eligible"
|
|
assert session.scalar(select(func.count()).select_from(EvaluationCaseResult)) == 2
|
|
session.rollback()
|
|
for model in (EvaluationRun, EvaluationCaseResult, EvaluationComparison):
|
|
assert session.scalar(select(func.count()).select_from(model)) == 0
|