Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/tests/test_evaluation_project_boundary.py

@@ -0,0 +1,120 @@
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import func, select, update
+
+from modelforge_api.domain.evaluation import (
+    EvaluationCaseResultCreate,
+    EvaluationComparisonCreate,
+    EvaluationRunComplete,
+    EvaluationRunCreate,
+)
+from modelforge_api.persistence.models import (
+    EvaluationCase,
+    EvaluationCaseResult,
+    EvaluationComparison,
+    EvaluationRun,
+    Project,
+)
+from modelforge_api.services.evaluation import EvaluationError, EvaluationService
+from tests.test_evaluation import _run, _session, _suite
+
+
+def test_run_creation_rejects_another_projects_suite_without_writing_run() -> None:
+    with _session() as session:
+        owner = Project(key="owner", name="Owner", description="test")
+        other = Project(key="other", name="Other", description="test")
+        session.add_all([owner, other])
+        session.commit()
+        service = EvaluationService(session)
+        suite = _suite(service, owner.id, uuid4())
+        request = EvaluationRunCreate(project_id=other.id, suite_revision_id=suite.latest_revision_id,
+            target_kind="current", target_index_ref="current", embedding_space_ref="space",
+            corpus_revision="corpus", retrieval_config_digest="a" * 64,
+            environment_fingerprint={"engine": "test"})
+        with pytest.raises(EvaluationError) as rejected:
+            service.create_run(request)
+        assert rejected.value.code == "suite_project_mismatch"
+        assert session.scalar(select(func.count()).select_from(EvaluationRun)) == 0
+
+
+def test_legacy_wrong_project_run_cannot_complete_or_become_eligible() -> None:
+    with _session() as session:
+        owner = Project(key="owner", name="Owner", description="test")
+        other = Project(key="other", name="Other", description="test")
+        session.add_all([owner, other])
+        session.commit()
+        service = EvaluationService(session)
+        relevant = uuid4()
+        suite = _suite(service, owner.id, relevant)
+        case = session.scalar(select(EvaluationCase))
+        assert case is not None
+        baseline = _run(service, owner.id, suite.latest_revision_id, case.id, relevant,
+                        target="current", rank=1)
+        candidate = _run(service, owner.id, suite.latest_revision_id, case.id, relevant,
+                         target="shadow", rank=1)
+        # Model historical rows admitted before the ownership check, without changing their suite.
+        session.execute(update(EvaluationRun).values(project_id=other.id))
+        session.commit()
+        comparison = service.compare(EvaluationComparisonCreate(
+            baseline_run_id=baseline.id, candidate_run_id=candidate.id))
+        assert comparison.comparability == "not_comparable"
+        assert comparison.promotion_eligibility == "requires_review"
+        assert comparison.comparability_evidence["baseline_suite_project"] is False
+        session.execute(update(EvaluationRun).where(EvaluationRun.id == candidate.id)
+                        .values(status="running"))
+        session.commit()
+        with pytest.raises(EvaluationError) as rejected:
+            service.complete_run(candidate.id, EvaluationRunComplete(results=[
+                EvaluationCaseResultCreate(case_id=case.id, ranked_results=[], latency_ms=1)]))
+        assert rejected.value.code == "suite_project_mismatch"
+
+
+def test_failed_baseline_remains_reportable_but_never_eligible_even_with_error_tolerance() -> None:
+    with _session() as session:
+        project = Project(key="owner", name="Owner", description="test")
+        session.add(project)
+        session.commit()
+        service = EvaluationService(session)
+        relevant = uuid4()
+        suite = _suite(service, project.id, relevant)
+        case = session.scalar(select(EvaluationCase))
+        assert case is not None
+        baseline = service.create_run(EvaluationRunCreate(project_id=project.id,
+            suite_revision_id=suite.latest_revision_id, target_kind="current", target_index_ref="current",
+            embedding_space_ref="space", corpus_revision="corpus-v1", retrieval_config_digest="a" * 64,
+            environment_fingerprint={"engine": "test-v1"}))
+        service.complete_run(baseline.id, EvaluationRunComplete(results=[EvaluationCaseResultCreate(
+            case_id=case.id, ranked_results=[], latency_ms=1, error_code="upstream_failed")]))
+        candidate = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
+                         target="shadow", rank=1)
+        comparison = service.compare(EvaluationComparisonCreate(baseline_run_id=baseline.id,
+            candidate_run_id=candidate.id, maximum_error_rate=1))
+        assert comparison.comparability == "comparable"
+        assert comparison.metric_deltas["recall_at_10"] == 1
+        assert comparison.promotion_eligibility == "not_eligible"
+        assert comparison.eligibility_evidence["baseline_error_count"] == 1
+        assert comparison.eligibility_evidence["baseline_error_free"] is False
+
+
+def test_caller_owned_transaction_rolls_back_runs_case_results_and_comparison() -> None:
+    with _session() as session:
+        project = Project(key="owner", name="Owner", description="test")
+        session.add(project)
+        session.commit()
+        relevant = uuid4()
+        suite = _suite(EvaluationService(session), project.id, relevant)
+        case = session.scalar(select(EvaluationCase))
+        assert case is not None
+        service = EvaluationService(session, commit=False)
+        baseline = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
+                        target="current", rank=2)
+        candidate = _run(service, project.id, suite.latest_revision_id, case.id, relevant,
+                         target="shadow", rank=1)
+        comparison = service.compare(EvaluationComparisonCreate(
+            baseline_run_id=baseline.id, candidate_run_id=candidate.id))
+        assert comparison.promotion_eligibility == "eligible"
+        assert session.scalar(select(func.count()).select_from(EvaluationCaseResult)) == 2
+        session.rollback()
+        for model in (EvaluationRun, EvaluationCaseResult, EvaluationComparison):
+            assert session.scalar(select(func.count()).select_from(model)) == 0