Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/tests/test_selected_comparison_postgres.py

@@ -0,0 +1,261 @@
+"""Owner/runtime PostgreSQL qualification using a complete offline foreign-key graph."""
+from __future__ import annotations
+
+import importlib
+import os
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+from uuid import UUID, uuid4
+
+import pytest
+from sqlalchemy import create_engine, event, inspect, select, text
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.evaluation import (
+    EvaluationCaseCreate,
+    EvaluationRevisionCreate,
+    EvaluationSuiteCreate,
+)
+from modelforge_api.domain.selected_model_comparison import ComparisonCreate
+from modelforge_api.domain.serving import CapabilityExperimentCreate
+from modelforge_api.persistence.models import (
+    AuditEvent,
+    Base,
+    CapabilityDeployment,
+    Project,
+    ProjectBinding,
+    SelectedModelComparison,
+    ServiceClient,
+)
+from modelforge_api.services.audit import AuditContext, AuditWriter
+from modelforge_api.services.evaluation import EvaluationError, EvaluationService
+from modelforge_api.services.recovery import RecoveryError, RecoveryService
+from modelforge_api.services.recovery_fingerprint import (
+    SELECTED_COMPARISON_RESTORE_ERROR,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    semantic_fingerprint,
+)
+from modelforge_api.services.selected_model_comparison import (
+    SelectedModelComparisonService,
+    _digest,
+)
+from modelforge_api.settings import Settings
+from tests.test_managed_identities_postgres import (
+    IdentityDatabase,
+    ManagedPostgresTools,
+    identity_database,
+)
+from tests.test_serving import approve_and_promote, setup_m5
+
+__all__ = ["identity_database"]
+PRIOR = "20260909_0032"
+CURRENT = "20260911_0035"
+
+
+def _service(session: Session) -> SelectedModelComparisonService:
+    return SelectedModelComparisonService(session, AuditContext("operator", "comparison-pg-fixture"))
+
+
+def _valid_selection(session: Session) -> tuple[ComparisonCreate, UUID]:
+    # These helpers persist explicit offline runtime evidence; they never contact a provider or GPU.
+    serving, candidate, _node = setup_m5(session)
+    _approval, stable = approve_and_promote(serving, candidate.id)
+    probe = serving.repo.probe(candidate.runtime_probe_id)
+    assert probe is not None
+    experiment = serving.create_experiment(candidate.id, CapabilityExperimentCreate(
+        route_key="selected-comparison-fixture", execution_approval_id=probe.execution_approval_id,
+        purpose="Isolated PostgreSQL graph qualification; synthetic evidence only",
+    ))
+    stable_row = session.get(CapabilityDeployment, stable.id)
+    assert stable_row is not None
+    project = Project(key="selected-comparison-fixture", name="Comparison qualification", description="Offline fixture")
+    session.add(project)
+    session.flush()
+    binding = ProjectBinding(project_id=project.id, capability_contract_id=stable_row.capability_contract_id,
+                             channel="stable", priority="production")
+    session.add(binding)
+    session.flush()
+    client = ServiceClient(name="selected-comparison-fixture", project_binding_id=binding.id,
+                           allowed_capabilities=["rag.embedding@1"])
+    session.add(client)
+    session.commit()
+    suite = EvaluationService(session).create_suite(EvaluationSuiteCreate(
+        project_id=project.id, key="selected-comparison-fixture", name="Reviewed fixture",
+        description="Synthetic owner-reviewed case; no consumer certification claim",
+        revision=EvaluationRevisionCreate(revision="v1", dataset_revision="fixture-v1",
+            retrieval_settings={"top_k": 10}, cases=[EvaluationCaseCreate(
+                case_key="fixture-question", query="Where is the fixture answer?", relevant_chunk_ids=[uuid4()],
+                label_provenance={"source": "owner_review", "corpus_revision": "a" * 64},
+                critical=True, review_status="reviewed",
+            )]),
+    ))
+    assert suite.latest_revision_id is not None
+    return ComparisonCreate(project_binding_id=binding.id, consumer_client_id=client.id,
+        suite_revision_id=suite.latest_revision_id, current_deployment_id=stable.id,
+        candidate_deployment_id=experiment.capability_deployment_id, candidate_route_id=experiment.id,
+        corpus_sha256="a" * 64, retrieval_config_sha256=_digest({"top_k": 10}),
+        current_index_ref="fixture-current-exact", candidate_index_ref="fixture-candidate-exact",
+        idempotency_key="fixture-one"), client.id
+
+
+def test_offline_selection_fixture_has_complete_foreign_keys() -> None:
+    engine = create_engine("sqlite://")
+    @event.listens_for(engine, "connect")
+    def foreign_keys(connection, _record):
+        connection.execute("PRAGMA foreign_keys=ON")
+    try:
+        Base.metadata.create_all(engine)
+        with Session(engine) as session:
+            assert session.scalar(text("PRAGMA foreign_keys")) == 1
+            request, client_id = _valid_selection(session)
+            created = _service(session).create(request)
+            client = session.get(ServiceClient, client_id)
+            assert client is not None
+            assert _service(session).claim(created.id, client).comparison.status == "running"
+            assert session.execute(text("PRAGMA foreign_key_check")).all() == []
+    finally:
+        engine.dispose()
+
+
+def test_0033_upgrade_prior_dump_restore_and_runtime_permissions(
+    identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+    database = identity_database
+    alembic = importlib.import_module("alembic.command")
+    alembic.upgrade(database.migration(), PRIOR)
+    with Session(database.runtime) as session:
+        assert not inspect(session.connection()).has_table("selected_model_comparisons")
+        AuditWriter(session, "operator", "prior-release").write("qualification.baseline", "comparison", None, {})
+        session.commit()
+        before = semantic_fingerprint(session.connection())
+    archive = tmp_path / "prior-comparisons.dump"
+    database.archive(archive)
+    alembic.upgrade(database.migration(), CURRENT)
+    with database.runtime.connect() as connection:
+        assert inspect(connection).has_table("selected_model_comparisons")
+        assert connection.scalar(text("SELECT count(*) FROM selected_model_comparisons")) == 0
+        after = semantic_fingerprint(connection)
+        contract = fingerprint_compatibility(before, after, source_schema_revision=PRIOR)
+        assert contract.compatible
+        assert audit_fingerprint_differences_are_compatible(before, after, contract)
+    alembic.downgrade(database.migration(), PRIOR)
+    alembic.upgrade(database.migration(), CURRENT)
+    with Session(database.runtime) as session:
+        request, _client_id = _valid_selection(session)
+        _service(session).create(request)
+        constraints = inspect(session.connection()).get_foreign_keys("selected_model_comparisons")
+        assert {item["constrained_columns"][0] for item in constraints} >= {
+            "project_binding_id", "consumer_client_id", "suite_revision_id", "current_deployment_id",
+            "candidate_deployment_id", "candidate_route_id",
+        }
+        assert session.scalar(text("SELECT count(*) FROM pg_constraint WHERE conrelid='selected_model_comparisons'::regclass AND contype='f' AND NOT convalidated")) == 0
+    with database.runtime.connect() as connection:
+        for statement in (
+            "DELETE FROM selected_model_comparisons",
+            "TRUNCATE selected_model_comparisons CASCADE",
+            "ALTER TABLE selected_model_comparisons ADD COLUMN forbidden integer",
+        ):
+            with pytest.raises(DBAPIError):
+                connection.execute(text(statement))
+            connection.rollback()
+    with pytest.raises(RuntimeError, match="history exists"):
+        alembic.downgrade(database.migration(), PRIOR)
+    database.archive(archive, restore=True)
+    alembic.upgrade(database.migration(restored=True), CURRENT)
+    with database.restored_runtime.connect() as connection:
+        restored = semantic_fingerprint(connection)
+        contract = fingerprint_compatibility(before, restored, source_schema_revision=PRIOR)
+        assert contract.compatible
+        assert audit_fingerprint_differences_are_compatible(before, restored, contract)
+
+
+def test_two_postgres_sessions_claim_exactly_once_with_one_audit_event(identity_database: IdentityDatabase) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), CURRENT)
+    with Session(database.runtime) as session:
+        request, client_id = _valid_selection(session)
+        comparison_id = _service(session).create(request).id
+    barrier = threading.Barrier(2)
+    def contender(_number: int) -> str:
+        with Session(database.runtime) as session:
+            client = session.get(ServiceClient, client_id)
+            assert client is not None
+            session.commit()
+            barrier.wait(timeout=30)
+            try:
+                claimed = _service(session).claim(comparison_id, client)
+                assert claimed.comparison.status == "running"
+                return "claimed"
+            except EvaluationError as error:
+                assert error.status_code == 409
+                return "conflict"
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        assert sorted(pool.map(contender, [0, 1])) == ["claimed", "conflict"]
+    with Session(database.runtime) as session:
+        row = session.get(SelectedModelComparison, comparison_id)
+        assert row is not None and row.status == "running" and row.version == 2
+        assert row.claim_token_hash and row.claim_expires_at
+        events = list(session.scalars(select(AuditEvent).where(AuditEvent.action == "SELECTED_MODEL_COMPARISON_CLAIMED")))
+        assert len(events) == 1
+
+
+def test_populated_postgres_restore_interrupts_claim_and_rejects_resurrection(
+    identity_database: IdentityDatabase, tmp_path: Path,
+) -> None:
+    database = identity_database
+    importlib.import_module("alembic.command").upgrade(database.migration(), CURRENT)
+    with Session(database.runtime) as session:
+        request, client_id = _valid_selection(session)
+        comparison_id = _service(session).create(request).id
+        client = session.get(ServiceClient, client_id)
+        assert client is not None
+        claimed = _service(session).claim(comparison_id, client)
+        source = semantic_fingerprint(session.connection())
+    archive = tmp_path / "active-comparison.dump"
+    database.archive(archive)
+    database.archive(archive, restore=True)
+    # The real restore migrates the destination before reconciliation. Keep the
+    # backup at the selected-comparison revision, then exercise recovery at head.
+    importlib.import_module("alembic.command").upgrade(
+        database.migration(restored=True), "head"
+    )
+    owner = database.restored_owner
+    with Session(database.runtime) as session:
+        recovery = RecoveryService(session, Settings(_env_file=None, env="test",
+            alembic_directory=Path(__file__).resolve().parents[1] / "alembic"))
+        recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
+        record = SimpleNamespace(id=uuid4(), source_fingerprint=source)
+        plan = SimpleNamespace(database_destination=owner.render_as_string(hide_password=False),
+            backup_set=SimpleNamespace(backup_id="selected-comparison-qualification", schema_revision=CURRENT,
+                                       completed_at=datetime.now(UTC)), validation_requirements={})
+        with pytest.raises(RecoveryError) as unready:
+            recovery._validate_restored(record, plan)
+        assert "gate" in unready.value.details, (unready.value.code, str(unready.value), unready.value.details)
+        assert unready.value.details["gate"]["current_truth_reconciled"] is False
+        session.rollback()
+        result = recovery._reconcile_restored(record, plan)
+        session.commit()
+        assert result["interrupted_selected_comparisons"] == 1
+        validated = recovery._validate_restored(record, plan)
+        assert all(validated["gate"].values())
+        session.commit()
+        assert recovery._reconcile_restored(record, plan)["reconciliation"] == "ALREADY_APPLIED"
+    with Session(database.restored_runtime) as session:
+        row = session.get(SelectedModelComparison, comparison_id)
+        assert row is not None and row.status == "interrupted"
+        assert row.version == claimed.comparison.version + 1
+        assert row.error_code == SELECTED_COMPARISON_RESTORE_ERROR
+        assert row.claim_token_hash is None and row.claim_expires_at is None
+        assert row.finished_at is None
+        restored = semantic_fingerprint(session.connection())
+        contract = fingerprint_compatibility(source, restored, source_schema_revision=CURRENT)
+        assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+        client = session.get(ServiceClient, client_id)
+        assert client is not None
+        with pytest.raises(EvaluationError):
+            _service(session).claim(comparison_id, client)