"""Owner/runtime PostgreSQL qualification using a complete offline foreign-key graph.""" from __future__ import annotations import importlib import os import threading from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime from pathlib import Path from types import SimpleNamespace from uuid import UUID, uuid4 import pytest from sqlalchemy import create_engine, event, inspect, select, text from sqlalchemy.exc import DBAPIError from sqlalchemy.orm import Session from modelforge_api.domain.evaluation import ( EvaluationCaseCreate, EvaluationRevisionCreate, EvaluationSuiteCreate, ) from modelforge_api.domain.selected_model_comparison import ComparisonCreate from modelforge_api.domain.serving import CapabilityExperimentCreate from modelforge_api.persistence.models import ( AuditEvent, Base, CapabilityDeployment, Project, ProjectBinding, SelectedModelComparison, ServiceClient, ) from modelforge_api.services.audit import AuditContext, AuditWriter from modelforge_api.services.evaluation import EvaluationError, EvaluationService from modelforge_api.services.recovery import RecoveryError, RecoveryService from modelforge_api.services.recovery_fingerprint import ( SELECTED_COMPARISON_RESTORE_ERROR, audit_fingerprint_differences_are_compatible, fingerprint_compatibility, semantic_fingerprint, ) from modelforge_api.services.selected_model_comparison import ( SelectedModelComparisonService, _digest, ) from modelforge_api.settings import Settings from tests.test_managed_identities_postgres import ( IdentityDatabase, ManagedPostgresTools, identity_database, ) from tests.test_serving import approve_and_promote, setup_m5 __all__ = ["identity_database"] PRIOR = "20260909_0032" CURRENT = "20260911_0035" def _service(session: Session) -> SelectedModelComparisonService: return SelectedModelComparisonService(session, AuditContext("operator", "comparison-pg-fixture")) def _valid_selection(session: Session) -> tuple[ComparisonCreate, UUID]: # These helpers persist explicit offline runtime evidence; they never contact a provider or GPU. serving, candidate, _node = setup_m5(session) _approval, stable = approve_and_promote(serving, candidate.id) probe = serving.repo.probe(candidate.runtime_probe_id) assert probe is not None experiment = serving.create_experiment(candidate.id, CapabilityExperimentCreate( route_key="selected-comparison-fixture", execution_approval_id=probe.execution_approval_id, purpose="Isolated PostgreSQL graph qualification; synthetic evidence only", )) stable_row = session.get(CapabilityDeployment, stable.id) assert stable_row is not None project = Project(key="selected-comparison-fixture", name="Comparison qualification", description="Offline fixture") session.add(project) session.flush() binding = ProjectBinding(project_id=project.id, capability_contract_id=stable_row.capability_contract_id, channel="stable", priority="production") session.add(binding) session.flush() client = ServiceClient(name="selected-comparison-fixture", project_binding_id=binding.id, allowed_capabilities=["rag.embedding@1"]) session.add(client) session.commit() suite = EvaluationService(session).create_suite(EvaluationSuiteCreate( project_id=project.id, key="selected-comparison-fixture", name="Reviewed fixture", description="Synthetic owner-reviewed case; no consumer certification claim", revision=EvaluationRevisionCreate(revision="v1", dataset_revision="fixture-v1", retrieval_settings={"top_k": 10}, cases=[EvaluationCaseCreate( case_key="fixture-question", query="Where is the fixture answer?", relevant_chunk_ids=[uuid4()], label_provenance={"source": "owner_review", "corpus_revision": "a" * 64}, critical=True, review_status="reviewed", )]), )) assert suite.latest_revision_id is not None return ComparisonCreate(project_binding_id=binding.id, consumer_client_id=client.id, suite_revision_id=suite.latest_revision_id, current_deployment_id=stable.id, candidate_deployment_id=experiment.capability_deployment_id, candidate_route_id=experiment.id, corpus_sha256="a" * 64, retrieval_config_sha256=_digest({"top_k": 10}), current_index_ref="fixture-current-exact", candidate_index_ref="fixture-candidate-exact", idempotency_key="fixture-one"), client.id def test_offline_selection_fixture_has_complete_foreign_keys() -> None: engine = create_engine("sqlite://") @event.listens_for(engine, "connect") def foreign_keys(connection, _record): connection.execute("PRAGMA foreign_keys=ON") try: Base.metadata.create_all(engine) with Session(engine) as session: assert session.scalar(text("PRAGMA foreign_keys")) == 1 request, client_id = _valid_selection(session) created = _service(session).create(request) client = session.get(ServiceClient, client_id) assert client is not None assert _service(session).claim(created.id, client).comparison.status == "running" assert session.execute(text("PRAGMA foreign_key_check")).all() == [] finally: engine.dispose() def test_0033_upgrade_prior_dump_restore_and_runtime_permissions( identity_database: IdentityDatabase, tmp_path: Path, ) -> None: database = identity_database alembic = importlib.import_module("alembic.command") alembic.upgrade(database.migration(), PRIOR) with Session(database.runtime) as session: assert not inspect(session.connection()).has_table("selected_model_comparisons") AuditWriter(session, "operator", "prior-release").write("qualification.baseline", "comparison", None, {}) session.commit() before = semantic_fingerprint(session.connection()) archive = tmp_path / "prior-comparisons.dump" database.archive(archive) alembic.upgrade(database.migration(), CURRENT) with database.runtime.connect() as connection: assert inspect(connection).has_table("selected_model_comparisons") assert connection.scalar(text("SELECT count(*) FROM selected_model_comparisons")) == 0 after = semantic_fingerprint(connection) contract = fingerprint_compatibility(before, after, source_schema_revision=PRIOR) assert contract.compatible assert audit_fingerprint_differences_are_compatible(before, after, contract) alembic.downgrade(database.migration(), PRIOR) alembic.upgrade(database.migration(), CURRENT) with Session(database.runtime) as session: request, _client_id = _valid_selection(session) _service(session).create(request) constraints = inspect(session.connection()).get_foreign_keys("selected_model_comparisons") assert {item["constrained_columns"][0] for item in constraints} >= { "project_binding_id", "consumer_client_id", "suite_revision_id", "current_deployment_id", "candidate_deployment_id", "candidate_route_id", } assert session.scalar(text("SELECT count(*) FROM pg_constraint WHERE conrelid='selected_model_comparisons'::regclass AND contype='f' AND NOT convalidated")) == 0 with database.runtime.connect() as connection: for statement in ( "DELETE FROM selected_model_comparisons", "TRUNCATE selected_model_comparisons CASCADE", "ALTER TABLE selected_model_comparisons ADD COLUMN forbidden integer", ): with pytest.raises(DBAPIError): connection.execute(text(statement)) connection.rollback() with pytest.raises(RuntimeError, match="history exists"): alembic.downgrade(database.migration(), PRIOR) database.archive(archive, restore=True) alembic.upgrade(database.migration(restored=True), CURRENT) with database.restored_runtime.connect() as connection: restored = semantic_fingerprint(connection) contract = fingerprint_compatibility(before, restored, source_schema_revision=PRIOR) assert contract.compatible assert audit_fingerprint_differences_are_compatible(before, restored, contract) def test_two_postgres_sessions_claim_exactly_once_with_one_audit_event(identity_database: IdentityDatabase) -> None: database = identity_database importlib.import_module("alembic.command").upgrade(database.migration(), CURRENT) with Session(database.runtime) as session: request, client_id = _valid_selection(session) comparison_id = _service(session).create(request).id barrier = threading.Barrier(2) def contender(_number: int) -> str: with Session(database.runtime) as session: client = session.get(ServiceClient, client_id) assert client is not None session.commit() barrier.wait(timeout=30) try: claimed = _service(session).claim(comparison_id, client) assert claimed.comparison.status == "running" return "claimed" except EvaluationError as error: assert error.status_code == 409 return "conflict" with ThreadPoolExecutor(max_workers=2) as pool: assert sorted(pool.map(contender, [0, 1])) == ["claimed", "conflict"] with Session(database.runtime) as session: row = session.get(SelectedModelComparison, comparison_id) assert row is not None and row.status == "running" and row.version == 2 assert row.claim_token_hash and row.claim_expires_at events = list(session.scalars(select(AuditEvent).where(AuditEvent.action == "SELECTED_MODEL_COMPARISON_CLAIMED"))) assert len(events) == 1 def test_populated_postgres_restore_interrupts_claim_and_rejects_resurrection( identity_database: IdentityDatabase, tmp_path: Path, ) -> None: database = identity_database importlib.import_module("alembic.command").upgrade(database.migration(), CURRENT) with Session(database.runtime) as session: request, client_id = _valid_selection(session) comparison_id = _service(session).create(request).id client = session.get(ServiceClient, client_id) assert client is not None claimed = _service(session).claim(comparison_id, client) source = semantic_fingerprint(session.connection()) archive = tmp_path / "active-comparison.dump" database.archive(archive) database.archive(archive, restore=True) # The real restore migrates the destination before reconciliation. Keep the # backup at the selected-comparison revision, then exercise recovery at head. importlib.import_module("alembic.command").upgrade( database.migration(restored=True), "head" ) owner = database.restored_owner with Session(database.runtime) as session: recovery = RecoveryService(session, Settings(_env_file=None, env="test", alembic_directory=Path(__file__).resolve().parents[1] / "alembic")) recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60) record = SimpleNamespace(id=uuid4(), source_fingerprint=source) plan = SimpleNamespace(database_destination=owner.render_as_string(hide_password=False), backup_set=SimpleNamespace(backup_id="selected-comparison-qualification", schema_revision=CURRENT, completed_at=datetime.now(UTC)), validation_requirements={}) with pytest.raises(RecoveryError) as unready: recovery._validate_restored(record, plan) assert "gate" in unready.value.details, (unready.value.code, str(unready.value), unready.value.details) assert unready.value.details["gate"]["current_truth_reconciled"] is False session.rollback() result = recovery._reconcile_restored(record, plan) session.commit() assert result["interrupted_selected_comparisons"] == 1 validated = recovery._validate_restored(record, plan) assert all(validated["gate"].values()) session.commit() assert recovery._reconcile_restored(record, plan)["reconciliation"] == "ALREADY_APPLIED" with Session(database.restored_runtime) as session: row = session.get(SelectedModelComparison, comparison_id) assert row is not None and row.status == "interrupted" assert row.version == claimed.comparison.version + 1 assert row.error_code == SELECTED_COMPARISON_RESTORE_ERROR assert row.claim_token_hash is None and row.claim_expires_at is None assert row.finished_at is None restored = semantic_fingerprint(session.connection()) contract = fingerprint_compatibility(source, restored, source_schema_revision=CURRENT) assert audit_fingerprint_differences_are_compatible(source, restored, contract) client = session.get(ServiceClient, client_id) assert client is not None with pytest.raises(EvaluationError): _service(session).claim(comparison_id, client)