from __future__ import annotations from pathlib import Path from uuid import uuid4 from alembic.config import Config from sqlalchemy import create_engine, inspect, text from alembic import command from modelforge_api.persistence.models import Base from modelforge_api.services.recovery_fingerprint import ( BENCHMARK_RUN_SCHEMA, audit_fingerprint_differences_are_compatible, fingerprint_compatibility, semantic_fingerprint, ) from tests.recovery_fixtures import recreate_0038_benchmark_runs _PRIOR = "20260924_0038" def _migration_config(database_url: str) -> Config: config = Config(str(Path(__file__).parents[1] / "alembic.ini")) config.set_main_option("sqlalchemy.url", database_url) return config def _create_0038_benchmark_table(engine) -> None: with engine.begin() as connection: recreate_0038_benchmark_runs(connection) connection.exec_driver_sql( "CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)" ) connection.exec_driver_sql( "INSERT INTO alembic_version (version_num) VALUES (?)", (_PRIOR,) ) def _insert_legacy_benchmark_run(engine) -> str: run_id = uuid4().hex with engine.begin() as connection: connection.execute( text( """ INSERT INTO benchmark_runs ( id, deployment_id, suite_id, model_revision_sha, artifact_sha256, runtime_type, runtime_version, runtime_image_digest, launch_arguments, suite_revision, dataset_revision, environment_fingerprint, environment_digest, seed, results, status, comparable, stale_reason, started_at, completed_at ) VALUES ( :id, :deployment_id, :suite_id, :model_revision_sha, :artifact_sha256, 'native', '1', NULL, '{}', 'suite-r1', 'dataset-r1', '{}', :environment_digest, 7, '{"cases":[]}', 'completed', 1, NULL, '2026-09-24T00:00:00', '2026-09-24T00:00:01' ) """ ), { "id": run_id, "deployment_id": uuid4().hex, "suite_id": uuid4().hex, "model_revision_sha": "a" * 64, "artifact_sha256": "b" * 64, "environment_digest": "c" * 64, }, ) return run_id def _fingerprint(engine): with engine.connect() as connection: return semantic_fingerprint(connection) def _migrated_database(tmp_path): database_path = tmp_path / "benchmark-recovery.db" database_url = f"sqlite:///{database_path.as_posix()}" engine = create_engine(database_url) Base.metadata.create_all(engine) _create_0038_benchmark_table(engine) run_id = _insert_legacy_benchmark_run(engine) prior_columns = {column["name"] for column in inspect(engine).get_columns("benchmark_runs")} assert "actor_subject" not in prior_columns source = _fingerprint(engine) assert set(source["tables"]["benchmark_runs"]) == { "row_count", "digest", "status", "redacted_columns", } command.upgrade(_migration_config(database_url), "head") assert _fingerprint(engine)["tables"]["benchmark_runs"]["row_count"] == 1 return engine, source, run_id def test_0038_benchmark_history_fingerprints_and_restores_through_0039(tmp_path) -> None: engine, source, _run_id = _migrated_database(tmp_path) try: restored = _fingerprint(engine) contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR) assert contract.compatible assert contract.mode.startswith("BENCHMARK_RUNS_UPGRADE|") assert audit_fingerprint_differences_are_compatible(source, restored, contract) finally: engine.dispose() def test_0039_upgrade_rejects_changed_legacy_benchmark_history(tmp_path) -> None: engine, source, run_id = _migrated_database(tmp_path) try: with engine.begin() as connection: connection.execute( text("UPDATE benchmark_runs SET model_revision_sha=:sha WHERE id=:id"), {"sha": "d" * 64, "id": run_id}, ) restored = _fingerprint(engine) contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR) assert not contract.compatible finally: engine.dispose() def test_0039_upgrade_rejects_operator_state_absent_from_0038_backup(tmp_path) -> None: engine, source, run_id = _migrated_database(tmp_path) try: with engine.begin() as connection: connection.execute( text( "UPDATE benchmark_runs SET actor_subject='operator-a', " "idempotency_key='run-1', request_fingerprint=:fingerprint, " "capability_deployment_id=:deployment WHERE id=:id" ), { "fingerprint": "e" * 64, "deployment": uuid4().hex, "id": run_id, }, ) restored = _fingerprint(engine) contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR) assert not contract.compatible finally: engine.dispose() def test_unversioned_current_database_keeps_operator_state_in_fingerprint(tmp_path) -> None: engine, source, run_id = _migrated_database(tmp_path) try: with engine.begin() as connection: connection.execute( text( "UPDATE benchmark_runs SET actor_subject='operator-a', " "idempotency_key='run-1', request_fingerprint=:fingerprint, " "capability_deployment_id=:deployment WHERE id=:id" ), { "fingerprint": "e" * 64, "deployment": uuid4().hex, "id": run_id, }, ) connection.execute(text("DROP TABLE alembic_version")) restored = _fingerprint(engine) projection = restored["tables"]["benchmark_runs"][ "benchmark_run_schema_projection" ] assert projection["non_legacy_row_count"] == 1 assert not fingerprint_compatibility( source, restored, source_schema_revision=_PRIOR ).compatible finally: engine.dispose() def test_unversioned_partial_benchmark_migration_fails_closed(tmp_path) -> None: engine, source, _run_id = _migrated_database(tmp_path) try: with engine.begin() as connection: connection.execute(text("DROP TABLE alembic_version")) connection.execute(text("ALTER TABLE benchmark_runs DROP COLUMN generation")) restored = _fingerprint(engine) assert restored["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True assert not fingerprint_compatibility( source, restored, source_schema_revision=_PRIOR ).compatible finally: engine.dispose() def test_0038_revision_with_0039_columns_fails_closed(tmp_path) -> None: engine, source, _run_id = _migrated_database(tmp_path) try: with engine.begin() as connection: connection.execute( text("UPDATE alembic_version SET version_num=:revision"), {"revision": _PRIOR}, ) restored = _fingerprint(engine) assert restored["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True assert not fingerprint_compatibility( source, restored, source_schema_revision=_PRIOR ).compatible finally: engine.dispose() def test_0039_revision_with_incomplete_benchmark_columns_fails_closed(tmp_path) -> None: engine, _source, _run_id = _migrated_database(tmp_path) try: with engine.begin() as connection: connection.execute(text("ALTER TABLE benchmark_runs DROP COLUMN generation")) current = _fingerprint(engine) assert current["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True assert not fingerprint_compatibility( current, current, source_schema_revision=BENCHMARK_RUN_SCHEMA ).compatible finally: engine.dispose() def test_0039_benchmark_history_is_exact_when_both_sides_are_current(tmp_path) -> None: engine, _legacy_source, run_id = _migrated_database(tmp_path) try: source = _fingerprint(engine) contract = fingerprint_compatibility( source, source, source_schema_revision=BENCHMARK_RUN_SCHEMA ) assert contract.compatible assert contract.mode.startswith("BENCHMARK_RUNS_STRICT|") assert audit_fingerprint_differences_are_compatible(source, source, contract) with engine.begin() as connection: connection.execute( text("UPDATE benchmark_runs SET generation=2 WHERE id=:id"), {"id": run_id} ) changed = _fingerprint(engine) assert not fingerprint_compatibility( source, changed, source_schema_revision=BENCHMARK_RUN_SCHEMA ).compatible finally: engine.dispose()