Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/tests/test_benchmark_run_recovery_compatibility.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+from pathlib import Path
+from uuid import uuid4
+
+from alembic.config import Config
+from sqlalchemy import create_engine, inspect, text
+
+from alembic import command
+from modelforge_api.persistence.models import Base
+from modelforge_api.services.recovery_fingerprint import (
+    BENCHMARK_RUN_SCHEMA,
+    audit_fingerprint_differences_are_compatible,
+    fingerprint_compatibility,
+    semantic_fingerprint,
+)
+from tests.recovery_fixtures import recreate_0038_benchmark_runs
+
+_PRIOR = "20260924_0038"
+
+
+def _migration_config(database_url: str) -> Config:
+    config = Config(str(Path(__file__).parents[1] / "alembic.ini"))
+    config.set_main_option("sqlalchemy.url", database_url)
+    return config
+
+
+def _create_0038_benchmark_table(engine) -> None:
+    with engine.begin() as connection:
+        recreate_0038_benchmark_runs(connection)
+        connection.exec_driver_sql(
+            "CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)"
+        )
+        connection.exec_driver_sql(
+            "INSERT INTO alembic_version (version_num) VALUES (?)", (_PRIOR,)
+        )
+
+
+def _insert_legacy_benchmark_run(engine) -> str:
+    run_id = uuid4().hex
+    with engine.begin() as connection:
+        connection.execute(
+            text(
+                """
+                INSERT INTO benchmark_runs (
+                    id, deployment_id, suite_id, model_revision_sha, artifact_sha256,
+                    runtime_type, runtime_version, runtime_image_digest, launch_arguments,
+                    suite_revision, dataset_revision, environment_fingerprint,
+                    environment_digest, seed, results, status, comparable, stale_reason,
+                    started_at, completed_at
+                ) VALUES (
+                    :id, :deployment_id, :suite_id, :model_revision_sha, :artifact_sha256,
+                    'native', '1', NULL, '{}', 'suite-r1', 'dataset-r1', '{}',
+                    :environment_digest, 7, '{"cases":[]}', 'completed', 1, NULL,
+                    '2026-09-24T00:00:00', '2026-09-24T00:00:01'
+                )
+                """
+            ),
+            {
+                "id": run_id,
+                "deployment_id": uuid4().hex,
+                "suite_id": uuid4().hex,
+                "model_revision_sha": "a" * 64,
+                "artifact_sha256": "b" * 64,
+                "environment_digest": "c" * 64,
+            },
+        )
+    return run_id
+
+
+def _fingerprint(engine):
+    with engine.connect() as connection:
+        return semantic_fingerprint(connection)
+
+
+def _migrated_database(tmp_path):
+    database_path = tmp_path / "benchmark-recovery.db"
+    database_url = f"sqlite:///{database_path.as_posix()}"
+    engine = create_engine(database_url)
+    Base.metadata.create_all(engine)
+    _create_0038_benchmark_table(engine)
+    run_id = _insert_legacy_benchmark_run(engine)
+
+    prior_columns = {column["name"] for column in inspect(engine).get_columns("benchmark_runs")}
+    assert "actor_subject" not in prior_columns
+    source = _fingerprint(engine)
+    assert set(source["tables"]["benchmark_runs"]) == {
+        "row_count",
+        "digest",
+        "status",
+        "redacted_columns",
+    }
+
+    command.upgrade(_migration_config(database_url), "head")
+    assert _fingerprint(engine)["tables"]["benchmark_runs"]["row_count"] == 1
+    return engine, source, run_id
+
+
+def test_0038_benchmark_history_fingerprints_and_restores_through_0039(tmp_path) -> None:
+    engine, source, _run_id = _migrated_database(tmp_path)
+    try:
+        restored = _fingerprint(engine)
+        contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
+
+        assert contract.compatible
+        assert contract.mode.startswith("BENCHMARK_RUNS_UPGRADE|")
+        assert audit_fingerprint_differences_are_compatible(source, restored, contract)
+    finally:
+        engine.dispose()
+
+
+def test_0039_upgrade_rejects_changed_legacy_benchmark_history(tmp_path) -> None:
+    engine, source, run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(
+                text("UPDATE benchmark_runs SET model_revision_sha=:sha WHERE id=:id"),
+                {"sha": "d" * 64, "id": run_id},
+            )
+        restored = _fingerprint(engine)
+
+        contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
+
+        assert not contract.compatible
+    finally:
+        engine.dispose()
+
+
+def test_0039_upgrade_rejects_operator_state_absent_from_0038_backup(tmp_path) -> None:
+    engine, source, run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(
+                text(
+                    "UPDATE benchmark_runs SET actor_subject='operator-a', "
+                    "idempotency_key='run-1', request_fingerprint=:fingerprint, "
+                    "capability_deployment_id=:deployment WHERE id=:id"
+                ),
+                {
+                    "fingerprint": "e" * 64,
+                    "deployment": uuid4().hex,
+                    "id": run_id,
+                },
+            )
+        restored = _fingerprint(engine)
+
+        contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
+
+        assert not contract.compatible
+    finally:
+        engine.dispose()
+
+
+def test_unversioned_current_database_keeps_operator_state_in_fingerprint(tmp_path) -> None:
+    engine, source, run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(
+                text(
+                    "UPDATE benchmark_runs SET actor_subject='operator-a', "
+                    "idempotency_key='run-1', request_fingerprint=:fingerprint, "
+                    "capability_deployment_id=:deployment WHERE id=:id"
+                ),
+                {
+                    "fingerprint": "e" * 64,
+                    "deployment": uuid4().hex,
+                    "id": run_id,
+                },
+            )
+            connection.execute(text("DROP TABLE alembic_version"))
+        restored = _fingerprint(engine)
+        projection = restored["tables"]["benchmark_runs"][
+            "benchmark_run_schema_projection"
+        ]
+
+        assert projection["non_legacy_row_count"] == 1
+        assert not fingerprint_compatibility(
+            source, restored, source_schema_revision=_PRIOR
+        ).compatible
+    finally:
+        engine.dispose()
+
+
+def test_unversioned_partial_benchmark_migration_fails_closed(tmp_path) -> None:
+    engine, source, _run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(text("DROP TABLE alembic_version"))
+            connection.execute(text("ALTER TABLE benchmark_runs DROP COLUMN generation"))
+        restored = _fingerprint(engine)
+
+        assert restored["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
+        assert not fingerprint_compatibility(
+            source, restored, source_schema_revision=_PRIOR
+        ).compatible
+    finally:
+        engine.dispose()
+
+
+def test_0038_revision_with_0039_columns_fails_closed(tmp_path) -> None:
+    engine, source, _run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(
+                text("UPDATE alembic_version SET version_num=:revision"),
+                {"revision": _PRIOR},
+            )
+        restored = _fingerprint(engine)
+
+        assert restored["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
+        assert not fingerprint_compatibility(
+            source, restored, source_schema_revision=_PRIOR
+        ).compatible
+    finally:
+        engine.dispose()
+
+
+def test_0039_revision_with_incomplete_benchmark_columns_fails_closed(tmp_path) -> None:
+    engine, _source, _run_id = _migrated_database(tmp_path)
+    try:
+        with engine.begin() as connection:
+            connection.execute(text("ALTER TABLE benchmark_runs DROP COLUMN generation"))
+        current = _fingerprint(engine)
+
+        assert current["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
+        assert not fingerprint_compatibility(
+            current, current, source_schema_revision=BENCHMARK_RUN_SCHEMA
+        ).compatible
+    finally:
+        engine.dispose()
+
+
+def test_0039_benchmark_history_is_exact_when_both_sides_are_current(tmp_path) -> None:
+    engine, _legacy_source, run_id = _migrated_database(tmp_path)
+    try:
+        source = _fingerprint(engine)
+        contract = fingerprint_compatibility(
+            source, source, source_schema_revision=BENCHMARK_RUN_SCHEMA
+        )
+        assert contract.compatible
+        assert contract.mode.startswith("BENCHMARK_RUNS_STRICT|")
+        assert audit_fingerprint_differences_are_compatible(source, source, contract)
+
+        with engine.begin() as connection:
+            connection.execute(
+                text("UPDATE benchmark_runs SET generation=2 WHERE id=:id"), {"id": run_id}
+            )
+        changed = _fingerprint(engine)
+        assert not fingerprint_compatibility(
+            source, changed, source_schema_revision=BENCHMARK_RUN_SCHEMA
+        ).compatible
+    finally:
+        engine.dispose()