254 lines
9.1 KiB
Python
254 lines
9.1 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
from alembic.config import Config
|
|
from sqlalchemy import create_engine, inspect, text
|
|
|
|
from alembic import command
|
|
from modelforge_api.persistence.models import Base
|
|
from modelforge_api.services.recovery_fingerprint import (
|
|
BENCHMARK_RUN_SCHEMA,
|
|
audit_fingerprint_differences_are_compatible,
|
|
fingerprint_compatibility,
|
|
semantic_fingerprint,
|
|
)
|
|
from tests.recovery_fixtures import recreate_0038_benchmark_runs
|
|
|
|
_PRIOR = "20260924_0038"
|
|
|
|
|
|
def _migration_config(database_url: str) -> Config:
|
|
config = Config(str(Path(__file__).parents[1] / "alembic.ini"))
|
|
config.set_main_option("sqlalchemy.url", database_url)
|
|
return config
|
|
|
|
|
|
def _create_0038_benchmark_table(engine) -> None:
|
|
with engine.begin() as connection:
|
|
recreate_0038_benchmark_runs(connection)
|
|
connection.exec_driver_sql(
|
|
"CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL)"
|
|
)
|
|
connection.exec_driver_sql(
|
|
"INSERT INTO alembic_version (version_num) VALUES (?)", (_PRIOR,)
|
|
)
|
|
|
|
|
|
def _insert_legacy_benchmark_run(engine) -> str:
|
|
run_id = uuid4().hex
|
|
with engine.begin() as connection:
|
|
connection.execute(
|
|
text(
|
|
"""
|
|
INSERT INTO benchmark_runs (
|
|
id, deployment_id, suite_id, model_revision_sha, artifact_sha256,
|
|
runtime_type, runtime_version, runtime_image_digest, launch_arguments,
|
|
suite_revision, dataset_revision, environment_fingerprint,
|
|
environment_digest, seed, results, status, comparable, stale_reason,
|
|
started_at, completed_at
|
|
) VALUES (
|
|
:id, :deployment_id, :suite_id, :model_revision_sha, :artifact_sha256,
|
|
'native', '1', NULL, '{}', 'suite-r1', 'dataset-r1', '{}',
|
|
:environment_digest, 7, '{"cases":[]}', 'completed', 1, NULL,
|
|
'2026-09-24T00:00:00', '2026-09-24T00:00:01'
|
|
)
|
|
"""
|
|
),
|
|
{
|
|
"id": run_id,
|
|
"deployment_id": uuid4().hex,
|
|
"suite_id": uuid4().hex,
|
|
"model_revision_sha": "a" * 64,
|
|
"artifact_sha256": "b" * 64,
|
|
"environment_digest": "c" * 64,
|
|
},
|
|
)
|
|
return run_id
|
|
|
|
|
|
def _fingerprint(engine):
|
|
with engine.connect() as connection:
|
|
return semantic_fingerprint(connection)
|
|
|
|
|
|
def _migrated_database(tmp_path):
|
|
database_path = tmp_path / "benchmark-recovery.db"
|
|
database_url = f"sqlite:///{database_path.as_posix()}"
|
|
engine = create_engine(database_url)
|
|
Base.metadata.create_all(engine)
|
|
_create_0038_benchmark_table(engine)
|
|
run_id = _insert_legacy_benchmark_run(engine)
|
|
|
|
prior_columns = {column["name"] for column in inspect(engine).get_columns("benchmark_runs")}
|
|
assert "actor_subject" not in prior_columns
|
|
source = _fingerprint(engine)
|
|
assert set(source["tables"]["benchmark_runs"]) == {
|
|
"row_count",
|
|
"digest",
|
|
"status",
|
|
"redacted_columns",
|
|
}
|
|
|
|
command.upgrade(_migration_config(database_url), "head")
|
|
assert _fingerprint(engine)["tables"]["benchmark_runs"]["row_count"] == 1
|
|
return engine, source, run_id
|
|
|
|
|
|
def test_0038_benchmark_history_fingerprints_and_restores_through_0039(tmp_path) -> None:
|
|
engine, source, _run_id = _migrated_database(tmp_path)
|
|
try:
|
|
restored = _fingerprint(engine)
|
|
contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
|
|
|
|
assert contract.compatible
|
|
assert contract.mode.startswith("BENCHMARK_RUNS_UPGRADE|")
|
|
assert audit_fingerprint_differences_are_compatible(source, restored, contract)
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
def test_0039_upgrade_rejects_changed_legacy_benchmark_history(tmp_path) -> None:
|
|
engine, source, run_id = _migrated_database(tmp_path)
|
|
try:
|
|
with engine.begin() as connection:
|
|
connection.execute(
|
|
text("UPDATE benchmark_runs SET model_revision_sha=:sha WHERE id=:id"),
|
|
{"sha": "d" * 64, "id": run_id},
|
|
)
|
|
restored = _fingerprint(engine)
|
|
|
|
contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
|
|
|
|
assert not contract.compatible
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
def test_0039_upgrade_rejects_operator_state_absent_from_0038_backup(tmp_path) -> None:
|
|
engine, source, run_id = _migrated_database(tmp_path)
|
|
try:
|
|
with engine.begin() as connection:
|
|
connection.execute(
|
|
text(
|
|
"UPDATE benchmark_runs SET actor_subject='operator-a', "
|
|
"idempotency_key='run-1', request_fingerprint=:fingerprint, "
|
|
"capability_deployment_id=:deployment WHERE id=:id"
|
|
),
|
|
{
|
|
"fingerprint": "e" * 64,
|
|
"deployment": uuid4().hex,
|
|
"id": run_id,
|
|
},
|
|
)
|
|
restored = _fingerprint(engine)
|
|
|
|
contract = fingerprint_compatibility(source, restored, source_schema_revision=_PRIOR)
|
|
|
|
assert not contract.compatible
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
def test_unversioned_current_database_keeps_operator_state_in_fingerprint(tmp_path) -> None:
|
|
engine, source, run_id = _migrated_database(tmp_path)
|
|
try:
|
|
with engine.begin() as connection:
|
|
connection.execute(
|
|
text(
|
|
"UPDATE benchmark_runs SET actor_subject='operator-a', "
|
|
"idempotency_key='run-1', request_fingerprint=:fingerprint, "
|
|
"capability_deployment_id=:deployment WHERE id=:id"
|
|
),
|
|
{
|
|
"fingerprint": "e" * 64,
|
|
"deployment": uuid4().hex,
|
|
"id": run_id,
|
|
},
|
|
)
|
|
connection.execute(text("DROP TABLE alembic_version"))
|
|
restored = _fingerprint(engine)
|
|
projection = restored["tables"]["benchmark_runs"][
|
|
"benchmark_run_schema_projection"
|
|
]
|
|
|
|
assert projection["non_legacy_row_count"] == 1
|
|
assert not fingerprint_compatibility(
|
|
source, restored, source_schema_revision=_PRIOR
|
|
).compatible
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
def test_unversioned_partial_benchmark_migration_fails_closed(tmp_path) -> None:
|
|
engine, source, _run_id = _migrated_database(tmp_path)
|
|
try:
|
|
with engine.begin() as connection:
|
|
connection.execute(text("DROP TABLE alembic_version"))
|
|
connection.execute(text("ALTER TABLE benchmark_runs DROP COLUMN generation"))
|
|
restored = _fingerprint(engine)
|
|
|
|
assert restored["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
|
|
assert not fingerprint_compatibility(
|
|
source, restored, source_schema_revision=_PRIOR
|
|
).compatible
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
def test_0038_revision_with_0039_columns_fails_closed(tmp_path) -> None:
|
|
engine, source, _run_id = _migrated_database(tmp_path)
|
|
try:
|
|
with engine.begin() as connection:
|
|
connection.execute(
|
|
text("UPDATE alembic_version SET version_num=:revision"),
|
|
{"revision": _PRIOR},
|
|
)
|
|
restored = _fingerprint(engine)
|
|
|
|
assert restored["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
|
|
assert not fingerprint_compatibility(
|
|
source, restored, source_schema_revision=_PRIOR
|
|
).compatible
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
def test_0039_revision_with_incomplete_benchmark_columns_fails_closed(tmp_path) -> None:
|
|
engine, _source, _run_id = _migrated_database(tmp_path)
|
|
try:
|
|
with engine.begin() as connection:
|
|
connection.execute(text("ALTER TABLE benchmark_runs DROP COLUMN generation"))
|
|
current = _fingerprint(engine)
|
|
|
|
assert current["tables"]["benchmark_runs"]["benchmark_runs_schema_observed"] is True
|
|
assert not fingerprint_compatibility(
|
|
current, current, source_schema_revision=BENCHMARK_RUN_SCHEMA
|
|
).compatible
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
def test_0039_benchmark_history_is_exact_when_both_sides_are_current(tmp_path) -> None:
|
|
engine, _legacy_source, run_id = _migrated_database(tmp_path)
|
|
try:
|
|
source = _fingerprint(engine)
|
|
contract = fingerprint_compatibility(
|
|
source, source, source_schema_revision=BENCHMARK_RUN_SCHEMA
|
|
)
|
|
assert contract.compatible
|
|
assert contract.mode.startswith("BENCHMARK_RUNS_STRICT|")
|
|
assert audit_fingerprint_differences_are_compatible(source, source, contract)
|
|
|
|
with engine.begin() as connection:
|
|
connection.execute(
|
|
text("UPDATE benchmark_runs SET generation=2 WHERE id=:id"), {"id": run_id}
|
|
)
|
|
changed = _fingerprint(engine)
|
|
assert not fingerprint_compatibility(
|
|
source, changed, source_schema_revision=BENCHMARK_RUN_SCHEMA
|
|
).compatible
|
|
finally:
|
|
engine.dispose()
|