171 lines
8.5 KiB
Python
171 lines
8.5 KiB
Python
"""Comparison history remains exact while restored claims become unusable."""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
from datetime import UTC, datetime, timedelta
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modelforge_api.persistence.models import Base, SelectedModelComparison
|
|
from modelforge_api.services.audit import AuditWriter
|
|
from modelforge_api.services.recovery import RecoveryService
|
|
from modelforge_api.services.recovery_fingerprint import (
|
|
SELECTED_COMPARISON_RESTORE_ERROR,
|
|
SELECTED_COMPARISON_SCHEMA,
|
|
_without_lab_execution,
|
|
_without_model_changes,
|
|
_without_portfolio,
|
|
_without_selected_comparisons,
|
|
audit_fingerprint_differences_are_compatible,
|
|
fingerprint_compatibility,
|
|
)
|
|
from modelforge_api.services.recovery_fingerprint import (
|
|
semantic_fingerprint as _current_semantic_fingerprint,
|
|
)
|
|
from modelforge_api.services.recovery_postgres import PostgresTarget
|
|
from modelforge_api.settings import Settings
|
|
|
|
TARGET = PostgresTarget("localhost", 5432, "fixture", "fixture", None)
|
|
|
|
|
|
def semantic_fingerprint(connection):
|
|
return _without_lab_execution(
|
|
_without_portfolio(_without_model_changes(_current_semantic_fingerprint(connection)))
|
|
)
|
|
|
|
|
|
class SQLiteRecoveryTarget:
|
|
"""Execute the actual bounded reconciliation SQL; emulate only PostgreSQL table discovery."""
|
|
def __init__(self, session: Session):
|
|
self.session = session
|
|
|
|
def scalar(self, _target: PostgresTarget, sql: str) -> str:
|
|
if "information_schema.tables" in sql:
|
|
return "1"
|
|
result = self.session.execute(text(sql))
|
|
return str(result.scalar() or 0) if result.returns_rows else ""
|
|
|
|
|
|
@pytest.fixture
|
|
def fixture():
|
|
engine = create_engine("sqlite://")
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
AuditWriter(session, "operator", "test").write("fixture.created", "test", None, {})
|
|
session.commit()
|
|
service = RecoveryService(session, Settings(_env_file=None))
|
|
service.engine = SQLiteRecoveryTarget(session) # type: ignore[assignment]
|
|
yield session, service
|
|
engine.dispose()
|
|
|
|
|
|
def comparison(session: Session, status: str) -> SelectedModelComparison:
|
|
moment = datetime(2026, 9, 9, 12, tzinfo=UTC)
|
|
item = SelectedModelComparison(
|
|
project_binding_id=uuid4(), consumer_client_id=uuid4(), suite_revision_id=uuid4(),
|
|
current_deployment_id=uuid4(), candidate_deployment_id=uuid4(), candidate_route_id=uuid4(),
|
|
idempotency_key=str(uuid4()), request_digest="a" * 64, actor_id="fixture-operator",
|
|
pins={"baseline": "a" * 40, "candidate": "b" * 40, "case_ids": ["one", "two"]},
|
|
status=status, version=4, expected_cases=2, current_completed_cases=1,
|
|
candidate_completed_cases=0, claim_token_hash="c" * 64,
|
|
claim_expires_at=moment + timedelta(minutes=5), claimed_at=moment,
|
|
deadline_at=moment + timedelta(minutes=10), cancel_requested_at=moment if status == "cancel_requested" else None,
|
|
finished_at=moment if status in {"completed", "failed", "cancelled", "interrupted"} else None,
|
|
baseline_run_id=uuid4(), candidate_run_id=uuid4(), evaluation_comparison_id=uuid4(),
|
|
result_digest="d" * 64, error_code="prior-evidence", created_at=moment, updated_at=moment,
|
|
)
|
|
session.add(item)
|
|
session.commit()
|
|
return item
|
|
|
|
|
|
@pytest.mark.parametrize("status", ["pending", "running", "cancel_requested", "completed", "failed", "cancelled", "interrupted"])
|
|
def test_restore_projects_only_active_state_and_never_replays_claims(fixture, status):
|
|
session, service = fixture
|
|
item = comparison(session, status)
|
|
before = semantic_fingerprint(session.connection())
|
|
previous = {name: getattr(item, name) for name in (
|
|
"pins", "current_completed_cases", "candidate_completed_cases", "baseline_run_id",
|
|
"candidate_run_id", "evaluation_comparison_id", "result_digest", "created_at", "updated_at",
|
|
"claimed_at", "finished_at", "deadline_at", "cancel_requested_at",
|
|
)}
|
|
active = status in {"pending", "running", "cancel_requested"}
|
|
assert service._selected_comparison_residuals(TARGET) == 1
|
|
assert service._reconcile_selected_comparisons(TARGET) == {"interrupted": int(active), "claims_cleared": 1}
|
|
session.expire_all()
|
|
assert item.status == ("interrupted" if active else status)
|
|
assert item.version == (5 if active else 4)
|
|
assert item.error_code == (SELECTED_COMPARISON_RESTORE_ERROR if active else "prior-evidence")
|
|
assert item.claim_token_hash is None and item.claim_expires_at is None
|
|
assert {name: getattr(item, name) for name in previous} == previous
|
|
restored = semantic_fingerprint(session.connection())
|
|
contract = fingerprint_compatibility(before, restored, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
|
|
assert contract.compatible
|
|
assert audit_fingerprint_differences_are_compatible(before, restored, contract)
|
|
assert service._selected_comparison_residuals(TARGET) == 0
|
|
assert service._reconcile_selected_comparisons(TARGET) == {"interrupted": 0, "claims_cleared": 0}
|
|
assert semantic_fingerprint(session.connection()) == restored
|
|
|
|
|
|
def test_active_rows_cannot_pass_restore_proof_without_reconciliation(fixture):
|
|
session, _ = fixture
|
|
comparison(session, "running")
|
|
before = semantic_fingerprint(session.connection())
|
|
contract = fingerprint_compatibility(before, before, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
|
|
assert contract.compatible
|
|
assert not audit_fingerprint_differences_are_compatible(before, before, contract)
|
|
|
|
|
|
def test_claim_secrets_and_expiry_are_excluded_but_claim_history_is_not(fixture):
|
|
session, _ = fixture
|
|
item = comparison(session, "running")
|
|
before = semantic_fingerprint(session.connection())
|
|
entry = before["tables"]["selected_model_comparisons"]
|
|
assert entry["redacted_columns"] == ["claim_token_hash"]
|
|
session.execute(text("update selected_model_comparisons set claim_token_hash=:token,claim_expires_at=NULL"), {"token": "e" * 64})
|
|
assert semantic_fingerprint(session.connection()) == before
|
|
session.execute(text("update selected_model_comparisons set claimed_at=NULL"))
|
|
assert semantic_fingerprint(session.connection())["tables"]["selected_model_comparisons"]["digest"] != entry["digest"]
|
|
assert item.id is not None
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["pins", "result", "delete", "terminal_status", "progress"])
|
|
def test_restore_rejects_changed_pins_results_missing_history_or_progress(fixture, mutation):
|
|
session, service = fixture
|
|
comparison(session, "completed")
|
|
before = semantic_fingerprint(session.connection())
|
|
service._reconcile_selected_comparisons(TARGET)
|
|
queries = {
|
|
"pins": "update selected_model_comparisons set pins='{}'",
|
|
"result": "update selected_model_comparisons set result_digest=NULL",
|
|
"delete": "delete from selected_model_comparisons",
|
|
"terminal_status": "update selected_model_comparisons set status='failed'",
|
|
"progress": "update selected_model_comparisons set current_completed_cases=0",
|
|
}
|
|
session.execute(text(queries[mutation]))
|
|
after = semantic_fingerprint(session.connection())
|
|
contract = fingerprint_compatibility(before, after, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
|
|
assert not audit_fingerprint_differences_are_compatible(before, after, contract)
|
|
|
|
|
|
def test_0032_upgrade_allows_only_empty_comparison_table_and_requires_current_history(fixture):
|
|
session, _ = fixture
|
|
empty = semantic_fingerprint(session.connection())
|
|
prior = _without_selected_comparisons(empty)
|
|
contract = fingerprint_compatibility(prior, empty, source_schema_revision="20260909_0032")
|
|
assert contract.compatible
|
|
assert audit_fingerprint_differences_are_compatible(prior, empty, contract)
|
|
comparison(session, "completed")
|
|
populated = semantic_fingerprint(session.connection())
|
|
assert not fingerprint_compatibility(prior, populated, source_schema_revision="20260909_0032").compatible
|
|
missing = copy.deepcopy(populated)
|
|
missing["tables"].pop("selected_model_comparisons")
|
|
assert not fingerprint_compatibility(populated, missing, source_schema_revision=SELECTED_COMPARISON_SCHEMA).compatible
|
|
missing_projection = copy.deepcopy(populated)
|
|
missing_projection["tables"]["selected_model_comparisons"].pop("restore_projection")
|
|
contract = fingerprint_compatibility(populated, missing_projection, source_schema_revision=SELECTED_COMPARISON_SCHEMA)
|
|
assert not audit_fingerprint_differences_are_compatible(populated, missing_projection, contract)
|