193 lines
11 KiB
Python
193 lines
11 KiB
Python
"""Fresh deployed-source proofs retain history across the explicit 0033 convergence."""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import importlib
|
|
import os
|
|
from datetime import UTC, datetime
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from alembic.migration import MigrationContext
|
|
from alembic.operations import Operations
|
|
from sqlalchemy import create_engine, select, text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modelforge_api.persistence.models import Base, ModelRevision, ProjectBinding
|
|
from modelforge_api.services.recovery import RecoveryService
|
|
from modelforge_api.services.recovery_fingerprint import (
|
|
CREATIVE_RESTORE_ERROR,
|
|
CREATIVE_TABLES,
|
|
_actual_table,
|
|
_without_model_changes,
|
|
_without_portfolio,
|
|
audit_fingerprint_differences_are_compatible,
|
|
fingerprint_compatibility,
|
|
)
|
|
from modelforge_api.services.recovery_fingerprint import (
|
|
semantic_fingerprint as _current_semantic_fingerprint,
|
|
)
|
|
from modelforge_api.services.recovery_postgres import PostgresTarget
|
|
from modelforge_api.settings import Settings
|
|
from tests.recovery_fixtures import recreate_0038_benchmark_runs
|
|
from tests.test_deployed_schema_convergence import _migration
|
|
from tests.test_managed_identities_postgres import IdentityDatabase, identity_database
|
|
from tests.test_selected_comparison_postgres import _valid_selection
|
|
from tests.test_selected_comparison_recovery import SQLiteRecoveryTarget
|
|
from tests.test_serving_generation_recovery import (
|
|
_remove_lab_serving_job_link,
|
|
legacy_generation_fixture,
|
|
)
|
|
|
|
__all__ = ["identity_database"]
|
|
TARGET = PostgresTarget("localhost", 5432, "fixture", "fixture", None)
|
|
|
|
|
|
def semantic_fingerprint(connection):
|
|
return _without_portfolio(_without_model_changes(_current_semantic_fingerprint(connection)))
|
|
|
|
|
|
def _populate(session: Session, state: str = "granted"):
|
|
request, client_id = _valid_selection(session)
|
|
connection = session.connection()
|
|
project_id = session.scalar(select(ProjectBinding.project_id).where(ProjectBinding.id == request.project_binding_id))
|
|
tables = {name: _actual_table(connection, name) for name in (*CREATIVE_TABLES, "model_update_watches")}
|
|
revision = session.execute(select(ModelRevision.id, ModelRevision.model_id, ModelRevision.resolved_commit_sha).order_by(ModelRevision.id).limit(1)).one()
|
|
from uuid import UUID
|
|
project_id = UUID(str(project_id))
|
|
revision_id, model_id = UUID(str(revision.id)), UUID(str(revision.model_id))
|
|
allocation_id, watch_id = uuid4(), uuid4()
|
|
connection.execute(tables["external_creative_allocations"].insert().values(id=allocation_id,
|
|
service_client_id=client_id, project_id=project_id, request_id="preserve", idempotency_key="preserve",
|
|
request_sha256="a" * 64, workload="audio", resource_ref="audio.fixture", provider_resource_id="fixture",
|
|
required_vram_bytes=1024, priority="background", ttl_seconds=600, state=state,
|
|
decision_code="PRIOR_DECISION", decision_evidence={"preserve": "exact"}, required_actions=[],
|
|
acquired_at=datetime.now(UTC)))
|
|
connection.execute(tables["external_creative_resources"].insert().values(id=uuid4(), project_id=project_id,
|
|
resource_ref="audio.fixture", workload="audio", provider_resource_id="fixture", approved=False,
|
|
configured_vram_bytes=1024))
|
|
connection.execute(tables["external_creative_action_attestations"].insert().values(id=uuid4(), allocation_id=allocation_id,
|
|
action="release_idle_comfy", before_vram_bytes=1024, after_vram_bytes=0,
|
|
observed_at=datetime.now(UTC), validated=True))
|
|
connection.execute(tables["external_creative_profile_observations"].insert().values(id=uuid4(), allocation_id=allocation_id,
|
|
resource_ref="audio.fixture", provider_resource_id="fixture", workload="audio", observed_peak_vram_bytes=512,
|
|
reserved_vram_bytes=1024, duration_ms=1, outcome="succeeded"))
|
|
connection.execute(tables["model_update_watches"].insert().values(id=watch_id, model_id=model_id,
|
|
baseline_revision_id=revision_id, baseline_commit_sha=revision.resolved_commit_sha, upstream_ref="main",
|
|
check_token=uuid4(), check_until=datetime.now(UTC)))
|
|
session.commit()
|
|
return allocation_id, watch_id
|
|
|
|
|
|
@pytest.fixture
|
|
def legacy():
|
|
engine = create_engine("sqlite://")
|
|
Base.metadata.create_all(engine)
|
|
with engine.begin() as connection:
|
|
# `create_all` uses today's metadata; this fixture represents schema 0032, before the
|
|
# notification outbox and LAB evidence existed, so remove those future physical artifacts.
|
|
connection.execute(text('DROP TABLE "alert_notification_deliveries"'))
|
|
connection.execute(text("DROP TABLE selected_model_comparisons"))
|
|
recreate_0038_benchmark_runs(connection)
|
|
_remove_lab_serving_job_link(connection)
|
|
connection.execute(text("ALTER TABLE model_update_watches DROP COLUMN enabled"))
|
|
connection.execute(text("ALTER TABLE model_update_watches DROP COLUMN version"))
|
|
connection.execute(text("CREATE TABLE alembic_version(version_num VARCHAR(32))"))
|
|
connection.execute(text("INSERT INTO alembic_version VALUES ('20260909_0032')"))
|
|
with Session(engine) as session:
|
|
_populate(session)
|
|
source = semantic_fingerprint(session.connection())
|
|
module = _migration()
|
|
module.op = Operations(MigrationContext.configure(session.connection()))
|
|
module.upgrade()
|
|
session.execute(text("UPDATE alembic_version SET version_num='20260909_0033'"))
|
|
service = RecoveryService(session, Settings(_env_file=None))
|
|
service.engine = SQLiteRecoveryTarget(session) # type: ignore[assignment]
|
|
yield session, source, service
|
|
engine.dispose()
|
|
|
|
|
|
def test_populated_source_projection_and_restore_preserve_all_other_identity(legacy):
|
|
session, source, service = legacy
|
|
assert "hub_controls_projection" in source["tables"]["model_update_watches"]
|
|
assert service._creative_allocation_residuals(TARGET) == 1
|
|
assert service._reconcile_creative_allocations(TARGET) == 1
|
|
session.execute(text("UPDATE model_update_watches SET check_token=NULL,check_until=NULL"))
|
|
restored = semantic_fingerprint(session.connection())
|
|
contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
|
|
assert contract.compatible
|
|
assert audit_fingerprint_differences_are_compatible(source, restored, contract)
|
|
row = session.execute(text("SELECT state,decision_code,uncertain_at,decision_evidence FROM external_creative_allocations")).one()
|
|
assert row.state == "uncertain" and row.decision_code == CREATIVE_RESTORE_ERROR
|
|
assert row.uncertain_at is None
|
|
assert service._creative_allocation_residuals(TARGET) == 0
|
|
assert service._reconcile_creative_allocations(TARGET) == 0
|
|
for name in CREATIVE_TABLES - {"external_creative_allocations"}:
|
|
assert source["tables"][name] == restored["tables"][name]
|
|
|
|
|
|
def test_legacy_opaque_populated_watch_digest_is_not_projectable(legacy):
|
|
session, source, _ = legacy
|
|
opaque = copy.deepcopy(source)
|
|
del opaque["tables"]["model_update_watches"]["hub_controls_projection"]
|
|
contract = fingerprint_compatibility(opaque, semantic_fingerprint(session.connection()), source_schema_revision="20260909_0032")
|
|
assert not contract.compatible and "opaque" in contract.reason
|
|
|
|
|
|
@pytest.mark.parametrize("table", sorted(CREATIVE_TABLES | {"model_update_watches"}))
|
|
def test_populated_state_tampering_and_missing_tables_fail_closed(legacy, table):
|
|
session, source, service = legacy
|
|
service._reconcile_creative_allocations(TARGET)
|
|
session.execute(text("UPDATE model_update_watches SET check_token=NULL,check_until=NULL"))
|
|
restored = semantic_fingerprint(session.connection())
|
|
contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
|
|
changed = copy.deepcopy(restored)
|
|
changed["tables"][table]["digest"] = "0" * 64
|
|
assert not audit_fingerprint_differences_are_compatible(source, changed, contract)
|
|
del changed["tables"][table]
|
|
assert not fingerprint_compatibility(source, changed, source_schema_revision="20260909_0032").compatible
|
|
|
|
|
|
def test_unreconciled_grant_and_unknown_physical_table_block_proof(legacy):
|
|
session, source, _ = legacy
|
|
restored = semantic_fingerprint(session.connection())
|
|
contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
|
|
assert not audit_fingerprint_differences_are_compatible(source, restored, contract)
|
|
restored["physical_tables"] = sorted([*restored["physical_tables"], "unknown_private_state"])
|
|
assert not fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032").compatible
|
|
|
|
|
|
def test_unknown_revision_and_forged_current_watch_projection_fail_closed(legacy):
|
|
session, source, _ = legacy
|
|
restored = semantic_fingerprint(session.connection())
|
|
assert not fingerprint_compatibility(source, restored, source_schema_revision="20260909_0031x").compatible
|
|
restored["tables"]["model_update_watches"]["hub_controls_projection"] = source["tables"]["model_update_watches"]["hub_controls_projection"]
|
|
assert not fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032").compatible
|
|
|
|
|
|
def test_actual_postgres_populated_source_dump_upgrade_restore_projection(identity_database: IdentityDatabase, tmp_path):
|
|
database = identity_database
|
|
alembic = importlib.import_module("alembic.command")
|
|
alembic.upgrade(database.migration(), "20260909_0032")
|
|
with legacy_generation_fixture(database.owner), Session(database.runtime) as session:
|
|
_populate(session, "queued")
|
|
with Session(database.runtime) as session:
|
|
source = semantic_fingerprint(session.connection())
|
|
archive = tmp_path / "populated-deployed0032.dump"
|
|
database.archive(archive)
|
|
database.archive(archive, restore=True)
|
|
alembic.upgrade(database.migration(restored=True), "20260910_0034")
|
|
with Session(database.restored_runtime) as session:
|
|
from tests.test_managed_identities_postgres import ManagedPostgresTools
|
|
recovery = RecoveryService(session, Settings(_env_file=None))
|
|
recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
|
|
target = PostgresTarget(database.restored_owner.host or "localhost", database.restored_owner.port or 5432,
|
|
database.restored_owner.database or "", database.restored_owner.username or "", database.restored_owner.password)
|
|
assert recovery._reconcile_creative_allocations(target) == 1
|
|
recovery.engine.scalar(target, "UPDATE model_update_watches SET check_token=NULL,check_until=NULL")
|
|
restored = semantic_fingerprint(session.connection())
|
|
contract = fingerprint_compatibility(source, restored, source_schema_revision="20260909_0032")
|
|
assert contract.compatible
|
|
assert audit_fingerprint_differences_are_compatible(source, restored, contract)
|
|
assert recovery._creative_allocation_residuals(target) == 0
|