"""Real PostgreSQL migration, backup recovery, role and concurrent-write qualification.""" from __future__ import annotations import importlib import os import re import subprocess import threading import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest from alembic.config import Config from sqlalchemy import create_engine, select, text from sqlalchemy.engine import make_url from sqlalchemy.exc import DBAPIError from sqlalchemy.orm import Session from modelforge_api.domain.configuration import ConfigurationApply, ConfigurationChange from modelforge_api.domain.release import TARGET_SCHEMA_REVISION from modelforge_api.persistence.models import AuditEvent, ConfigurationRevision, Project from modelforge_api.services.audit import AuditContext, AuditWriter from modelforge_api.services.configuration_changes import ( ConfigurationError, ConfigurationService, configuration_state, ) from modelforge_api.services.recovery_fingerprint import ( audit_fingerprint_differences_are_compatible, fingerprint_compatibility, semantic_fingerprint, ) from modelforge_api.settings import Settings def _database_tool(tool: str, arguments: list[str], password: str | None, archive: Path) -> None: container = os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER") environment = dict(os.environ) if password: environment["PGPASSWORD"] = password command = [tool, *arguments] if container: assert re.fullmatch(r"[0-9a-f]{12,64}", container) command = ["docker", "exec", "-i", "-e", "PGPASSWORD", container, *command] with archive.open("wb" if tool == "pg_dump" else "rb") as handle: result = subprocess.run( # noqa: S603 - bounded test tool and UUID database arguments command, env=environment, timeout=120, check=False, stdin=handle if tool == "pg_restore" else subprocess.DEVNULL, stdout=handle if tool == "pg_dump" else subprocess.DEVNULL, stderr=subprocess.PIPE, ) assert result.returncode == 0, f"{tool} failed during isolated configuration qualification" def test_postgres_configuration_upgrade_backup_recovery_and_concurrency( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL") if not admin_url: pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL") owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL") runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL") assert owner_url and runtime_url, ( "managed qualification requires separate owner and runtime roles" ) database = f"test_config_{uuid.uuid4().hex}" restored_database = f"test_config_restore_{uuid.uuid4().hex}" admin = create_engine(admin_url, isolation_level="AUTOCOMMIT") owner = make_url(owner_url).set(database=database) runtime = create_engine(make_url(runtime_url).set(database=database)) restored_runtime = create_engine(make_url(runtime_url).set(database=restored_database)) migration = Config() migration.set_main_option( "script_location", str(Path(__file__).resolve().parents[1] / "alembic") ) migration.set_main_option( "sqlalchemy.url", owner.render_as_string(hide_password=False).replace("%", "%%") ) alembic = importlib.import_module("alembic.command") with admin.connect() as connection: connection.execute(text(f'CREATE DATABASE "{database}" OWNER modelforge')) connection.execute(text(f'CREATE DATABASE "{restored_database}" OWNER modelforge')) try: alembic.upgrade(migration, "20260830_0024") with Session(runtime) as session: session.add( Project( key="existing-consumer", name="Existing consumer", description="Prior-release fixture", ) ) AuditWriter(session, "operator", "prior-release").write( "qualification.baseline", "project", None, {"source_schema": "20260830_0024"} ) session.commit() baseline = session.scalar(select(AuditEvent.event_hash)) archive = tmp_path / "pre-configuration.dump" host = ( "127.0.0.1" if os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER") else (owner.host or "localhost") ) args = [ "--host", host, "--port", str(owner.port or 5432), "--username", owner.username or "modelforge", "--dbname", database, ] _database_tool("pg_dump", [*args, "--format=custom"], owner.password, archive) assert archive.stat().st_size > 0 alembic.upgrade(migration, "head") with Session(runtime) as session: assert session.scalar(select(Project.key)) == "existing-consumer" assert session.scalar(select(AuditEvent.event_hash)) == baseline assert configuration_state(session) == (0, {}) # Mechanical round-trip is qualified before operator changes. Once changes exist, # rollback uses the retained backup rather than destroying revision history. alembic.downgrade(migration, "20260830_0024") alembic.upgrade(migration, "head") apply_preview_barrier = threading.Barrier(2) synchronize_apply_preview = threading.local() original_preview = ConfigurationService.preview def preview_at_cas( subject: ConfigurationService, change: ConfigurationChange ): preview = original_preview(subject, change) if getattr(synchronize_apply_preview, "enabled", False): synchronize_apply_preview.enabled = False # Both real PostgreSQL transactions have validated the same revision and digest; # release them together immediately before their conditional UPDATE. apply_preview_barrier.wait(timeout=30) return preview monkeypatch.setattr(ConfigurationService, "preview", preview_at_cas) def contender(expected_revision: int, value: int) -> str: with Session(runtime) as session: subject = ConfigurationService( session, Settings(_env_file=None, env="test"), AuditContext("operator", f"contender-{value}", "configuration-race"), ) change = ConfigurationChange( expected_revision=expected_revision, changes={"gateway_max_batch_size": value}, ) preview = subject.preview(change) session.rollback() # Start both writes from independent fresh transactions. synchronize_apply_preview.enabled = True try: subject.apply( ConfigurationApply( **change.model_dump(), preview_digest=preview.preview_digest, confirm=True, ) ) return "applied" except ConfigurationError as error: assert error.code == "CONFIGURATION_CONFLICT" return "conflict" with ThreadPoolExecutor(max_workers=2) as pool: assert sorted(pool.map(lambda value: contender(0, value), [2, 3])) == [ "applied", "conflict", ] with Session(runtime) as session: assert configuration_state(session)[0] == 1 assert len(list(session.scalars(select(ConfigurationRevision)))) == 1 assert ( len( list( session.scalars( select(AuditEvent).where(AuditEvent.action == "configuration.applied") ) ) ) == 1 ) # Exercise the existing-row UPDATE path as well as the empty-singleton insert race. with ThreadPoolExecutor(max_workers=2) as pool: assert sorted(pool.map(lambda value: contender(1, value), [4, 5])) == [ "applied", "conflict", ] with Session(runtime) as session: assert configuration_state(session)[0] == 2 assert [row.revision for row in session.scalars( select(ConfigurationRevision).order_by(ConfigurationRevision.revision) )] == [1, 2] assert len(list(session.scalars( select(AuditEvent).where(AuditEvent.action == "configuration.applied") ))) == 2 with pytest.raises(RuntimeError, match="Configuration history exists"): alembic.downgrade(migration, "20260830_0024") with Session(runtime) as session: assert configuration_state(session)[0] == 2 assert ( session.execute(text("SELECT version_num FROM alembic_version")).scalar_one() == TARGET_SCHEMA_REVISION ) with runtime.connect() as connection: for statement in ( "UPDATE configuration_revisions SET actor = 'tampered'", "DELETE FROM configuration_revisions", "TRUNCATE configuration_revisions", ): with pytest.raises(DBAPIError): connection.execute(text(statement)) connection.rollback() args[-1] = restored_database _database_tool("pg_restore", [*args, "--exit-on-error"], owner.password, archive) with Session(restored_runtime) as session: assert session.scalar(select(Project.key)) == "existing-consumer" assert session.scalar(select(AuditEvent.event_hash)) == baseline assert ( session.execute(text("SELECT version_num FROM alembic_version")).scalar_one() == "20260830_0024" ) migration.set_main_option( "sqlalchemy.url", owner.set(database=restored_database) .render_as_string(hide_password=False) .replace("%", "%%"), ) alembic.upgrade(migration, "head") with Session(restored_runtime) as session: assert configuration_state(session) == (0, {}) with runtime.connect() as connection: current = semantic_fingerprint(connection) contract = fingerprint_compatibility( current, current, source_schema_revision=TARGET_SCHEMA_REVISION ) assert contract.compatible assert audit_fingerprint_differences_are_compatible(current, current, contract) finally: runtime.dispose() restored_runtime.dispose() with admin.connect() as connection: for target in (database, restored_database): connection.execute(text(f'DROP DATABASE IF EXISTS "{target}" WITH (FORCE)')) admin.dispose()