255 lines
11 KiB
Python
255 lines
11 KiB
Python
"""Real PostgreSQL migration, backup recovery, role and concurrent-write qualification."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from alembic.config import Config
|
|
from sqlalchemy import create_engine, select, text
|
|
from sqlalchemy.engine import make_url
|
|
from sqlalchemy.exc import DBAPIError
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modelforge_api.domain.configuration import ConfigurationApply, ConfigurationChange
|
|
from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
|
|
from modelforge_api.persistence.models import AuditEvent, ConfigurationRevision, Project
|
|
from modelforge_api.services.audit import AuditContext, AuditWriter
|
|
from modelforge_api.services.configuration_changes import (
|
|
ConfigurationError,
|
|
ConfigurationService,
|
|
configuration_state,
|
|
)
|
|
from modelforge_api.services.recovery_fingerprint import (
|
|
audit_fingerprint_differences_are_compatible,
|
|
fingerprint_compatibility,
|
|
semantic_fingerprint,
|
|
)
|
|
from modelforge_api.settings import Settings
|
|
|
|
|
|
def _database_tool(tool: str, arguments: list[str], password: str | None, archive: Path) -> None:
|
|
container = os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER")
|
|
environment = dict(os.environ)
|
|
if password:
|
|
environment["PGPASSWORD"] = password
|
|
command = [tool, *arguments]
|
|
if container:
|
|
assert re.fullmatch(r"[0-9a-f]{12,64}", container)
|
|
command = ["docker", "exec", "-i", "-e", "PGPASSWORD", container, *command]
|
|
with archive.open("wb" if tool == "pg_dump" else "rb") as handle:
|
|
result = subprocess.run( # noqa: S603 - bounded test tool and UUID database arguments
|
|
command,
|
|
env=environment,
|
|
timeout=120,
|
|
check=False,
|
|
stdin=handle if tool == "pg_restore" else subprocess.DEVNULL,
|
|
stdout=handle if tool == "pg_dump" else subprocess.DEVNULL,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
assert result.returncode == 0, f"{tool} failed during isolated configuration qualification"
|
|
|
|
|
|
def test_postgres_configuration_upgrade_backup_recovery_and_concurrency(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
admin_url = os.getenv("MODELFORGE_TEST_POSTGRES_URL")
|
|
if not admin_url:
|
|
pytest.skip("requires managed MODELFORGE_TEST_POSTGRES_URL")
|
|
owner_url = os.getenv("MODELFORGE_TEST_POSTGRES_OWNER_URL")
|
|
runtime_url = os.getenv("MODELFORGE_TEST_POSTGRES_RUNTIME_URL")
|
|
assert owner_url and runtime_url, (
|
|
"managed qualification requires separate owner and runtime roles"
|
|
)
|
|
database = f"test_config_{uuid.uuid4().hex}"
|
|
restored_database = f"test_config_restore_{uuid.uuid4().hex}"
|
|
admin = create_engine(admin_url, isolation_level="AUTOCOMMIT")
|
|
owner = make_url(owner_url).set(database=database)
|
|
runtime = create_engine(make_url(runtime_url).set(database=database))
|
|
restored_runtime = create_engine(make_url(runtime_url).set(database=restored_database))
|
|
migration = Config()
|
|
migration.set_main_option(
|
|
"script_location", str(Path(__file__).resolve().parents[1] / "alembic")
|
|
)
|
|
migration.set_main_option(
|
|
"sqlalchemy.url", owner.render_as_string(hide_password=False).replace("%", "%%")
|
|
)
|
|
alembic = importlib.import_module("alembic.command")
|
|
with admin.connect() as connection:
|
|
connection.execute(text(f'CREATE DATABASE "{database}" OWNER modelforge'))
|
|
connection.execute(text(f'CREATE DATABASE "{restored_database}" OWNER modelforge'))
|
|
try:
|
|
alembic.upgrade(migration, "20260830_0024")
|
|
with Session(runtime) as session:
|
|
session.add(
|
|
Project(
|
|
key="existing-consumer",
|
|
name="Existing consumer",
|
|
description="Prior-release fixture",
|
|
)
|
|
)
|
|
AuditWriter(session, "operator", "prior-release").write(
|
|
"qualification.baseline", "project", None, {"source_schema": "20260830_0024"}
|
|
)
|
|
session.commit()
|
|
baseline = session.scalar(select(AuditEvent.event_hash))
|
|
archive = tmp_path / "pre-configuration.dump"
|
|
host = (
|
|
"127.0.0.1"
|
|
if os.getenv("MODELFORGE_TEST_POSTGRES_CONTAINER")
|
|
else (owner.host or "localhost")
|
|
)
|
|
args = [
|
|
"--host",
|
|
host,
|
|
"--port",
|
|
str(owner.port or 5432),
|
|
"--username",
|
|
owner.username or "modelforge",
|
|
"--dbname",
|
|
database,
|
|
]
|
|
_database_tool("pg_dump", [*args, "--format=custom"], owner.password, archive)
|
|
assert archive.stat().st_size > 0
|
|
alembic.upgrade(migration, "head")
|
|
with Session(runtime) as session:
|
|
assert session.scalar(select(Project.key)) == "existing-consumer"
|
|
assert session.scalar(select(AuditEvent.event_hash)) == baseline
|
|
assert configuration_state(session) == (0, {})
|
|
# Mechanical round-trip is qualified before operator changes. Once changes exist,
|
|
# rollback uses the retained backup rather than destroying revision history.
|
|
alembic.downgrade(migration, "20260830_0024")
|
|
alembic.upgrade(migration, "head")
|
|
apply_preview_barrier = threading.Barrier(2)
|
|
synchronize_apply_preview = threading.local()
|
|
original_preview = ConfigurationService.preview
|
|
|
|
def preview_at_cas(
|
|
subject: ConfigurationService, change: ConfigurationChange
|
|
):
|
|
preview = original_preview(subject, change)
|
|
if getattr(synchronize_apply_preview, "enabled", False):
|
|
synchronize_apply_preview.enabled = False
|
|
# Both real PostgreSQL transactions have validated the same revision and digest;
|
|
# release them together immediately before their conditional UPDATE.
|
|
apply_preview_barrier.wait(timeout=30)
|
|
return preview
|
|
|
|
monkeypatch.setattr(ConfigurationService, "preview", preview_at_cas)
|
|
|
|
def contender(expected_revision: int, value: int) -> str:
|
|
with Session(runtime) as session:
|
|
subject = ConfigurationService(
|
|
session,
|
|
Settings(_env_file=None, env="test"),
|
|
AuditContext("operator", f"contender-{value}", "configuration-race"),
|
|
)
|
|
change = ConfigurationChange(
|
|
expected_revision=expected_revision,
|
|
changes={"gateway_max_batch_size": value},
|
|
)
|
|
preview = subject.preview(change)
|
|
session.rollback() # Start both writes from independent fresh transactions.
|
|
synchronize_apply_preview.enabled = True
|
|
try:
|
|
subject.apply(
|
|
ConfigurationApply(
|
|
**change.model_dump(),
|
|
preview_digest=preview.preview_digest,
|
|
confirm=True,
|
|
)
|
|
)
|
|
return "applied"
|
|
except ConfigurationError as error:
|
|
assert error.code == "CONFIGURATION_CONFLICT"
|
|
return "conflict"
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
assert sorted(pool.map(lambda value: contender(0, value), [2, 3])) == [
|
|
"applied",
|
|
"conflict",
|
|
]
|
|
with Session(runtime) as session:
|
|
assert configuration_state(session)[0] == 1
|
|
assert len(list(session.scalars(select(ConfigurationRevision)))) == 1
|
|
assert (
|
|
len(
|
|
list(
|
|
session.scalars(
|
|
select(AuditEvent).where(AuditEvent.action == "configuration.applied")
|
|
)
|
|
)
|
|
)
|
|
== 1
|
|
)
|
|
# Exercise the existing-row UPDATE path as well as the empty-singleton insert race.
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
assert sorted(pool.map(lambda value: contender(1, value), [4, 5])) == [
|
|
"applied",
|
|
"conflict",
|
|
]
|
|
with Session(runtime) as session:
|
|
assert configuration_state(session)[0] == 2
|
|
assert [row.revision for row in session.scalars(
|
|
select(ConfigurationRevision).order_by(ConfigurationRevision.revision)
|
|
)] == [1, 2]
|
|
assert len(list(session.scalars(
|
|
select(AuditEvent).where(AuditEvent.action == "configuration.applied")
|
|
))) == 2
|
|
with pytest.raises(RuntimeError, match="Configuration history exists"):
|
|
alembic.downgrade(migration, "20260830_0024")
|
|
with Session(runtime) as session:
|
|
assert configuration_state(session)[0] == 2
|
|
assert (
|
|
session.execute(text("SELECT version_num FROM alembic_version")).scalar_one()
|
|
== TARGET_SCHEMA_REVISION
|
|
)
|
|
with runtime.connect() as connection:
|
|
for statement in (
|
|
"UPDATE configuration_revisions SET actor = 'tampered'",
|
|
"DELETE FROM configuration_revisions",
|
|
"TRUNCATE configuration_revisions",
|
|
):
|
|
with pytest.raises(DBAPIError):
|
|
connection.execute(text(statement))
|
|
connection.rollback()
|
|
args[-1] = restored_database
|
|
_database_tool("pg_restore", [*args, "--exit-on-error"], owner.password, archive)
|
|
with Session(restored_runtime) as session:
|
|
assert session.scalar(select(Project.key)) == "existing-consumer"
|
|
assert session.scalar(select(AuditEvent.event_hash)) == baseline
|
|
assert (
|
|
session.execute(text("SELECT version_num FROM alembic_version")).scalar_one()
|
|
== "20260830_0024"
|
|
)
|
|
migration.set_main_option(
|
|
"sqlalchemy.url",
|
|
owner.set(database=restored_database)
|
|
.render_as_string(hide_password=False)
|
|
.replace("%", "%%"),
|
|
)
|
|
alembic.upgrade(migration, "head")
|
|
with Session(restored_runtime) as session:
|
|
assert configuration_state(session) == (0, {})
|
|
with runtime.connect() as connection:
|
|
current = semantic_fingerprint(connection)
|
|
contract = fingerprint_compatibility(
|
|
current, current, source_schema_revision=TARGET_SCHEMA_REVISION
|
|
)
|
|
assert contract.compatible
|
|
assert audit_fingerprint_differences_are_compatible(current, current, contract)
|
|
finally:
|
|
runtime.dispose()
|
|
restored_runtime.dispose()
|
|
with admin.connect() as connection:
|
|
for target in (database, restored_database):
|
|
connection.execute(text(f'DROP DATABASE IF EXISTS "{target}" WITH (FORCE)'))
|
|
admin.dispose()
|