196 lines
9.7 KiB
Python
196 lines
9.7 KiB
Python
# ruff: noqa: S106 - isolated synthetic persistence fixtures
|
|
"""Managed PostgreSQL qualification for encrypted secret lifecycle and restore admission."""
|
|
import base64
|
|
import importlib
|
|
import os
|
|
import threading
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from pydantic import SecretStr
|
|
from sqlalchemy import select, text
|
|
from sqlalchemy.exc import DBAPIError
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modelforge_api.domain.release import TARGET_SCHEMA_REVISION
|
|
from modelforge_api.domain.secret_slots import SecretSlotSet, SecretSlotVersionRequest
|
|
from modelforge_api.persistence.models import SecretSlotState
|
|
from modelforge_api.services import secret_slots
|
|
from modelforge_api.services.recovery_fingerprint import (
|
|
SECRET_TABLES,
|
|
audit_fingerprint_differences_are_compatible,
|
|
fingerprint_compatibility,
|
|
semantic_fingerprint,
|
|
)
|
|
from modelforge_api.settings import Settings
|
|
from tests.test_managed_identities_postgres import (
|
|
EXTERNAL_ADMIN,
|
|
IdentityDatabase,
|
|
ManagedPostgresTools,
|
|
identity_database,
|
|
)
|
|
|
|
__all__ = ["identity_database"]
|
|
PRIOR = "20260908_0027"
|
|
CURRENT = TARGET_SCHEMA_REVISION
|
|
REASON = "Isolated secret lifecycle qualification"
|
|
|
|
|
|
def _settings() -> Settings:
|
|
return Settings(_env_file=None, env="test", secret_slot_root_key=SecretStr(base64.b64encode(b"q" * 32).decode()),
|
|
secret_slot_root_key_id="qualification", alembic_directory=Path(__file__).resolve().parents[1] / "alembic")
|
|
|
|
|
|
def _service(session: Session) -> secret_slots.SecretSlotService:
|
|
return secret_slots.SecretSlotService(session, _settings(), EXTERNAL_ADMIN, "secret-pg-qualification")
|
|
|
|
|
|
def _set(session: Session, revision: int = 0):
|
|
return _service(session).set(SecretSlotSet(expected_revision=revision, reason=REASON, value=SecretStr("hf_qualification_only")))
|
|
|
|
|
|
def test_secret_upgrade_dump_restore_and_immutable_ciphertext_grants(identity_database: IdentityDatabase, tmp_path: Path) -> None:
|
|
database = identity_database
|
|
alembic = importlib.import_module("alembic.command")
|
|
alembic.upgrade(database.migration(), PRIOR)
|
|
with database.runtime.connect() as connection:
|
|
source = semantic_fingerprint(connection)
|
|
archive = tmp_path / "pre-secret.dump"
|
|
database.archive(archive)
|
|
alembic.upgrade(database.migration(), "head")
|
|
with database.runtime.connect() as connection:
|
|
assert connection.scalar(text("SELECT version_num FROM alembic_version")) == CURRENT
|
|
restored = semantic_fingerprint(connection)
|
|
contract = fingerprint_compatibility(source, restored, source_schema_revision=PRIOR)
|
|
assert audit_fingerprint_differences_are_compatible(source, restored, contract)
|
|
alembic.downgrade(database.migration(), PRIOR)
|
|
alembic.upgrade(database.migration(), "head")
|
|
with Session(database.runtime) as session:
|
|
_set(session)
|
|
with database.runtime.connect() as connection:
|
|
for sql in (
|
|
"UPDATE platform_secret_versions SET secret_ciphertext='changed'",
|
|
"UPDATE platform_secret_versions SET secret_nonce='changed'",
|
|
"UPDATE platform_secret_versions SET key_id='changed'",
|
|
"UPDATE platform_secret_versions SET version=2",
|
|
"UPDATE platform_secret_versions SET created_at=now()",
|
|
"DELETE FROM platform_secret_versions", "TRUNCATE platform_secret_versions CASCADE",
|
|
"DELETE FROM platform_secret_state", "ALTER TABLE platform_secret_versions ADD COLUMN forbidden integer",
|
|
):
|
|
with pytest.raises(DBAPIError):
|
|
connection.execute(text(sql))
|
|
connection.rollback()
|
|
connection.execute(text("UPDATE platform_secret_versions SET revoked_at=now()"))
|
|
connection.commit()
|
|
with pytest.raises(RuntimeError, match="history exists"):
|
|
alembic.downgrade(database.migration(), PRIOR)
|
|
database.archive(archive, restore=True)
|
|
alembic.upgrade(database.migration(restored=True), "head")
|
|
with database.restored_runtime.connect() as connection:
|
|
restored = semantic_fingerprint(connection)
|
|
assert audit_fingerprint_differences_are_compatible(source, restored, fingerprint_compatibility(source, restored, source_schema_revision=PRIOR))
|
|
|
|
|
|
def test_secret_concurrent_set_has_one_cas_winner(identity_database: IdentityDatabase) -> None:
|
|
database = identity_database
|
|
importlib.import_module("alembic.command").upgrade(database.migration(), "head")
|
|
barrier = threading.Barrier(2)
|
|
def attempt() -> str:
|
|
with Session(database.runtime) as session:
|
|
barrier.wait(timeout=10)
|
|
try:
|
|
_set(session)
|
|
return "success"
|
|
except secret_slots.SecretSlotError as error:
|
|
return error.code
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
results = list(pool.map(lambda _: attempt(), range(2)))
|
|
assert results.count("success") == 1
|
|
assert len(set(results)) == 2
|
|
with database.runtime.connect() as connection:
|
|
assert connection.scalar(select(SecretSlotState.revision)) == 1
|
|
assert connection.scalar(text("SELECT count(*) FROM platform_secret_versions")) == 1
|
|
|
|
|
|
def test_supported_restore_clears_secret_test_and_activation_until_retest(identity_database: IdentityDatabase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from modelforge_api.services.recovery import RecoveryError, RecoveryService
|
|
database = identity_database
|
|
importlib.import_module("alembic.command").upgrade(database.migration(), "head")
|
|
monkeypatch.setattr(secret_slots, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
|
|
with Session(database.runtime) as session:
|
|
_set(session)
|
|
_service(session).test(SecretSlotVersionRequest(version=1, expected_revision=1, reason=REASON))
|
|
_service(session).activate(SecretSlotVersionRequest(version=1, expected_revision=2, reason=REASON))
|
|
assert secret_slots.resolve_huggingface_token(session, _settings()) == "hf_qualification_only"
|
|
source = semantic_fingerprint(session.connection())
|
|
archive = tmp_path / "secret.dump"
|
|
database.archive(archive)
|
|
database.archive(archive, restore=True)
|
|
with Session(database.runtime) as session:
|
|
recovery = RecoveryService(session, _settings())
|
|
recovery.engine = ManagedPostgresTools(base_environment=dict(os.environ), timeout_seconds=60)
|
|
record = SimpleNamespace(id=uuid.uuid4(), source_fingerprint=source)
|
|
plan = SimpleNamespace(database_destination=database.restored_owner.render_as_string(hide_password=False),
|
|
backup_set=SimpleNamespace(backup_id="secret-qualification", schema_revision=CURRENT, completed_at=datetime.now(UTC)), validation_requirements={})
|
|
with pytest.raises(RecoveryError) as refused:
|
|
recovery._validate_restored(record, plan)
|
|
assert refused.value.details["gate"]["current_truth_reconciled"] is False
|
|
session.rollback()
|
|
result = recovery._reconcile_restored(record, plan)
|
|
session.commit()
|
|
for table in ("platform_secret_tests", "platform_secret_activations"):
|
|
assert result["cleared_current_truth"][table] == 1
|
|
ready = recovery._validate_restored(record, plan)
|
|
assert ready["gate"]["current_truth_reconciled"] is True
|
|
session.commit()
|
|
with Session(database.restored_runtime) as session:
|
|
restored = semantic_fingerprint(session.connection())
|
|
for table in SECRET_TABLES:
|
|
assert restored["tables"][table] == source["tables"][table]
|
|
with pytest.raises(secret_slots.SecretSlotError):
|
|
secret_slots.resolve_huggingface_token(session, _settings())
|
|
session.rollback()
|
|
with pytest.raises(secret_slots.SecretSlotError) as refused_activation:
|
|
_service(session).activate(SecretSlotVersionRequest(version=1, expected_revision=3, reason=REASON))
|
|
assert refused_activation.value.code == "SECRET_SLOT_TEST_REQUIRED"
|
|
_service(session).test(SecretSlotVersionRequest(version=1, expected_revision=3, reason=REASON))
|
|
_service(session).activate(SecretSlotVersionRequest(version=1, expected_revision=4, reason=REASON))
|
|
assert secret_slots.resolve_huggingface_token(session, _settings()) == "hf_qualification_only"
|
|
|
|
def test_secret_network_result_cannot_bless_concurrently_replaced_candidate(
|
|
identity_database: IdentityDatabase, monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
database = identity_database
|
|
importlib.import_module("alembic.command").upgrade(database.migration(), "head")
|
|
with Session(database.runtime) as session:
|
|
_set(session)
|
|
entered, resume = threading.Event(), threading.Event()
|
|
def provider(_token: str) -> tuple[bool, str]:
|
|
entered.set()
|
|
assert resume.wait(timeout=20)
|
|
return True, "HF_AUTHENTICATED"
|
|
monkeypatch.setattr(secret_slots, "test_huggingface_token", provider)
|
|
def attempt() -> str:
|
|
with Session(database.runtime) as session:
|
|
try:
|
|
_service(session).test(SecretSlotVersionRequest(version=1, expected_revision=1, reason=REASON))
|
|
return "success"
|
|
except secret_slots.SecretSlotError as error:
|
|
return error.code
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
future = pool.submit(attempt)
|
|
try:
|
|
assert entered.wait(timeout=20)
|
|
with Session(database.runtime) as session:
|
|
_set(session, 1)
|
|
finally:
|
|
resume.set()
|
|
assert future.result(timeout=20) != "success"
|
|
with database.runtime.connect() as connection:
|
|
assert connection.scalar(text("SELECT count(*) FROM platform_secret_tests WHERE passed")) == 0
|
|
assert connection.scalar(select(SecretSlotState.revision)) == 2
|