283 lines
11 KiB
Python
283 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from pydantic import SecretStr
|
|
from sqlalchemy import create_engine, func, select
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from modelforge_api.api.authorization import (
|
|
Principal,
|
|
PrincipalRole,
|
|
authenticate_operator_principal,
|
|
)
|
|
from modelforge_api.db import get_session
|
|
from modelforge_api.domain.configuration import (
|
|
ConfigurationApply,
|
|
ConfigurationChange,
|
|
ConfigurationImport,
|
|
)
|
|
from modelforge_api.domain.serving import EmbeddingInvokeRequest
|
|
from modelforge_api.main import app
|
|
from modelforge_api.persistence.models import (
|
|
AuditEvent,
|
|
Base,
|
|
ConfigurationRevision,
|
|
)
|
|
from modelforge_api.services.audit import AuditContext, AuditWriter
|
|
from modelforge_api.services.configuration_changes import (
|
|
ConfigurationError,
|
|
ConfigurationService,
|
|
configuration_state,
|
|
)
|
|
from modelforge_api.services.manifest_registry import ManifestRegistry
|
|
from modelforge_api.services.serving import ServingError, ServingService
|
|
from modelforge_api.services.transient_payloads import MemoryPayloadStore
|
|
from modelforge_api.settings import Settings, get_settings
|
|
|
|
TOKEN = "rc01-synthetic-administrator" # noqa: S105
|
|
|
|
|
|
@pytest.fixture
|
|
def session() -> Iterator[Session]:
|
|
engine = create_engine(
|
|
"sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False}
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as database:
|
|
yield database
|
|
engine.dispose()
|
|
|
|
|
|
def settings() -> Settings:
|
|
return Settings(
|
|
_env_file=None,
|
|
env="test",
|
|
operator_api_key=SecretStr(TOKEN),
|
|
config_root=Path(__file__).resolve().parents[2] / "config",
|
|
)
|
|
|
|
|
|
def service(session: Session) -> ConfigurationService:
|
|
return ConfigurationService(
|
|
session, settings(), AuditContext("operator", "reviewer", "config-test")
|
|
)
|
|
|
|
|
|
def reviewed_apply(
|
|
subject: ConfigurationService, change: ConfigurationChange
|
|
) -> ConfigurationApply:
|
|
preview = subject.preview(change)
|
|
return ConfigurationApply(
|
|
**change.model_dump(), preview_digest=preview.preview_digest, confirm=True
|
|
)
|
|
|
|
|
|
def test_preview_apply_rollback_and_real_serving_consumer(session: Session) -> None:
|
|
subject = service(session)
|
|
old_request = ServingService(
|
|
session, settings(), ManifestRegistry(settings().config_root), MemoryPayloadStore()
|
|
)
|
|
assert old_request._validate_inputs(EmbeddingInvokeRequest(input=["one", "two"])) == ["one", "two"]
|
|
change = ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 1})
|
|
reviewed = reviewed_apply(subject, change)
|
|
assert configuration_state(session) == (0, {})
|
|
assert session.scalar(select(func.count()).select_from(AuditEvent)) == 0
|
|
result = subject.apply(reviewed)
|
|
assert result.status == "accepted_for_new_requests"
|
|
session.expire_all()
|
|
assert configuration_state(session) == (1, {"gateway_max_batch_size": 1})
|
|
request = EmbeddingInvokeRequest(input=["one", "two"])
|
|
assert old_request._validate_inputs(request) == ["one", "two"]
|
|
new_request = ServingService(
|
|
session, settings(), ManifestRegistry(settings().config_root), MemoryPayloadStore()
|
|
)
|
|
with pytest.raises(ServingError, match="configured maximum"):
|
|
new_request._validate_inputs(request)
|
|
event = session.scalar(select(AuditEvent).where(AuditEvent.action == "configuration.applied"))
|
|
assert (
|
|
event is not None and event.actor_id == "reviewer" and event.correlation_id == "config-test"
|
|
)
|
|
assert TOKEN not in str(event.details)
|
|
rollback = subject.rollback_change(0, 1)
|
|
subject.apply(reviewed_apply(subject, rollback))
|
|
assert configuration_state(session) == (2, {})
|
|
assert [row.revision for row in subject.history()] == [2, 1]
|
|
|
|
|
|
def test_preview_reports_the_exact_resolved_source_across_override_changes(
|
|
session: Session, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
monkeypatch.setenv("MODELFORGE_GATEWAY_MAX_BATCH_SIZE", "16")
|
|
base_settings = Settings(_env_file=None, env="test")
|
|
subject = ConfigurationService(
|
|
session, base_settings, AuditContext("operator", "reviewer", "config-source-test")
|
|
)
|
|
|
|
add_override = ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 12})
|
|
add_preview = subject.preview(add_override)
|
|
assert [
|
|
(item.source_before, item.source_after) for item in add_preview.differences
|
|
] == [("process_environment", "operator_override")]
|
|
subject.apply(reviewed_apply(subject, add_override))
|
|
|
|
remove_override = ConfigurationChange(expected_revision=1, changes={"gateway_max_batch_size": None})
|
|
remove_preview = subject.preview(remove_override)
|
|
assert [
|
|
(item.source_before, item.source_after) for item in remove_preview.differences
|
|
] == [("operator_override", "process_environment")]
|
|
|
|
|
|
def test_preview_cannot_be_reused_or_changed_and_baseline_drift_conflicts(session: Session) -> None:
|
|
subject = service(session)
|
|
reviewed = reviewed_apply(
|
|
subject, ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
|
|
)
|
|
tampered = reviewed.model_copy(update={"changes": {"gateway_max_batch_size": 3}})
|
|
with pytest.raises(ConfigurationError, match="proposal changed"):
|
|
subject.apply(tampered)
|
|
changed_baseline = ConfigurationService(
|
|
session,
|
|
settings().model_copy(update={"gateway_request_timeout_seconds": 90}),
|
|
subject.context,
|
|
)
|
|
with pytest.raises(ConfigurationError, match="proposal changed"):
|
|
changed_baseline.apply(reviewed)
|
|
subject.apply(reviewed)
|
|
with pytest.raises(ConfigurationError, match="Configuration changed"):
|
|
subject.apply(reviewed)
|
|
assert len(subject.history()) == 1
|
|
|
|
|
|
def test_audit_failure_rolls_back_pointer_and_history(
|
|
session: Session, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
subject = service(session)
|
|
reviewed = reviewed_apply(
|
|
subject, ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
|
|
)
|
|
|
|
def fail(*args: object, **kwargs: object) -> None:
|
|
raise RuntimeError("audit unavailable")
|
|
|
|
monkeypatch.setattr(AuditWriter, "write", fail)
|
|
with pytest.raises(RuntimeError, match="audit unavailable"):
|
|
subject.apply(reviewed)
|
|
assert configuration_state(session) == (0, {})
|
|
assert session.scalar(select(func.count()).select_from(ConfigurationRevision)) == 0
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"changes",
|
|
[
|
|
{"operator_api_key": 10},
|
|
{"allow_remote_code": 1},
|
|
{"scheduler_safety_reserve_bytes": 1},
|
|
{"gateway_max_batch_size": 0},
|
|
{"gateway_max_batch_size": 2.5},
|
|
{"gateway_queue_timeout_seconds": 50},
|
|
{"gateway_max_input_characters": 10**310},
|
|
{"gateway_max_batch_size": float("inf")},
|
|
],
|
|
)
|
|
def test_unsafe_and_invalid_changes_fail_before_persistence(
|
|
session: Session, changes: dict[str, int | float]
|
|
) -> None:
|
|
with pytest.raises(ConfigurationError):
|
|
service(session).preview(ConfigurationChange(expected_revision=0, changes=changes))
|
|
assert configuration_state(session) == (0, {})
|
|
|
|
|
|
def test_export_import_compatibility_and_replacement(session: Session) -> None:
|
|
subject = service(session)
|
|
subject.apply(
|
|
reviewed_apply(
|
|
subject, ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2})
|
|
)
|
|
)
|
|
exported = subject.export()
|
|
assert TOKEN not in exported.model_dump_json()
|
|
assert "database_url" in exported.secret_slots
|
|
assert "postgresql" not in exported.model_dump_json()
|
|
exported.overrides = {"gateway_max_input_characters": 300}
|
|
change = subject.import_change(ConfigurationImport(expected_revision=1, document=exported))
|
|
assert change.changes == {"gateway_max_batch_size": None, "gateway_max_input_characters": 300}
|
|
subject.apply(reviewed_apply(subject, change))
|
|
assert configuration_state(session)[1] == exported.overrides
|
|
for replacement in (
|
|
{"format": "unknown"},
|
|
{"source_version": "2.0.0"},
|
|
{"secret_slots": []},
|
|
{"overrides": {"database_url": 1}},
|
|
):
|
|
incompatible = exported.model_copy(update=replacement)
|
|
with pytest.raises(ConfigurationError):
|
|
subject.import_change(ConfigurationImport(expected_revision=2, document=incompatible))
|
|
|
|
|
|
def test_api_authorization_validation_redaction_and_audited_apply(session: Session) -> None:
|
|
app.dependency_overrides[get_settings] = settings
|
|
app.dependency_overrides[get_session] = lambda: session
|
|
client = TestClient(app)
|
|
headers = {"X-ModelForge-Admin-Token": TOKEN}
|
|
change = {"expected_revision": 0, "changes": {"gateway_max_batch_size": 2}}
|
|
try:
|
|
assert client.post("/api/v1/configuration/preview", json=change).status_code == 401
|
|
for role in (PrincipalRole.VIEWER, PrincipalRole.OPERATOR):
|
|
app.dependency_overrides[authenticate_operator_principal] = lambda role=role: Principal(
|
|
"limited", role, "test"
|
|
)
|
|
assert (
|
|
client.post("/api/v1/configuration/apply", headers=headers, json={}).status_code
|
|
== 403
|
|
)
|
|
app.dependency_overrides.pop(authenticate_operator_principal)
|
|
preview = client.post("/api/v1/configuration/preview", headers=headers, json=change)
|
|
assert preview.status_code == 200
|
|
body = {**change, "preview_digest": preview.json()["preview_digest"], "confirm": True}
|
|
assert (
|
|
client.post("/api/v1/configuration/apply", headers=headers, json=body).status_code
|
|
== 200
|
|
)
|
|
catalog = client.get("/api/v1/configuration/settings", headers=headers).json()
|
|
entry = next(row for row in catalog["settings"] if row["key"] == "gateway_max_batch_size")
|
|
assert (catalog["revision"], entry["value"], entry["source"]) == (1, 2, "operator_override")
|
|
log_level = next(row for row in catalog["settings"] if row["key"] == "log_level")
|
|
assert log_level["source"] == "pydantic_default"
|
|
for malicious in (
|
|
{"operator_api_key": "secret-must-not-leak"},
|
|
{"secret-must-not-leak": "secret-must-not-leak"},
|
|
):
|
|
rejected = client.post(
|
|
"/api/v1/configuration/preview",
|
|
headers=headers,
|
|
json={"expected_revision": 1, "changes": malicious},
|
|
)
|
|
assert rejected.status_code == 422
|
|
assert "secret-must-not-leak" not in rejected.text
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
def test_configuration_state_survives_a_new_connection(tmp_path: Path) -> None:
|
|
database_url = f"sqlite:///{tmp_path / 'config.db'}"
|
|
first_engine = create_engine(database_url)
|
|
Base.metadata.create_all(first_engine)
|
|
with Session(first_engine) as first:
|
|
subject = service(first)
|
|
subject.apply(
|
|
reviewed_apply(
|
|
subject,
|
|
ConfigurationChange(expected_revision=0, changes={"gateway_max_batch_size": 2}),
|
|
)
|
|
)
|
|
first_engine.dispose()
|
|
second_engine = create_engine(database_url)
|
|
with Session(second_engine) as second:
|
|
assert configuration_state(second) == (1, {"gateway_max_batch_size": 2})
|
|
second_engine.dispose()
|