242 lines
9.0 KiB
Python
242 lines
9.0 KiB
Python
from typing import Any
|
|
|
|
from fastapi.testclient import TestClient
|
|
from pydantic import SecretStr
|
|
from sqlalchemy import create_engine, event
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from modelforge_api.api.routes.catalog import list_capabilities, list_projects
|
|
from modelforge_api.api.routes.registry import get_registry_service
|
|
from modelforge_api.db import get_session
|
|
from modelforge_api.domain.release import PRODUCT_VERSION, RELEASE_CHANNEL
|
|
from modelforge_api.main import app
|
|
from modelforge_api.persistence.models import Base, Capability, CapabilityContract
|
|
from modelforge_api.services.manifest_registry import ManifestRegistry
|
|
from modelforge_api.services.project_registry import sync_project_registry
|
|
from modelforge_api.services.registry import RegistryService, seed_candidate_registry
|
|
from modelforge_api.settings import Settings, get_settings
|
|
|
|
client = TestClient(app)
|
|
|
|
|
|
def _test_operator_credential() -> str:
|
|
return "modelforge-test-operator"
|
|
|
|
|
|
CONTROL_PLANE_HEADERS = {"X-ModelForge-Admin-Token": _test_operator_credential()}
|
|
|
|
|
|
def _configure_operator_auth() -> None:
|
|
settings = Settings(
|
|
_env_file=None,
|
|
operator_api_key=SecretStr(_test_operator_credential()),
|
|
)
|
|
app.dependency_overrides[get_settings] = lambda: settings
|
|
|
|
|
|
def test_liveness_and_correlation_id() -> None:
|
|
response = client.get("/api/v1/health/live", headers={"x-correlation-id": "test-correlation"})
|
|
assert response.status_code == 200
|
|
assert response.headers["x-correlation-id"] == "test-correlation"
|
|
assert response.json()["status"] == "ok"
|
|
|
|
|
|
def test_readiness_validates_all_manifests() -> None:
|
|
response = client.get("/api/v1/health/ready")
|
|
assert response.status_code == 200
|
|
assert response.json()["checks"] == {"manifests": "healthy"}
|
|
|
|
|
|
def test_public_version_contract_supplies_console_release_identity_fields() -> None:
|
|
commit = "e472d444dd007f22db9f2eac0b97e4e97d0a2993"
|
|
built_at = "2026-09-24T12:00:00Z"
|
|
image_digest = f"sha256:{'f' * 64}"
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
_env_file=None,
|
|
build_commit=commit,
|
|
build_timestamp=built_at,
|
|
build_image_digest=image_digest,
|
|
)
|
|
try:
|
|
response = client.get("/api/v1/version")
|
|
assert response.status_code == 200
|
|
payload = response.json()
|
|
|
|
assert payload["version"] == "1.3.0-rc.1"
|
|
assert payload["release_channel"] == "rc"
|
|
assert payload["source_commit"] == commit
|
|
assert payload["built_at"] == built_at
|
|
assert payload["image_digest"] == image_digest
|
|
assert payload["compatibility"]["schema_revision"] == "20260924_0039"
|
|
assert payload["compatibility"]["agent_protocol_version"] == 1
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
def test_system_metadata_reports_the_release_without_unpromoted_inference() -> None:
|
|
"""A released product reports its version and channel, not the milestone that built it."""
|
|
|
|
engine = create_engine(
|
|
"sqlite+pysqlite:///:memory:",
|
|
connect_args={"check_same_thread": False},
|
|
poolclass=StaticPool,
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
_configure_operator_auth()
|
|
app.dependency_overrides[get_session] = lambda: session
|
|
try:
|
|
payload = client.get(
|
|
"/api/v1/system",
|
|
headers=CONTROL_PLANE_HEADERS,
|
|
).json()
|
|
assert payload["version"] == PRODUCT_VERSION
|
|
assert payload["release_channel"] == RELEASE_CHANNEL
|
|
assert "milestone" not in payload
|
|
assert payload["production_inference_available"] is False
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
def test_seed_catalog_is_unverified_candidate_metadata_only() -> None:
|
|
engine = create_engine(
|
|
"sqlite+pysqlite:///:memory:",
|
|
connect_args={"check_same_thread": False},
|
|
poolclass=StaticPool,
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
seed_candidate_registry(session, ManifestRegistry())
|
|
_configure_operator_auth()
|
|
app.dependency_overrides[get_registry_service] = lambda: RegistryService(session)
|
|
try:
|
|
response = client.get("/api/v1/models", headers=CONTROL_PLANE_HEADERS)
|
|
assert response.status_code == 200
|
|
models = response.json()["items"]
|
|
assert len(models) == 15
|
|
assert all(model["lifecycle"] == "candidate" for model in models)
|
|
assert all(model["verification_status"] == "unverified" for model in models)
|
|
assert all(model["deployment_status"] == "not_deployed" for model in models)
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
def test_capability_and_project_contract_endpoints() -> None:
|
|
"""Runs against an in-memory database, like its neighbours.
|
|
|
|
It used to call the endpoint with no session override, so `get_session` handed it the real
|
|
engine and the test quietly required a live PostgreSQL on localhost. It passed on a developer
|
|
machine that happened to have one running and failed anywhere else — the same class of hidden
|
|
environment dependency the release gate exists to find.
|
|
"""
|
|
|
|
engine = create_engine(
|
|
"sqlite+pysqlite:///:memory:",
|
|
connect_args={"check_same_thread": False},
|
|
poolclass=StaticPool,
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
_configure_operator_auth()
|
|
app.dependency_overrides[get_session] = lambda: session
|
|
try:
|
|
capabilities = client.get(
|
|
"/api/v1/capabilities",
|
|
headers=CONTROL_PLANE_HEADERS,
|
|
).json()
|
|
projects = client.get(
|
|
"/api/v1/projects",
|
|
headers=CONTROL_PLANE_HEADERS,
|
|
).json()
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
assert {item["key"] for item in capabilities} >= {"rag.embedding", "vision.embedding"}
|
|
assert {item["id"] for item in projects} == {"examplerag", "examplevision", "example-ops"}
|
|
assert all("model_id" not in binding for project in projects for binding in project["bindings"])
|
|
|
|
|
|
def test_capability_catalog_select_count_does_not_grow_with_contracts() -> None:
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(engine)
|
|
manifests = ManifestRegistry()
|
|
expected = manifests.capabilities()
|
|
assert len(expected) > 1
|
|
selects = 0
|
|
|
|
def count_selects(
|
|
_connection: Any,
|
|
_cursor: Any,
|
|
statement: str,
|
|
_parameters: Any,
|
|
_context: Any,
|
|
_executemany: bool,
|
|
) -> None:
|
|
nonlocal selects
|
|
if statement.lstrip().upper().startswith("SELECT"):
|
|
selects += 1
|
|
|
|
event.listen(engine, "before_cursor_execute", count_selects)
|
|
try:
|
|
with Session(engine) as session:
|
|
result = list_capabilities(registry=manifests, session=session)
|
|
finally:
|
|
event.remove(engine, "before_cursor_execute", count_selects)
|
|
engine.dispose()
|
|
|
|
assert [(item.key, item.version) for item in result] == [
|
|
(item.capability, item.version) for item in expected
|
|
]
|
|
assert selects == 1
|
|
|
|
|
|
def test_project_catalog_select_count_does_not_grow_with_bindings() -> None:
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(engine)
|
|
manifests = ManifestRegistry()
|
|
with Session(engine) as session:
|
|
for manifest in manifests.capabilities():
|
|
if manifest.capability not in {"rag.embedding", "rag.reranking", "vision.embedding"}:
|
|
continue
|
|
capability = Capability(key=manifest.capability, description=manifest.description)
|
|
session.add(capability)
|
|
session.flush()
|
|
session.add(
|
|
CapabilityContract(
|
|
capability_id=capability.id,
|
|
version=manifest.version,
|
|
input_schema=manifest.input_schema,
|
|
output_schema=manifest.output_schema,
|
|
contract=manifest.model_dump(mode="json"),
|
|
upgrade_class=manifest.upgrade_class.value,
|
|
)
|
|
)
|
|
session.commit()
|
|
assert sync_project_registry(session, manifests).bindings_created == 3
|
|
|
|
selects = 0
|
|
|
|
def count_selects(
|
|
_connection: Any,
|
|
_cursor: Any,
|
|
statement: str,
|
|
_parameters: Any,
|
|
_context: Any,
|
|
_executemany: bool,
|
|
) -> None:
|
|
nonlocal selects
|
|
if statement.lstrip().upper().startswith("SELECT"):
|
|
selects += 1
|
|
|
|
event.listen(engine, "before_cursor_execute", count_selects)
|
|
try:
|
|
with Session(engine) as session:
|
|
result = list_projects(registry=manifests, session=session)
|
|
finally:
|
|
event.remove(engine, "before_cursor_execute", count_selects)
|
|
engine.dispose()
|
|
|
|
assert {item.id for item in result} == {"examplerag", "examplevision", "example-ops"}
|
|
assert sum(len(item.bindings) for item in result) == 3
|
|
assert selects == 4
|