306 lines
16 KiB
Python
306 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import uuid
|
|
from collections.abc import Iterator
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from pydantic import SecretStr, ValidationError
|
|
from sqlalchemy import create_engine, event, select
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from modelforge_api.api.routes.serving import get_serving_service
|
|
from modelforge_api.db import get_session
|
|
from modelforge_api.domain.projects import ProjectCreate, ProjectRequirementCreate
|
|
from modelforge_api.domain.serving import (
|
|
ServiceClientCreate,
|
|
ServiceCredentialRevoke,
|
|
ServiceCredentialRotate,
|
|
)
|
|
from modelforge_api.main import app
|
|
from modelforge_api.persistence.models import (
|
|
AuditEvent,
|
|
Base,
|
|
Project,
|
|
ProjectOnboarding,
|
|
ServiceCredential,
|
|
)
|
|
from modelforge_api.services.manifest_registry import ManifestRegistry
|
|
from modelforge_api.services.projects import ProjectService
|
|
from modelforge_api.services.serving import ServingError, ServingService, _aware
|
|
from modelforge_api.services.transient_payloads import MemoryPayloadStore
|
|
from modelforge_api.settings import Settings, get_settings
|
|
|
|
|
|
@pytest.fixture
|
|
def serving() -> Iterator[ServingService]:
|
|
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
yield ServingService(session, Settings(_env_file=None, env="test"), ManifestRegistry(), MemoryPayloadStore())
|
|
engine.dispose()
|
|
|
|
|
|
def onboard(serving: ServingService):
|
|
projects = ProjectService(serving)
|
|
projects.create(ProjectCreate(key="example-app", name="Example", description="Integration", owner="Alice"))
|
|
project = projects.bind("example-app", ProjectRequirementCreate(capability="rag.embedding@1", migration_support="reindex"))
|
|
return serving.create_client(ServiceClientCreate(
|
|
name="example-reader", project_key=project.key, project_binding_id=project.requirements[0].id,
|
|
allowed_capabilities=["rag.embedding@1"], integration_environment="lab", purpose="Integration check",
|
|
credential_expires_at=datetime.now(UTC) + timedelta(days=30), workload_priority="interactive",
|
|
))
|
|
|
|
|
|
def test_project_identity_binding_safety_and_readiness_are_persistent(serving: ServingService) -> None:
|
|
created = onboard(serving)
|
|
projects = ProjectService(serving)
|
|
project = projects.detail("example-app")
|
|
assert project.owner == "Alice" and project.environment == "lab"
|
|
assert project.production_eligible is None and project.state == "needs_attention"
|
|
requirement = project.requirements[0]
|
|
assert requirement.credential_ready and requirement.deployment_id is None
|
|
assert requirement.capability_checked_at is None
|
|
assert len(requirement.blockers) == 3
|
|
serving.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
|
|
assert projects.detail("example-app").requirements[0].connectivity_checked_at is not None
|
|
assert projects.detail("example-app").state == "needs_attention"
|
|
serving.session.expire_all()
|
|
listed = projects.list()
|
|
assert listed == [projects.detail("example-app")]
|
|
with pytest.raises(ServingError, match="already exists"):
|
|
projects.create(ProjectCreate(key="example-app", name="Duplicate", description="Duplicate", owner="Alice"))
|
|
with pytest.raises(ServingError, match="reindex"):
|
|
projects.bind("example-app", ProjectRequirementCreate(capability="vision.embedding@1"))
|
|
with pytest.raises(ValidationError):
|
|
ProjectRequirementCreate(capability="rag.embedding@1", model_path="/weights")
|
|
|
|
|
|
def test_project_workspace_list_batches_base_reads_as_projects_grow(serving: ServingService) -> None:
|
|
now = datetime.now(UTC)
|
|
|
|
def add_projects(start: int, end: int) -> None:
|
|
for index in range(start, end):
|
|
project = Project(
|
|
key=f"workspace-{index:02d}",
|
|
name=f"Workspace {index}",
|
|
description="Representative project workspace",
|
|
)
|
|
serving.session.add(project)
|
|
serving.session.flush()
|
|
serving.session.add(
|
|
ProjectOnboarding(
|
|
project_id=project.id,
|
|
owner="Owner",
|
|
environment="lab",
|
|
labels=["representative"],
|
|
created_at=now,
|
|
)
|
|
)
|
|
serving.session.commit()
|
|
|
|
def list_with_select_count() -> tuple[int, list[object]]:
|
|
selects: list[str] = []
|
|
|
|
def count_selects(
|
|
_connection: object,
|
|
_cursor: object,
|
|
statement: str,
|
|
_parameters: object,
|
|
_context: object,
|
|
_executemany: bool,
|
|
) -> None:
|
|
if statement.lstrip().upper().startswith("SELECT"):
|
|
selects.append(statement)
|
|
|
|
serving.session.expire_all()
|
|
engine = serving.session.get_bind()
|
|
event.listen(engine, "before_cursor_execute", count_selects)
|
|
try:
|
|
result = ProjectService(serving).list()
|
|
finally:
|
|
event.remove(engine, "before_cursor_execute", count_selects)
|
|
return len(selects), result
|
|
|
|
add_projects(1, 2)
|
|
one_project_selects, one_project_result = list_with_select_count()
|
|
assert len(one_project_result) == 1
|
|
assert one_project_result[0].owner == "Owner"
|
|
assert one_project_result[0] == ProjectService(serving).detail("workspace-01")
|
|
|
|
add_projects(2, 21)
|
|
twenty_project_selects, twenty_project_result = list_with_select_count()
|
|
assert len(twenty_project_result) == 20
|
|
assert [item.key for item in twenty_project_result] == [
|
|
f"workspace-{index:02d}" for index in range(1, 21)
|
|
]
|
|
assert all(item.owner == "Owner" and item.environment == "lab" for item in twenty_project_result)
|
|
assert one_project_selects == twenty_project_selects == 3
|
|
assert [item.key for item in ProjectService(serving).list(limit=2, after_key="workspace-18")] == [
|
|
"workspace-19",
|
|
"workspace-20",
|
|
]
|
|
|
|
|
|
def test_explicit_binding_cannot_cross_projects_and_inactive_projects_stop_serving(serving: ServingService) -> None:
|
|
created = onboard(serving)
|
|
projects = ProjectService(serving)
|
|
projects.create(ProjectCreate(key="other", name="Other", description="Other", owner="Bob"))
|
|
with pytest.raises(ServingError, match="active binding"):
|
|
serving.create_client(ServiceClientCreate(
|
|
name="cross-project", project_key="other", project_binding_id=created.project_binding_id,
|
|
allowed_capabilities=["rag.embedding@1"], integration_environment="lab", purpose="Rejected identity",
|
|
))
|
|
project = serving.session.scalar(select(Project).where(Project.key == "example-app"))
|
|
assert project
|
|
project.active = False
|
|
serving.session.commit()
|
|
with pytest.raises(ServingError, match="inactive"):
|
|
serving.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
|
|
|
|
|
|
def test_rotation_overlap_stale_requests_expiry_and_revoke_all(serving: ServingService) -> None:
|
|
first = onboard(serving)
|
|
assert first.credential_id
|
|
expires = datetime.now(UTC) + timedelta(days=7)
|
|
request = ServiceCredentialRotate(expected_credential_id=first.credential_id, overlap_seconds=900, expires_at=expires)
|
|
second = serving.rotate_client_credential(first.id, request)
|
|
assert serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1").id == first.id
|
|
assert serving.authenticate(f"Bearer {second.credential}", "rag.embedding@1").id == first.id
|
|
assert second.credential_expires_at and _aware(second.credential_expires_at) == expires
|
|
with pytest.raises(ServingError, match="changed"):
|
|
serving.rotate_client_credential(first.id, request)
|
|
assert second.credential_id
|
|
third = serving.rotate_client_credential(first.id, ServiceCredentialRotate(expected_credential_id=second.credential_id, overlap_seconds=900, expires_at=expires))
|
|
with pytest.raises(ServingError, match="revoked"):
|
|
serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1")
|
|
assert third.credential_id
|
|
with pytest.raises(ServingError, match="exact"):
|
|
serving.revoke_client_credential(first.id, ServiceCredentialRevoke(expected_credential_id=third.credential_id, confirmation="wrong"))
|
|
serving.revoke_client_credential(first.id, ServiceCredentialRevoke(expected_credential_id=third.credential_id, confirmation=first.name))
|
|
for token in (first.credential, second.credential, third.credential):
|
|
with pytest.raises(ServingError, match="revoked"):
|
|
serving.authenticate(f"Bearer {token}", "rag.embedding@1")
|
|
assert serving.clients()[0].credential_id is None
|
|
exposed = str(serving.credential_history(first.id)) + str([event.details for event in serving.session.scalars(select(AuditEvent))])
|
|
assert all(token not in exposed for token in (first.credential, second.credential, third.credential))
|
|
|
|
|
|
def test_overlap_never_extends_expiry_and_expired_or_disabled_rotation_refuses(serving: ServingService) -> None:
|
|
first = onboard(serving)
|
|
current = serving.session.get(ServiceCredential, first.credential_id)
|
|
assert current and first.credential_id
|
|
deadline = datetime.now(UTC) + timedelta(seconds=1)
|
|
current.expires_at = deadline
|
|
serving.session.commit()
|
|
serving.rotate_client_credential(first.id, ServiceCredentialRotate(expected_credential_id=first.credential_id, overlap_seconds=86400, expires_at=datetime.now(UTC) + timedelta(days=1)))
|
|
serving.session.refresh(current)
|
|
assert _aware(current.expires_at) == deadline
|
|
current.expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
serving.session.commit()
|
|
with pytest.raises(ServingError, match="expired"):
|
|
serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1")
|
|
|
|
|
|
def test_http_roles_audit_no_store_and_exact_machine_scope(serving: ServingService) -> None:
|
|
tokens = {role: "mfop_" + role[0] * 43 for role in ("viewer", "operator", "admin")}
|
|
settings = Settings(_env_file=None, env="test", operator_identities=SecretStr(json.dumps([
|
|
{"subject": role, "role": role, "credential_id": role, "token_sha256": hashlib.sha256(token.encode()).hexdigest(), "expires_at": (datetime.now(UTC) + timedelta(days=1)).isoformat()}
|
|
for role, token in tokens.items()
|
|
])))
|
|
app.dependency_overrides[get_settings] = lambda: settings
|
|
app.dependency_overrides[get_session] = lambda: serving.session
|
|
app.dependency_overrides[get_serving_service] = lambda: serving
|
|
try:
|
|
http = TestClient(app)
|
|
body = {"key": "api-project", "name": "API", "description": "API project", "owner": "Alice"}
|
|
path = "/api/v1/project-workspace"
|
|
assert http.get(path).status_code == 401
|
|
assert http.get(f"{path}?limit=1").status_code == 401
|
|
empty_page = http.get(f"{path}?limit=1", headers={"X-ModelForge-Admin-Token": tokens["viewer"]})
|
|
assert empty_page.status_code == 200 and empty_page.json() == []
|
|
assert http.post(path, json=body).status_code == 401
|
|
assert http.post(path, json=body, headers={"X-ModelForge-Admin-Token": tokens["viewer"]}).status_code == 403
|
|
response = http.post(path, json=body, headers={"X-ModelForge-Admin-Token": tokens["operator"], "X-Correlation-ID": "project-create"})
|
|
assert response.status_code == 201 and response.headers["cache-control"] == "no-store"
|
|
event = serving.session.scalar(select(AuditEvent).where(AuditEvent.action == "PROJECT_CREATED"))
|
|
assert event and event.actor_id == "operator" and event.correlation_id == "project-create"
|
|
created = onboard(serving)
|
|
check = "/api/v1/capabilities/rag.embedding@1/check"
|
|
headers = {"Authorization": f"Bearer {created.credential}"}
|
|
assert http.get(check, headers=headers).json()["client_id"] == str(created.id)
|
|
assert http.get(check.replace("rag.embedding", "vision.embedding"), headers=headers).status_code == 403
|
|
assert http.get(path, headers=headers).status_code == 401
|
|
assert http.get(check, headers={"X-ModelForge-Admin-Token": tokens["admin"]}).status_code == 401
|
|
assert http.post(f"/api/v1/admin/service-clients/{created.id}/credentials/rotate", json={}, headers={"X-ModelForge-Admin-Token": tokens["operator"]}).status_code == 403
|
|
assert http.get("/api/v1/projects", headers={"X-ModelForge-Admin-Token": tokens["viewer"]}).status_code == 200
|
|
viewer_headers = {"X-ModelForge-Admin-Token": tokens["viewer"]}
|
|
changes_path = f"{path}/changes"
|
|
assert http.get(changes_path).status_code == 401
|
|
empty_changes = http.get(
|
|
f"{changes_path}?project_key=example-app&project_binding_id={created.project_binding_id}"
|
|
"&open_only=true&limit=1&offset=0",
|
|
headers=viewer_headers,
|
|
)
|
|
assert empty_changes.status_code == 200
|
|
assert empty_changes.json() == {"items": [], "total": 0}
|
|
assert http.get(f"{changes_path}?limit=201", headers=viewer_headers).status_code == 422
|
|
assert http.get(f"{changes_path}?offset=-1", headers=viewer_headers).status_code == 422
|
|
legacy_index = http.get(path, headers=viewer_headers)
|
|
first_page = http.get(f"{path}?limit=1", headers=viewer_headers)
|
|
first_key = first_page.json()[0]["key"]
|
|
second_page = http.get(f"{path}?limit=1&after_key={first_key}", headers=viewer_headers)
|
|
last_key = second_page.json()[0]["key"]
|
|
empty_page = http.get(f"{path}?limit=1&after_key={last_key}", headers=viewer_headers)
|
|
assert legacy_index.status_code == 200
|
|
assert [item["key"] for item in legacy_index.json()] == ["api-project", "example-app"]
|
|
assert [item["key"] for item in first_page.json() + second_page.json()] == ["api-project", "example-app"]
|
|
assert set(first_page.json()[0]) == {
|
|
"id", "key", "name", "description", "active", "owner", "environment", "labels",
|
|
"source", "production_eligible", "state", "requirements", "clients",
|
|
}
|
|
assert empty_page.status_code == 200 and empty_page.json() == []
|
|
assert http.get(f"{path}?limit=201", headers=viewer_headers).status_code == 422
|
|
assert http.get(f"{path}?after_key={first_key}", headers=viewer_headers).status_code == 422
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
def test_readiness_requires_current_credential_current_deployment_and_fresh_success(serving: ServingService) -> None:
|
|
from modelforge_api.persistence.models import GatewayRequest
|
|
from tests.test_serving import approve_and_promote, setup_m5
|
|
|
|
service, candidate, _node = setup_m5(serving.session)
|
|
_, deployment = approve_and_promote(service, candidate.id)
|
|
first = onboard(service)
|
|
projects = ProjectService(service)
|
|
service.authenticate("Bearer " + first.credential, "rag.embedding@1")
|
|
record = GatewayRequest(
|
|
request_id=uuid.uuid4(), service_client_id=first.id,
|
|
capability_key="rag.embedding", capability_version=1,
|
|
capability_deployment_id=deployment.id, status="completed", priority="interactive",
|
|
input_sha256="0" * 64, input_count=1,
|
|
decision_evidence={"service_credential_id": str(first.credential_id)}, created_at=datetime.now(UTC),
|
|
)
|
|
service.session.add(record); service.session.commit()
|
|
assert projects.detail("example-app").state == "ready"
|
|
assert first.credential_id
|
|
second = service.rotate_client_credential(first.id, ServiceCredentialRotate(
|
|
expected_credential_id=first.credential_id, overlap_seconds=900,
|
|
expires_at=datetime.now(UTC) + timedelta(days=1),
|
|
))
|
|
service.authenticate("Bearer " + second.credential, "rag.embedding@1")
|
|
record.created_at = datetime.now(UTC)
|
|
service.session.commit()
|
|
assert projects.detail("example-app").state == "needs_attention" # Old overlap cannot qualify new key.
|
|
record.decision_evidence = {"service_credential_id": str(second.credential_id)}
|
|
service.session.commit()
|
|
assert projects.detail("example-app").state == "ready"
|
|
record.created_at = datetime.now(UTC) - timedelta(days=2)
|
|
service.session.commit()
|
|
assert projects.detail("example-app").state == "needs_attention"
|