from __future__ import annotations import hashlib import json import uuid from collections.abc import Iterator from datetime import UTC, datetime, timedelta import pytest from fastapi.testclient import TestClient from pydantic import SecretStr, ValidationError from sqlalchemy import create_engine, event, select from sqlalchemy.orm import Session from sqlalchemy.pool import StaticPool from modelforge_api.api.routes.serving import get_serving_service from modelforge_api.db import get_session from modelforge_api.domain.projects import ProjectCreate, ProjectRequirementCreate from modelforge_api.domain.serving import ( ServiceClientCreate, ServiceCredentialRevoke, ServiceCredentialRotate, ) from modelforge_api.main import app from modelforge_api.persistence.models import ( AuditEvent, Base, Project, ProjectOnboarding, ServiceCredential, ) from modelforge_api.services.manifest_registry import ManifestRegistry from modelforge_api.services.projects import ProjectService from modelforge_api.services.serving import ServingError, ServingService, _aware from modelforge_api.services.transient_payloads import MemoryPayloadStore from modelforge_api.settings import Settings, get_settings @pytest.fixture def serving() -> Iterator[ServingService]: engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) Base.metadata.create_all(engine) with Session(engine) as session: yield ServingService(session, Settings(_env_file=None, env="test"), ManifestRegistry(), MemoryPayloadStore()) engine.dispose() def onboard(serving: ServingService): projects = ProjectService(serving) projects.create(ProjectCreate(key="example-app", name="Example", description="Integration", owner="Alice")) project = projects.bind("example-app", ProjectRequirementCreate(capability="rag.embedding@1", migration_support="reindex")) return serving.create_client(ServiceClientCreate( name="example-reader", project_key=project.key, project_binding_id=project.requirements[0].id, allowed_capabilities=["rag.embedding@1"], integration_environment="lab", purpose="Integration check", credential_expires_at=datetime.now(UTC) + timedelta(days=30), workload_priority="interactive", )) def test_project_identity_binding_safety_and_readiness_are_persistent(serving: ServingService) -> None: created = onboard(serving) projects = ProjectService(serving) project = projects.detail("example-app") assert project.owner == "Alice" and project.environment == "lab" assert project.production_eligible is None and project.state == "needs_attention" requirement = project.requirements[0] assert requirement.credential_ready and requirement.deployment_id is None assert requirement.capability_checked_at is None assert len(requirement.blockers) == 3 serving.authenticate(f"Bearer {created.credential}", "rag.embedding@1") assert projects.detail("example-app").requirements[0].connectivity_checked_at is not None assert projects.detail("example-app").state == "needs_attention" serving.session.expire_all() listed = projects.list() assert listed == [projects.detail("example-app")] with pytest.raises(ServingError, match="already exists"): projects.create(ProjectCreate(key="example-app", name="Duplicate", description="Duplicate", owner="Alice")) with pytest.raises(ServingError, match="reindex"): projects.bind("example-app", ProjectRequirementCreate(capability="vision.embedding@1")) with pytest.raises(ValidationError): ProjectRequirementCreate(capability="rag.embedding@1", model_path="/weights") def test_project_workspace_list_batches_base_reads_as_projects_grow(serving: ServingService) -> None: now = datetime.now(UTC) def add_projects(start: int, end: int) -> None: for index in range(start, end): project = Project( key=f"workspace-{index:02d}", name=f"Workspace {index}", description="Representative project workspace", ) serving.session.add(project) serving.session.flush() serving.session.add( ProjectOnboarding( project_id=project.id, owner="Owner", environment="lab", labels=["representative"], created_at=now, ) ) serving.session.commit() def list_with_select_count() -> tuple[int, list[object]]: selects: list[str] = [] def count_selects( _connection: object, _cursor: object, statement: str, _parameters: object, _context: object, _executemany: bool, ) -> None: if statement.lstrip().upper().startswith("SELECT"): selects.append(statement) serving.session.expire_all() engine = serving.session.get_bind() event.listen(engine, "before_cursor_execute", count_selects) try: result = ProjectService(serving).list() finally: event.remove(engine, "before_cursor_execute", count_selects) return len(selects), result add_projects(1, 2) one_project_selects, one_project_result = list_with_select_count() assert len(one_project_result) == 1 assert one_project_result[0].owner == "Owner" assert one_project_result[0] == ProjectService(serving).detail("workspace-01") add_projects(2, 21) twenty_project_selects, twenty_project_result = list_with_select_count() assert len(twenty_project_result) == 20 assert [item.key for item in twenty_project_result] == [ f"workspace-{index:02d}" for index in range(1, 21) ] assert all(item.owner == "Owner" and item.environment == "lab" for item in twenty_project_result) assert one_project_selects == twenty_project_selects == 3 assert [item.key for item in ProjectService(serving).list(limit=2, after_key="workspace-18")] == [ "workspace-19", "workspace-20", ] def test_explicit_binding_cannot_cross_projects_and_inactive_projects_stop_serving(serving: ServingService) -> None: created = onboard(serving) projects = ProjectService(serving) projects.create(ProjectCreate(key="other", name="Other", description="Other", owner="Bob")) with pytest.raises(ServingError, match="active binding"): serving.create_client(ServiceClientCreate( name="cross-project", project_key="other", project_binding_id=created.project_binding_id, allowed_capabilities=["rag.embedding@1"], integration_environment="lab", purpose="Rejected identity", )) project = serving.session.scalar(select(Project).where(Project.key == "example-app")) assert project project.active = False serving.session.commit() with pytest.raises(ServingError, match="inactive"): serving.authenticate(f"Bearer {created.credential}", "rag.embedding@1") def test_rotation_overlap_stale_requests_expiry_and_revoke_all(serving: ServingService) -> None: first = onboard(serving) assert first.credential_id expires = datetime.now(UTC) + timedelta(days=7) request = ServiceCredentialRotate(expected_credential_id=first.credential_id, overlap_seconds=900, expires_at=expires) second = serving.rotate_client_credential(first.id, request) assert serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1").id == first.id assert serving.authenticate(f"Bearer {second.credential}", "rag.embedding@1").id == first.id assert second.credential_expires_at and _aware(second.credential_expires_at) == expires with pytest.raises(ServingError, match="changed"): serving.rotate_client_credential(first.id, request) assert second.credential_id third = serving.rotate_client_credential(first.id, ServiceCredentialRotate(expected_credential_id=second.credential_id, overlap_seconds=900, expires_at=expires)) with pytest.raises(ServingError, match="revoked"): serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1") assert third.credential_id with pytest.raises(ServingError, match="exact"): serving.revoke_client_credential(first.id, ServiceCredentialRevoke(expected_credential_id=third.credential_id, confirmation="wrong")) serving.revoke_client_credential(first.id, ServiceCredentialRevoke(expected_credential_id=third.credential_id, confirmation=first.name)) for token in (first.credential, second.credential, third.credential): with pytest.raises(ServingError, match="revoked"): serving.authenticate(f"Bearer {token}", "rag.embedding@1") assert serving.clients()[0].credential_id is None exposed = str(serving.credential_history(first.id)) + str([event.details for event in serving.session.scalars(select(AuditEvent))]) assert all(token not in exposed for token in (first.credential, second.credential, third.credential)) def test_overlap_never_extends_expiry_and_expired_or_disabled_rotation_refuses(serving: ServingService) -> None: first = onboard(serving) current = serving.session.get(ServiceCredential, first.credential_id) assert current and first.credential_id deadline = datetime.now(UTC) + timedelta(seconds=1) current.expires_at = deadline serving.session.commit() serving.rotate_client_credential(first.id, ServiceCredentialRotate(expected_credential_id=first.credential_id, overlap_seconds=86400, expires_at=datetime.now(UTC) + timedelta(days=1))) serving.session.refresh(current) assert _aware(current.expires_at) == deadline current.expires_at = datetime.now(UTC) - timedelta(seconds=1) serving.session.commit() with pytest.raises(ServingError, match="expired"): serving.authenticate(f"Bearer {first.credential}", "rag.embedding@1") def test_http_roles_audit_no_store_and_exact_machine_scope(serving: ServingService) -> None: tokens = {role: "mfop_" + role[0] * 43 for role in ("viewer", "operator", "admin")} settings = Settings(_env_file=None, env="test", operator_identities=SecretStr(json.dumps([ {"subject": role, "role": role, "credential_id": role, "token_sha256": hashlib.sha256(token.encode()).hexdigest(), "expires_at": (datetime.now(UTC) + timedelta(days=1)).isoformat()} for role, token in tokens.items() ]))) app.dependency_overrides[get_settings] = lambda: settings app.dependency_overrides[get_session] = lambda: serving.session app.dependency_overrides[get_serving_service] = lambda: serving try: http = TestClient(app) body = {"key": "api-project", "name": "API", "description": "API project", "owner": "Alice"} path = "/api/v1/project-workspace" assert http.get(path).status_code == 401 assert http.get(f"{path}?limit=1").status_code == 401 empty_page = http.get(f"{path}?limit=1", headers={"X-ModelForge-Admin-Token": tokens["viewer"]}) assert empty_page.status_code == 200 and empty_page.json() == [] assert http.post(path, json=body).status_code == 401 assert http.post(path, json=body, headers={"X-ModelForge-Admin-Token": tokens["viewer"]}).status_code == 403 response = http.post(path, json=body, headers={"X-ModelForge-Admin-Token": tokens["operator"], "X-Correlation-ID": "project-create"}) assert response.status_code == 201 and response.headers["cache-control"] == "no-store" event = serving.session.scalar(select(AuditEvent).where(AuditEvent.action == "PROJECT_CREATED")) assert event and event.actor_id == "operator" and event.correlation_id == "project-create" created = onboard(serving) check = "/api/v1/capabilities/rag.embedding@1/check" headers = {"Authorization": f"Bearer {created.credential}"} assert http.get(check, headers=headers).json()["client_id"] == str(created.id) assert http.get(check.replace("rag.embedding", "vision.embedding"), headers=headers).status_code == 403 assert http.get(path, headers=headers).status_code == 401 assert http.get(check, headers={"X-ModelForge-Admin-Token": tokens["admin"]}).status_code == 401 assert http.post(f"/api/v1/admin/service-clients/{created.id}/credentials/rotate", json={}, headers={"X-ModelForge-Admin-Token": tokens["operator"]}).status_code == 403 assert http.get("/api/v1/projects", headers={"X-ModelForge-Admin-Token": tokens["viewer"]}).status_code == 200 viewer_headers = {"X-ModelForge-Admin-Token": tokens["viewer"]} changes_path = f"{path}/changes" assert http.get(changes_path).status_code == 401 empty_changes = http.get( f"{changes_path}?project_key=example-app&project_binding_id={created.project_binding_id}" "&open_only=true&limit=1&offset=0", headers=viewer_headers, ) assert empty_changes.status_code == 200 assert empty_changes.json() == {"items": [], "total": 0} assert http.get(f"{changes_path}?limit=201", headers=viewer_headers).status_code == 422 assert http.get(f"{changes_path}?offset=-1", headers=viewer_headers).status_code == 422 legacy_index = http.get(path, headers=viewer_headers) first_page = http.get(f"{path}?limit=1", headers=viewer_headers) first_key = first_page.json()[0]["key"] second_page = http.get(f"{path}?limit=1&after_key={first_key}", headers=viewer_headers) last_key = second_page.json()[0]["key"] empty_page = http.get(f"{path}?limit=1&after_key={last_key}", headers=viewer_headers) assert legacy_index.status_code == 200 assert [item["key"] for item in legacy_index.json()] == ["api-project", "example-app"] assert [item["key"] for item in first_page.json() + second_page.json()] == ["api-project", "example-app"] assert set(first_page.json()[0]) == { "id", "key", "name", "description", "active", "owner", "environment", "labels", "source", "production_eligible", "state", "requirements", "clients", } assert empty_page.status_code == 200 and empty_page.json() == [] assert http.get(f"{path}?limit=201", headers=viewer_headers).status_code == 422 assert http.get(f"{path}?after_key={first_key}", headers=viewer_headers).status_code == 422 finally: app.dependency_overrides.clear() def test_readiness_requires_current_credential_current_deployment_and_fresh_success(serving: ServingService) -> None: from modelforge_api.persistence.models import GatewayRequest from tests.test_serving import approve_and_promote, setup_m5 service, candidate, _node = setup_m5(serving.session) _, deployment = approve_and_promote(service, candidate.id) first = onboard(service) projects = ProjectService(service) service.authenticate("Bearer " + first.credential, "rag.embedding@1") record = GatewayRequest( request_id=uuid.uuid4(), service_client_id=first.id, capability_key="rag.embedding", capability_version=1, capability_deployment_id=deployment.id, status="completed", priority="interactive", input_sha256="0" * 64, input_count=1, decision_evidence={"service_credential_id": str(first.credential_id)}, created_at=datetime.now(UTC), ) service.session.add(record); service.session.commit() assert projects.detail("example-app").state == "ready" assert first.credential_id second = service.rotate_client_credential(first.id, ServiceCredentialRotate( expected_credential_id=first.credential_id, overlap_seconds=900, expires_at=datetime.now(UTC) + timedelta(days=1), )) service.authenticate("Bearer " + second.credential, "rag.embedding@1") record.created_at = datetime.now(UTC) service.session.commit() assert projects.detail("example-app").state == "needs_attention" # Old overlap cannot qualify new key. record.decision_evidence = {"service_credential_id": str(second.credential_id)} service.session.commit() assert projects.detail("example-app").state == "ready" record.created_at = datetime.now(UTC) - timedelta(days=2) service.session.commit() assert projects.detail("example-app").state == "needs_attention"