from __future__ import annotations import uuid from collections.abc import Iterator from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import pytest from pydantic import ValidationError from sqlalchemy import create_engine, func, select, update from sqlalchemy.orm import Session from modelforge_api.domain.lab import LabPreflightRequest from modelforge_api.persistence.models import Base, DeploymentCandidate, RuntimeProfile from modelforge_api.services.lab import FAMILY_MODALITIES, LabPreflightService from tests.test_serving import setup_m5 @pytest.fixture def session() -> Iterator[Session]: engine = create_engine("sqlite+pysqlite:///:memory:") Base.metadata.create_all(engine) with Session(engine) as db: yield db engine.dispose() INPUTS: dict[str, dict[str, Any]] = { "rag.embedding": {"input": "private input content"}, "rag.reranking": { "query": "private query", "documents": [{"id": "a", "text": "private document"}], "top_n": 1, }, "document.ocr": {"content_base64": "cHJpdmF0ZQ==", "media_type": "image/png"}, "vision.embedding": {"items": [{"text": "private visual text"}]}, "speech.transcription": {"audio_base64": "cHJpdmF0ZQ=="}, } def setup_lab( db: Session, capability: str = "rag.embedding" ) -> tuple[LabPreflightService, LabPreflightRequest]: serving, candidate_response, _ = setup_m5(db) svc = LabPreflightService(db, serving.settings) candidate = db.get(DeploymentCandidate, candidate_response.id) assert candidate is not None profile = svc.runtime.repo.profile(candidate.runtime_profile_id) probe = svc.runtime.repo.probe(candidate.runtime_probe_id) assert profile is not None and probe is not None # Synthetic completed family evidence, not a hardware qualification. # Construct synthetic family fixture below ORM sealing; production immutability remains tested elsewhere. db.execute( update(RuntimeProfile) .where(RuntimeProfile.id == profile.id) .values(modality=FAMILY_MODALITIES[capability]) ) if capability == "rag.reranking": probe.inference_result = {"output_type": "ranked_scores", "finite": True} probe.runtime_facts = {**probe.runtime_facts, "adapter": "qwen3_reranker"} elif capability in {"document.ocr", "speech.transcription"}: probe.inference_result = { "output_type": "ocr_text" if capability == "document.ocr" else "transcription", "output_valid": True, } db.commit() request = LabPreflightRequest.model_validate( { "target": { "deployment_candidate_id": candidate.id, "artifact_set_id": candidate.artifact_set_id, "runtime_profile_id": candidate.runtime_profile_id, "compute_node_id": candidate.compute_node_id, "execution_approval_id": probe.execution_approval_id, }, "input": {"capability": capability, "payload": INPUTS[capability]}, } ) return svc, request def table_counts(db: Session) -> dict[str, int]: return { name: db.scalar(select(func.count()).select_from(table)) or 0 for name, table in Base.metadata.tables.items() } @pytest.mark.parametrize("capability", INPUTS) def test_all_families_return_redacted_read_only_preflight( session: Session, capability: str ) -> None: svc, request = setup_lab(session, capability) before = table_counts(session) response = svc.preflight(request) assert response.checks_passed, response.blockers assert response.binding is not None assert response.binding.deployment_candidate_id == request.target.deployment_candidate_id assert response.binding.capability == capability assert response.permission_granted is False and response.execution_started is False assert len(response.input_summary.sha256) == 64 assert response.input_summary.item_count == 1 assert response.input_summary.encoded_bytes > 0 encoded = response.model_dump_json() assert "private" not in encoded and "cHJpdmF0ZQ==" not in encoded assert "Reviewed exact" not in encoded assert table_counts(session) == before assert not session.new and not session.dirty and not session.deleted @pytest.mark.parametrize( "field", [ "deployment_candidate_id", "artifact_set_id", "runtime_profile_id", "compute_node_id", "execution_approval_id", ], ) def test_target_ids_never_select_another_entity(session: Session, field: str) -> None: svc, request = setup_lab(session) data = request.model_dump() data["target"][field] = uuid.uuid4() response = svc.preflight(LabPreflightRequest.model_validate(data)) assert not response.checks_passed and response.binding is None @pytest.mark.parametrize( "change,code", [ ("revoked", "LAB_APPROVAL_STALE"), ("expired", "LAB_APPROVAL_STALE"), ("approval_fingerprint", "LAB_APPROVAL_STALE"), ("assessment", "ASSESSMENT_STALE_OR_BLOCKED"), ("profile_fingerprint", "ASSESSMENT_STALE_OR_BLOCKED"), ("production", "CANDIDATE_NOT_LAB_READY"), ("channel", "CANDIDATE_NOT_LAB_READY"), ("offline", "NODE_NOT_READY"), ("lab_ineligible", "NODE_NOT_READY"), ("assessment_blocker", "ASSESSMENT_STALE_OR_BLOCKED"), ("assessment_status", "ASSESSMENT_STALE_OR_BLOCKED"), ("node_protocol", "NODE_NOT_READY"), ("modality", "CAPABILITY_PROFILE_MISMATCH"), ("probe", "PROBE_NOT_COMPLETED"), ("probe_identity", "EVIDENCE_IDENTITY_MISMATCH"), ("invalid_output", "PROBE_OUTPUT_INVALID"), ("wrong_dimension", "PROBE_OUTPUT_INVALID"), ("gpu", "PROBED_ACCELERATOR_MISSING"), ("remote_code", "RUNTIME_POLICY_UNSUPPORTED"), ], ) def test_stale_and_incoherent_evidence_blocks(session: Session, change: str, code: str) -> None: svc, request = setup_lab(session) candidate = session.get(DeploymentCandidate, request.target.deployment_candidate_id) assert candidate is not None repo = svc.runtime.repo approval = repo.approval(request.target.execution_approval_id) assessment = repo.assessment(candidate.compatibility_assessment_id) profile = repo.profile(candidate.runtime_profile_id) node = repo.node(candidate.compute_node_id) probe = repo.probe(candidate.runtime_probe_id) assert approval and assessment and profile and node and probe if change == "revoked": approval.revoked_at = datetime.now(UTC) elif change == "expired": approval.expires_at = datetime.now(UTC) - timedelta(seconds=1) elif change == "approval_fingerprint": approval.evidence_fingerprint = "0" * 64 elif change == "assessment": assessment.stale = True elif change == "profile_fingerprint": session.execute( update(RuntimeProfile) .where(RuntimeProfile.id == profile.id) .values(fingerprint="0" * 64) ) elif change == "production": candidate.production = True elif change == "channel": candidate.channel = "stable" elif change == "offline": node.liveness_state = "offline" elif change == "lab_ineligible": node.lab_eligible = False elif change == "assessment_blocker": assessment.blockers = ["private reason"] elif change == "assessment_status": assessment.status = "future_unknown" elif change == "node_protocol": node.agent_capabilities = [] elif change == "modality": session.execute( update(RuntimeProfile).where(RuntimeProfile.id == profile.id).values(modality="audio") ) elif change == "probe": probe.status = "running" elif change == "probe_identity": probe.execution_approval_id = uuid.uuid4() elif change == "invalid_output": probe.inference_result = {"dimension": True, "finite": True} elif change == "wrong_dimension": probe.inference_result = {"dimension": 768, "finite": True} elif change == "gpu": probe.runtime_facts = {"gpu_uuid": "missing"} elif change == "remote_code": session.execute( update(RuntimeProfile) .where(RuntimeProfile.id == profile.id) .values(trust_remote_code=True) ) session.commit() response = svc.preflight(request) assert not response.checks_passed and code in response.blockers assert not response.execution_started and not response.permission_granted @pytest.mark.parametrize( "capability,payload", [ ("rag.embedding", {"input": []}), ("rag.reranking", {"query": "x", "documents": [], "top_n": 1}), ("document.ocr", {"content_base64": "!!!", "media_type": "image/png"}), ( "vision.embedding", {"items": [{"text": "x", "image_base64": "eA==", "media_type": "image/png"}]}, ), ("speech.transcription", {"audio_base64": "!!!"}), ], ) def test_malformed_family_input_rejected( session: Session, capability: str, payload: dict[str, Any] ) -> None: _, request = setup_lab(session) data = request.model_dump() data["input"] = {"capability": capability, "payload": payload} with pytest.raises(ValidationError): LabPreflightRequest.model_validate(data) @pytest.mark.parametrize( "limits", [ {"timeout_seconds": 301}, {"timeout_seconds": 4}, {"timeout_seconds": 5, "queue_timeout_seconds": 6}, {"queue_timeout_seconds": 0}, {"retention_seconds": 3601}, {"max_output_bytes": 1048577}, {"concurrency": 2}, {"timeout_seconds": True}, ], ) def test_budget_limits_are_bounded(session: Session, limits: dict[str, Any]) -> None: _, request = setup_lab(session) data = request.model_dump() data["limits"].update(limits) with pytest.raises(ValidationError): LabPreflightRequest.model_validate(data) def test_embedding_server_limits_and_payload_limit(session: Session) -> None: svc, request = setup_lab(session) data = request.model_dump() data["input"]["payload"]["input"] = ["x" * 9000] * 9 response = svc.preflight(LabPreflightRequest.model_validate(data)) assert {"INPUT_LIMIT_EXCEEDED", "PAYLOAD_LIMIT_EXCEEDED"}.issubset(response.blockers) def test_preflight_rechecks_evidence_and_does_not_accept_reason(session: Session) -> None: svc, request = setup_lab(session) assert svc.preflight(request).checks_passed approval = svc.runtime.repo.approval(request.target.execution_approval_id) assert approval is not None approval.revoked_at = datetime.now(UTC) session.commit() assert "LAB_APPROVAL_STALE" in svc.preflight(request).blockers with pytest.raises(ValidationError): LabPreflightRequest.model_validate({**request.model_dump(), "reason": "secret reason"}) def test_preflight_uses_injected_manifest_root(session: Session, tmp_path: Path) -> None: svc, request = setup_lab(session) isolated_settings = svc.settings.model_copy(update={"config_root": tmp_path}) response = LabPreflightService(session, isolated_settings).preflight(request) assert "CAPABILITY_CONTRACT_UNAVAILABLE" in response.blockers assert not response.checks_passed