Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/tests/test_lab_preflight.py

@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+import uuid
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import pytest
+from pydantic import ValidationError
+from sqlalchemy import create_engine, func, select, update
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.lab import LabPreflightRequest
+from modelforge_api.persistence.models import Base, DeploymentCandidate, RuntimeProfile
+from modelforge_api.services.lab import FAMILY_MODALITIES, LabPreflightService
+from tests.test_serving import setup_m5
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    with Session(engine) as db:
+        yield db
+    engine.dispose()
+
+
+INPUTS: dict[str, dict[str, Any]] = {
+    "rag.embedding": {"input": "private input content"},
+    "rag.reranking": {
+        "query": "private query",
+        "documents": [{"id": "a", "text": "private document"}],
+        "top_n": 1,
+    },
+    "document.ocr": {"content_base64": "cHJpdmF0ZQ==", "media_type": "image/png"},
+    "vision.embedding": {"items": [{"text": "private visual text"}]},
+    "speech.transcription": {"audio_base64": "cHJpdmF0ZQ=="},
+}
+
+
+def setup_lab(
+    db: Session, capability: str = "rag.embedding"
+) -> tuple[LabPreflightService, LabPreflightRequest]:
+    serving, candidate_response, _ = setup_m5(db)
+    svc = LabPreflightService(db, serving.settings)
+    candidate = db.get(DeploymentCandidate, candidate_response.id)
+    assert candidate is not None
+    profile = svc.runtime.repo.profile(candidate.runtime_profile_id)
+    probe = svc.runtime.repo.probe(candidate.runtime_probe_id)
+    assert profile is not None and probe is not None
+    # Synthetic completed family evidence, not a hardware qualification.
+    # Construct synthetic family fixture below ORM sealing; production immutability remains tested elsewhere.
+    db.execute(
+        update(RuntimeProfile)
+        .where(RuntimeProfile.id == profile.id)
+        .values(modality=FAMILY_MODALITIES[capability])
+    )
+    if capability == "rag.reranking":
+        probe.inference_result = {"output_type": "ranked_scores", "finite": True}
+        probe.runtime_facts = {**probe.runtime_facts, "adapter": "qwen3_reranker"}
+    elif capability in {"document.ocr", "speech.transcription"}:
+        probe.inference_result = {
+            "output_type": "ocr_text" if capability == "document.ocr" else "transcription",
+            "output_valid": True,
+        }
+    db.commit()
+    request = LabPreflightRequest.model_validate(
+        {
+            "target": {
+                "deployment_candidate_id": candidate.id,
+                "artifact_set_id": candidate.artifact_set_id,
+                "runtime_profile_id": candidate.runtime_profile_id,
+                "compute_node_id": candidate.compute_node_id,
+                "execution_approval_id": probe.execution_approval_id,
+            },
+            "input": {"capability": capability, "payload": INPUTS[capability]},
+        }
+    )
+    return svc, request
+
+
+def table_counts(db: Session) -> dict[str, int]:
+    return {
+        name: db.scalar(select(func.count()).select_from(table)) or 0
+        for name, table in Base.metadata.tables.items()
+    }
+
+
+@pytest.mark.parametrize("capability", INPUTS)
+def test_all_families_return_redacted_read_only_preflight(
+    session: Session, capability: str
+) -> None:
+    svc, request = setup_lab(session, capability)
+    before = table_counts(session)
+    response = svc.preflight(request)
+    assert response.checks_passed, response.blockers
+    assert response.binding is not None
+    assert response.binding.deployment_candidate_id == request.target.deployment_candidate_id
+    assert response.binding.capability == capability
+    assert response.permission_granted is False and response.execution_started is False
+    assert len(response.input_summary.sha256) == 64
+    assert response.input_summary.item_count == 1
+    assert response.input_summary.encoded_bytes > 0
+    encoded = response.model_dump_json()
+    assert "private" not in encoded and "cHJpdmF0ZQ==" not in encoded
+    assert "Reviewed exact" not in encoded
+    assert table_counts(session) == before
+    assert not session.new and not session.dirty and not session.deleted
+
+
+@pytest.mark.parametrize(
+    "field",
+    [
+        "deployment_candidate_id",
+        "artifact_set_id",
+        "runtime_profile_id",
+        "compute_node_id",
+        "execution_approval_id",
+    ],
+)
+def test_target_ids_never_select_another_entity(session: Session, field: str) -> None:
+    svc, request = setup_lab(session)
+    data = request.model_dump()
+    data["target"][field] = uuid.uuid4()
+    response = svc.preflight(LabPreflightRequest.model_validate(data))
+    assert not response.checks_passed and response.binding is None
+
+
+@pytest.mark.parametrize(
+    "change,code",
+    [
+        ("revoked", "LAB_APPROVAL_STALE"),
+        ("expired", "LAB_APPROVAL_STALE"),
+        ("approval_fingerprint", "LAB_APPROVAL_STALE"),
+        ("assessment", "ASSESSMENT_STALE_OR_BLOCKED"),
+        ("profile_fingerprint", "ASSESSMENT_STALE_OR_BLOCKED"),
+        ("production", "CANDIDATE_NOT_LAB_READY"),
+        ("channel", "CANDIDATE_NOT_LAB_READY"),
+        ("offline", "NODE_NOT_READY"),
+        ("lab_ineligible", "NODE_NOT_READY"),
+        ("assessment_blocker", "ASSESSMENT_STALE_OR_BLOCKED"),
+        ("assessment_status", "ASSESSMENT_STALE_OR_BLOCKED"),
+        ("node_protocol", "NODE_NOT_READY"),
+        ("modality", "CAPABILITY_PROFILE_MISMATCH"),
+        ("probe", "PROBE_NOT_COMPLETED"),
+        ("probe_identity", "EVIDENCE_IDENTITY_MISMATCH"),
+        ("invalid_output", "PROBE_OUTPUT_INVALID"),
+        ("wrong_dimension", "PROBE_OUTPUT_INVALID"),
+        ("gpu", "PROBED_ACCELERATOR_MISSING"),
+        ("remote_code", "RUNTIME_POLICY_UNSUPPORTED"),
+    ],
+)
+def test_stale_and_incoherent_evidence_blocks(session: Session, change: str, code: str) -> None:
+    svc, request = setup_lab(session)
+    candidate = session.get(DeploymentCandidate, request.target.deployment_candidate_id)
+    assert candidate is not None
+    repo = svc.runtime.repo
+    approval = repo.approval(request.target.execution_approval_id)
+    assessment = repo.assessment(candidate.compatibility_assessment_id)
+    profile = repo.profile(candidate.runtime_profile_id)
+    node = repo.node(candidate.compute_node_id)
+    probe = repo.probe(candidate.runtime_probe_id)
+    assert approval and assessment and profile and node and probe
+    if change == "revoked":
+        approval.revoked_at = datetime.now(UTC)
+    elif change == "expired":
+        approval.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    elif change == "approval_fingerprint":
+        approval.evidence_fingerprint = "0" * 64
+    elif change == "assessment":
+        assessment.stale = True
+    elif change == "profile_fingerprint":
+        session.execute(
+            update(RuntimeProfile)
+            .where(RuntimeProfile.id == profile.id)
+            .values(fingerprint="0" * 64)
+        )
+    elif change == "production":
+        candidate.production = True
+    elif change == "channel":
+        candidate.channel = "stable"
+    elif change == "offline":
+        node.liveness_state = "offline"
+    elif change == "lab_ineligible":
+        node.lab_eligible = False
+    elif change == "assessment_blocker":
+        assessment.blockers = ["private reason"]
+    elif change == "assessment_status":
+        assessment.status = "future_unknown"
+    elif change == "node_protocol":
+        node.agent_capabilities = []
+    elif change == "modality":
+        session.execute(
+            update(RuntimeProfile).where(RuntimeProfile.id == profile.id).values(modality="audio")
+        )
+    elif change == "probe":
+        probe.status = "running"
+    elif change == "probe_identity":
+        probe.execution_approval_id = uuid.uuid4()
+    elif change == "invalid_output":
+        probe.inference_result = {"dimension": True, "finite": True}
+    elif change == "wrong_dimension":
+        probe.inference_result = {"dimension": 768, "finite": True}
+    elif change == "gpu":
+        probe.runtime_facts = {"gpu_uuid": "missing"}
+    elif change == "remote_code":
+        session.execute(
+            update(RuntimeProfile)
+            .where(RuntimeProfile.id == profile.id)
+            .values(trust_remote_code=True)
+        )
+    session.commit()
+    response = svc.preflight(request)
+    assert not response.checks_passed and code in response.blockers
+    assert not response.execution_started and not response.permission_granted
+
+
+@pytest.mark.parametrize(
+    "capability,payload",
+    [
+        ("rag.embedding", {"input": []}),
+        ("rag.reranking", {"query": "x", "documents": [], "top_n": 1}),
+        ("document.ocr", {"content_base64": "!!!", "media_type": "image/png"}),
+        (
+            "vision.embedding",
+            {"items": [{"text": "x", "image_base64": "eA==", "media_type": "image/png"}]},
+        ),
+        ("speech.transcription", {"audio_base64": "!!!"}),
+    ],
+)
+def test_malformed_family_input_rejected(
+    session: Session, capability: str, payload: dict[str, Any]
+) -> None:
+    _, request = setup_lab(session)
+    data = request.model_dump()
+    data["input"] = {"capability": capability, "payload": payload}
+    with pytest.raises(ValidationError):
+        LabPreflightRequest.model_validate(data)
+
+
+@pytest.mark.parametrize(
+    "limits",
+    [
+        {"timeout_seconds": 301},
+        {"timeout_seconds": 4},
+        {"timeout_seconds": 5, "queue_timeout_seconds": 6},
+        {"queue_timeout_seconds": 0},
+        {"retention_seconds": 3601},
+        {"max_output_bytes": 1048577},
+        {"concurrency": 2},
+        {"timeout_seconds": True},
+    ],
+)
+def test_budget_limits_are_bounded(session: Session, limits: dict[str, Any]) -> None:
+    _, request = setup_lab(session)
+    data = request.model_dump()
+    data["limits"].update(limits)
+    with pytest.raises(ValidationError):
+        LabPreflightRequest.model_validate(data)
+
+
+def test_embedding_server_limits_and_payload_limit(session: Session) -> None:
+    svc, request = setup_lab(session)
+    data = request.model_dump()
+    data["input"]["payload"]["input"] = ["x" * 9000] * 9
+    response = svc.preflight(LabPreflightRequest.model_validate(data))
+    assert {"INPUT_LIMIT_EXCEEDED", "PAYLOAD_LIMIT_EXCEEDED"}.issubset(response.blockers)
+
+
+def test_preflight_rechecks_evidence_and_does_not_accept_reason(session: Session) -> None:
+    svc, request = setup_lab(session)
+    assert svc.preflight(request).checks_passed
+    approval = svc.runtime.repo.approval(request.target.execution_approval_id)
+    assert approval is not None
+    approval.revoked_at = datetime.now(UTC)
+    session.commit()
+    assert "LAB_APPROVAL_STALE" in svc.preflight(request).blockers
+    with pytest.raises(ValidationError):
+        LabPreflightRequest.model_validate({**request.model_dump(), "reason": "secret reason"})
+
+
+def test_preflight_uses_injected_manifest_root(session: Session, tmp_path: Path) -> None:
+    svc, request = setup_lab(session)
+    isolated_settings = svc.settings.model_copy(update={"config_root": tmp_path})
+    response = LabPreflightService(session, isolated_settings).preflight(request)
+    assert "CAPABILITY_CONTRACT_UNAVAILABLE" in response.blockers
+    assert not response.checks_passed