289 lines
11 KiB
Python
289 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import uuid
|
|
from collections.abc import Iterator
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
from sqlalchemy import create_engine, func, select, update
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modelforge_api.domain.lab import LabPreflightRequest
|
|
from modelforge_api.persistence.models import Base, DeploymentCandidate, RuntimeProfile
|
|
from modelforge_api.services.lab import FAMILY_MODALITIES, LabPreflightService
|
|
from tests.test_serving import setup_m5
|
|
|
|
|
|
@pytest.fixture
|
|
def session() -> Iterator[Session]:
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as db:
|
|
yield db
|
|
engine.dispose()
|
|
|
|
|
|
INPUTS: dict[str, dict[str, Any]] = {
|
|
"rag.embedding": {"input": "private input content"},
|
|
"rag.reranking": {
|
|
"query": "private query",
|
|
"documents": [{"id": "a", "text": "private document"}],
|
|
"top_n": 1,
|
|
},
|
|
"document.ocr": {"content_base64": "cHJpdmF0ZQ==", "media_type": "image/png"},
|
|
"vision.embedding": {"items": [{"text": "private visual text"}]},
|
|
"speech.transcription": {"audio_base64": "cHJpdmF0ZQ=="},
|
|
}
|
|
|
|
|
|
def setup_lab(
|
|
db: Session, capability: str = "rag.embedding"
|
|
) -> tuple[LabPreflightService, LabPreflightRequest]:
|
|
serving, candidate_response, _ = setup_m5(db)
|
|
svc = LabPreflightService(db, serving.settings)
|
|
candidate = db.get(DeploymentCandidate, candidate_response.id)
|
|
assert candidate is not None
|
|
profile = svc.runtime.repo.profile(candidate.runtime_profile_id)
|
|
probe = svc.runtime.repo.probe(candidate.runtime_probe_id)
|
|
assert profile is not None and probe is not None
|
|
# Synthetic completed family evidence, not a hardware qualification.
|
|
# Construct synthetic family fixture below ORM sealing; production immutability remains tested elsewhere.
|
|
db.execute(
|
|
update(RuntimeProfile)
|
|
.where(RuntimeProfile.id == profile.id)
|
|
.values(modality=FAMILY_MODALITIES[capability])
|
|
)
|
|
if capability == "rag.reranking":
|
|
probe.inference_result = {"output_type": "ranked_scores", "finite": True}
|
|
probe.runtime_facts = {**probe.runtime_facts, "adapter": "qwen3_reranker"}
|
|
elif capability in {"document.ocr", "speech.transcription"}:
|
|
probe.inference_result = {
|
|
"output_type": "ocr_text" if capability == "document.ocr" else "transcription",
|
|
"output_valid": True,
|
|
}
|
|
db.commit()
|
|
request = LabPreflightRequest.model_validate(
|
|
{
|
|
"target": {
|
|
"deployment_candidate_id": candidate.id,
|
|
"artifact_set_id": candidate.artifact_set_id,
|
|
"runtime_profile_id": candidate.runtime_profile_id,
|
|
"compute_node_id": candidate.compute_node_id,
|
|
"execution_approval_id": probe.execution_approval_id,
|
|
},
|
|
"input": {"capability": capability, "payload": INPUTS[capability]},
|
|
}
|
|
)
|
|
return svc, request
|
|
|
|
|
|
def table_counts(db: Session) -> dict[str, int]:
|
|
return {
|
|
name: db.scalar(select(func.count()).select_from(table)) or 0
|
|
for name, table in Base.metadata.tables.items()
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("capability", INPUTS)
|
|
def test_all_families_return_redacted_read_only_preflight(
|
|
session: Session, capability: str
|
|
) -> None:
|
|
svc, request = setup_lab(session, capability)
|
|
before = table_counts(session)
|
|
response = svc.preflight(request)
|
|
assert response.checks_passed, response.blockers
|
|
assert response.binding is not None
|
|
assert response.binding.deployment_candidate_id == request.target.deployment_candidate_id
|
|
assert response.binding.capability == capability
|
|
assert response.permission_granted is False and response.execution_started is False
|
|
assert len(response.input_summary.sha256) == 64
|
|
assert response.input_summary.item_count == 1
|
|
assert response.input_summary.encoded_bytes > 0
|
|
encoded = response.model_dump_json()
|
|
assert "private" not in encoded and "cHJpdmF0ZQ==" not in encoded
|
|
assert "Reviewed exact" not in encoded
|
|
assert table_counts(session) == before
|
|
assert not session.new and not session.dirty and not session.deleted
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"field",
|
|
[
|
|
"deployment_candidate_id",
|
|
"artifact_set_id",
|
|
"runtime_profile_id",
|
|
"compute_node_id",
|
|
"execution_approval_id",
|
|
],
|
|
)
|
|
def test_target_ids_never_select_another_entity(session: Session, field: str) -> None:
|
|
svc, request = setup_lab(session)
|
|
data = request.model_dump()
|
|
data["target"][field] = uuid.uuid4()
|
|
response = svc.preflight(LabPreflightRequest.model_validate(data))
|
|
assert not response.checks_passed and response.binding is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change,code",
|
|
[
|
|
("revoked", "LAB_APPROVAL_STALE"),
|
|
("expired", "LAB_APPROVAL_STALE"),
|
|
("approval_fingerprint", "LAB_APPROVAL_STALE"),
|
|
("assessment", "ASSESSMENT_STALE_OR_BLOCKED"),
|
|
("profile_fingerprint", "ASSESSMENT_STALE_OR_BLOCKED"),
|
|
("production", "CANDIDATE_NOT_LAB_READY"),
|
|
("channel", "CANDIDATE_NOT_LAB_READY"),
|
|
("offline", "NODE_NOT_READY"),
|
|
("lab_ineligible", "NODE_NOT_READY"),
|
|
("assessment_blocker", "ASSESSMENT_STALE_OR_BLOCKED"),
|
|
("assessment_status", "ASSESSMENT_STALE_OR_BLOCKED"),
|
|
("node_protocol", "NODE_NOT_READY"),
|
|
("modality", "CAPABILITY_PROFILE_MISMATCH"),
|
|
("probe", "PROBE_NOT_COMPLETED"),
|
|
("probe_identity", "EVIDENCE_IDENTITY_MISMATCH"),
|
|
("invalid_output", "PROBE_OUTPUT_INVALID"),
|
|
("wrong_dimension", "PROBE_OUTPUT_INVALID"),
|
|
("gpu", "PROBED_ACCELERATOR_MISSING"),
|
|
("remote_code", "RUNTIME_POLICY_UNSUPPORTED"),
|
|
],
|
|
)
|
|
def test_stale_and_incoherent_evidence_blocks(session: Session, change: str, code: str) -> None:
|
|
svc, request = setup_lab(session)
|
|
candidate = session.get(DeploymentCandidate, request.target.deployment_candidate_id)
|
|
assert candidate is not None
|
|
repo = svc.runtime.repo
|
|
approval = repo.approval(request.target.execution_approval_id)
|
|
assessment = repo.assessment(candidate.compatibility_assessment_id)
|
|
profile = repo.profile(candidate.runtime_profile_id)
|
|
node = repo.node(candidate.compute_node_id)
|
|
probe = repo.probe(candidate.runtime_probe_id)
|
|
assert approval and assessment and profile and node and probe
|
|
if change == "revoked":
|
|
approval.revoked_at = datetime.now(UTC)
|
|
elif change == "expired":
|
|
approval.expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
elif change == "approval_fingerprint":
|
|
approval.evidence_fingerprint = "0" * 64
|
|
elif change == "assessment":
|
|
assessment.stale = True
|
|
elif change == "profile_fingerprint":
|
|
session.execute(
|
|
update(RuntimeProfile)
|
|
.where(RuntimeProfile.id == profile.id)
|
|
.values(fingerprint="0" * 64)
|
|
)
|
|
elif change == "production":
|
|
candidate.production = True
|
|
elif change == "channel":
|
|
candidate.channel = "stable"
|
|
elif change == "offline":
|
|
node.liveness_state = "offline"
|
|
elif change == "lab_ineligible":
|
|
node.lab_eligible = False
|
|
elif change == "assessment_blocker":
|
|
assessment.blockers = ["private reason"]
|
|
elif change == "assessment_status":
|
|
assessment.status = "future_unknown"
|
|
elif change == "node_protocol":
|
|
node.agent_capabilities = []
|
|
elif change == "modality":
|
|
session.execute(
|
|
update(RuntimeProfile).where(RuntimeProfile.id == profile.id).values(modality="audio")
|
|
)
|
|
elif change == "probe":
|
|
probe.status = "running"
|
|
elif change == "probe_identity":
|
|
probe.execution_approval_id = uuid.uuid4()
|
|
elif change == "invalid_output":
|
|
probe.inference_result = {"dimension": True, "finite": True}
|
|
elif change == "wrong_dimension":
|
|
probe.inference_result = {"dimension": 768, "finite": True}
|
|
elif change == "gpu":
|
|
probe.runtime_facts = {"gpu_uuid": "missing"}
|
|
elif change == "remote_code":
|
|
session.execute(
|
|
update(RuntimeProfile)
|
|
.where(RuntimeProfile.id == profile.id)
|
|
.values(trust_remote_code=True)
|
|
)
|
|
session.commit()
|
|
response = svc.preflight(request)
|
|
assert not response.checks_passed and code in response.blockers
|
|
assert not response.execution_started and not response.permission_granted
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"capability,payload",
|
|
[
|
|
("rag.embedding", {"input": []}),
|
|
("rag.reranking", {"query": "x", "documents": [], "top_n": 1}),
|
|
("document.ocr", {"content_base64": "!!!", "media_type": "image/png"}),
|
|
(
|
|
"vision.embedding",
|
|
{"items": [{"text": "x", "image_base64": "eA==", "media_type": "image/png"}]},
|
|
),
|
|
("speech.transcription", {"audio_base64": "!!!"}),
|
|
],
|
|
)
|
|
def test_malformed_family_input_rejected(
|
|
session: Session, capability: str, payload: dict[str, Any]
|
|
) -> None:
|
|
_, request = setup_lab(session)
|
|
data = request.model_dump()
|
|
data["input"] = {"capability": capability, "payload": payload}
|
|
with pytest.raises(ValidationError):
|
|
LabPreflightRequest.model_validate(data)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"limits",
|
|
[
|
|
{"timeout_seconds": 301},
|
|
{"timeout_seconds": 4},
|
|
{"timeout_seconds": 5, "queue_timeout_seconds": 6},
|
|
{"queue_timeout_seconds": 0},
|
|
{"retention_seconds": 3601},
|
|
{"max_output_bytes": 1048577},
|
|
{"concurrency": 2},
|
|
{"timeout_seconds": True},
|
|
],
|
|
)
|
|
def test_budget_limits_are_bounded(session: Session, limits: dict[str, Any]) -> None:
|
|
_, request = setup_lab(session)
|
|
data = request.model_dump()
|
|
data["limits"].update(limits)
|
|
with pytest.raises(ValidationError):
|
|
LabPreflightRequest.model_validate(data)
|
|
|
|
|
|
def test_embedding_server_limits_and_payload_limit(session: Session) -> None:
|
|
svc, request = setup_lab(session)
|
|
data = request.model_dump()
|
|
data["input"]["payload"]["input"] = ["x" * 9000] * 9
|
|
response = svc.preflight(LabPreflightRequest.model_validate(data))
|
|
assert {"INPUT_LIMIT_EXCEEDED", "PAYLOAD_LIMIT_EXCEEDED"}.issubset(response.blockers)
|
|
|
|
|
|
def test_preflight_rechecks_evidence_and_does_not_accept_reason(session: Session) -> None:
|
|
svc, request = setup_lab(session)
|
|
assert svc.preflight(request).checks_passed
|
|
approval = svc.runtime.repo.approval(request.target.execution_approval_id)
|
|
assert approval is not None
|
|
approval.revoked_at = datetime.now(UTC)
|
|
session.commit()
|
|
assert "LAB_APPROVAL_STALE" in svc.preflight(request).blockers
|
|
with pytest.raises(ValidationError):
|
|
LabPreflightRequest.model_validate({**request.model_dump(), "reason": "secret reason"})
|
|
|
|
|
|
def test_preflight_uses_injected_manifest_root(session: Session, tmp_path: Path) -> None:
|
|
svc, request = setup_lab(session)
|
|
isolated_settings = svc.settings.model_copy(update={"config_root": tmp_path})
|
|
response = LabPreflightService(session, isolated_settings).preflight(request)
|
|
assert "CAPABILITY_CONTRACT_UNAVAILABLE" in response.blockers
|
|
assert not response.checks_passed
|