Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/tests/test_dispatch_context.py

@@ -0,0 +1,472 @@
+import hashlib
+import json
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from pydantic import ValidationError
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+
+from modelforge_api.domain.dispatch_context import DispatchContext
+from modelforge_api.domain.lab import LabRerankingInput
+from modelforge_api.domain.serving import (
+    CapabilityExperimentCreate,
+    EmbeddingInvokeRequest,
+    RerankingInvokeRequest,
+    ServiceClientCreate,
+)
+from modelforge_api.persistence.models import (
+    Base,
+    DeploymentCandidate,
+    GatewayRequest,
+    SchedulerPolicyRevision,
+    ServiceClient,
+    ServingJob,
+)
+from modelforge_api.services.manifest_registry import ManifestRegistry
+from modelforge_api.services.serving import ServingError
+from modelforge_api.services.transient_payloads import MemoryPayloadStore
+from tests.test_lab_preflight import setup_lab
+from tests.test_serving import InlineWorkerServingService, approve_and_promote, setup_m5
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+    engine = create_engine("sqlite://")
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+    engine.dispose()
+
+
+@pytest.mark.parametrize("priority", ["production", "interactive", "background", "benchmark"])
+def test_persisted_service_priority_and_lease_owner_are_preserved(priority: str) -> None:
+    client_id = uuid4()
+    context = DispatchContext.for_service_client(client_id, "project-client", priority)
+    assert context.service_client_id == client_id
+    assert context.workload_priority == priority
+    assert context.lease_owner == "project-client"
+    assert context.provenance()["actor_id"] == str(client_id)
+    assert context.operation_id is None
+    with pytest.raises(ValidationError, match="frozen"):
+        context.workload_priority = "lab"
+
+
+@pytest.mark.parametrize("priority", ["lab", "maintenance", "admin", "", "PRODUCTION"])
+def test_service_dispatch_rejects_unconfigured_priority_domain(priority: str) -> None:
+    with pytest.raises(ValidationError):
+        DispatchContext.for_service_client(uuid4(), "project-client", priority)
+
+
+def test_operator_identity_is_separate_and_cannot_elevate_priority() -> None:
+    context = DispatchContext.for_operator("jens", uuid4())
+    assert context.service_client_id is None
+    assert context.workload_priority == "lab"
+    assert context.lease_owner == "operator:jens"
+    for changes in (
+        {"workload_priority": "production"}, {"service_client_id": uuid4()},
+        {"operation_id": None}, {"actor_kind": "service_client"},
+        {"lease_owner": "a-different-owner"}, {"token": "not-accepted"},
+    ):
+        with pytest.raises(ValidationError):
+            DispatchContext.model_validate({**context.model_dump(), **changes})
+    assert set(context.provenance()) == {
+        "actor_kind", "actor_id", "service_client_id", "operation_id", "workload_priority",
+    }
+
+
+def test_operator_scheduler_admission_requires_no_fabricated_service_client(session: Session) -> None:
+    service, candidate, _ = setup_m5(session)
+    _, response = approve_and_promote(service, candidate.id)
+    deployment = service.repo.deployment(response.id)
+    assert deployment is not None
+    context = DispatchContext.for_operator("operator-test", uuid4())
+    request = GatewayRequest(request_id=uuid4(), service_client_id=None,
+        capability_key="rag.embedding", capability_version=1,
+        capability_deployment_id=deployment.id, compute_node_id=deployment.compute_node_id,
+        status="queued", priority="lab", input_sha256="a" * 64, input_count=1)
+    session.add(request)
+    session.commit()
+    lease = service._acquire_request_lease(deployment, request, context)
+    assert lease.priority == "lab"
+    assert lease.owner == "operator:operator-test"
+    assert lease.request_id == request.request_id
+    assert request.decision_evidence["dispatch_context"] == context.provenance()
+    assert session.scalar(select(func.count()).select_from(ServiceClient)) == 0
+
+
+def test_operator_context_cannot_bypass_paused_lab_policy(session: Session) -> None:
+    service, candidate, _ = setup_m5(session)
+    _, response = approve_and_promote(service, candidate.id)
+    deployment = service.repo.deployment(response.id)
+    assert deployment is not None
+    session.add(SchedulerPolicyRevision(revision="lab-paused-test", active=True,
+                                        configuration={"lab_paused": True}))
+    context = DispatchContext.for_operator("operator-test", uuid4())
+    request = GatewayRequest(request_id=uuid4(), service_client_id=None,
+        capability_key="rag.embedding", capability_version=1,
+        capability_deployment_id=deployment.id, compute_node_id=deployment.compute_node_id,
+        status="queued", priority="lab", input_sha256="a" * 64, input_count=1)
+    session.add(request)
+    session.commit()
+    with pytest.raises(ServingError) as raised:
+        service._acquire_request_lease(deployment, request, context)
+    assert raised.value.code == "POLICY_BLOCKED"
+    assert service.repo.active_leases(deployment.accelerator_id, datetime.now(UTC)) == []
+
+
+@pytest.mark.parametrize("mismatch", ["client", "priority", "deployment"])
+def test_context_mismatch_fails_before_scheduler_mutation(session: Session, mismatch: str) -> None:
+    service, candidate, _ = setup_m5(session)
+    _, response = approve_and_promote(service, candidate.id)
+    deployment = service.repo.deployment(response.id)
+    assert deployment is not None
+    context = DispatchContext.for_operator("operator-test", uuid4())
+    request = GatewayRequest(request_id=uuid4(), service_client_id=uuid4() if mismatch == "client" else None,
+        capability_key="rag.embedding", capability_version=1,
+        capability_deployment_id=uuid4() if mismatch == "deployment" else deployment.id,
+        compute_node_id=deployment.compute_node_id, status="queued",
+        priority="production" if mismatch == "priority" else "lab", input_sha256="a" * 64, input_count=1)
+    with pytest.raises(ServingError) as raised:
+        service._acquire_request_lease(deployment, request, context)
+    assert raised.value.code == "DISPATCH_CONTEXT_MISMATCH"
+    assert request.status == "queued"
+    assert service.repo.active_leases(deployment.accelerator_id, datetime.now(UTC)) == []
+
+
+def operator_fixture(session: Session):
+    preflight, request = setup_lab(session)
+    service = InlineWorkerServingService(session, preflight.settings,
+        ManifestRegistry(), MemoryPayloadStore())
+    experiment = service.create_experiment(request.target.deployment_candidate_id,
+        CapabilityExperimentCreate(route_key="operator-selected", purpose="Bounded exact-target unit test",
+                                  execution_approval_id=request.target.execution_approval_id))
+    return service, request, experiment
+
+
+def test_operator_embedding_executes_exact_selected_experiment_without_client(session: Session) -> None:
+    service, request, selected = operator_fixture(session)
+    service.synthetic_worker_metrics = {
+        "gpu_uuid": service.repo.accelerator(selected.deployment.accelerator_id).device_uuid,
+        "used_vram_bytes": 4_096,
+        "process_used_vram_bytes": 2_048,
+        "free_vram_bytes": 8_192,
+        "gpu_utilization_percent": 32,
+        "temperature_c": 63,
+        "power_draw_w": 101.25,
+        "private_metric_canary": "must-not-be-persisted",
+    }
+    other = service.create_experiment(request.target.deployment_candidate_id,
+        CapabilityExperimentCreate(route_key="operator-other", purpose="Unselected comparison fixture",
+                                  execution_approval_id=request.target.execution_approval_id))
+    service.authenticated_credential_id = uuid4()  # stale service-instance state must not cross domains
+    operation_id = uuid4()
+    context = DispatchContext.for_operator("operator-test", operation_id)
+    result = service.invoke_operator_lab(request.input, context,
+        deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
+    assert record is not None
+    assert record.capability_deployment_id == selected.capability_deployment_id != other.capability_deployment_id
+    assert record.service_client_id is None and record.priority == "lab"
+    assert record.decision_evidence["service_credential_id"] is None
+    assert record.decision_evidence["dispatch_context"] == context.provenance()
+    assert record.decision_evidence["operator_route_id"] == str(selected.id)
+    assert result.embedding_space_id == selected.deployment.embedding_space.id
+    evidence = record.decision_evidence["lab_run_evidence"]
+    candidate = session.get(DeploymentCandidate, request.target.deployment_candidate_id)
+    profile = service.repo.profile(request.target.runtime_profile_id)
+    deployment = service.repo.deployment(selected.capability_deployment_id)
+    assert candidate is not None and profile is not None and deployment is not None
+    accelerator = service.repo.accelerator(deployment.accelerator_id)
+    assert accelerator is not None
+    assert evidence["schema_version"] == 1
+    assert evidence["status"] == "completed"
+    assert evidence["capability"] == "rag.embedding@1"
+    assert evidence["operation_id"] == str(operation_id)
+    assert evidence["input"] == {
+        "sha256": record.input_sha256,
+        "item_count": 1,
+        "content_logged": False,
+    }
+    assert evidence["target"] == {
+        "deployment_id": str(selected.capability_deployment_id),
+        "candidate_id": str(candidate.id),
+        "experiment_route_id": str(selected.id),
+        "capability_contract_id": str(deployment.capability_contract_id),
+        "contract_version": 1,
+        "artifact_set_id": str(deployment.artifact_set_id),
+        "artifact_sha256": profile.artifact_sha256,
+        "runtime_profile_id": str(profile.id),
+        "runtime_profile_fingerprint": profile.fingerprint,
+        "compatibility_assessment_id": str(candidate.compatibility_assessment_id),
+        "runtime_probe_id": str(candidate.runtime_probe_id),
+        "compute_node_id": str(deployment.compute_node_id),
+        "accelerator_id": str(deployment.accelerator_id),
+        "accelerator_uuid": accelerator.device_uuid,
+        "execution_approval_id": str(request.target.execution_approval_id),
+        "embedding_space_id": str(deployment.embedding_space_id),
+        "deployment_fingerprint": deployment.config_fingerprint,
+    }
+    expected_output_hash = hashlib.sha256(
+        json.dumps(result.data, sort_keys=True, separators=(",", ":")).encode()
+    ).hexdigest()
+    assert evidence["output"] == {
+        "type": "float_vector",
+        "dimension": result.dimension,
+        "item_count": len(result.data),
+        "embedding_space_id": str(result.embedding_space_id),
+        "sha256": expected_output_hash,
+        "content_logged": False,
+    }
+    serving_job = session.scalar(
+        select(ServingJob).where(
+            ServingJob.gateway_request_id == record.id,
+            ServingJob.operation == "invoke",
+        )
+    )
+    expected_observations = {
+        "source": "runtime-worker",
+        "phase": "after_inference",
+        "availability": "complete",
+        "device_matches_target": True,
+        "metrics": {
+            "gpu_uuid": accelerator.device_uuid,
+            "used_vram_bytes": 4_096,
+            "process_used_vram_bytes": 2_048,
+            "free_vram_bytes": 8_192,
+            "gpu_utilization_percent": 32,
+            "temperature_c": 63,
+            "power_draw_w": 101.25,
+        },
+    }
+    assert evidence["worker_observations"] == expected_observations
+    assert serving_job is not None
+    assert serving_job.result_summary["resource_observation"] == expected_observations
+    assert "vectors" not in evidence["output"] and "data" not in evidence["output"]
+    assert "private input content" not in json.dumps(evidence)
+    assert "must-not-be-persisted" not in json.dumps(evidence)
+    assert session.scalar(select(func.count()).select_from(ServiceClient)) == 0
+
+
+def test_operator_lab_without_worker_resource_metrics_does_not_invent_gpu_evidence(
+    session: Session,
+) -> None:
+    service, request, selected = operator_fixture(session)
+    result = service.invoke_operator_lab(
+        request.input,
+        DispatchContext.for_operator("operator-test", uuid4()),
+        deployment_id=selected.capability_deployment_id,
+        experiment_route_id=selected.id,
+    )
+    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
+    assert record is not None
+    observations = record.decision_evidence["lab_run_evidence"]["worker_observations"]
+    assert observations == {
+        "source": "runtime-worker",
+        "phase": "after_inference",
+        "availability": "unavailable",
+        "device_matches_target": None,
+        "metrics": {},
+    }
+
+
+def test_operator_lab_rejects_unbounded_worker_resource_metrics(session: Session) -> None:
+    service, request, selected = operator_fixture(session)
+    deployment = service.repo.deployment(selected.capability_deployment_id)
+    assert deployment is not None
+    accelerator = service.repo.accelerator(deployment.accelerator_id)
+    assert accelerator is not None
+    service.synthetic_worker_metrics = {
+        "gpu_uuid": accelerator.device_uuid,
+        "used_vram_bytes": True,
+        "process_used_vram_bytes": 2**100,
+        "free_vram_bytes": 8_192,
+        "gpu_utilization_percent": 101,
+        "temperature_c": 10**1000,
+        "power_draw_w": 10**1000,
+        "private_metric_canary": "must-not-be-persisted",
+    }
+    result = service.invoke_operator_lab(
+        request.input,
+        DispatchContext.for_operator("operator-test", uuid4()),
+        deployment_id=selected.capability_deployment_id,
+        experiment_route_id=selected.id,
+    )
+    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
+    assert record is not None
+    observations = record.decision_evidence["lab_run_evidence"]["worker_observations"]
+    assert observations == {
+        "source": "runtime-worker",
+        "phase": "after_inference",
+        "availability": "partial",
+        "device_matches_target": True,
+        "metrics": {"gpu_uuid": accelerator.device_uuid, "free_vram_bytes": 8_192},
+    }
+
+
+def test_operator_lab_scheduler_failure_is_durable_and_redacted(
+    session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    service, request, selected = operator_fixture(session)
+
+    def reject_admission(*_args, **_kwargs):
+        raise ServingError(429, "LAB_QUEUE_REJECTED", "private scheduler detail")
+
+    monkeypatch.setattr(service, "_acquire_request_lease", reject_admission)
+    with pytest.raises(ServingError, match="private scheduler detail"):
+        service.invoke_operator_lab(
+            request.input,
+            DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=selected.capability_deployment_id,
+            experiment_route_id=selected.id,
+        )
+
+    record = session.scalar(select(GatewayRequest).where(GatewayRequest.priority == "lab"))
+    assert record is not None and record.status == "failed"
+    assert record.failure_code == "LAB_QUEUE_REJECTED"
+    evidence = record.decision_evidence["lab_run_evidence"]
+    assert evidence["status"] == "failed"
+    assert evidence["failure_code"] == "LAB_QUEUE_REJECTED"
+    assert "output" not in evidence and "worker_observations" not in evidence
+    assert "private input content" not in json.dumps(evidence)
+    assert "private scheduler detail" not in json.dumps(evidence)
+
+
+@pytest.mark.parametrize("bad_target", ["missing_deployment", "missing_route", "inactive_route", "stable"])
+def test_operator_embedding_rejects_non_exact_or_non_lab_target(session: Session, bad_target: str) -> None:
+    service, request, selected = operator_fixture(session)
+    deployment_id, route_id = selected.capability_deployment_id, selected.id
+    if bad_target == "missing_deployment":
+        deployment_id = uuid4()
+    elif bad_target == "missing_route":
+        route_id = uuid4()
+    elif bad_target == "inactive_route":
+        service.deactivate_experiment(route_id)
+    else:
+        _, stable = approve_and_promote(service, request.target.deployment_candidate_id)
+        deployment_id = stable.id
+    with pytest.raises(ServingError) as raised:
+        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=deployment_id, experiment_route_id=route_id)
+    assert raised.value.code == "LAB_TARGET_NOT_ROUTABLE"
+    assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
+
+
+def test_operator_entry_rejects_service_context_and_unimplemented_family(session: Session) -> None:
+    service, request, selected = operator_fixture(session)
+    with pytest.raises(ServingError) as denied:
+        service.invoke_operator_lab(request.input, DispatchContext.for_service_client(uuid4(), "test-client", "production"),
+            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    assert denied.value.code == "OPERATOR_CONTEXT_REQUIRED"
+    reranking = LabRerankingInput(capability="rag.reranking", payload=RerankingInvokeRequest(
+        query="test", documents=[{"id": "a", "text": "test"}], top_n=1))
+    with pytest.raises(ServingError) as unsupported:
+        service.invoke_operator_lab(reranking, DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    assert unsupported.value.code == "LAB_INPUT_NOT_SUPPORTED"
+    assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
+
+
+@pytest.mark.parametrize("invalid", ["approval_expired", "wrong_contract"])
+def test_operator_entry_rechecks_exact_approval_and_contract(session: Session, invalid: str) -> None:
+    service, request, selected = operator_fixture(session)
+    if invalid == "approval_expired":
+        approval = service.repo.execution_approval(request.target.execution_approval_id)
+        assert approval is not None
+        approval.expires_at = datetime.now(UTC) - timedelta(seconds=1)
+    else:
+        route = service.repo.experiment_route_by_id(selected.id)
+        assert route is not None
+        route.capability_contract_id = service.ensure_contract("rag.reranking", 1).id
+    session.commit()
+    with pytest.raises(ServingError) as denied:
+        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    assert denied.value.code == ("LAB_ADMISSION_BLOCKED" if invalid == "approval_expired" else "LAB_TARGET_NOT_ROUTABLE")
+    assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
+
+
+def test_route_deactivation_after_load_prevents_operator_inference(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+    service, request, selected = operator_fixture(session)
+    original = service._ensure_loaded
+    original_put = service.payload_store.put
+    input_references = []
+    def track_put(payload, ttl_seconds):
+        reference = original_put(payload, ttl_seconds)
+        if "input" in payload:
+            input_references.append(reference)
+        return reference
+    monkeypatch.setattr(service.payload_store, "put", track_put)
+    def load_then_deactivate(deployment, deadline, *, priority):
+        loaded = original(deployment, deadline, priority=priority)
+        route = service.repo.experiment_route_by_id(selected.id)
+        assert route is not None
+        route.status = "inactive"
+        session.commit()
+        return loaded
+    monkeypatch.setattr(service, "_ensure_loaded", load_then_deactivate)
+    with pytest.raises(ServingError) as denied:
+        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    assert denied.value.code == "LAB_TARGET_NOT_ROUTABLE"
+    assert session.scalar(select(func.count()).select_from(ServingJob).where(ServingJob.operation == "invoke")) == 0
+    record = session.scalar(select(GatewayRequest))
+    assert record is not None and record.status == "failed"
+    assert len(input_references) == 1
+    assert service.payload_store.get(input_references[0]) is None
+
+
+def test_gateway_times_validation_once_before_stable_resolution(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+    service, request, _ = operator_fixture(session)
+    approve_and_promote(service, request.target.deployment_candidate_id)
+    created = service.create_client(ServiceClientCreate(name="timing-client", allowed_capabilities=["rag.embedding@1"]))
+    client = service.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
+    events = []
+    clock = [1000.0]
+    original_validate = service._validate_inputs
+    original_resolve = service._stable_embedding_deployment
+    def validate(value):
+        events.append("validate")
+        clock[0] += 0.125
+        return original_validate(value)
+    def resolve():
+        events.append("resolve")
+        clock[0] += 0.375
+        return original_resolve()
+    monkeypatch.setattr(service, "_validate_inputs", validate)
+    monkeypatch.setattr(service, "_stable_embedding_deployment", resolve)
+    monkeypatch.setattr("modelforge_api.services.serving.time.perf_counter", lambda: clock[0])
+    response = service.invoke(EmbeddingInvokeRequest(input="timed request"), client)
+    assert events == ["validate", "resolve"]
+    record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == response.request_id))
+    assert record is not None
+    latency = record.decision_evidence["latency_ms"]
+    assert latency["validation"] == 125.0
+    assert latency["resolution"] == 375.0
+    assert latency["total"] >= 500.0
+
+
+def test_failed_enqueue_deletes_input_without_a_job_owner(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
+    service, request, selected = operator_fixture(session)
+    original_enqueue = service._enqueue_job
+    original_put = service.payload_store.put
+    input_references = []
+    def track_put(payload, ttl_seconds):
+        reference = original_put(payload, ttl_seconds)
+        if "input" in payload:
+            input_references.append(reference)
+        return reference
+    def enqueue(deployment, operation, **kwargs):
+        if operation == "invoke":
+            raise ServingError(409, "ENQUEUE_REJECTED", "isolated enqueue failure")
+        return original_enqueue(deployment, operation, **kwargs)
+    monkeypatch.setattr(service.payload_store, "put", track_put)
+    monkeypatch.setattr(service, "_enqueue_job", enqueue)
+    with pytest.raises(ServingError, match="isolated enqueue failure"):
+        service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
+            deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
+    assert len(input_references) == 1
+    assert service.payload_store.get(input_references[0]) is None
+    assert session.scalar(select(func.count()).select_from(ServingJob).where(ServingJob.operation == "invoke")) == 0