473 lines
23 KiB
Python
473 lines
23 KiB
Python
import hashlib
|
|
import json
|
|
from collections.abc import Iterator
|
|
from datetime import UTC, datetime, timedelta
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
from sqlalchemy import create_engine, func, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modelforge_api.domain.dispatch_context import DispatchContext
|
|
from modelforge_api.domain.lab import LabRerankingInput
|
|
from modelforge_api.domain.serving import (
|
|
CapabilityExperimentCreate,
|
|
EmbeddingInvokeRequest,
|
|
RerankingInvokeRequest,
|
|
ServiceClientCreate,
|
|
)
|
|
from modelforge_api.persistence.models import (
|
|
Base,
|
|
DeploymentCandidate,
|
|
GatewayRequest,
|
|
SchedulerPolicyRevision,
|
|
ServiceClient,
|
|
ServingJob,
|
|
)
|
|
from modelforge_api.services.manifest_registry import ManifestRegistry
|
|
from modelforge_api.services.serving import ServingError
|
|
from modelforge_api.services.transient_payloads import MemoryPayloadStore
|
|
from tests.test_lab_preflight import setup_lab
|
|
from tests.test_serving import InlineWorkerServingService, approve_and_promote, setup_m5
|
|
|
|
|
|
@pytest.fixture
|
|
def session() -> Iterator[Session]:
|
|
engine = create_engine("sqlite://")
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as value:
|
|
yield value
|
|
engine.dispose()
|
|
|
|
|
|
@pytest.mark.parametrize("priority", ["production", "interactive", "background", "benchmark"])
|
|
def test_persisted_service_priority_and_lease_owner_are_preserved(priority: str) -> None:
|
|
client_id = uuid4()
|
|
context = DispatchContext.for_service_client(client_id, "project-client", priority)
|
|
assert context.service_client_id == client_id
|
|
assert context.workload_priority == priority
|
|
assert context.lease_owner == "project-client"
|
|
assert context.provenance()["actor_id"] == str(client_id)
|
|
assert context.operation_id is None
|
|
with pytest.raises(ValidationError, match="frozen"):
|
|
context.workload_priority = "lab"
|
|
|
|
|
|
@pytest.mark.parametrize("priority", ["lab", "maintenance", "admin", "", "PRODUCTION"])
|
|
def test_service_dispatch_rejects_unconfigured_priority_domain(priority: str) -> None:
|
|
with pytest.raises(ValidationError):
|
|
DispatchContext.for_service_client(uuid4(), "project-client", priority)
|
|
|
|
|
|
def test_operator_identity_is_separate_and_cannot_elevate_priority() -> None:
|
|
context = DispatchContext.for_operator("jens", uuid4())
|
|
assert context.service_client_id is None
|
|
assert context.workload_priority == "lab"
|
|
assert context.lease_owner == "operator:jens"
|
|
for changes in (
|
|
{"workload_priority": "production"}, {"service_client_id": uuid4()},
|
|
{"operation_id": None}, {"actor_kind": "service_client"},
|
|
{"lease_owner": "a-different-owner"}, {"token": "not-accepted"},
|
|
):
|
|
with pytest.raises(ValidationError):
|
|
DispatchContext.model_validate({**context.model_dump(), **changes})
|
|
assert set(context.provenance()) == {
|
|
"actor_kind", "actor_id", "service_client_id", "operation_id", "workload_priority",
|
|
}
|
|
|
|
|
|
def test_operator_scheduler_admission_requires_no_fabricated_service_client(session: Session) -> None:
|
|
service, candidate, _ = setup_m5(session)
|
|
_, response = approve_and_promote(service, candidate.id)
|
|
deployment = service.repo.deployment(response.id)
|
|
assert deployment is not None
|
|
context = DispatchContext.for_operator("operator-test", uuid4())
|
|
request = GatewayRequest(request_id=uuid4(), service_client_id=None,
|
|
capability_key="rag.embedding", capability_version=1,
|
|
capability_deployment_id=deployment.id, compute_node_id=deployment.compute_node_id,
|
|
status="queued", priority="lab", input_sha256="a" * 64, input_count=1)
|
|
session.add(request)
|
|
session.commit()
|
|
lease = service._acquire_request_lease(deployment, request, context)
|
|
assert lease.priority == "lab"
|
|
assert lease.owner == "operator:operator-test"
|
|
assert lease.request_id == request.request_id
|
|
assert request.decision_evidence["dispatch_context"] == context.provenance()
|
|
assert session.scalar(select(func.count()).select_from(ServiceClient)) == 0
|
|
|
|
|
|
def test_operator_context_cannot_bypass_paused_lab_policy(session: Session) -> None:
|
|
service, candidate, _ = setup_m5(session)
|
|
_, response = approve_and_promote(service, candidate.id)
|
|
deployment = service.repo.deployment(response.id)
|
|
assert deployment is not None
|
|
session.add(SchedulerPolicyRevision(revision="lab-paused-test", active=True,
|
|
configuration={"lab_paused": True}))
|
|
context = DispatchContext.for_operator("operator-test", uuid4())
|
|
request = GatewayRequest(request_id=uuid4(), service_client_id=None,
|
|
capability_key="rag.embedding", capability_version=1,
|
|
capability_deployment_id=deployment.id, compute_node_id=deployment.compute_node_id,
|
|
status="queued", priority="lab", input_sha256="a" * 64, input_count=1)
|
|
session.add(request)
|
|
session.commit()
|
|
with pytest.raises(ServingError) as raised:
|
|
service._acquire_request_lease(deployment, request, context)
|
|
assert raised.value.code == "POLICY_BLOCKED"
|
|
assert service.repo.active_leases(deployment.accelerator_id, datetime.now(UTC)) == []
|
|
|
|
|
|
@pytest.mark.parametrize("mismatch", ["client", "priority", "deployment"])
|
|
def test_context_mismatch_fails_before_scheduler_mutation(session: Session, mismatch: str) -> None:
|
|
service, candidate, _ = setup_m5(session)
|
|
_, response = approve_and_promote(service, candidate.id)
|
|
deployment = service.repo.deployment(response.id)
|
|
assert deployment is not None
|
|
context = DispatchContext.for_operator("operator-test", uuid4())
|
|
request = GatewayRequest(request_id=uuid4(), service_client_id=uuid4() if mismatch == "client" else None,
|
|
capability_key="rag.embedding", capability_version=1,
|
|
capability_deployment_id=uuid4() if mismatch == "deployment" else deployment.id,
|
|
compute_node_id=deployment.compute_node_id, status="queued",
|
|
priority="production" if mismatch == "priority" else "lab", input_sha256="a" * 64, input_count=1)
|
|
with pytest.raises(ServingError) as raised:
|
|
service._acquire_request_lease(deployment, request, context)
|
|
assert raised.value.code == "DISPATCH_CONTEXT_MISMATCH"
|
|
assert request.status == "queued"
|
|
assert service.repo.active_leases(deployment.accelerator_id, datetime.now(UTC)) == []
|
|
|
|
|
|
def operator_fixture(session: Session):
|
|
preflight, request = setup_lab(session)
|
|
service = InlineWorkerServingService(session, preflight.settings,
|
|
ManifestRegistry(), MemoryPayloadStore())
|
|
experiment = service.create_experiment(request.target.deployment_candidate_id,
|
|
CapabilityExperimentCreate(route_key="operator-selected", purpose="Bounded exact-target unit test",
|
|
execution_approval_id=request.target.execution_approval_id))
|
|
return service, request, experiment
|
|
|
|
|
|
def test_operator_embedding_executes_exact_selected_experiment_without_client(session: Session) -> None:
|
|
service, request, selected = operator_fixture(session)
|
|
service.synthetic_worker_metrics = {
|
|
"gpu_uuid": service.repo.accelerator(selected.deployment.accelerator_id).device_uuid,
|
|
"used_vram_bytes": 4_096,
|
|
"process_used_vram_bytes": 2_048,
|
|
"free_vram_bytes": 8_192,
|
|
"gpu_utilization_percent": 32,
|
|
"temperature_c": 63,
|
|
"power_draw_w": 101.25,
|
|
"private_metric_canary": "must-not-be-persisted",
|
|
}
|
|
other = service.create_experiment(request.target.deployment_candidate_id,
|
|
CapabilityExperimentCreate(route_key="operator-other", purpose="Unselected comparison fixture",
|
|
execution_approval_id=request.target.execution_approval_id))
|
|
service.authenticated_credential_id = uuid4() # stale service-instance state must not cross domains
|
|
operation_id = uuid4()
|
|
context = DispatchContext.for_operator("operator-test", operation_id)
|
|
result = service.invoke_operator_lab(request.input, context,
|
|
deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
|
|
record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
|
|
assert record is not None
|
|
assert record.capability_deployment_id == selected.capability_deployment_id != other.capability_deployment_id
|
|
assert record.service_client_id is None and record.priority == "lab"
|
|
assert record.decision_evidence["service_credential_id"] is None
|
|
assert record.decision_evidence["dispatch_context"] == context.provenance()
|
|
assert record.decision_evidence["operator_route_id"] == str(selected.id)
|
|
assert result.embedding_space_id == selected.deployment.embedding_space.id
|
|
evidence = record.decision_evidence["lab_run_evidence"]
|
|
candidate = session.get(DeploymentCandidate, request.target.deployment_candidate_id)
|
|
profile = service.repo.profile(request.target.runtime_profile_id)
|
|
deployment = service.repo.deployment(selected.capability_deployment_id)
|
|
assert candidate is not None and profile is not None and deployment is not None
|
|
accelerator = service.repo.accelerator(deployment.accelerator_id)
|
|
assert accelerator is not None
|
|
assert evidence["schema_version"] == 1
|
|
assert evidence["status"] == "completed"
|
|
assert evidence["capability"] == "rag.embedding@1"
|
|
assert evidence["operation_id"] == str(operation_id)
|
|
assert evidence["input"] == {
|
|
"sha256": record.input_sha256,
|
|
"item_count": 1,
|
|
"content_logged": False,
|
|
}
|
|
assert evidence["target"] == {
|
|
"deployment_id": str(selected.capability_deployment_id),
|
|
"candidate_id": str(candidate.id),
|
|
"experiment_route_id": str(selected.id),
|
|
"capability_contract_id": str(deployment.capability_contract_id),
|
|
"contract_version": 1,
|
|
"artifact_set_id": str(deployment.artifact_set_id),
|
|
"artifact_sha256": profile.artifact_sha256,
|
|
"runtime_profile_id": str(profile.id),
|
|
"runtime_profile_fingerprint": profile.fingerprint,
|
|
"compatibility_assessment_id": str(candidate.compatibility_assessment_id),
|
|
"runtime_probe_id": str(candidate.runtime_probe_id),
|
|
"compute_node_id": str(deployment.compute_node_id),
|
|
"accelerator_id": str(deployment.accelerator_id),
|
|
"accelerator_uuid": accelerator.device_uuid,
|
|
"execution_approval_id": str(request.target.execution_approval_id),
|
|
"embedding_space_id": str(deployment.embedding_space_id),
|
|
"deployment_fingerprint": deployment.config_fingerprint,
|
|
}
|
|
expected_output_hash = hashlib.sha256(
|
|
json.dumps(result.data, sort_keys=True, separators=(",", ":")).encode()
|
|
).hexdigest()
|
|
assert evidence["output"] == {
|
|
"type": "float_vector",
|
|
"dimension": result.dimension,
|
|
"item_count": len(result.data),
|
|
"embedding_space_id": str(result.embedding_space_id),
|
|
"sha256": expected_output_hash,
|
|
"content_logged": False,
|
|
}
|
|
serving_job = session.scalar(
|
|
select(ServingJob).where(
|
|
ServingJob.gateway_request_id == record.id,
|
|
ServingJob.operation == "invoke",
|
|
)
|
|
)
|
|
expected_observations = {
|
|
"source": "runtime-worker",
|
|
"phase": "after_inference",
|
|
"availability": "complete",
|
|
"device_matches_target": True,
|
|
"metrics": {
|
|
"gpu_uuid": accelerator.device_uuid,
|
|
"used_vram_bytes": 4_096,
|
|
"process_used_vram_bytes": 2_048,
|
|
"free_vram_bytes": 8_192,
|
|
"gpu_utilization_percent": 32,
|
|
"temperature_c": 63,
|
|
"power_draw_w": 101.25,
|
|
},
|
|
}
|
|
assert evidence["worker_observations"] == expected_observations
|
|
assert serving_job is not None
|
|
assert serving_job.result_summary["resource_observation"] == expected_observations
|
|
assert "vectors" not in evidence["output"] and "data" not in evidence["output"]
|
|
assert "private input content" not in json.dumps(evidence)
|
|
assert "must-not-be-persisted" not in json.dumps(evidence)
|
|
assert session.scalar(select(func.count()).select_from(ServiceClient)) == 0
|
|
|
|
|
|
def test_operator_lab_without_worker_resource_metrics_does_not_invent_gpu_evidence(
|
|
session: Session,
|
|
) -> None:
|
|
service, request, selected = operator_fixture(session)
|
|
result = service.invoke_operator_lab(
|
|
request.input,
|
|
DispatchContext.for_operator("operator-test", uuid4()),
|
|
deployment_id=selected.capability_deployment_id,
|
|
experiment_route_id=selected.id,
|
|
)
|
|
record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
|
|
assert record is not None
|
|
observations = record.decision_evidence["lab_run_evidence"]["worker_observations"]
|
|
assert observations == {
|
|
"source": "runtime-worker",
|
|
"phase": "after_inference",
|
|
"availability": "unavailable",
|
|
"device_matches_target": None,
|
|
"metrics": {},
|
|
}
|
|
|
|
|
|
def test_operator_lab_rejects_unbounded_worker_resource_metrics(session: Session) -> None:
|
|
service, request, selected = operator_fixture(session)
|
|
deployment = service.repo.deployment(selected.capability_deployment_id)
|
|
assert deployment is not None
|
|
accelerator = service.repo.accelerator(deployment.accelerator_id)
|
|
assert accelerator is not None
|
|
service.synthetic_worker_metrics = {
|
|
"gpu_uuid": accelerator.device_uuid,
|
|
"used_vram_bytes": True,
|
|
"process_used_vram_bytes": 2**100,
|
|
"free_vram_bytes": 8_192,
|
|
"gpu_utilization_percent": 101,
|
|
"temperature_c": 10**1000,
|
|
"power_draw_w": 10**1000,
|
|
"private_metric_canary": "must-not-be-persisted",
|
|
}
|
|
result = service.invoke_operator_lab(
|
|
request.input,
|
|
DispatchContext.for_operator("operator-test", uuid4()),
|
|
deployment_id=selected.capability_deployment_id,
|
|
experiment_route_id=selected.id,
|
|
)
|
|
record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == result.request_id))
|
|
assert record is not None
|
|
observations = record.decision_evidence["lab_run_evidence"]["worker_observations"]
|
|
assert observations == {
|
|
"source": "runtime-worker",
|
|
"phase": "after_inference",
|
|
"availability": "partial",
|
|
"device_matches_target": True,
|
|
"metrics": {"gpu_uuid": accelerator.device_uuid, "free_vram_bytes": 8_192},
|
|
}
|
|
|
|
|
|
def test_operator_lab_scheduler_failure_is_durable_and_redacted(
|
|
session: Session, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
service, request, selected = operator_fixture(session)
|
|
|
|
def reject_admission(*_args, **_kwargs):
|
|
raise ServingError(429, "LAB_QUEUE_REJECTED", "private scheduler detail")
|
|
|
|
monkeypatch.setattr(service, "_acquire_request_lease", reject_admission)
|
|
with pytest.raises(ServingError, match="private scheduler detail"):
|
|
service.invoke_operator_lab(
|
|
request.input,
|
|
DispatchContext.for_operator("operator-test", uuid4()),
|
|
deployment_id=selected.capability_deployment_id,
|
|
experiment_route_id=selected.id,
|
|
)
|
|
|
|
record = session.scalar(select(GatewayRequest).where(GatewayRequest.priority == "lab"))
|
|
assert record is not None and record.status == "failed"
|
|
assert record.failure_code == "LAB_QUEUE_REJECTED"
|
|
evidence = record.decision_evidence["lab_run_evidence"]
|
|
assert evidence["status"] == "failed"
|
|
assert evidence["failure_code"] == "LAB_QUEUE_REJECTED"
|
|
assert "output" not in evidence and "worker_observations" not in evidence
|
|
assert "private input content" not in json.dumps(evidence)
|
|
assert "private scheduler detail" not in json.dumps(evidence)
|
|
|
|
|
|
@pytest.mark.parametrize("bad_target", ["missing_deployment", "missing_route", "inactive_route", "stable"])
|
|
def test_operator_embedding_rejects_non_exact_or_non_lab_target(session: Session, bad_target: str) -> None:
|
|
service, request, selected = operator_fixture(session)
|
|
deployment_id, route_id = selected.capability_deployment_id, selected.id
|
|
if bad_target == "missing_deployment":
|
|
deployment_id = uuid4()
|
|
elif bad_target == "missing_route":
|
|
route_id = uuid4()
|
|
elif bad_target == "inactive_route":
|
|
service.deactivate_experiment(route_id)
|
|
else:
|
|
_, stable = approve_and_promote(service, request.target.deployment_candidate_id)
|
|
deployment_id = stable.id
|
|
with pytest.raises(ServingError) as raised:
|
|
service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
|
|
deployment_id=deployment_id, experiment_route_id=route_id)
|
|
assert raised.value.code == "LAB_TARGET_NOT_ROUTABLE"
|
|
assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
|
|
|
|
|
|
def test_operator_entry_rejects_service_context_and_unimplemented_family(session: Session) -> None:
|
|
service, request, selected = operator_fixture(session)
|
|
with pytest.raises(ServingError) as denied:
|
|
service.invoke_operator_lab(request.input, DispatchContext.for_service_client(uuid4(), "test-client", "production"),
|
|
deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
|
|
assert denied.value.code == "OPERATOR_CONTEXT_REQUIRED"
|
|
reranking = LabRerankingInput(capability="rag.reranking", payload=RerankingInvokeRequest(
|
|
query="test", documents=[{"id": "a", "text": "test"}], top_n=1))
|
|
with pytest.raises(ServingError) as unsupported:
|
|
service.invoke_operator_lab(reranking, DispatchContext.for_operator("operator-test", uuid4()),
|
|
deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
|
|
assert unsupported.value.code == "LAB_INPUT_NOT_SUPPORTED"
|
|
assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
|
|
|
|
|
|
@pytest.mark.parametrize("invalid", ["approval_expired", "wrong_contract"])
|
|
def test_operator_entry_rechecks_exact_approval_and_contract(session: Session, invalid: str) -> None:
|
|
service, request, selected = operator_fixture(session)
|
|
if invalid == "approval_expired":
|
|
approval = service.repo.execution_approval(request.target.execution_approval_id)
|
|
assert approval is not None
|
|
approval.expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
else:
|
|
route = service.repo.experiment_route_by_id(selected.id)
|
|
assert route is not None
|
|
route.capability_contract_id = service.ensure_contract("rag.reranking", 1).id
|
|
session.commit()
|
|
with pytest.raises(ServingError) as denied:
|
|
service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
|
|
deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
|
|
assert denied.value.code == ("LAB_ADMISSION_BLOCKED" if invalid == "approval_expired" else "LAB_TARGET_NOT_ROUTABLE")
|
|
assert session.scalar(select(func.count()).select_from(GatewayRequest)) == 0
|
|
|
|
|
|
def test_route_deactivation_after_load_prevents_operator_inference(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
service, request, selected = operator_fixture(session)
|
|
original = service._ensure_loaded
|
|
original_put = service.payload_store.put
|
|
input_references = []
|
|
def track_put(payload, ttl_seconds):
|
|
reference = original_put(payload, ttl_seconds)
|
|
if "input" in payload:
|
|
input_references.append(reference)
|
|
return reference
|
|
monkeypatch.setattr(service.payload_store, "put", track_put)
|
|
def load_then_deactivate(deployment, deadline, *, priority):
|
|
loaded = original(deployment, deadline, priority=priority)
|
|
route = service.repo.experiment_route_by_id(selected.id)
|
|
assert route is not None
|
|
route.status = "inactive"
|
|
session.commit()
|
|
return loaded
|
|
monkeypatch.setattr(service, "_ensure_loaded", load_then_deactivate)
|
|
with pytest.raises(ServingError) as denied:
|
|
service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
|
|
deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
|
|
assert denied.value.code == "LAB_TARGET_NOT_ROUTABLE"
|
|
assert session.scalar(select(func.count()).select_from(ServingJob).where(ServingJob.operation == "invoke")) == 0
|
|
record = session.scalar(select(GatewayRequest))
|
|
assert record is not None and record.status == "failed"
|
|
assert len(input_references) == 1
|
|
assert service.payload_store.get(input_references[0]) is None
|
|
|
|
|
|
def test_gateway_times_validation_once_before_stable_resolution(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
service, request, _ = operator_fixture(session)
|
|
approve_and_promote(service, request.target.deployment_candidate_id)
|
|
created = service.create_client(ServiceClientCreate(name="timing-client", allowed_capabilities=["rag.embedding@1"]))
|
|
client = service.authenticate(f"Bearer {created.credential}", "rag.embedding@1")
|
|
events = []
|
|
clock = [1000.0]
|
|
original_validate = service._validate_inputs
|
|
original_resolve = service._stable_embedding_deployment
|
|
def validate(value):
|
|
events.append("validate")
|
|
clock[0] += 0.125
|
|
return original_validate(value)
|
|
def resolve():
|
|
events.append("resolve")
|
|
clock[0] += 0.375
|
|
return original_resolve()
|
|
monkeypatch.setattr(service, "_validate_inputs", validate)
|
|
monkeypatch.setattr(service, "_stable_embedding_deployment", resolve)
|
|
monkeypatch.setattr("modelforge_api.services.serving.time.perf_counter", lambda: clock[0])
|
|
response = service.invoke(EmbeddingInvokeRequest(input="timed request"), client)
|
|
assert events == ["validate", "resolve"]
|
|
record = session.scalar(select(GatewayRequest).where(GatewayRequest.request_id == response.request_id))
|
|
assert record is not None
|
|
latency = record.decision_evidence["latency_ms"]
|
|
assert latency["validation"] == 125.0
|
|
assert latency["resolution"] == 375.0
|
|
assert latency["total"] >= 500.0
|
|
|
|
|
|
def test_failed_enqueue_deletes_input_without_a_job_owner(session: Session, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
service, request, selected = operator_fixture(session)
|
|
original_enqueue = service._enqueue_job
|
|
original_put = service.payload_store.put
|
|
input_references = []
|
|
def track_put(payload, ttl_seconds):
|
|
reference = original_put(payload, ttl_seconds)
|
|
if "input" in payload:
|
|
input_references.append(reference)
|
|
return reference
|
|
def enqueue(deployment, operation, **kwargs):
|
|
if operation == "invoke":
|
|
raise ServingError(409, "ENQUEUE_REJECTED", "isolated enqueue failure")
|
|
return original_enqueue(deployment, operation, **kwargs)
|
|
monkeypatch.setattr(service.payload_store, "put", track_put)
|
|
monkeypatch.setattr(service, "_enqueue_job", enqueue)
|
|
with pytest.raises(ServingError, match="isolated enqueue failure"):
|
|
service.invoke_operator_lab(request.input, DispatchContext.for_operator("operator-test", uuid4()),
|
|
deployment_id=selected.capability_deployment_id, experiment_route_id=selected.id)
|
|
assert len(input_references) == 1
|
|
assert service.payload_store.get(input_references[0]) is None
|
|
assert session.scalar(select(func.count()).select_from(ServingJob).where(ServingJob.operation == "invoke")) == 0
|