"""Redis write outages fail closed on non-text Gateway invocation routes.""" from __future__ import annotations import pytest import redis from fastapi.testclient import TestClient from sqlalchemy import create_engine, select from sqlalchemy.orm import Session from sqlalchemy.pool import StaticPool from modelforge_api.api.routes.serving import get_serving_service from modelforge_api.domain.serving import CapabilityExperimentCreate, ServiceClientCreate from modelforge_api.main import app from modelforge_api.persistence.models import Base, GatewayRequest, ServingGpuLease, ServingJob from modelforge_api.settings import get_settings from tests.test_serving import InlineWorkerServingService, setup_m5 @pytest.mark.parametrize( ("capability", "body"), [ ( "rag.reranking", { "query": "private query", "documents": [{"id": "one", "text": "private document"}], "top_n": 1, }, ), ("vision.embedding", {"items": [{"text": "private visual query"}]}), ], ) def test_payload_write_outage_fails_closed_for_non_text_routes( capability: str, body: dict[str, object], monkeypatch: pytest.MonkeyPatch ) -> None: engine = create_engine( "sqlite+pysqlite:///:memory:", connect_args={"check_same_thread": False}, poolclass=StaticPool, ) Base.metadata.create_all(engine) with Session(engine) as session: base, candidate, _node = setup_m5(session) probe = base.repo.probe(candidate.runtime_probe_id) assert probe is not None if capability == "rag.reranking": probe.inference_result = { "status": "passed", "output_type": "ranked_scores", "count": 1, "finite": True, "latency_ms": 30.0, } probe.runtime_facts = {**probe.runtime_facts, "adapter": "qwen3_reranker"} session.commit() base.create_experiment( candidate.id, CapabilityExperimentCreate( route_key=f"storage-fault-{capability.replace('.', '-')}", execution_approval_id=probe.execution_approval_id, capability=capability, purpose="Test the exact local Gateway route during a Redis payload write outage.", ), ) service = InlineWorkerServingService( session, base.settings, base.manifests, base.payload_store ) created = service.create_client( ServiceClientCreate( name=f"storage-fault-{capability.replace('.', '-')}", allowed_capabilities=[f"{capability}@1"], requests_per_minute=10, ) ) def unavailable(*_args: object, **_kwargs: object) -> str: raise redis.ConnectionError("private redis host and credential") monkeypatch.setattr(service.payload_store, "put", unavailable) app.dependency_overrides[get_settings] = lambda: service.settings app.dependency_overrides[get_serving_service] = lambda: service try: response = TestClient(app).post( f"/api/v1/capabilities/{capability}@1/invoke", headers={"Authorization": f"Bearer {created.credential}"}, json=body, ) finally: app.dependency_overrides.clear() assert response.status_code == 503 assert response.json()["error"]["code"] == "PAYLOAD_STORE_UNAVAILABLE" assert "private redis host" not in response.text request = session.scalars(select(GatewayRequest)).one() lease = session.scalars(select(ServingGpuLease)).one() assert request.status == "failed" assert request.failure_code == "PAYLOAD_STORE_UNAVAILABLE" assert request.finished_at is not None assert request.decision_evidence["input_persisted"] is False assert "private" not in str(request.decision_evidence) assert lease.state == "failed" assert lease.failure_code == "PAYLOAD_STORE_UNAVAILABLE" assert lease.released_at is not None assert not session.scalars(select(ServingJob).where(ServingJob.operation == "invoke")).all() assert service.payload_store.values == {} residency = service.repo.residency(request.capability_deployment_id) assert residency is not None and residency.active_requests == 0