Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/tests/test_rc_failure_storage_modalities.py

@@ -0,0 +1,108 @@
+"""Redis write outages fail closed on non-text Gateway invocation routes."""
+
+from __future__ import annotations
+
+import pytest
+import redis
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.api.routes.serving import get_serving_service
+from modelforge_api.domain.serving import CapabilityExperimentCreate, ServiceClientCreate
+from modelforge_api.main import app
+from modelforge_api.persistence.models import Base, GatewayRequest, ServingGpuLease, ServingJob
+from modelforge_api.settings import get_settings
+from tests.test_serving import InlineWorkerServingService, setup_m5
+
+
+@pytest.mark.parametrize(
+    ("capability", "body"),
+    [
+        (
+            "rag.reranking",
+            {
+                "query": "private query",
+                "documents": [{"id": "one", "text": "private document"}],
+                "top_n": 1,
+            },
+        ),
+        ("vision.embedding", {"items": [{"text": "private visual query"}]}),
+    ],
+)
+def test_payload_write_outage_fails_closed_for_non_text_routes(
+    capability: str, body: dict[str, object], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        base, candidate, _node = setup_m5(session)
+        probe = base.repo.probe(candidate.runtime_probe_id)
+        assert probe is not None
+        if capability == "rag.reranking":
+            probe.inference_result = {
+                "status": "passed",
+                "output_type": "ranked_scores",
+                "count": 1,
+                "finite": True,
+                "latency_ms": 30.0,
+            }
+            probe.runtime_facts = {**probe.runtime_facts, "adapter": "qwen3_reranker"}
+        session.commit()
+        base.create_experiment(
+            candidate.id,
+            CapabilityExperimentCreate(
+                route_key=f"storage-fault-{capability.replace('.', '-')}",
+                execution_approval_id=probe.execution_approval_id,
+                capability=capability,
+                purpose="Test the exact local Gateway route during a Redis payload write outage.",
+            ),
+        )
+        service = InlineWorkerServingService(
+            session, base.settings, base.manifests, base.payload_store
+        )
+        created = service.create_client(
+            ServiceClientCreate(
+                name=f"storage-fault-{capability.replace('.', '-')}",
+                allowed_capabilities=[f"{capability}@1"],
+                requests_per_minute=10,
+            )
+        )
+
+        def unavailable(*_args: object, **_kwargs: object) -> str:
+            raise redis.ConnectionError("private redis host and credential")
+
+        monkeypatch.setattr(service.payload_store, "put", unavailable)
+        app.dependency_overrides[get_settings] = lambda: service.settings
+        app.dependency_overrides[get_serving_service] = lambda: service
+        try:
+            response = TestClient(app).post(
+                f"/api/v1/capabilities/{capability}@1/invoke",
+                headers={"Authorization": f"Bearer {created.credential}"},
+                json=body,
+            )
+        finally:
+            app.dependency_overrides.clear()
+
+        assert response.status_code == 503
+        assert response.json()["error"]["code"] == "PAYLOAD_STORE_UNAVAILABLE"
+        assert "private redis host" not in response.text
+        request = session.scalars(select(GatewayRequest)).one()
+        lease = session.scalars(select(ServingGpuLease)).one()
+        assert request.status == "failed"
+        assert request.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
+        assert request.finished_at is not None
+        assert request.decision_evidence["input_persisted"] is False
+        assert "private" not in str(request.decision_evidence)
+        assert lease.state == "failed"
+        assert lease.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
+        assert lease.released_at is not None
+        assert not session.scalars(select(ServingJob).where(ServingJob.operation == "invoke")).all()
+        assert service.payload_store.values == {}
+        residency = service.repo.residency(request.capability_deployment_id)
+        assert residency is not None and residency.active_requests == 0