109 lines
4.4 KiB
Python
109 lines
4.4 KiB
Python
"""Redis write outages fail closed on non-text Gateway invocation routes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
import redis
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from modelforge_api.api.routes.serving import get_serving_service
|
|
from modelforge_api.domain.serving import CapabilityExperimentCreate, ServiceClientCreate
|
|
from modelforge_api.main import app
|
|
from modelforge_api.persistence.models import Base, GatewayRequest, ServingGpuLease, ServingJob
|
|
from modelforge_api.settings import get_settings
|
|
from tests.test_serving import InlineWorkerServingService, setup_m5
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("capability", "body"),
|
|
[
|
|
(
|
|
"rag.reranking",
|
|
{
|
|
"query": "private query",
|
|
"documents": [{"id": "one", "text": "private document"}],
|
|
"top_n": 1,
|
|
},
|
|
),
|
|
("vision.embedding", {"items": [{"text": "private visual query"}]}),
|
|
],
|
|
)
|
|
def test_payload_write_outage_fails_closed_for_non_text_routes(
|
|
capability: str, body: dict[str, object], monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
engine = create_engine(
|
|
"sqlite+pysqlite:///:memory:",
|
|
connect_args={"check_same_thread": False},
|
|
poolclass=StaticPool,
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
base, candidate, _node = setup_m5(session)
|
|
probe = base.repo.probe(candidate.runtime_probe_id)
|
|
assert probe is not None
|
|
if capability == "rag.reranking":
|
|
probe.inference_result = {
|
|
"status": "passed",
|
|
"output_type": "ranked_scores",
|
|
"count": 1,
|
|
"finite": True,
|
|
"latency_ms": 30.0,
|
|
}
|
|
probe.runtime_facts = {**probe.runtime_facts, "adapter": "qwen3_reranker"}
|
|
session.commit()
|
|
base.create_experiment(
|
|
candidate.id,
|
|
CapabilityExperimentCreate(
|
|
route_key=f"storage-fault-{capability.replace('.', '-')}",
|
|
execution_approval_id=probe.execution_approval_id,
|
|
capability=capability,
|
|
purpose="Test the exact local Gateway route during a Redis payload write outage.",
|
|
),
|
|
)
|
|
service = InlineWorkerServingService(
|
|
session, base.settings, base.manifests, base.payload_store
|
|
)
|
|
created = service.create_client(
|
|
ServiceClientCreate(
|
|
name=f"storage-fault-{capability.replace('.', '-')}",
|
|
allowed_capabilities=[f"{capability}@1"],
|
|
requests_per_minute=10,
|
|
)
|
|
)
|
|
|
|
def unavailable(*_args: object, **_kwargs: object) -> str:
|
|
raise redis.ConnectionError("private redis host and credential")
|
|
|
|
monkeypatch.setattr(service.payload_store, "put", unavailable)
|
|
app.dependency_overrides[get_settings] = lambda: service.settings
|
|
app.dependency_overrides[get_serving_service] = lambda: service
|
|
try:
|
|
response = TestClient(app).post(
|
|
f"/api/v1/capabilities/{capability}@1/invoke",
|
|
headers={"Authorization": f"Bearer {created.credential}"},
|
|
json=body,
|
|
)
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
assert response.status_code == 503
|
|
assert response.json()["error"]["code"] == "PAYLOAD_STORE_UNAVAILABLE"
|
|
assert "private redis host" not in response.text
|
|
request = session.scalars(select(GatewayRequest)).one()
|
|
lease = session.scalars(select(ServingGpuLease)).one()
|
|
assert request.status == "failed"
|
|
assert request.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
|
|
assert request.finished_at is not None
|
|
assert request.decision_evidence["input_persisted"] is False
|
|
assert "private" not in str(request.decision_evidence)
|
|
assert lease.state == "failed"
|
|
assert lease.failure_code == "PAYLOAD_STORE_UNAVAILABLE"
|
|
assert lease.released_at is not None
|
|
assert not session.scalars(select(ServingJob).where(ServingJob.operation == "invoke")).all()
|
|
assert service.payload_store.values == {}
|
|
residency = service.repo.residency(request.capability_deployment_id)
|
|
assert residency is not None and residency.active_requests == 0
|