Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/tests/test_rc_failure_node.py

@@ -0,0 +1,124 @@
+"""Deterministic node and NVML failure injection for RC-QUAL-03.
+
+These tests exercise the serving boundary without a live GPU. The isolated
+candidate still needs the corresponding real-stack fault evidence.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+from sqlalchemy.pool import StaticPool
+
+from modelforge_api.domain.serving import (
+    EmbeddingInvokeRequest,
+    PlacementPlanRequest,
+    ServiceClientCreate,
+)
+from modelforge_api.persistence.models import (
+    Base,
+    GatewayRequest,
+    ServingGpuLease,
+    ServingJob,
+)
+from modelforge_api.services.serving import ServingError
+from tests.test_serving import approve_and_promote, setup_m5
+
+
+@pytest.fixture
+def session() -> Session:
+    engine = create_engine(
+        "sqlite+pysqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+    with Session(engine) as value:
+        yield value
+    engine.dispose()
+
+
+def _count(session: Session, model: type[GatewayRequest | ServingGpuLease | ServingJob]) -> int:
+    return session.scalar(select(func.count()).select_from(model)) or 0
+
+
+def test_stale_node_refuses_route_and_placement_without_allocating_work(
+    session: Session,
+) -> None:
+    service, candidate, node = setup_m5(session)
+    _approval, promoted = approve_and_promote(service, candidate.id)
+    client = service.create_client(
+        ServiceClientCreate(name="node-loss", allowed_capabilities=["rag.embedding@1"])
+    )
+    authenticated = service.authenticate(f"Bearer {client.credential}", "rag.embedding@1")
+    counts = tuple(
+        _count(session, model) for model in (GatewayRequest, ServingGpuLease, ServingJob)
+    )
+
+    node.liveness_state = "stale"
+    node.last_heartbeat_at = datetime.now(UTC) - timedelta(minutes=5)
+    session.commit()
+
+    plan = service.dry_run_placement(promoted.id, PlacementPlanRequest(priority="production"))
+    assert plan.verdict == "REJECT_HEALTH"
+    assert plan.reason_codes == ["NODE_UNAVAILABLE"]
+    assert plan.evictions == []
+    with pytest.raises(ServingError) as caught:
+        service.invoke(EmbeddingInvokeRequest(input="must stay local"), authenticated)
+    assert caught.value.status_code == 503
+    assert caught.value.code == "NO_ELIGIBLE_NODE"
+    assert (
+        tuple(_count(session, model) for model in (GatewayRequest, ServingGpuLease, ServingJob))
+        == counts
+    )
+
+
+@pytest.mark.parametrize("telemetry_failure", ["missing", "stale"])
+def test_nvml_unknown_refuses_invoke_and_cleans_request_without_worker_fallback(
+    session: Session,
+    telemetry_failure: str,
+) -> None:
+    service, candidate, _node = setup_m5(session)
+    _approval, promoted = approve_and_promote(service, candidate.id)
+    client = service.create_client(
+        ServiceClientCreate(
+            name=f"nvml-{telemetry_failure}", allowed_capabilities=["rag.embedding@1"]
+        )
+    )
+    authenticated = service.authenticate(f"Bearer {client.credential}", "rag.embedding@1")
+    jobs_before = _count(session, ServingJob)
+    leases_before = _count(session, ServingGpuLease)
+    telemetry = service.repo.telemetry(promoted.accelerator_id)
+    assert telemetry is not None
+    if telemetry_failure == "missing":
+        telemetry.used_vram_bytes = None
+        telemetry.free_vram_bytes = None
+    else:
+        telemetry.observed_at = datetime.now(UTC) - timedelta(hours=1)
+    session.commit()
+
+    budget = service.scheduler_overview()[0]
+    assert budget.attribution_confidence == "UNKNOWN"
+    assert budget.schedulable_free_vram_bytes == 0
+    plan = service.dry_run_placement(promoted.id, PlacementPlanRequest(priority="production"))
+    assert plan.verdict == "REJECT_HEALTH"
+    assert plan.reason_codes == ["SCHEDULER_STATE_STALE"]
+    assert plan.evictions == []
+
+    with pytest.raises(ServingError) as caught:
+        service.invoke(
+            EmbeddingInvokeRequest(input="never infer on guessed capacity"), authenticated
+        )
+    assert caught.value.status_code == 503
+    assert caught.value.code == "SCHEDULER_STATE_STALE"
+    request = session.scalar(select(GatewayRequest).order_by(GatewayRequest.created_at.desc()))
+    assert request is not None
+    assert request.status == "failed"
+    assert request.failure_code == "SCHEDULER_STATE_STALE"
+    assert request.finished_at is not None
+    assert request.compute_node_id == promoted.compute_node_id
+    assert _count(session, ServingGpuLease) == leases_before
+    assert _count(session, ServingJob) == jobs_before