125 lines
4.6 KiB
Python
125 lines
4.6 KiB
Python
"""Deterministic node and NVML failure injection for RC-QUAL-03.
|
|
|
|
These tests exercise the serving boundary without a live GPU. The isolated
|
|
candidate still needs the corresponding real-stack fault evidence.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, func, select
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from modelforge_api.domain.serving import (
|
|
EmbeddingInvokeRequest,
|
|
PlacementPlanRequest,
|
|
ServiceClientCreate,
|
|
)
|
|
from modelforge_api.persistence.models import (
|
|
Base,
|
|
GatewayRequest,
|
|
ServingGpuLease,
|
|
ServingJob,
|
|
)
|
|
from modelforge_api.services.serving import ServingError
|
|
from tests.test_serving import approve_and_promote, setup_m5
|
|
|
|
|
|
@pytest.fixture
|
|
def session() -> Session:
|
|
engine = create_engine(
|
|
"sqlite+pysqlite:///:memory:",
|
|
connect_args={"check_same_thread": False},
|
|
poolclass=StaticPool,
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as value:
|
|
yield value
|
|
engine.dispose()
|
|
|
|
|
|
def _count(session: Session, model: type[GatewayRequest | ServingGpuLease | ServingJob]) -> int:
|
|
return session.scalar(select(func.count()).select_from(model)) or 0
|
|
|
|
|
|
def test_stale_node_refuses_route_and_placement_without_allocating_work(
|
|
session: Session,
|
|
) -> None:
|
|
service, candidate, node = setup_m5(session)
|
|
_approval, promoted = approve_and_promote(service, candidate.id)
|
|
client = service.create_client(
|
|
ServiceClientCreate(name="node-loss", allowed_capabilities=["rag.embedding@1"])
|
|
)
|
|
authenticated = service.authenticate(f"Bearer {client.credential}", "rag.embedding@1")
|
|
counts = tuple(
|
|
_count(session, model) for model in (GatewayRequest, ServingGpuLease, ServingJob)
|
|
)
|
|
|
|
node.liveness_state = "stale"
|
|
node.last_heartbeat_at = datetime.now(UTC) - timedelta(minutes=5)
|
|
session.commit()
|
|
|
|
plan = service.dry_run_placement(promoted.id, PlacementPlanRequest(priority="production"))
|
|
assert plan.verdict == "REJECT_HEALTH"
|
|
assert plan.reason_codes == ["NODE_UNAVAILABLE"]
|
|
assert plan.evictions == []
|
|
with pytest.raises(ServingError) as caught:
|
|
service.invoke(EmbeddingInvokeRequest(input="must stay local"), authenticated)
|
|
assert caught.value.status_code == 503
|
|
assert caught.value.code == "NO_ELIGIBLE_NODE"
|
|
assert (
|
|
tuple(_count(session, model) for model in (GatewayRequest, ServingGpuLease, ServingJob))
|
|
== counts
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("telemetry_failure", ["missing", "stale"])
|
|
def test_nvml_unknown_refuses_invoke_and_cleans_request_without_worker_fallback(
|
|
session: Session,
|
|
telemetry_failure: str,
|
|
) -> None:
|
|
service, candidate, _node = setup_m5(session)
|
|
_approval, promoted = approve_and_promote(service, candidate.id)
|
|
client = service.create_client(
|
|
ServiceClientCreate(
|
|
name=f"nvml-{telemetry_failure}", allowed_capabilities=["rag.embedding@1"]
|
|
)
|
|
)
|
|
authenticated = service.authenticate(f"Bearer {client.credential}", "rag.embedding@1")
|
|
jobs_before = _count(session, ServingJob)
|
|
leases_before = _count(session, ServingGpuLease)
|
|
telemetry = service.repo.telemetry(promoted.accelerator_id)
|
|
assert telemetry is not None
|
|
if telemetry_failure == "missing":
|
|
telemetry.used_vram_bytes = None
|
|
telemetry.free_vram_bytes = None
|
|
else:
|
|
telemetry.observed_at = datetime.now(UTC) - timedelta(hours=1)
|
|
session.commit()
|
|
|
|
budget = service.scheduler_overview()[0]
|
|
assert budget.attribution_confidence == "UNKNOWN"
|
|
assert budget.schedulable_free_vram_bytes == 0
|
|
plan = service.dry_run_placement(promoted.id, PlacementPlanRequest(priority="production"))
|
|
assert plan.verdict == "REJECT_HEALTH"
|
|
assert plan.reason_codes == ["SCHEDULER_STATE_STALE"]
|
|
assert plan.evictions == []
|
|
|
|
with pytest.raises(ServingError) as caught:
|
|
service.invoke(
|
|
EmbeddingInvokeRequest(input="never infer on guessed capacity"), authenticated
|
|
)
|
|
assert caught.value.status_code == 503
|
|
assert caught.value.code == "SCHEDULER_STATE_STALE"
|
|
request = session.scalar(select(GatewayRequest).order_by(GatewayRequest.created_at.desc()))
|
|
assert request is not None
|
|
assert request.status == "failed"
|
|
assert request.failure_code == "SCHEDULER_STATE_STALE"
|
|
assert request.finished_at is not None
|
|
assert request.compute_node_id == promoted.compute_node_id
|
|
assert _count(session, ServingGpuLease) == leases_before
|
|
assert _count(session, ServingJob) == jobs_before
|