430 lines
15 KiB
Python
430 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Iterator
|
|
from datetime import UTC, datetime, timedelta
|
|
from uuid import UUID
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from pydantic import SecretStr
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.exc import SQLAlchemyError
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from modelforge_api.api.routes.observability import get_service
|
|
from modelforge_api.main import app
|
|
from modelforge_api.persistence.models import (
|
|
Accelerator,
|
|
Base,
|
|
ComputeNode,
|
|
GatewayRequest,
|
|
OperationalIncident,
|
|
ProjectBinding,
|
|
ServiceClient,
|
|
ServingGpuLease,
|
|
)
|
|
from modelforge_api.services.observability import ObservabilityService
|
|
from modelforge_api.settings import Settings, get_settings
|
|
|
|
|
|
@pytest.fixture
|
|
def session() -> Iterator[Session]:
|
|
engine = create_engine(
|
|
"sqlite+pysqlite:///:memory:",
|
|
connect_args={"check_same_thread": False},
|
|
poolclass=StaticPool,
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as value:
|
|
yield value
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def clean_overrides() -> Iterator[None]:
|
|
yield
|
|
app.dependency_overrides.clear()
|
|
|
|
|
|
def add_active_lease(
|
|
session: Session,
|
|
*,
|
|
lease_id: str,
|
|
lease_created_at: datetime,
|
|
expires_at: datetime,
|
|
state: str = "active",
|
|
request_status: str = "running",
|
|
heartbeat_at: datetime | None = None,
|
|
include_request: bool = True,
|
|
request_deployment_id: UUID | None = None,
|
|
) -> tuple[str, str, str, str]:
|
|
node_id = UUID(f"00000000-0000-0000-0000-{lease_id[-12:]}")
|
|
accelerator_id = UUID(f"10000000-0000-0000-0000-{lease_id[-12:]}")
|
|
deployment_id = UUID(f"20000000-0000-0000-0000-{lease_id[-12:]}")
|
|
lease_uuid = UUID(f"30000000-0000-0000-0000-{lease_id[-12:]}")
|
|
request_uuid = UUID(f"40000000-0000-0000-0000-{lease_id[-12:]}")
|
|
client_uuid = UUID(f"50000000-0000-0000-0000-{lease_id[-12:]}")
|
|
project_binding_uuid = UUID(f"60000000-0000-0000-0000-{lease_id[-12:]}")
|
|
project_uuid = UUID(f"70000000-0000-0000-0000-{lease_id[-12:]}")
|
|
contract_uuid = UUID(f"80000000-0000-0000-0000-{lease_id[-12:]}")
|
|
node = ComputeNode(id=node_id, key=f"node-{lease_id}", hostname=f"{lease_id}.local")
|
|
accelerator = Accelerator(
|
|
id=accelerator_id,
|
|
compute_node_id=node_id,
|
|
device_index=0,
|
|
device_uuid=f"GPU-{lease_id}",
|
|
name="Test GPU",
|
|
)
|
|
binding = ProjectBinding(
|
|
id=project_binding_uuid,
|
|
project_id=project_uuid,
|
|
capability_contract_id=contract_uuid,
|
|
channel="stable",
|
|
priority="production",
|
|
)
|
|
client = ServiceClient(
|
|
id=client_uuid,
|
|
name=f"private-client-{lease_id}",
|
|
project_binding_id=project_binding_uuid,
|
|
)
|
|
session.add_all([node, accelerator, binding, client])
|
|
if include_request:
|
|
session.add(
|
|
GatewayRequest(
|
|
request_id=request_uuid,
|
|
service_client_id=client_uuid,
|
|
capability_key="rag.embedding",
|
|
capability_version=1,
|
|
capability_deployment_id=request_deployment_id or deployment_id,
|
|
compute_node_id=node_id,
|
|
status=request_status,
|
|
priority="production",
|
|
input_sha256="f" * 64,
|
|
input_count=1,
|
|
)
|
|
)
|
|
session.add(
|
|
ServingGpuLease(
|
|
id=lease_uuid,
|
|
accelerator_id=accelerator_id,
|
|
capability_deployment_id=deployment_id,
|
|
request_id=request_uuid if include_request else None,
|
|
reserved_vram_bytes=2_048,
|
|
priority="production",
|
|
state=state,
|
|
owner=f"private-owner-{lease_id}",
|
|
created_at=lease_created_at,
|
|
expires_at=expires_at,
|
|
acquired_at=lease_created_at,
|
|
heartbeat_at=heartbeat_at or lease_created_at,
|
|
lease_type="request_execution",
|
|
materialized_vram_bytes=512,
|
|
)
|
|
)
|
|
return str(lease_uuid), str(project_uuid), str(node_id), str(deployment_id)
|
|
|
|
|
|
def test_operations_routes_require_the_operator_credential(session: Session) -> None:
|
|
subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
|
|
subject.ensure_defaults()
|
|
app.dependency_overrides[get_service] = lambda: subject
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
operator_api_key="operator-secret", observability_monitor_enabled=False
|
|
)
|
|
client = TestClient(app)
|
|
|
|
assert client.get("/api/v1/admin/operations/overview").status_code == 401
|
|
assert client.get(
|
|
"/api/v1/admin/operations/overview",
|
|
headers={"Authorization": "Bearer capability-client-secret"},
|
|
).status_code == 401
|
|
response = client.get(
|
|
"/api/v1/admin/operations/overview",
|
|
headers={"X-ModelForge-Admin-Token": "operator-secret"},
|
|
)
|
|
assert response.status_code == 200
|
|
assert response.json()["status"] == "HEALTHY"
|
|
|
|
|
|
def test_incident_list_keeps_legacy_shape_and_supports_bounded_pages(session: Session) -> None:
|
|
observed_at = datetime.now(UTC)
|
|
session.add_all([
|
|
OperationalIncident(
|
|
fingerprint=f"{index:064x}",
|
|
title=f"Incident {index}",
|
|
state="OPEN",
|
|
severity="CRITICAL",
|
|
root_subject_type="node",
|
|
root_subject_ref=f"node-{index}",
|
|
correlation="LIKELY_ROOT",
|
|
first_seen_at=observed_at + timedelta(seconds=index),
|
|
last_seen_at=observed_at + timedelta(seconds=index),
|
|
)
|
|
for index in range(3)
|
|
])
|
|
session.commit()
|
|
|
|
service = ObservabilityService(session, Settings(observability_monitor_enabled=False))
|
|
app.dependency_overrides[get_service] = lambda: service
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
operator_api_key="operator-secret", observability_monitor_enabled=False
|
|
)
|
|
client = TestClient(app)
|
|
headers = {"X-ModelForge-Admin-Token": "operator-secret"}
|
|
path = "/api/v1/admin/operations/incidents"
|
|
|
|
legacy = client.get(path, headers=headers)
|
|
first_page = client.get(f"{path}?limit=2&offset=0", headers=headers)
|
|
second_page = client.get(f"{path}?limit=2&offset=2", headers=headers)
|
|
empty_page = client.get(f"{path}?limit=2&offset=3", headers=headers)
|
|
|
|
assert legacy.status_code == 200 and len(legacy.json()) == 3
|
|
assert first_page.status_code == 200 and len(first_page.json()) == 2
|
|
assert second_page.status_code == 200 and len(second_page.json()) == 1
|
|
assert empty_page.status_code == 200 and empty_page.json() == []
|
|
assert [row["title"] for row in first_page.json() + second_page.json()] == [
|
|
"Incident 2", "Incident 1", "Incident 0",
|
|
]
|
|
assert set(first_page.json()[0]) == {
|
|
"id", "fingerprint", "title", "state", "severity", "root_subject_type",
|
|
"root_subject_ref", "correlation", "first_seen_at", "last_seen_at", "resolved_at",
|
|
}
|
|
assert client.get(f"{path}?limit=501&offset=0", headers=headers).status_code == 422
|
|
assert client.get(f"{path}?offset=2", headers=headers).status_code == 422
|
|
|
|
|
|
def test_active_lease_inventory_is_admin_only_and_returns_safe_current_identity(
|
|
session: Session,
|
|
) -> None:
|
|
now = datetime.now(UTC)
|
|
lease_id, project_id, node_id, deployment_id = add_active_lease(
|
|
session,
|
|
lease_id="lease-000000000001",
|
|
lease_created_at=now - timedelta(seconds=2),
|
|
expires_at=now + timedelta(seconds=30),
|
|
)
|
|
subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
|
|
app.dependency_overrides[get_service] = lambda: subject
|
|
tokens = {role: f"mfop_{role[0] * 43}" for role in ("viewer", "operator", "admin")}
|
|
identities = [
|
|
{
|
|
"credential_id": f"active-leases-{role}",
|
|
"subject": f"active-leases-{role}",
|
|
"role": role,
|
|
"token_sha256": hashlib.sha256(token.encode()).hexdigest(),
|
|
"expires_at": (now + timedelta(hours=1)).isoformat(),
|
|
}
|
|
for role, token in tokens.items()
|
|
]
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
_env_file=None,
|
|
operator_api_key=None,
|
|
operator_identities=SecretStr(json.dumps(identities)),
|
|
observability_monitor_enabled=False,
|
|
)
|
|
client = TestClient(app)
|
|
path = "/api/v1/admin/operations/active-leases"
|
|
|
|
assert client.get(path).status_code == 401
|
|
|
|
for role in ("viewer", "operator"):
|
|
response = client.get(path, headers={"X-ModelForge-Admin-Token": tokens[role]})
|
|
assert response.status_code == 403
|
|
|
|
response = client.get(path, headers={"X-ModelForge-Admin-Token": tokens["admin"]})
|
|
|
|
assert response.status_code == 200
|
|
payload = response.json()
|
|
assert set(payload) == {"observed_at", "completeness", "limit", "returned_count", "leases"}
|
|
assert payload["completeness"] == "COMPLETE"
|
|
assert payload["limit"] == 100
|
|
assert payload["returned_count"] == 1
|
|
assert set(payload["leases"][0]) == {
|
|
"lease_id",
|
|
"project_id",
|
|
"node_id",
|
|
"accelerator_id",
|
|
"deployment_id",
|
|
"lease_state",
|
|
"workload_status",
|
|
"priority",
|
|
"lease_type",
|
|
"reserved_vram_bytes",
|
|
"materialized_vram_bytes",
|
|
"created_at",
|
|
"acquired_at",
|
|
"heartbeat_at",
|
|
"expires_at",
|
|
}
|
|
lease = payload["leases"][0]
|
|
assert lease["lease_id"] == lease_id
|
|
assert lease["project_id"] == project_id
|
|
assert lease["node_id"] == node_id
|
|
assert lease["deployment_id"] == deployment_id
|
|
assert lease["lease_state"] == "ACTIVE"
|
|
assert lease["workload_status"] == "RUNNING"
|
|
for forbidden in (
|
|
"owner",
|
|
"request_id",
|
|
"private-owner",
|
|
"private-client",
|
|
"input_sha256",
|
|
"payload",
|
|
):
|
|
assert forbidden not in response.text
|
|
assert not session.new
|
|
assert not session.dirty
|
|
assert not session.deleted
|
|
|
|
|
|
def test_active_lease_inventory_excludes_expired_rows_and_bounds_sorted_results(
|
|
session: Session,
|
|
) -> None:
|
|
now = datetime.now(UTC)
|
|
first_id, *_ = add_active_lease(
|
|
session,
|
|
lease_id="lease-000000000011",
|
|
lease_created_at=now - timedelta(seconds=10),
|
|
expires_at=now + timedelta(seconds=60),
|
|
)
|
|
second_id, *_ = add_active_lease(
|
|
session,
|
|
lease_id="lease-000000000012",
|
|
lease_created_at=now - timedelta(seconds=5),
|
|
expires_at=now + timedelta(seconds=60),
|
|
)
|
|
expired_id, *_ = add_active_lease(
|
|
session,
|
|
lease_id="lease-000000000013",
|
|
lease_created_at=now - timedelta(seconds=30),
|
|
expires_at=now - timedelta(seconds=1),
|
|
)
|
|
terminal_id, *_ = add_active_lease(
|
|
session,
|
|
lease_id="lease-000000000014",
|
|
lease_created_at=now - timedelta(seconds=3),
|
|
expires_at=now + timedelta(seconds=60),
|
|
state="released",
|
|
)
|
|
app.dependency_overrides[get_service] = lambda: ObservabilityService(
|
|
session, Settings(observability_monitor_enabled=False)
|
|
)
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
_env_file=None,
|
|
operator_api_key=SecretStr("operator-secret"),
|
|
observability_monitor_enabled=False,
|
|
)
|
|
|
|
response = TestClient(app).get(
|
|
"/api/v1/admin/operations/active-leases?limit=1&sort_by=created_at&order=desc",
|
|
headers={"X-ModelForge-Admin-Token": "operator-secret"},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
payload = response.json()
|
|
assert payload["completeness"] == "TRUNCATED"
|
|
assert payload["limit"] == payload["returned_count"] == 1
|
|
assert [lease["lease_id"] for lease in payload["leases"]] == [second_id]
|
|
assert expired_id not in response.text
|
|
assert terminal_id not in response.text
|
|
assert first_id not in response.text
|
|
assert TestClient(app).get(
|
|
"/api/v1/admin/operations/active-leases?limit=501",
|
|
headers={"X-ModelForge-Admin-Token": "operator-secret"},
|
|
).status_code == 422
|
|
|
|
|
|
def test_active_lease_inventory_marks_unresolved_workload_status_unknown(
|
|
session: Session,
|
|
) -> None:
|
|
now = datetime.now(UTC)
|
|
lease_id, project_id, *_ = add_active_lease(
|
|
session,
|
|
lease_id="lease-000000000021",
|
|
lease_created_at=now - timedelta(seconds=2),
|
|
expires_at=now + timedelta(seconds=30),
|
|
request_status="completed",
|
|
heartbeat_at=now - timedelta(minutes=5),
|
|
)
|
|
mismatched_lease_id, *_ = add_active_lease(
|
|
session,
|
|
lease_id="lease-000000000022",
|
|
lease_created_at=now - timedelta(seconds=1),
|
|
expires_at=now + timedelta(seconds=30),
|
|
state="pending",
|
|
request_status="running",
|
|
)
|
|
app.dependency_overrides[get_service] = lambda: ObservabilityService(
|
|
session, Settings(observability_monitor_enabled=False)
|
|
)
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
_env_file=None,
|
|
operator_api_key=SecretStr("operator-secret"),
|
|
observability_monitor_enabled=False,
|
|
)
|
|
|
|
response = TestClient(app).get(
|
|
"/api/v1/admin/operations/active-leases",
|
|
headers={"X-ModelForge-Admin-Token": "operator-secret"},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
leases = {item["lease_id"]: item for item in response.json()["leases"]}
|
|
lease = leases[lease_id]
|
|
assert lease["project_id"] == project_id
|
|
assert lease["workload_status"] == "UNKNOWN"
|
|
assert lease["lease_state"] == "ACTIVE"
|
|
assert lease["heartbeat_at"] == (now - timedelta(minutes=5)).isoformat().replace("+00:00", "Z")
|
|
assert leases[mismatched_lease_id]["lease_state"] == "PENDING"
|
|
assert leases[mismatched_lease_id]["workload_status"] == "UNKNOWN"
|
|
|
|
|
|
def test_prometheus_endpoint_is_bounded_and_contains_no_request_payload(
|
|
session: Session,
|
|
) -> None:
|
|
subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
|
|
subject.ensure_defaults()
|
|
app.dependency_overrides[get_service] = lambda: subject
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
operator_api_key="operator-secret", observability_monitor_enabled=False
|
|
)
|
|
response = TestClient(app).get(
|
|
"/metrics", headers={"X-ModelForge-Admin-Token": "operator-secret"}
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.headers["content-type"].startswith("text/plain")
|
|
assert "modelforge_api_requests_total" in response.text
|
|
assert "modelforge_api_request_duration_seconds_bucket" in response.text
|
|
for forbidden in ("request_id=", "input_sha256=", "prompt=", "payload="):
|
|
assert forbidden not in response.text
|
|
|
|
|
|
def test_process_metrics_remain_available_when_observability_storage_fails() -> None:
|
|
class BrokenSession:
|
|
rolled_back = False
|
|
|
|
def rollback(self) -> None:
|
|
self.rolled_back = True
|
|
|
|
class BrokenService:
|
|
session = BrokenSession()
|
|
|
|
def prometheus(self) -> str:
|
|
raise SQLAlchemyError("simulated observability database outage")
|
|
|
|
broken = BrokenService()
|
|
app.dependency_overrides[get_service] = lambda: broken
|
|
app.dependency_overrides[get_settings] = lambda: Settings(
|
|
operator_api_key="operator-secret", observability_monitor_enabled=False
|
|
)
|
|
response = TestClient(app).get(
|
|
"/metrics", headers={"X-ModelForge-Admin-Token": "operator-secret"}
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert broken.session.rolled_back is True
|
|
assert "modelforge_observability_degraded 1" in response.text
|