Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

backend/tests/test_observability_api_m14.py

@@ -1,9 +1,14 @@
 from __future__ import annotations
 
+import hashlib
+import json
 from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from uuid import UUID
 
 import pytest
 from fastapi.testclient import TestClient
+from pydantic import SecretStr
 from sqlalchemy import create_engine
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import Session
@@ -11,7 +16,16 @@ from sqlalchemy.pool import StaticPool
 
 from modelforge_api.api.routes.observability import get_service
 from modelforge_api.main import app
-from modelforge_api.persistence.models import Base
+from modelforge_api.persistence.models import (
+    Accelerator,
+    Base,
+    ComputeNode,
+    GatewayRequest,
+    OperationalIncident,
+    ProjectBinding,
+    ServiceClient,
+    ServingGpuLease,
+)
 from modelforge_api.services.observability import ObservabilityService
 from modelforge_api.settings import Settings, get_settings
 
@@ -34,6 +48,84 @@ def clean_overrides() -> Iterator[None]:
     app.dependency_overrides.clear()
 
 
+def add_active_lease(
+    session: Session,
+    *,
+    lease_id: str,
+    lease_created_at: datetime,
+    expires_at: datetime,
+    state: str = "active",
+    request_status: str = "running",
+    heartbeat_at: datetime | None = None,
+    include_request: bool = True,
+    request_deployment_id: UUID | None = None,
+) -> tuple[str, str, str, str]:
+    node_id = UUID(f"00000000-0000-0000-0000-{lease_id[-12:]}")
+    accelerator_id = UUID(f"10000000-0000-0000-0000-{lease_id[-12:]}")
+    deployment_id = UUID(f"20000000-0000-0000-0000-{lease_id[-12:]}")
+    lease_uuid = UUID(f"30000000-0000-0000-0000-{lease_id[-12:]}")
+    request_uuid = UUID(f"40000000-0000-0000-0000-{lease_id[-12:]}")
+    client_uuid = UUID(f"50000000-0000-0000-0000-{lease_id[-12:]}")
+    project_binding_uuid = UUID(f"60000000-0000-0000-0000-{lease_id[-12:]}")
+    project_uuid = UUID(f"70000000-0000-0000-0000-{lease_id[-12:]}")
+    contract_uuid = UUID(f"80000000-0000-0000-0000-{lease_id[-12:]}")
+    node = ComputeNode(id=node_id, key=f"node-{lease_id}", hostname=f"{lease_id}.local")
+    accelerator = Accelerator(
+        id=accelerator_id,
+        compute_node_id=node_id,
+        device_index=0,
+        device_uuid=f"GPU-{lease_id}",
+        name="Test GPU",
+    )
+    binding = ProjectBinding(
+        id=project_binding_uuid,
+        project_id=project_uuid,
+        capability_contract_id=contract_uuid,
+        channel="stable",
+        priority="production",
+    )
+    client = ServiceClient(
+        id=client_uuid,
+        name=f"private-client-{lease_id}",
+        project_binding_id=project_binding_uuid,
+    )
+    session.add_all([node, accelerator, binding, client])
+    if include_request:
+        session.add(
+            GatewayRequest(
+                request_id=request_uuid,
+                service_client_id=client_uuid,
+                capability_key="rag.embedding",
+                capability_version=1,
+                capability_deployment_id=request_deployment_id or deployment_id,
+                compute_node_id=node_id,
+                status=request_status,
+                priority="production",
+                input_sha256="f" * 64,
+                input_count=1,
+            )
+        )
+    session.add(
+        ServingGpuLease(
+            id=lease_uuid,
+            accelerator_id=accelerator_id,
+            capability_deployment_id=deployment_id,
+            request_id=request_uuid if include_request else None,
+            reserved_vram_bytes=2_048,
+            priority="production",
+            state=state,
+            owner=f"private-owner-{lease_id}",
+            created_at=lease_created_at,
+            expires_at=expires_at,
+            acquired_at=lease_created_at,
+            heartbeat_at=heartbeat_at or lease_created_at,
+            lease_type="request_execution",
+            materialized_vram_bytes=512,
+        )
+    )
+    return str(lease_uuid), str(project_uuid), str(node_id), str(deployment_id)
+
+
 def test_operations_routes_require_the_operator_credential(session: Session) -> None:
     subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
     subject.ensure_defaults()
@@ -56,6 +148,239 @@ def test_operations_routes_require_the_operator_credential(session: Session) ->
     assert response.json()["status"] == "HEALTHY"
 
 
+def test_incident_list_keeps_legacy_shape_and_supports_bounded_pages(session: Session) -> None:
+    observed_at = datetime.now(UTC)
+    session.add_all([
+        OperationalIncident(
+            fingerprint=f"{index:064x}",
+            title=f"Incident {index}",
+            state="OPEN",
+            severity="CRITICAL",
+            root_subject_type="node",
+            root_subject_ref=f"node-{index}",
+            correlation="LIKELY_ROOT",
+            first_seen_at=observed_at + timedelta(seconds=index),
+            last_seen_at=observed_at + timedelta(seconds=index),
+        )
+        for index in range(3)
+    ])
+    session.commit()
+
+    service = ObservabilityService(session, Settings(observability_monitor_enabled=False))
+    app.dependency_overrides[get_service] = lambda: service
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        operator_api_key="operator-secret", observability_monitor_enabled=False
+    )
+    client = TestClient(app)
+    headers = {"X-ModelForge-Admin-Token": "operator-secret"}
+    path = "/api/v1/admin/operations/incidents"
+
+    legacy = client.get(path, headers=headers)
+    first_page = client.get(f"{path}?limit=2&offset=0", headers=headers)
+    second_page = client.get(f"{path}?limit=2&offset=2", headers=headers)
+    empty_page = client.get(f"{path}?limit=2&offset=3", headers=headers)
+
+    assert legacy.status_code == 200 and len(legacy.json()) == 3
+    assert first_page.status_code == 200 and len(first_page.json()) == 2
+    assert second_page.status_code == 200 and len(second_page.json()) == 1
+    assert empty_page.status_code == 200 and empty_page.json() == []
+    assert [row["title"] for row in first_page.json() + second_page.json()] == [
+        "Incident 2", "Incident 1", "Incident 0",
+    ]
+    assert set(first_page.json()[0]) == {
+        "id", "fingerprint", "title", "state", "severity", "root_subject_type",
+        "root_subject_ref", "correlation", "first_seen_at", "last_seen_at", "resolved_at",
+    }
+    assert client.get(f"{path}?limit=501&offset=0", headers=headers).status_code == 422
+    assert client.get(f"{path}?offset=2", headers=headers).status_code == 422
+
+
+def test_active_lease_inventory_is_admin_only_and_returns_safe_current_identity(
+    session: Session,
+) -> None:
+    now = datetime.now(UTC)
+    lease_id, project_id, node_id, deployment_id = add_active_lease(
+        session,
+        lease_id="lease-000000000001",
+        lease_created_at=now - timedelta(seconds=2),
+        expires_at=now + timedelta(seconds=30),
+    )
+    subject = ObservabilityService(session, Settings(observability_monitor_enabled=False))
+    app.dependency_overrides[get_service] = lambda: subject
+    tokens = {role: f"mfop_{role[0] * 43}" for role in ("viewer", "operator", "admin")}
+    identities = [
+        {
+            "credential_id": f"active-leases-{role}",
+            "subject": f"active-leases-{role}",
+            "role": role,
+            "token_sha256": hashlib.sha256(token.encode()).hexdigest(),
+            "expires_at": (now + timedelta(hours=1)).isoformat(),
+        }
+        for role, token in tokens.items()
+    ]
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        _env_file=None,
+        operator_api_key=None,
+        operator_identities=SecretStr(json.dumps(identities)),
+        observability_monitor_enabled=False,
+    )
+    client = TestClient(app)
+    path = "/api/v1/admin/operations/active-leases"
+
+    assert client.get(path).status_code == 401
+
+    for role in ("viewer", "operator"):
+        response = client.get(path, headers={"X-ModelForge-Admin-Token": tokens[role]})
+        assert response.status_code == 403
+
+    response = client.get(path, headers={"X-ModelForge-Admin-Token": tokens["admin"]})
+
+    assert response.status_code == 200
+    payload = response.json()
+    assert set(payload) == {"observed_at", "completeness", "limit", "returned_count", "leases"}
+    assert payload["completeness"] == "COMPLETE"
+    assert payload["limit"] == 100
+    assert payload["returned_count"] == 1
+    assert set(payload["leases"][0]) == {
+        "lease_id",
+        "project_id",
+        "node_id",
+        "accelerator_id",
+        "deployment_id",
+        "lease_state",
+        "workload_status",
+        "priority",
+        "lease_type",
+        "reserved_vram_bytes",
+        "materialized_vram_bytes",
+        "created_at",
+        "acquired_at",
+        "heartbeat_at",
+        "expires_at",
+    }
+    lease = payload["leases"][0]
+    assert lease["lease_id"] == lease_id
+    assert lease["project_id"] == project_id
+    assert lease["node_id"] == node_id
+    assert lease["deployment_id"] == deployment_id
+    assert lease["lease_state"] == "ACTIVE"
+    assert lease["workload_status"] == "RUNNING"
+    for forbidden in (
+        "owner",
+        "request_id",
+        "private-owner",
+        "private-client",
+        "input_sha256",
+        "payload",
+    ):
+        assert forbidden not in response.text
+    assert not session.new
+    assert not session.dirty
+    assert not session.deleted
+
+
+def test_active_lease_inventory_excludes_expired_rows_and_bounds_sorted_results(
+    session: Session,
+) -> None:
+    now = datetime.now(UTC)
+    first_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000011",
+        lease_created_at=now - timedelta(seconds=10),
+        expires_at=now + timedelta(seconds=60),
+    )
+    second_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000012",
+        lease_created_at=now - timedelta(seconds=5),
+        expires_at=now + timedelta(seconds=60),
+    )
+    expired_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000013",
+        lease_created_at=now - timedelta(seconds=30),
+        expires_at=now - timedelta(seconds=1),
+    )
+    terminal_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000014",
+        lease_created_at=now - timedelta(seconds=3),
+        expires_at=now + timedelta(seconds=60),
+        state="released",
+    )
+    app.dependency_overrides[get_service] = lambda: ObservabilityService(
+        session, Settings(observability_monitor_enabled=False)
+    )
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        _env_file=None,
+        operator_api_key=SecretStr("operator-secret"),
+        observability_monitor_enabled=False,
+    )
+
+    response = TestClient(app).get(
+        "/api/v1/admin/operations/active-leases?limit=1&sort_by=created_at&order=desc",
+        headers={"X-ModelForge-Admin-Token": "operator-secret"},
+    )
+
+    assert response.status_code == 200
+    payload = response.json()
+    assert payload["completeness"] == "TRUNCATED"
+    assert payload["limit"] == payload["returned_count"] == 1
+    assert [lease["lease_id"] for lease in payload["leases"]] == [second_id]
+    assert expired_id not in response.text
+    assert terminal_id not in response.text
+    assert first_id not in response.text
+    assert TestClient(app).get(
+        "/api/v1/admin/operations/active-leases?limit=501",
+        headers={"X-ModelForge-Admin-Token": "operator-secret"},
+    ).status_code == 422
+
+
+def test_active_lease_inventory_marks_unresolved_workload_status_unknown(
+    session: Session,
+) -> None:
+    now = datetime.now(UTC)
+    lease_id, project_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000021",
+        lease_created_at=now - timedelta(seconds=2),
+        expires_at=now + timedelta(seconds=30),
+        request_status="completed",
+        heartbeat_at=now - timedelta(minutes=5),
+    )
+    mismatched_lease_id, *_ = add_active_lease(
+        session,
+        lease_id="lease-000000000022",
+        lease_created_at=now - timedelta(seconds=1),
+        expires_at=now + timedelta(seconds=30),
+        state="pending",
+        request_status="running",
+    )
+    app.dependency_overrides[get_service] = lambda: ObservabilityService(
+        session, Settings(observability_monitor_enabled=False)
+    )
+    app.dependency_overrides[get_settings] = lambda: Settings(
+        _env_file=None,
+        operator_api_key=SecretStr("operator-secret"),
+        observability_monitor_enabled=False,
+    )
+
+    response = TestClient(app).get(
+        "/api/v1/admin/operations/active-leases",
+        headers={"X-ModelForge-Admin-Token": "operator-secret"},
+    )
+
+    assert response.status_code == 200
+    leases = {item["lease_id"]: item for item in response.json()["leases"]}
+    lease = leases[lease_id]
+    assert lease["project_id"] == project_id
+    assert lease["workload_status"] == "UNKNOWN"
+    assert lease["lease_state"] == "ACTIVE"
+    assert lease["heartbeat_at"] == (now - timedelta(minutes=5)).isoformat().replace("+00:00", "Z")
+    assert leases[mismatched_lease_id]["lease_state"] == "PENDING"
+    assert leases[mismatched_lease_id]["workload_status"] == "UNKNOWN"
+
+
 def test_prometheus_endpoint_is_bounded_and_contains_no_request_payload(
     session: Session,
 ) -> None: