236 lines
8.4 KiB
Python
236 lines
8.4 KiB
Python
from datetime import datetime, timezone
|
|
from functools import partial
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from backend.app.connectors import gpu_metrics
|
|
from backend.app.connectors.gpu_metrics import (
|
|
GpuMetricsConnector,
|
|
normalize_gpu_snapshot,
|
|
)
|
|
|
|
NOW = datetime(2026, 10, 1, 12, 0, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
def snapshot(**device_values):
|
|
return {
|
|
'gpus': {
|
|
'0': {
|
|
'index': '0',
|
|
'name': 'NVIDIA GeForce RTX 4080 SUPER',
|
|
'utilization': 27.5,
|
|
'memory_used': 1559.0,
|
|
'memory_total': 16376.0,
|
|
'temperature': 47.0,
|
|
'timestamp': '2026-10-01T11:59:59+00:00',
|
|
'uuid': 'private-device-uuid',
|
|
'serial': 'private-device-serial',
|
|
'compute_processes_count': 1,
|
|
**device_values,
|
|
},
|
|
},
|
|
'timestamp': 'async',
|
|
'processes': [{'pid': '42', 'name': 'private-process', 'command': 'private-command'}],
|
|
'system': {'hostname': 'private-host'},
|
|
}
|
|
|
|
|
|
def test_gpu_hot_snapshot_normalizes_allowlisted_device_metrics_only():
|
|
result = normalize_gpu_snapshot(snapshot(), now=NOW)
|
|
|
|
assert result == {
|
|
'status': 'ok',
|
|
'observedAt': '2026-10-01T11:59:59Z',
|
|
'devices': [{
|
|
'id': 'gpu-0',
|
|
'name': 'NVIDIA GeForce RTX 4080 SUPER',
|
|
'utilizationPercent': 27.5,
|
|
'memoryUsedMiB': 1559.0,
|
|
'memoryTotalMiB': 16376.0,
|
|
'temperatureC': 47.0,
|
|
}],
|
|
'reason': None,
|
|
'source': 'gpu-hot',
|
|
}
|
|
assert 'private' not in str(result)
|
|
|
|
|
|
def test_gpu_hot_numeric_zero_is_a_real_idle_reading():
|
|
result = normalize_gpu_snapshot(snapshot(utilization=0, memory_used=0), now=NOW)
|
|
|
|
assert result['status'] == 'ok'
|
|
assert result['devices'][0]['utilizationPercent'] == 0.0
|
|
assert result['devices'][0]['memoryUsedMiB'] == 0.0
|
|
|
|
|
|
@pytest.mark.parametrize('value', [None, True, 'n/a', float('nan'), float('inf'), -1, 101])
|
|
def test_missing_or_invalid_utilization_remains_null(value):
|
|
result = normalize_gpu_snapshot(snapshot(utilization=value), now=NOW)
|
|
|
|
assert result['status'] == 'partial'
|
|
assert result['devices'][0]['utilizationPercent'] is None
|
|
assert result['devices'][0]['memoryUsedMiB'] == 1559.0
|
|
|
|
|
|
@pytest.mark.parametrize('field,value,normalized', [
|
|
('memory_used', -1, 'memoryUsedMiB'),
|
|
('memory_used', float('inf'), 'memoryUsedMiB'),
|
|
('memory_total', 0, 'memoryTotalMiB'),
|
|
('temperature', 151, 'temperatureC'),
|
|
('temperature', float('nan'), 'temperatureC'),
|
|
])
|
|
def test_invalid_secondary_metrics_do_not_replace_valid_utilization(field, value, normalized):
|
|
result = normalize_gpu_snapshot(snapshot(**{field: value}), now=NOW)
|
|
|
|
assert result['status'] == 'partial'
|
|
assert result['devices'][0][normalized] is None
|
|
assert result['devices'][0]['utilizationPercent'] == 27.5
|
|
|
|
|
|
def test_used_memory_larger_than_capacity_is_not_clamped_to_capacity():
|
|
result = normalize_gpu_snapshot(snapshot(memory_used=20000), now=NOW)
|
|
|
|
assert result['status'] == 'partial'
|
|
assert result['devices'][0]['memoryUsedMiB'] is None
|
|
assert result['devices'][0]['memoryTotalMiB'] == 16376.0
|
|
|
|
|
|
@pytest.mark.parametrize('timestamp', [None, 'async', 'not-a-date', '2026-10-01T11:59:59'])
|
|
def test_absent_or_timezone_naive_source_time_is_not_invented(timestamp):
|
|
result = normalize_gpu_snapshot(snapshot(timestamp=timestamp), now=NOW)
|
|
|
|
assert result['status'] == 'partial'
|
|
assert result['observedAt'] is None
|
|
assert result['devices'][0]['utilizationPercent'] == 27.5
|
|
|
|
|
|
@pytest.mark.parametrize('timestamp', ['2026-10-01T11:59:00Z', '2026-10-01T12:01:00Z'])
|
|
def test_stale_or_future_device_values_are_not_presented_as_live(timestamp):
|
|
result = normalize_gpu_snapshot(snapshot(timestamp=timestamp), now=NOW)
|
|
|
|
assert result['status'] == 'unavailable'
|
|
assert result['devices'] == []
|
|
assert result['observedAt'] is None
|
|
|
|
|
|
def test_one_stale_gpu_does_not_hide_another_fresh_gpu():
|
|
payload = snapshot()
|
|
payload['gpus']['1'] = {
|
|
**payload['gpus']['0'],
|
|
'index': '1',
|
|
'timestamp': '2026-10-01T11:50:00Z',
|
|
'utilization': 99,
|
|
}
|
|
|
|
result = normalize_gpu_snapshot(payload, now=NOW)
|
|
|
|
assert result['status'] == 'partial'
|
|
assert [device['id'] for device in result['devices']] == ['gpu-0']
|
|
assert result['observedAt'] == '2026-10-01T11:59:59Z'
|
|
|
|
|
|
@pytest.mark.parametrize('payload', [{'gpus': {}, 'timestamp': 'async'}, {'gpus': []}, None, [], {'gpus': 'bad'}, {'gpus': {'bad': 'bad'}}])
|
|
def test_empty_or_malformed_source_never_reports_zero_gpu_usage(payload):
|
|
result = normalize_gpu_snapshot(payload, now=NOW)
|
|
|
|
assert result['status'] in {'unavailable', 'error'}
|
|
assert result['devices'] == []
|
|
assert result['observedAt'] is None
|
|
|
|
|
|
def install_transport(monkeypatch, handler):
|
|
monkeypatch.setattr(gpu_metrics.httpx, 'Client', partial(httpx.Client, transport=httpx.MockTransport(handler)))
|
|
|
|
|
|
def test_missing_config_performs_no_network_request(monkeypatch):
|
|
def unexpected_request(_request):
|
|
pytest.fail('missing config must not perform a request')
|
|
|
|
install_transport(monkeypatch, unexpected_request)
|
|
result = GpuMetricsConnector(snapshot_url='').get_metrics()
|
|
|
|
assert result['status'] == 'missing_config'
|
|
assert result['devices'] == []
|
|
|
|
|
|
@pytest.mark.parametrize('url', ['file:///private/path', 'http://user:secret@metrics/api/gpu-data', 'invalid-url'])
|
|
def test_invalid_or_embedded_credential_urls_fail_without_leaking_values(monkeypatch, url):
|
|
def unexpected_request(_request):
|
|
pytest.fail('invalid config must not perform a request')
|
|
|
|
install_transport(monkeypatch, unexpected_request)
|
|
result = GpuMetricsConnector(snapshot_url=url).get_metrics()
|
|
|
|
assert result['status'] == 'error'
|
|
assert result['devices'] == []
|
|
assert url not in str(result)
|
|
assert 'secret' not in str(result)
|
|
|
|
|
|
def test_collector_uses_read_only_get_and_caps_timeout(monkeypatch):
|
|
requests = []
|
|
|
|
def handler(request):
|
|
requests.append(request)
|
|
return httpx.Response(200, json=snapshot(timestamp=None))
|
|
|
|
install_transport(monkeypatch, handler)
|
|
result = GpuMetricsConnector(snapshot_url='http://metrics:1312/api/gpu-data', timeout_seconds=100).get_metrics()
|
|
|
|
assert result['status'] == 'partial'
|
|
assert result['devices'][0]['utilizationPercent'] == 27.5
|
|
assert len(requests) == 1
|
|
assert requests[0].method == 'GET'
|
|
assert str(requests[0].url) == 'http://metrics:1312/api/gpu-data'
|
|
assert all(value <= 3.0 for value in requests[0].extensions['timeout'].values())
|
|
|
|
|
|
@pytest.mark.parametrize('error', [httpx.ConnectError('private-host private-token'), httpx.ReadTimeout('private-host private-token')])
|
|
def test_network_failures_return_fixed_safe_reasons(monkeypatch, error):
|
|
def handler(_request):
|
|
raise error
|
|
|
|
install_transport(monkeypatch, handler)
|
|
result = GpuMetricsConnector(snapshot_url='http://metrics/api/gpu-data').get_metrics()
|
|
|
|
assert result['status'] == 'unavailable'
|
|
assert result['devices'] == []
|
|
assert 'private' not in str(result)
|
|
|
|
|
|
@pytest.mark.parametrize('status', [301, 302, 401, 403, 500])
|
|
def test_http_failures_do_not_follow_redirects_or_expose_response_bodies(monkeypatch, status):
|
|
requests = []
|
|
|
|
def handler(request):
|
|
requests.append(request)
|
|
return httpx.Response(status, text='private-token', headers={'Location': 'http://private-host'})
|
|
|
|
install_transport(monkeypatch, handler)
|
|
result = GpuMetricsConnector(snapshot_url='http://metrics/api/gpu-data').get_metrics()
|
|
|
|
assert result['status'] in {'error', 'unavailable'}
|
|
assert len(requests) == 1
|
|
assert result['devices'] == []
|
|
assert 'private' not in str(result)
|
|
|
|
|
|
def test_invalid_json_is_reported_without_source_content(monkeypatch):
|
|
install_transport(monkeypatch, lambda _request: httpx.Response(200, text='private-token invalid-json'))
|
|
|
|
result = GpuMetricsConnector(snapshot_url='http://metrics/api/gpu-data').get_metrics()
|
|
|
|
assert result['status'] == 'error'
|
|
assert result['devices'] == []
|
|
assert 'private' not in str(result)
|
|
|
|
|
|
def test_oversized_response_is_rejected(monkeypatch):
|
|
install_transport(monkeypatch, lambda _request: httpx.Response(200, content=b' ' * 1_048_577))
|
|
|
|
result = GpuMetricsConnector(snapshot_url='http://metrics/api/gpu-data').get_metrics()
|
|
|
|
assert result['status'] == 'error'
|
|
assert result['devices'] == []
|