59 lines
2.0 KiB
Python
59 lines
2.0 KiB
Python
"""Transient infrastructure errors remain visible and do not stop reconciliation."""
|
|
|
|
import uuid
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
import redis
|
|
from sqlalchemy.exc import OperationalError
|
|
|
|
from modelforge_api.services import serving_reconciliation as module
|
|
from modelforge_api.settings import Settings
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("error", [
|
|
OperationalError("private SQL", {}, Exception("private connection data")),
|
|
redis.ConnectionError("private Redis address"),
|
|
RuntimeError("private runtime details"),
|
|
])
|
|
async def test_transient_failure_logged_safely_and_next_cycle_runs(monkeypatch, error):
|
|
settings = Settings(_env_file=None, env="test")
|
|
loop = module.ServingReconciliationService(settings, MagicMock())
|
|
session = MagicMock()
|
|
monkeypatch.setattr(module, "Session", session)
|
|
monkeypatch.setattr(module, "ManifestRegistry", MagicMock())
|
|
monkeypatch.setattr(module, "RedisPayloadStore", MagicMock())
|
|
logger = MagicMock()
|
|
monkeypatch.setattr(module, "logger", logger)
|
|
calls = 0
|
|
|
|
def reconcile():
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == 1:
|
|
raise error
|
|
loop.stop()
|
|
|
|
worker = MagicMock()
|
|
worker.return_value.reconcile.side_effect = reconcile
|
|
monkeypatch.setattr(module, "ServingService", worker)
|
|
# Skip the timer without changing stop semantics or creating a real Redis connection.
|
|
async def wait_for(awaitable, timeout):
|
|
del timeout
|
|
if loop._stop.is_set():
|
|
return await awaitable
|
|
awaitable.close()
|
|
raise TimeoutError
|
|
|
|
monkeypatch.setattr(module.asyncio, "wait_for", wait_for)
|
|
await loop.run()
|
|
assert calls == 2
|
|
logger.error.assert_called_once()
|
|
args, details = logger.error.call_args
|
|
assert args == ("serving_reconciliation_failed",)
|
|
assert details["error_type"] == type(error).__name__
|
|
assert uuid.UUID(details["correlation_id"])
|
|
assert "private" not in repr(logger.error.call_args)
|
|
assert session.return_value.__exit__.call_count == 2
|