346 lines
16 KiB
Python
346 lines
16 KiB
Python
from datetime import UTC, datetime, timedelta
|
|
from unittest.mock import Mock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from fastapi import FastAPI, HTTPException
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import create_engine, func, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from modelforge_api.api.authorization import (
|
|
Principal,
|
|
PrincipalRole,
|
|
authenticate_operator_principal,
|
|
)
|
|
from modelforge_api.api.routes.model_updates import get_model_update_service, router
|
|
from modelforge_api.domain.model_updates import ModelWatchCreate, ModelWatchUpdate
|
|
from modelforge_api.persistence.models import (
|
|
ArtifactJob,
|
|
AuditEvent,
|
|
Base,
|
|
CapabilityDeployment,
|
|
Model,
|
|
ModelRevision,
|
|
ModelUpdateWatch,
|
|
)
|
|
from modelforge_api.services.acquisition import AcquisitionService, ProviderBoundaryError
|
|
from modelforge_api.services.audit import AuditContext
|
|
from modelforge_api.services.model_updates import ModelUpdateService
|
|
from modelforge_api.services.registry import RegistryConflict, RegistryService
|
|
|
|
|
|
@pytest.fixture
|
|
def setup():
|
|
engine = create_engine("sqlite://")
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
model = Model(key="selected", display_name="Selected", upstream_provider="huggingface", upstream_source="org/selected")
|
|
session.add(model)
|
|
session.flush()
|
|
revision = ModelRevision(model_id=model.id, upstream_revision="main", resolved_commit_sha="a" * 40)
|
|
session.add(revision)
|
|
session.commit()
|
|
acquisition = Mock(spec=AcquisitionService)
|
|
def refresh(model_id, ref):
|
|
candidate = session.scalar(select(ModelRevision).where(ModelRevision.resolved_commit_sha == "b" * 40))
|
|
if candidate is None:
|
|
session.add(ModelRevision(model_id=model_id, upstream_revision=ref, resolved_commit_sha="b" * 40))
|
|
session.commit()
|
|
return Mock(resolved_commit_sha="b" * 40)
|
|
acquisition.refresh_model.side_effect = refresh
|
|
service = ModelUpdateService(session, acquisition, AuditContext("operator", "test"))
|
|
request = ModelWatchCreate(model_id=model.id, baseline_revision_id=revision.id)
|
|
yield session, service, acquisition, request
|
|
engine.dispose()
|
|
|
|
|
|
def test_tracking_and_notice_history_explain_registry_delete_refusal(setup):
|
|
session, service, _, request = setup
|
|
watch = service.create(request)
|
|
service.check(watch.id)
|
|
notice, = service.notices()
|
|
service.dismiss(notice.id)
|
|
service.change(watch.id, ModelWatchUpdate(
|
|
expected_version=watch.version,
|
|
expected_baseline_revision_id=watch.baseline_revision_id,
|
|
baseline_revision_id=watch.baseline_revision_id, enabled=False,
|
|
reason="Keep historical tracking while paused",
|
|
))
|
|
registry = RegistryService(session)
|
|
for revision_id, relation in (
|
|
(watch.baseline_revision_id, "update_tracking_baseline"),
|
|
(notice.candidate_revision_id, "update_candidate_history"),
|
|
):
|
|
with pytest.raises(RegistryConflict) as blocked:
|
|
registry.delete("model_revision", revision_id)
|
|
assert any(item["relation"] == relation for item in blocked.value.details["dependencies"])
|
|
assert session.get(ModelRevision, revision_id) is not None
|
|
with pytest.raises(RegistryConflict) as blocked_model:
|
|
registry.delete("model", request.model_id)
|
|
assert {"update_tracking_baseline", "update_candidate_history"}.issubset(
|
|
{item["relation"] for item in blocked_model.value.details["dependencies"]})
|
|
|
|
|
|
def test_check_deduplicates_dismissed_notice_and_never_deploys(setup):
|
|
session, service, acquisition, request = setup
|
|
watch = service.create(request)
|
|
assert service.create(request).id == watch.id
|
|
for _ in range(2):
|
|
assert service.check(watch.id).last_error_code is None
|
|
notice, = service.notices()
|
|
assert notice.baseline_commit_sha == "a" * 40
|
|
assert notice.candidate_commit_sha == "b" * 40
|
|
service.dismiss(notice.id)
|
|
service.check(watch.id)
|
|
assert len(service.notices()) == 1
|
|
assert service.notices()[0].status == "dismissed"
|
|
assert session.scalar(select(func.count()).select_from(ArtifactJob)) == 0
|
|
assert session.scalar(select(func.count()).select_from(CapabilityDeployment)) == 0
|
|
acquisition.create_plan.assert_not_called()
|
|
acquisition.execute_plan.assert_not_called()
|
|
|
|
|
|
def test_unchanged_and_provider_failure_preserve_baseline(setup):
|
|
session, service, acquisition, request = setup
|
|
watch = service.create(request)
|
|
acquisition.refresh_model.side_effect = None
|
|
acquisition.refresh_model.return_value = Mock(resolved_commit_sha="a" * 40)
|
|
service.check(watch.id)
|
|
assert service.notices() == []
|
|
acquisition.refresh_model.side_effect = ProviderBoundaryError("private provider details")
|
|
failed = service.check(watch.id)
|
|
assert failed.last_error_code == "upstream_unavailable"
|
|
assert failed.baseline_commit_sha == watch.baseline_commit_sha
|
|
assert failed.last_checked_at is not None
|
|
assert service.notices() == []
|
|
assert "private provider details" not in failed.model_dump_json()
|
|
assert session.get(ModelUpdateWatch, watch.id).check_token is None
|
|
|
|
|
|
def test_concurrent_check_refused_and_expired_lease_recoverable(setup):
|
|
session, service, acquisition, request = setup
|
|
watch = service.create(request)
|
|
item = session.get(ModelUpdateWatch, watch.id)
|
|
item.check_until = datetime.now(UTC) + timedelta(minutes=1)
|
|
item.check_token = uuid4()
|
|
session.commit()
|
|
with pytest.raises(RegistryConflict):
|
|
service.check(watch.id)
|
|
acquisition.refresh_model.assert_not_called()
|
|
item.check_until = datetime.now(UTC) - timedelta(seconds=1)
|
|
session.commit()
|
|
service.check(watch.id)
|
|
assert len(service.notices()) == 1
|
|
|
|
|
|
def test_unexpected_failure_records_failure_and_releases_lease(setup):
|
|
session, service, acquisition, request = setup
|
|
watch = service.create(request)
|
|
acquisition.refresh_model.side_effect = RuntimeError("test crash")
|
|
with pytest.raises(RuntimeError):
|
|
service.check(watch.id)
|
|
item = session.get(ModelUpdateWatch, watch.id)
|
|
assert item.last_error_code == "check_failed"
|
|
assert item.check_token is None
|
|
assert item.baseline_commit_sha == watch.baseline_commit_sha
|
|
|
|
|
|
def test_baseline_mismatch_and_stale_baseline_refused(setup):
|
|
session, service, _acquisition, request = setup
|
|
watch = service.create(request)
|
|
service.check(watch.id)
|
|
revision = session.scalar(select(ModelRevision).where(ModelRevision.resolved_commit_sha == "b" * 40))
|
|
with pytest.raises(RegistryConflict, match="explicit tracking update"):
|
|
service.create(request.model_copy(update={"baseline_revision_id": revision.id}))
|
|
other = Model(key="other", display_name="Other", upstream_provider="huggingface", upstream_source="org/other")
|
|
session.add(other)
|
|
session.commit()
|
|
with pytest.raises(RegistryConflict, match="own exact"):
|
|
service.create(request.model_copy(update={"model_id": other.id}))
|
|
|
|
|
|
def test_watch_bound_and_pagination(setup):
|
|
_, service, _, request = setup
|
|
for index in range(100):
|
|
service.create(request.model_copy(update={"upstream_ref": f"branch-{index}"}))
|
|
with pytest.raises(RegistryConflict, match="100"):
|
|
service.create(request)
|
|
assert len(service.watches(limit=100)) == 100
|
|
assert len(service.watches(limit=10, offset=95)) == 5
|
|
|
|
|
|
@pytest.mark.parametrize("failure", [False, True])
|
|
def test_superseded_check_cannot_clear_successor_or_publish_notice(setup, failure):
|
|
session, service, acquisition, request = setup
|
|
watch = service.create(request)
|
|
successor = uuid4()
|
|
def supersede(model_id, ref):
|
|
item = session.get(ModelUpdateWatch, watch.id)
|
|
item.check_token = successor
|
|
item.check_until = datetime.now(UTC) + timedelta(minutes=5)
|
|
session.commit()
|
|
if failure:
|
|
raise RuntimeError("superseded failure")
|
|
return Mock(resolved_commit_sha="a" * 40)
|
|
acquisition.refresh_model.side_effect = supersede
|
|
with pytest.raises(RuntimeError if failure else RegistryConflict):
|
|
service.check(watch.id)
|
|
item = session.get(ModelUpdateWatch, watch.id)
|
|
assert item.check_token == successor
|
|
assert item.last_checked_at is None
|
|
assert service.notices() == []
|
|
|
|
|
|
def test_metadata_audit_uses_authenticated_context(setup):
|
|
_, service, acquisition, _ = setup
|
|
assert acquisition.audit is service.audit
|
|
assert acquisition.audit.context.actor_id == "test"
|
|
|
|
|
|
@pytest.mark.parametrize("role,status", [(None,401),(PrincipalRole.VIEWER,403),(PrincipalRole.OPERATOR,200),(PrincipalRole.ADMIN,200)])
|
|
@pytest.mark.parametrize("action", ["create", "check", "dismiss", "change"])
|
|
def test_mutation_roles(role, status, action, setup):
|
|
_, service, _, request = setup
|
|
watch = service.create(request)
|
|
service.check(watch.id)
|
|
notice = service.notices()[0]
|
|
app = FastAPI()
|
|
app.include_router(router)
|
|
def principal():
|
|
if role is None:
|
|
raise HTTPException(401)
|
|
return Principal("test", role, "test")
|
|
mocked = Mock()
|
|
mocked.create.return_value = watch
|
|
mocked.check.return_value = watch
|
|
mocked.dismiss.return_value = notice
|
|
mocked.change.return_value = watch
|
|
app.dependency_overrides[authenticate_operator_principal] = principal
|
|
app.dependency_overrides[get_model_update_service] = lambda: mocked
|
|
with TestClient(app) as client:
|
|
if action == "create":
|
|
response = client.post("/api/v1/model-updates/watches", json=request.model_dump(mode="json"))
|
|
elif action == "change":
|
|
response = client.patch(f"/api/v1/model-updates/watches/{watch.id}", json={
|
|
"expected_version": watch.version, "expected_baseline_revision_id": str(watch.baseline_revision_id),
|
|
"baseline_revision_id": str(watch.baseline_revision_id), "enabled": False, "reason": "Pause tracking",
|
|
})
|
|
else:
|
|
path = f"watches/{watch.id}/check" if action == "check" else f"notices/{notice.id}/dismiss"
|
|
response = client.post(f"/api/v1/model-updates/{path}")
|
|
assert response.status_code == status
|
|
assert getattr(mocked, action).call_count == int(status == 200)
|
|
|
|
|
|
def test_list_bounds_require_viewer():
|
|
app = FastAPI()
|
|
app.include_router(router)
|
|
mocked = Mock()
|
|
mocked.watches.return_value = []
|
|
app.dependency_overrides[get_model_update_service] = lambda: mocked
|
|
app.dependency_overrides[authenticate_operator_principal] = lambda: Principal("viewer", PrincipalRole.VIEWER, "test")
|
|
with TestClient(app) as client:
|
|
assert client.get("/api/v1/model-updates/watches").status_code == 200
|
|
assert client.get("/api/v1/model-updates/watches?limit=101").status_code == 422
|
|
assert client.get("/api/v1/model-updates/watches?offset=-1").status_code == 422
|
|
|
|
|
|
def tracking_change(watch, **changes):
|
|
return ModelWatchUpdate(expected_version=watch.version,
|
|
expected_baseline_revision_id=watch.baseline_revision_id,
|
|
baseline_revision_id=changes.get("baseline_revision_id", watch.baseline_revision_id),
|
|
enabled=changes.get("enabled", watch.enabled), reason="Reviewed tracking change")
|
|
|
|
|
|
def test_pause_resume_fences_in_flight_check_and_stale_changes(setup):
|
|
session, service, acquisition, request = setup
|
|
watch = service.create(request)
|
|
original = acquisition.refresh_model.side_effect
|
|
def interrupted(model_id, ref):
|
|
candidate = original(model_id, ref)
|
|
paused = service.change(watch.id, tracking_change(watch, enabled=False))
|
|
assert not paused.enabled and paused.version == 2
|
|
assert session.get(ModelUpdateWatch, watch.id).check_token is None
|
|
with pytest.raises(RegistryConflict, match="changed"):
|
|
service.change(watch.id, tracking_change(watch, enabled=True))
|
|
resumed = service.change(watch.id, tracking_change(paused, enabled=True))
|
|
assert resumed.enabled and resumed.version == 3
|
|
return candidate
|
|
acquisition.refresh_model.side_effect = interrupted
|
|
with pytest.raises(RegistryConflict, match="superseded"):
|
|
service.check(watch.id)
|
|
assert service.notices() == []
|
|
assert service.watches()[0].last_checked_at is None
|
|
|
|
|
|
def test_paused_watch_and_not_due_check_do_not_call_provider(setup):
|
|
_, service, acquisition, request = setup
|
|
watch = service.create(request)
|
|
paused = service.change(watch.id, tracking_change(watch, enabled=False))
|
|
with pytest.raises(RegistryConflict, match="paused"):
|
|
service.check(watch.id)
|
|
acquisition.refresh_model.assert_not_called()
|
|
service.change(watch.id, tracking_change(paused, enabled=True))
|
|
service.check(watch.id)
|
|
acquisition.refresh_model.reset_mock()
|
|
with pytest.raises(RegistryConflict, match="not due"):
|
|
service.check(watch.id, if_checked_before=datetime.now(UTC) - timedelta(hours=1))
|
|
acquisition.refresh_model.assert_not_called()
|
|
|
|
|
|
def test_baseline_advance_is_audited_cas_and_preserves_notice_provenance(setup):
|
|
session, service, _, request = setup
|
|
watch = service.create(request)
|
|
service.check(watch.id)
|
|
notice, = service.notices()
|
|
advanced = service.change(watch.id, tracking_change(watch, baseline_revision_id=notice.candidate_revision_id))
|
|
assert advanced.baseline_commit_sha == notice.candidate_commit_sha
|
|
assert advanced.version == watch.version + 1
|
|
assert advanced.last_checked_at is None
|
|
assert service.notices()[0] == notice
|
|
audit = session.scalar(select(AuditEvent).where(AuditEvent.action == "MODEL_WATCH_TRACKING_CHANGED"))
|
|
assert audit.details["previous"]["baseline_revision_id"] == str(watch.baseline_revision_id)
|
|
assert audit.details["baseline_revision_id"] == str(notice.candidate_revision_id)
|
|
assert audit.details["reason"] == "Reviewed tracking change"
|
|
with pytest.raises(RegistryConflict, match="changed"):
|
|
service.change(watch.id, tracking_change(watch, enabled=False))
|
|
assert session.scalar(select(func.count()).select_from(CapabilityDeployment)) == 0
|
|
|
|
|
|
def test_baseline_advance_rejects_other_models_and_active_check(setup):
|
|
session, service, _, request = setup
|
|
watch = service.create(request)
|
|
service.check(watch.id)
|
|
notice = service.notices()[0]
|
|
item = session.get(ModelUpdateWatch, watch.id)
|
|
item.check_token = uuid4()
|
|
item.check_until = datetime.now(UTC) + timedelta(minutes=1)
|
|
session.commit()
|
|
with pytest.raises(RegistryConflict, match="active check"):
|
|
service.change(watch.id, tracking_change(watch, baseline_revision_id=notice.candidate_revision_id))
|
|
other = Model(key="other", display_name="Other", upstream_provider="huggingface", upstream_source="org/other")
|
|
session.add(other)
|
|
session.flush()
|
|
other_revision = ModelRevision(model_id=other.id, upstream_revision="main", resolved_commit_sha="c" * 40)
|
|
session.add(other_revision)
|
|
session.commit()
|
|
with pytest.raises(RegistryConflict, match="same model"):
|
|
service.change(watch.id, tracking_change(watch, baseline_revision_id=other_revision.id))
|
|
paused = service.change(watch.id, tracking_change(watch, enabled=False))
|
|
updated = service.change(watch.id, tracking_change(paused, baseline_revision_id=notice.candidate_revision_id))
|
|
assert not updated.enabled
|
|
assert session.get(ModelUpdateWatch, watch.id).check_token is None
|
|
|
|
|
|
@pytest.mark.parametrize("env,enabled,active", [("production",True,True),("production",False,False),("development",True,False)])
|
|
def test_monitor_status_reports_actual_environment_gate(env, enabled, active):
|
|
from modelforge_api.settings import Settings, get_settings
|
|
app = FastAPI()
|
|
app.include_router(router)
|
|
app.dependency_overrides[authenticate_operator_principal] = lambda: Principal("viewer", PrincipalRole.VIEWER, "test")
|
|
app.dependency_overrides[get_settings] = lambda: Settings(env=env, model_update_monitor_enabled=enabled,
|
|
model_update_check_interval_seconds=7200)
|
|
with TestClient(app) as client:
|
|
assert client.get("/api/v1/model-updates/monitor").json() == {"enabled": active, "check_interval_seconds": 7200}
|