284 lines
13 KiB
Python
284 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import httpx
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from pydantic import SecretStr
|
|
from sqlalchemy import create_engine, delete, select
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from modelforge_api.api.authorization import Principal, PrincipalRole
|
|
from modelforge_api.db import get_session
|
|
from modelforge_api.domain.secret_slots import (
|
|
SLOT,
|
|
SecretSlotRevoke,
|
|
SecretSlotSet,
|
|
SecretSlotVersionRequest,
|
|
)
|
|
from modelforge_api.main import app
|
|
from modelforge_api.persistence.models import (
|
|
AuditEvent,
|
|
Base,
|
|
SecretSlotActivation,
|
|
SecretSlotTest,
|
|
SecretSlotVersion,
|
|
)
|
|
from modelforge_api.services import secret_slots as module
|
|
from modelforge_api.services.audit import AuditContext
|
|
from modelforge_api.services.configuration_changes import ConfigurationService
|
|
from modelforge_api.settings import Settings, get_settings
|
|
|
|
ADMIN = Principal("recovery-admin", PrincipalRole.ADMIN, "legacy_admin_token")
|
|
TOKEN = "hf_disposable_unit_test_only" # noqa: S105 - synthetic offline test data
|
|
REASON = "Reviewed credential lifecycle"
|
|
|
|
|
|
@pytest.fixture
|
|
def session():
|
|
engine = create_engine("sqlite://", poolclass=StaticPool,
|
|
connect_args={"check_same_thread": False})
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as connection:
|
|
yield connection
|
|
engine.dispose()
|
|
|
|
|
|
@pytest.fixture
|
|
def settings():
|
|
return Settings(_env_file=None, env="test", hf_token="hf_deployment_test_only", # noqa: S106
|
|
secret_slot_root_key=base64.b64encode(bytes(range(32))).decode())
|
|
|
|
|
|
def service(session, settings, principal=ADMIN):
|
|
return module.SecretSlotService(session, settings, principal, "slot-test-42")
|
|
|
|
|
|
def candidate(subject, value=TOKEN):
|
|
return subject.set(SecretSlotSet(expected_revision=subject.metadata().revision,
|
|
reason=REASON, value=value))
|
|
|
|
|
|
def request(metadata):
|
|
return SecretSlotVersionRequest(expected_revision=metadata.revision,
|
|
version=metadata.candidate_version, reason=REASON)
|
|
|
|
|
|
def test_full_lifecycle_encryption_and_no_fallback(session, settings, monkeypatch):
|
|
subject = service(session, settings)
|
|
assert module.resolve_huggingface_token(session, settings) == settings.hf_token.get_secret_value()
|
|
first = candidate(subject)
|
|
assert first.state == "pending" and first.active_version is None
|
|
row = session.get(SecretSlotVersion, 1)
|
|
assert TOKEN not in row.secret_ciphertext
|
|
with pytest.raises(module.SecretSlotError, match="explicitly activated"):
|
|
module.resolve_huggingface_token(session, settings)
|
|
with pytest.raises(module.SecretSlotError, match="successful test"):
|
|
subject.activate(request(first))
|
|
monkeypatch.setattr(module, "test_huggingface_token", lambda token: (token == TOKEN, "HF_AUTHENTICATED"))
|
|
tested = subject.test(request(first))
|
|
active = subject.activate(request(tested))
|
|
assert active.state == "active"
|
|
assert module.resolve_huggingface_token(session, settings) == TOKEN
|
|
rotation = candidate(subject, "hf_replacement_test_only")
|
|
assert module.resolve_huggingface_token(session, settings) == TOKEN
|
|
monkeypatch.setattr(module, "test_huggingface_token", lambda token: (False, "HF_AUTHENTICATION_REJECTED"))
|
|
failed = subject.test(request(rotation))
|
|
assert failed.versions[0].test_status == "failed" and failed.active_version == 1
|
|
revoked = subject.revoke(SecretSlotRevoke(expected_revision=failed.revision,
|
|
confirm_slot=SLOT, reason=REASON))
|
|
assert revoked.state == "revoked"
|
|
with pytest.raises(module.SecretSlotError):
|
|
module.resolve_huggingface_token(session, settings)
|
|
exported = ConfigurationService(session, settings, AuditContext("operator", "test", "c")).export()
|
|
evidence = subject.catalog().model_dump_json() + exported.model_dump_json()
|
|
evidence += str(list(session.scalars(select(AuditEvent.details))))
|
|
assert SLOT in exported.secret_slots
|
|
assert TOKEN not in evidence and row.secret_ciphertext not in evidence
|
|
|
|
|
|
@pytest.mark.parametrize("role", [PrincipalRole.VIEWER, PrincipalRole.OPERATOR])
|
|
def test_non_admin_cannot_modify(session, settings, role):
|
|
subject = service(session, settings, Principal("person", role, "named_operator_key"))
|
|
assert subject.catalog().slot.source == "deployment"
|
|
with pytest.raises(module.SecretSlotError) as denied:
|
|
candidate(subject)
|
|
assert denied.value.status_code == 403
|
|
assert subject.metadata().revision == 0
|
|
|
|
|
|
def test_test_result_cannot_bless_concurrent_replacement(session, settings, monkeypatch):
|
|
subject = service(session, settings)
|
|
first = candidate(subject)
|
|
def changed(token):
|
|
candidate(subject, "hf_newer_candidate")
|
|
return True, "HF_AUTHENTICATED"
|
|
monkeypatch.setattr(module, "test_huggingface_token", changed)
|
|
with pytest.raises(module.SecretSlotError) as conflict:
|
|
subject.test(request(first))
|
|
assert conflict.value.code == "SECRET_SLOT_VERSION_CONFLICT"
|
|
assert subject.metadata().versions[0].test_status == "untested"
|
|
|
|
|
|
def test_root_key_binding_and_recovery_quarantine(session, settings, monkeypatch):
|
|
subject = service(session, settings)
|
|
first = candidate(subject)
|
|
monkeypatch.setattr(module, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
|
|
active = subject.activate(request(subject.test(request(first))))
|
|
for replacement in (None, SecretStr(base64.b64encode(b"x" * 32).decode())):
|
|
changed = settings.model_copy(update={"secret_slot_root_key": replacement})
|
|
with pytest.raises(module.SecretSlotError):
|
|
module.resolve_huggingface_token(session, changed)
|
|
wrong_id = settings.model_copy(update={"secret_slot_root_key_id": "different-key"})
|
|
with pytest.raises(module.SecretSlotError):
|
|
module.resolve_huggingface_token(session, wrong_id)
|
|
# Supported recovery clears these exact current-truth tables; real restore is PG-qualified.
|
|
session.execute(delete(SecretSlotActivation))
|
|
session.execute(delete(SecretSlotTest))
|
|
session.commit()
|
|
assert subject.metadata().state == "quarantined"
|
|
with pytest.raises(module.SecretSlotError):
|
|
module.resolve_huggingface_token(session, settings)
|
|
with pytest.raises(module.SecretSlotError):
|
|
subject.activate(request(active))
|
|
refreshed = subject.test(request(subject.metadata()))
|
|
subject.activate(request(refreshed))
|
|
assert module.resolve_huggingface_token(session, settings) == TOKEN
|
|
|
|
|
|
def test_expired_evidence_and_audit_failure_do_not_activate(session, settings, monkeypatch):
|
|
subject = service(session, settings)
|
|
first = candidate(subject)
|
|
monkeypatch.setattr(module, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
|
|
tested = subject.test(request(first))
|
|
session.get(SecretSlotTest, 1).tested_at = datetime.now(UTC) - timedelta(minutes=6)
|
|
session.commit()
|
|
with pytest.raises(module.SecretSlotError):
|
|
subject.activate(request(tested))
|
|
tested = subject.test(request(tested))
|
|
def unavailable(*args, **kwargs):
|
|
raise RuntimeError("audit unavailable")
|
|
monkeypatch.setattr(subject.audit, "write", unavailable)
|
|
with pytest.raises(RuntimeError):
|
|
subject.activate(request(tested))
|
|
assert subject.metadata().active_version is None
|
|
assert subject.metadata().revision == tested.revision
|
|
|
|
|
|
class Body(httpx.SyncByteStream):
|
|
def __init__(self, chunks):
|
|
self.chunks = chunks
|
|
def __iter__(self):
|
|
yield from self.chunks
|
|
|
|
|
|
@pytest.mark.parametrize("status,headers,chunks,expected", [
|
|
(200, {}, [b'{"name":"local-test"}'], (True, "HF_AUTHENTICATED")),
|
|
(401, {}, [], (False, "HF_AUTHENTICATION_REJECTED")),
|
|
(302, {"location": "https://untrusted.invalid"}, [], (False, "HF_PROVIDER_UNAVAILABLE")),
|
|
(200, {"content-encoding": "gzip"}, [], (False, "HF_RESPONSE_INVALID")),
|
|
(200, {}, [b"x" * 65537], (False, "HF_RESPONSE_INVALID")),
|
|
(200, {}, [b'{}'], (False, "HF_RESPONSE_INVALID")),
|
|
(200, {}, [b"[" * 2000 + b"0" + b"]" * 2000], (False, "HF_RESPONSE_INVALID")),
|
|
])
|
|
def test_provider_check_is_fixed_bounded_and_redacted(monkeypatch, status, headers, chunks, expected):
|
|
actual_client = httpx.Client
|
|
calls = []
|
|
def send(request):
|
|
calls.append(request)
|
|
return httpx.Response(status, headers=headers, stream=Body(chunks))
|
|
def factory(**kwargs):
|
|
assert kwargs["follow_redirects"] is False and kwargs["trust_env"] is False
|
|
return actual_client(transport=httpx.MockTransport(send), **kwargs)
|
|
monkeypatch.setattr(module.httpx, "Client", factory)
|
|
assert module.test_huggingface_token(TOKEN) == expected
|
|
assert len(calls) == 1 and str(calls[0].url) == "https://huggingface.co/api/whoami-v2"
|
|
assert calls[0].headers["authorization"] == f"Bearer {TOKEN}"
|
|
|
|
|
|
def test_provider_stream_has_total_deadline(monkeypatch):
|
|
actual_client = httpx.Client
|
|
def send(request):
|
|
return httpx.Response(200, stream=Body([b'{"name":"local-test"}']))
|
|
monkeypatch.setattr(module.httpx, "Client", lambda **kwargs: actual_client(
|
|
transport=httpx.MockTransport(send), **kwargs))
|
|
moments = iter([0.0, 11.0])
|
|
monkeypatch.setattr(module.time, "monotonic", lambda: next(moments))
|
|
assert module.test_huggingface_token(TOKEN) == (False, "HF_PROVIDER_TIMEOUT")
|
|
|
|
|
|
def test_http_write_only_validation_and_no_store(session, settings):
|
|
settings = settings.model_copy(update={"operator_api_key": SecretStr("local-anchor")})
|
|
prior = dict(app.dependency_overrides)
|
|
app.dependency_overrides[get_settings] = lambda: settings
|
|
app.dependency_overrides[get_session] = lambda: session
|
|
try:
|
|
client = TestClient(app)
|
|
url = "/api/v1/configuration/secret-slots"
|
|
assert client.get(url).status_code == 401
|
|
headers = {"X-ModelForge-Admin-Token": "local-anchor"}
|
|
response = client.post(url + "/huggingface_control_plane/set", headers=headers,
|
|
json={"expected_revision": 0, "reason": REASON, "value": TOKEN})
|
|
assert response.status_code == 200, response.text
|
|
assert response.headers["cache-control"] == "no-store"
|
|
assert TOKEN not in response.text
|
|
invalid = client.post(url + "/huggingface_control_plane/set", headers=headers,
|
|
json={"expected_revision": 1, "reason": TOKEN, "value": TOKEN + " "})
|
|
assert invalid.status_code == 422 and TOKEN not in invalid.text
|
|
metadata = client.get(url, headers=headers)
|
|
assert metadata.status_code == 200 and TOKEN not in metadata.text
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
app.dependency_overrides.update(prior)
|
|
|
|
|
|
def test_both_control_plane_consumers_resolve_active_snapshot(session, settings, monkeypatch):
|
|
from modelforge_api.api.routes.acquisition import get_acquisition_service
|
|
from modelforge_api.api.routes.agent import get_agent_acquisition_service
|
|
from modelforge_api.services.registry import RegistryNotFound
|
|
|
|
subject = service(session, settings)
|
|
first = candidate(subject)
|
|
for consumer in (get_acquisition_service, get_agent_acquisition_service):
|
|
local_service = consumer(session, settings)
|
|
# Read-only persisted history and node progress must not require upstream credentials.
|
|
with pytest.raises(RegistryNotFound, match="model revision not found"):
|
|
local_service.artifact_sets(uuid.uuid4())
|
|
with pytest.raises(module.SecretSlotError):
|
|
local_service.provider.resolve()
|
|
monkeypatch.setattr(module, "test_huggingface_token", lambda token: (True, "HF_AUTHENTICATED"))
|
|
subject.activate(request(subject.test(request(first))))
|
|
for consumer in (get_acquisition_service, get_agent_acquisition_service):
|
|
resolved = consumer(session, settings)
|
|
assert resolved.provider.resolve().token == TOKEN
|
|
assert resolved.credential_configured is True
|
|
|
|
|
|
def test_revoked_managed_admin_cannot_start_external_test(session, settings, monkeypatch):
|
|
from modelforge_api.domain.managed_identity import ManagedIdentityCreate, ManagedIdentityRevoke
|
|
from modelforge_api.services.operator_identities import (
|
|
ManagedIdentityService,
|
|
authenticate_managed_operator,
|
|
)
|
|
|
|
manager = ManagedIdentityService(session, ADMIN, "identity-check")
|
|
issued = manager.create(ManagedIdentityCreate(slug="alpha", role="admin",
|
|
expires_in_days=1, expected_revision=0, reason=REASON))
|
|
manager.create(ManagedIdentityCreate(slug="beta", role="admin",
|
|
expires_in_days=1, expected_revision=1, reason=REASON))
|
|
principal = authenticate_managed_operator(session, issued.one_time_key)
|
|
session.commit()
|
|
first = candidate(service(session, settings))
|
|
manager.revoke("alpha", ManagedIdentityRevoke(expected_version=1,
|
|
confirm_subject="managed:alpha", reason=REASON))
|
|
def forbidden(token):
|
|
pytest.fail("Revoked administrator must not trigger provider I/O")
|
|
monkeypatch.setattr(module, "test_huggingface_token", forbidden)
|
|
with pytest.raises(module.SecretSlotError) as rejected:
|
|
service(session, settings, principal).test(request(first))
|
|
assert rejected.value.status_code == 403
|