511 lines
20 KiB
Python
511 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from alembic.migration import MigrationContext
|
|
from alembic.operations import Operations
|
|
from fastapi import HTTPException
|
|
from fastapi.testclient import TestClient
|
|
from pydantic import SecretStr, ValidationError
|
|
from sqlalchemy import create_engine, delete, event, select, update
|
|
from sqlalchemy.exc import OperationalError
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from modelforge_api.api.authorization import Principal, PrincipalRole, authenticate_operator_token
|
|
from modelforge_api.db import get_session
|
|
from modelforge_api.domain.managed_identity import (
|
|
ManagedIdentityChange,
|
|
ManagedIdentityCreate,
|
|
ManagedIdentityRevoke,
|
|
ManagedIdentityRotate,
|
|
)
|
|
from modelforge_api.main import app
|
|
from modelforge_api.persistence.models import (
|
|
AuditEvent,
|
|
Base,
|
|
ManagedOperatorCredential,
|
|
ManagedOperatorCredentialActivation,
|
|
ManagedOperatorIdentity,
|
|
ManagedOperatorState,
|
|
)
|
|
from modelforge_api.services import audit as audit_module
|
|
from modelforge_api.services.operator_identities import (
|
|
ManagedIdentityError,
|
|
ManagedIdentityService,
|
|
authenticate_managed_operator,
|
|
)
|
|
from modelforge_api.services.recovery_fingerprint import (
|
|
CURRENT_TRUTH_TABLES,
|
|
MANAGED_ACTIVATION_TABLE,
|
|
MANAGED_IDENTITY_SCHEMA,
|
|
SECRET_SCHEMA,
|
|
_without_creative,
|
|
_without_lab_execution,
|
|
_without_managed_identities,
|
|
_without_model_changes,
|
|
_without_model_updates,
|
|
_without_secrets,
|
|
_without_selected_comparisons,
|
|
audit_fingerprint_differences_are_compatible,
|
|
fingerprint_compatibility,
|
|
fingerprint_session,
|
|
)
|
|
from modelforge_api.settings import Settings, get_settings
|
|
|
|
ANCHOR = Principal("legacy-operator", PrincipalRole.ADMIN, "legacy_admin_token")
|
|
REASON = "Reviewed operator identity lifecycle"
|
|
|
|
|
|
@pytest.fixture
|
|
def session():
|
|
engine = create_engine(
|
|
"sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False}
|
|
)
|
|
Base.metadata.create_all(engine)
|
|
with Session(engine) as session:
|
|
yield session
|
|
engine.dispose()
|
|
|
|
|
|
def service(session: Session, principal: Principal = ANCHOR) -> ManagedIdentityService:
|
|
return ManagedIdentityService(session, principal, "managed-request-42")
|
|
|
|
|
|
def create(session: Session, slug: str = "alice", role: str = "admin"):
|
|
return service(session).create(
|
|
ManagedIdentityCreate(
|
|
slug=slug,
|
|
role=role,
|
|
expires_in_days=30,
|
|
expected_revision=service(session).list().revision,
|
|
reason=REASON,
|
|
)
|
|
)
|
|
|
|
|
|
def test_issue_metadata_auth_and_audit(session: Session) -> None:
|
|
issued = create(session)
|
|
assert issued.one_time_key.startswith("mfopm_") and len(issued.one_time_key) == 49
|
|
assert issued.revision == 1 and issued.identity.subject == "managed:alice"
|
|
principal = authenticate_managed_operator(session, issued.one_time_key)
|
|
session.commit()
|
|
assert principal.subject == "managed:alice" and principal.role == PrincipalRole.ADMIN
|
|
metadata = service(session).list().model_dump_json()
|
|
digest = hashlib.sha256(issued.one_time_key.encode()).hexdigest()
|
|
assert (
|
|
issued.one_time_key not in metadata
|
|
and digest not in metadata
|
|
and "token_sha256" not in metadata
|
|
)
|
|
assert service(session).list().identities[0].credentials[0].last_used_at is not None
|
|
audit = session.scalar(
|
|
select(AuditEvent).where(AuditEvent.action == "managed_identity.created")
|
|
)
|
|
assert audit is not None and audit.actor_id == "legacy-operator"
|
|
assert audit.correlation_id == "managed-request-42" and issued.one_time_key not in str(
|
|
audit.details
|
|
)
|
|
|
|
|
|
def test_credential_metadata_preserves_utc_after_database_reload(session: Session) -> None:
|
|
create(session)
|
|
instant = datetime(2026, 9, 8, 10, 30, tzinfo=UTC)
|
|
session.execute(update(ManagedOperatorCredential).values(
|
|
revoked_at=instant, superseded_at=instant, last_used_at=instant,
|
|
))
|
|
session.commit()
|
|
session.expire_all()
|
|
credential = service(session).list().identities[0].credentials[0]
|
|
assert credential.revoked_at == instant
|
|
assert credential.superseded_at == instant
|
|
assert credential.last_used_at == instant
|
|
|
|
|
|
@pytest.mark.parametrize("schema", ["20260908_0026", MANAGED_IDENTITY_SCHEMA])
|
|
def test_recovery_allows_one_marker_but_preserves_history(session: Session, schema: str) -> None:
|
|
create(session)
|
|
before = _without_lab_execution(_without_model_changes(
|
|
_without_secrets(
|
|
_without_model_updates(
|
|
_without_selected_comparisons(_without_creative(fingerprint_session(session)))
|
|
)
|
|
)
|
|
))
|
|
audit_module.AuditWriter(session, "operator", "recovery").write(
|
|
"RECOVERY_RECONCILIATION_COMPLETED", "restore", None, {"restored": True},
|
|
)
|
|
session.commit()
|
|
after = _without_lab_execution(_without_model_changes(
|
|
_without_secrets(
|
|
_without_model_updates(
|
|
_without_selected_comparisons(_without_creative(fingerprint_session(session)))
|
|
)
|
|
)
|
|
))
|
|
if schema == "20260908_0026":
|
|
before, after = _without_managed_identities(before), _without_managed_identities(after)
|
|
contract = fingerprint_compatibility(before, after, source_schema_revision=schema)
|
|
assert audit_fingerprint_differences_are_compatible(before, after, contract)
|
|
for table in before["tables"]:
|
|
altered = copy.deepcopy(after)
|
|
if table == "audit_events":
|
|
altered["tables"][table]["row_count"] += 1
|
|
elif table == "audit_chain_heads":
|
|
altered["tables"][table]["row_count"] = 0
|
|
elif table == "operational_alerts":
|
|
altered["tables"][table]["row_count"] += 1
|
|
else:
|
|
altered["tables"][table]["digest"] = "0" * 64
|
|
assert not audit_fingerprint_differences_are_compatible(before, altered, contract), table
|
|
|
|
|
|
@pytest.mark.parametrize("operation", ["revoke", "disable", "demote"])
|
|
def test_last_admin_is_preserved_atomically(session: Session, operation: str) -> None:
|
|
issued = create(session)
|
|
before = service(session).list().model_dump_json()
|
|
events = list(session.scalars(select(AuditEvent.id)))
|
|
with pytest.raises(ManagedIdentityError) as refused:
|
|
if operation == "revoke":
|
|
service(session).revoke(
|
|
"alice",
|
|
ManagedIdentityRevoke(
|
|
expected_version=1, confirm_subject="managed:alice", reason=REASON
|
|
),
|
|
)
|
|
else:
|
|
service(session).change(
|
|
"alice",
|
|
ManagedIdentityChange(
|
|
expected_version=1,
|
|
role="viewer" if operation == "demote" else "admin",
|
|
enabled=operation != "disable",
|
|
reason=REASON,
|
|
),
|
|
)
|
|
assert refused.value.code == "LAST_MANAGED_ADMIN"
|
|
assert service(session).list().model_dump_json() == before
|
|
assert list(session.scalars(select(AuditEvent.id))) == events
|
|
assert authenticate_managed_operator(session, issued.one_time_key).role == PrincipalRole.ADMIN
|
|
|
|
|
|
def test_initial_admin_and_revision_conflicts(session: Session) -> None:
|
|
with pytest.raises(ManagedIdentityError, match="first managed"):
|
|
create(session, role="viewer")
|
|
assert service(session).list().revision == 0
|
|
create(session)
|
|
with pytest.raises(ManagedIdentityError) as conflict:
|
|
service(session).create(
|
|
ManagedIdentityCreate(
|
|
slug="bob", role="viewer", expires_in_days=1, expected_revision=0, reason=REASON
|
|
)
|
|
)
|
|
assert conflict.value.code == "MANAGED_IDENTITY_VERSION_CONFLICT"
|
|
assert session.get(ManagedOperatorIdentity, "bob") is None
|
|
|
|
|
|
def test_bounded_overlap_revoke_and_no_fallback(session: Session) -> None:
|
|
first = create(session)
|
|
rotated = service(session).rotate(
|
|
"alice",
|
|
ManagedIdentityRotate(
|
|
expected_version=1, expires_in_days=30, overlap_seconds=900, reason=REASON
|
|
),
|
|
)
|
|
assert len(rotated.identity.credentials) == 2
|
|
assert authenticate_managed_operator(session, first.one_time_key).subject == "managed:alice"
|
|
latest = service(session).rotate(
|
|
"alice",
|
|
ManagedIdentityRotate(
|
|
expected_version=2, expires_in_days=30, overlap_seconds=0, reason=REASON
|
|
),
|
|
)
|
|
for token in (first.one_time_key, rotated.one_time_key):
|
|
with pytest.raises(ManagedIdentityError):
|
|
authenticate_managed_operator(session, token)
|
|
assert authenticate_managed_operator(session, latest.one_time_key)
|
|
with pytest.raises(HTTPException):
|
|
authenticate_operator_token(
|
|
Settings(_env_file=None, operator_api_key=SecretStr(latest.one_time_key)),
|
|
latest.one_time_key,
|
|
)
|
|
|
|
|
|
def test_initiator_revoked_after_authentication_is_rechecked(session: Session) -> None:
|
|
alice = create(session)
|
|
create(session, "bob")
|
|
stale = authenticate_managed_operator(session, alice.one_time_key)
|
|
session.commit()
|
|
service(session).revoke(
|
|
"alice",
|
|
ManagedIdentityRevoke(expected_version=1, confirm_subject="managed:alice", reason=REASON),
|
|
)
|
|
with pytest.raises(ManagedIdentityError) as denied:
|
|
service(session, stale).change(
|
|
"bob",
|
|
ManagedIdentityChange(expected_version=1, role="admin", enabled=True, reason=REASON),
|
|
)
|
|
assert denied.value.code == "MANAGED_ADMIN_NO_LONGER_AUTHORIZED"
|
|
assert session.get(ManagedOperatorIdentity, "bob").version == 1
|
|
|
|
|
|
def test_recovery_activation_quarantine_preserves_authoritative_history(session: Session) -> None:
|
|
original = create(session)
|
|
before = _without_lab_execution(_without_model_changes(
|
|
_without_model_updates(
|
|
_without_selected_comparisons(_without_creative(fingerprint_session(session)))
|
|
)
|
|
))
|
|
assert MANAGED_ACTIVATION_TABLE in CURRENT_TRUTH_TABLES
|
|
session.execute(delete(ManagedOperatorCredentialActivation))
|
|
session.commit()
|
|
after = _without_lab_execution(_without_model_changes(
|
|
_without_model_updates(
|
|
_without_selected_comparisons(_without_creative(fingerprint_session(session)))
|
|
)
|
|
))
|
|
contract = fingerprint_compatibility(
|
|
before, after, source_schema_revision=SECRET_SCHEMA
|
|
)
|
|
assert audit_fingerprint_differences_are_compatible(before, after, contract)
|
|
with pytest.raises(ManagedIdentityError):
|
|
authenticate_managed_operator(session, original.one_time_key)
|
|
metadata = service(session).list().identities[0].credentials[0]
|
|
assert not metadata.activated and not metadata.usable
|
|
recovered = service(session).rotate(
|
|
"alice",
|
|
ManagedIdentityRotate(
|
|
expected_version=1, expires_in_days=30, overlap_seconds=900, reason=REASON
|
|
),
|
|
)
|
|
assert authenticate_managed_operator(session, recovered.one_time_key)
|
|
with pytest.raises(ManagedIdentityError):
|
|
authenticate_managed_operator(session, original.one_time_key)
|
|
|
|
|
|
def test_audit_failure_rolls_back_key_and_revision(
|
|
session: Session, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
def failed(*args, **kwargs):
|
|
raise RuntimeError("audit unavailable")
|
|
|
|
monkeypatch.setattr(audit_module, "_append_canonical_audit_event", failed)
|
|
with pytest.raises(RuntimeError):
|
|
create(session)
|
|
assert service(session).list().revision == 0
|
|
assert session.scalar(select(ManagedOperatorCredential)) is None
|
|
|
|
|
|
def test_metadata_failure_preserves_previous_key_and_audit(
|
|
session: Session, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
issued = create(session)
|
|
before = list(session.scalars(select(AuditEvent.id)))
|
|
|
|
def unavailable(*args, **kwargs):
|
|
raise RuntimeError("metadata unavailable")
|
|
|
|
subject = service(session)
|
|
monkeypatch.setattr(subject, "_metadata", unavailable)
|
|
with pytest.raises(RuntimeError, match="metadata unavailable"):
|
|
subject.rotate(
|
|
"alice",
|
|
ManagedIdentityRotate(
|
|
expected_version=1, expires_in_days=30, overlap_seconds=0, reason=REASON
|
|
),
|
|
)
|
|
assert list(session.scalars(select(AuditEvent.id))) == before
|
|
assert service(session).list().revision == 1
|
|
assert authenticate_managed_operator(session, issued.one_time_key)
|
|
|
|
|
|
def test_issuance_returns_snapshot_without_reads_after_commit(session: Session) -> None:
|
|
committed = False
|
|
|
|
def mark_commit(session):
|
|
nonlocal committed
|
|
committed = True
|
|
|
|
def reject_late_read(orm_execute_state):
|
|
if committed:
|
|
raise AssertionError("database read after commit")
|
|
|
|
event.listen(session, "after_commit", mark_commit)
|
|
event.listen(session, "do_orm_execute", reject_late_read)
|
|
try:
|
|
issued = create(session)
|
|
assert issued.revision == 1 and issued.identity.credentials[0].usable
|
|
assert "one_time_key" in issued.model_dump_json()
|
|
finally:
|
|
event.remove(session, "after_commit", mark_commit)
|
|
event.remove(session, "do_orm_execute", reject_late_read)
|
|
|
|
|
|
@pytest.mark.parametrize("alteration", ["expiry", "disabled", "revoked", "quarantined"])
|
|
def test_unusable_credential_rejected(session: Session, alteration: str) -> None:
|
|
issued = create(session)
|
|
credential_id = uuid.UUID(issued.identity.credentials[0].id)
|
|
if alteration == "expiry":
|
|
session.execute(
|
|
update(ManagedOperatorCredential)
|
|
.where(ManagedOperatorCredential.id == credential_id)
|
|
.values(expires_at=datetime.now(UTC) - timedelta(seconds=1))
|
|
)
|
|
elif alteration == "revoked":
|
|
session.execute(
|
|
update(ManagedOperatorCredential)
|
|
.where(ManagedOperatorCredential.id == credential_id)
|
|
.values(revoked_at=datetime.now(UTC))
|
|
)
|
|
elif alteration == "disabled":
|
|
session.execute(update(ManagedOperatorIdentity).values(enabled=False))
|
|
else:
|
|
session.execute(delete(ManagedOperatorCredentialActivation))
|
|
session.commit()
|
|
with pytest.raises(ManagedIdentityError) as error:
|
|
authenticate_managed_operator(session, issued.one_time_key)
|
|
assert error.value.status_code == 401
|
|
assert not service(session).list().identities[0].credentials[0].usable
|
|
|
|
|
|
def test_non_admin_management_and_exact_revoke_confirmation(session: Session) -> None:
|
|
create(session)
|
|
viewer = create(session, "viewer", "viewer")
|
|
principal = authenticate_managed_operator(session, viewer.one_time_key)
|
|
session.commit()
|
|
with pytest.raises(ManagedIdentityError) as denied:
|
|
service(session, principal).list()
|
|
assert denied.value.status_code == 403
|
|
with pytest.raises(ManagedIdentityError) as denied:
|
|
service(session).revoke(
|
|
"viewer",
|
|
ManagedIdentityRevoke(expected_version=1, confirm_subject="viewer", reason=REASON),
|
|
)
|
|
assert denied.value.code == "MANAGED_IDENTITY_CONFIRMATION_REQUIRED"
|
|
result = service(session).revoke(
|
|
"viewer",
|
|
ManagedIdentityRevoke(expected_version=1, confirm_subject="managed:viewer", reason=REASON),
|
|
)
|
|
assert result.identity.version == 2 and not result.identity.credentials[0].usable
|
|
|
|
|
|
def test_metadata_history_is_bounded(session: Session) -> None:
|
|
create(session)
|
|
for version in range(1, 12):
|
|
service(session).rotate(
|
|
"alice",
|
|
ManagedIdentityRotate(
|
|
expected_version=version, expires_in_days=1, overlap_seconds=0, reason=REASON
|
|
),
|
|
)
|
|
assert len(service(session).list().identities[0].credentials) == 10
|
|
|
|
|
|
def test_migration_empty_downgrade_and_history_refusal() -> None:
|
|
path = (
|
|
Path(__file__).resolve().parents[1]
|
|
/ "alembic/versions/20260908_0027_managed_operator_identities.py"
|
|
)
|
|
specification = importlib.util.spec_from_file_location("managed_identity_migration", path)
|
|
assert specification is not None and specification.loader is not None
|
|
migration = importlib.util.module_from_spec(specification)
|
|
specification.loader.exec_module(migration)
|
|
engine = create_engine("sqlite://")
|
|
with engine.begin() as connection, Operations.context(MigrationContext.configure(connection)):
|
|
migration.upgrade()
|
|
assert connection.scalar(select(ManagedOperatorState.revision)) == 0
|
|
migration.downgrade()
|
|
migration.upgrade()
|
|
connection.execute(
|
|
ManagedOperatorIdentity.__table__.insert().values(
|
|
slug="retained",
|
|
role="admin",
|
|
enabled=True,
|
|
version=1,
|
|
created_at=datetime.now(UTC),
|
|
)
|
|
)
|
|
with pytest.raises(RuntimeError, match="history exists"):
|
|
migration.downgrade()
|
|
engine.dispose()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"field,value",
|
|
[
|
|
("expires_in_days", True),
|
|
("expires_in_days", "30"),
|
|
("expires_in_days", 366),
|
|
("expected_revision", False),
|
|
("reason", " "),
|
|
("reason", "copied mfopm_" + "a" * 43),
|
|
("slug", "managed:alice"),
|
|
],
|
|
)
|
|
def test_strict_inputs_and_secret_safe_reason(field: str, value: object) -> None:
|
|
body = {"slug": "alice", "role": "admin", "expires_in_days": 30, "expected_revision": 0, "reason": REASON}
|
|
body[field] = value
|
|
with pytest.raises(ValidationError):
|
|
ManagedIdentityCreate.model_validate(body)
|
|
|
|
|
|
def test_http_boundaries_no_store_and_dependency_failure(
|
|
session: Session, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
settings = Settings(_env_file=None, env="test", operator_api_key="anchor-secret")
|
|
app.dependency_overrides[get_settings] = lambda: settings
|
|
app.dependency_overrides[get_session] = lambda: session
|
|
try:
|
|
client = TestClient(app)
|
|
url = "/api/v1/operator/managed-identities"
|
|
headers = {
|
|
"X-ModelForge-Admin-Token": "anchor-secret",
|
|
"X-Correlation-ID": "http-managed-42",
|
|
}
|
|
assert client.post(url, content="invalid").status_code == 401
|
|
response = client.post(
|
|
url,
|
|
json={
|
|
"slug": "alice", "role": "admin", "expires_in_days": 30, "expected_revision": 0, "reason": REASON
|
|
},
|
|
headers=headers,
|
|
)
|
|
assert response.status_code == 201, response.text
|
|
token = response.json()["one_time_key"]
|
|
assert (
|
|
response.headers["cache-control"] == "no-store" and "set-cookie" not in response.headers
|
|
)
|
|
own = {"X-ModelForge-Admin-Token": token}
|
|
identity = client.get("/api/v1/operator/identity", headers=own)
|
|
assert identity.json()["subject"] == "managed:alice"
|
|
assert token not in client.get(url, headers=own).text
|
|
failure = client.post(
|
|
url,
|
|
json={
|
|
"slug": "bob",
|
|
"role": "admin",
|
|
"expires_in_days": 30,
|
|
"expected_revision": 1,
|
|
"reason": "copied " + token,
|
|
},
|
|
headers=own,
|
|
)
|
|
assert failure.status_code == 422 and token not in failure.text
|
|
|
|
def unavailable(*args, **kwargs):
|
|
raise OperationalError("hidden", {}, Exception("private database detail"))
|
|
|
|
monkeypatch.setattr("modelforge_api.main.authenticate_managed_operator", unavailable)
|
|
response = client.post(url, content="not JSON", headers=own)
|
|
assert response.status_code == 503
|
|
assert "private database detail" not in response.text
|
|
assert client.get("/api/v1/operator/identity", headers=headers).status_code == 200
|
|
finally:
|
|
app.dependency_overrides.clear()
|