Jens/ModelForge-Public
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Publish curated ITWorx-ModelForge source from 990401737f96c8d46a9f7b0d84994516cadcb6a0

Browse Source
This commit is contained in:
Public Source Publisher committed 2026-09-30 09:23:17 +00:00
1 parent 7082ab955a
commit 848f0d8f8a
434 files changed
+67298 -4315
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/Dockerfile
+14 -12
View File
@@ -25,22 +25,24 @@ LABEL org.opencontainers.image.description="Capability-first local AI ModelOps a
LABEL org.opencontainers.image.version="${MODELFORGE_VERSION}"
LABEL org.opencontainers.image.revision="${MODELFORGE_COMMIT}"
LABEL org.opencontainers.image.created="${MODELFORGE_BUILT_AT}"
LABEL org.opencontainers.image.source="https://git.example.com/example/modelforge.git"
LABEL org.opencontainers.image.source="https://git.example.com/example/modelforge"
LABEL org.opencontainers.image.vendor="ITWorx"
LABEL org.opencontainers.image.licenses="AGPL-3.0-or-later"
WORKDIR /app
COPY pyproject.toml ./
COPY src ./src
COPY alembic.ini ./
COPY alembic ./alembic
# Upgrade the installer before it resolves anything: the pinned base image ships a pip
# carrying archive-extraction advisories. pip never runs at runtime, but a release image
# should not carry a known-vulnerable installer.
RUN pip install --no-cache-dir --upgrade pip "setuptools>=78.1.1" "msgpack>=1.2.1" \
&& pip install --no-cache-dir . \
COPY config/security/build-toolchain.lock /tmp/modelforge-build-toolchain.lock
COPY backend/pyproject.toml ./
COPY backend/src ./src
COPY backend/alembic.ini ./
COPY backend/alembic ./alembic
# Build tooling is preinstalled from a fully resolved hash lock. Disabling PEP 517 isolation
# makes pip use those exact tools rather than fetching a second, unverified build environment.
RUN python -m pip install --no-cache-dir --require-hashes --only-binary=:all: \
--index-url https://pypi.org/simple -r /tmp/modelforge-build-toolchain.lock \
&& python -m pip install --no-cache-dir --no-build-isolation . \
&& pip check \
&& python -m pip uninstall --yes pip setuptools
&& python -m pip uninstall --yes pip setuptools wheel editables hatchling pathspec pluggy \
tomlkit trove-classifiers
RUN addgroup -S modelforge && adduser -S -G modelforge -h /app modelforge \
&& mkdir -p /data/state /data/hf-cache /data/artifacts /data/quarantine \
@@ -49,4 +51,4 @@ RUN addgroup -S modelforge && adduser -S -G modelforge -h /app modelforge \
USER modelforge
EXPOSE 8000
CMD ["uvicorn", "modelforge_api.main:app", "--host", "0.0.0.0", "--port", "8000"]
CMD ["uvicorn", "modelforge_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-proxy-headers"]
backend/alembic/versions/20260824_0001_m0_foundation.py
+69
View File
@@ -19,6 +19,49 @@ def upgrade() -> None:
# The declarative metadata is the reviewed M0 schema contract. create_all is
# used only from this baseline migration; later changes require explicit ops.
Base.metadata.create_all(bind=op.get_bind(), checkfirst=False)
# BenchmarkRun was part of M0. The evolving metadata also contains RC-LAB-02's later
# capability-target, actor/idempotency and environment fields, so rebuild its original
# shape here; migration 0039 adds those fields at the correct historical boundary.
op.drop_table("benchmark_runs")
op.create_table(
"benchmark_runs",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("deployment_id", sa.Uuid(), nullable=False),
sa.Column("suite_id", sa.Uuid(), nullable=False),
sa.Column("model_revision_sha", sa.String(length=64), nullable=False),
sa.Column("artifact_sha256", sa.String(length=64), nullable=False),
sa.Column("runtime_type", sa.String(length=32), nullable=False),
sa.Column("runtime_version", sa.String(length=128), nullable=False),
sa.Column("runtime_image_digest", sa.String(length=71), nullable=True),
sa.Column("launch_arguments", sa.JSON(), nullable=False),
sa.Column("suite_revision", sa.String(length=128), nullable=False),
sa.Column("dataset_revision", sa.String(length=255), nullable=False),
sa.Column("environment_fingerprint", sa.JSON(), nullable=False),
sa.Column("environment_digest", sa.String(length=64), nullable=False),
sa.Column("seed", sa.Integer(), nullable=True),
sa.Column("results", sa.JSON(), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False),
sa.Column("comparable", sa.Boolean(), nullable=False),
sa.Column("stale_reason", sa.Text(), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(["deployment_id"], ["deployments.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["suite_id"], ["benchmark_suites.id"], ondelete="RESTRICT"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_benchmark_runs_deployment_id", "benchmark_runs", ["deployment_id"])
op.create_index("ix_benchmark_runs_suite_id", "benchmark_runs", ["suite_id"])
op.create_index("ix_benchmark_runs_artifact_sha256", "benchmark_runs", ["artifact_sha256"])
op.create_index(
"ix_benchmark_runs_environment_digest", "benchmark_runs", ["environment_digest"]
)
op.create_index(
"ix_benchmark_runs_deployment_completed",
"benchmark_runs",
["deployment_id", "completed_at"],
)
op.drop_constraint("ck_capability_serving_generation", "capability_contracts", type_="check")
op.drop_column("capability_contracts", "serving_generation")
# Freeze the original M0 shape even though the declarative metadata evolves.
# Remove newer foreign-key columns before dropping the tables they target.
for column in (
@@ -36,6 +79,27 @@ def upgrade() -> None:
# Later milestone tables are present in the evolving declarative metadata
# used to build this baseline. Drop dependants before their M5/M0 targets
# so a fresh upgrade still reconstructs each historical revision exactly.
"model_change_cases",
"selected_model_comparisons",
"external_creative_action_attestations",
"external_creative_profile_observations",
"external_creative_resources",
"external_creative_allocations",
"model_update_notices",
"model_update_watches",
"portfolio_repository_observations",
"portfolio_reconciliation_runs",
"platform_secret_activations",
"platform_secret_tests",
"platform_secret_versions",
"platform_secret_state",
"managed_operator_credential_activations",
"managed_operator_credentials",
"managed_operator_identities",
"managed_operator_state",
"configuration_revisions",
"project_onboarding",
"configuration_state",
"audit_chain_heads",
"node_decommission_operations",
# M15 recovery tables come first: artifact_recovery_operations references
@@ -50,6 +114,8 @@ def upgrade() -> None:
"recovery_asset_records",
"recovery_policy_revisions",
"incident_timeline_events",
# The M37 outbox restricts deletion of its alert history parent.
"alert_notification_deliveries",
"alert_history_events",
"operational_alerts",
"operational_incidents",
@@ -107,6 +173,9 @@ def upgrade() -> None:
"project_evaluation_bindings",
"serving_gpu_leases",
"serving_jobs",
# M38 durable LAB history owns a restrictive reference to gateway_requests;
# drop it before the M0 gateway table while reconstructing the historical baseline.
"lab_execution_runs",
"gateway_requests",
"residency_allocations",
"service_credentials",
backend/alembic/versions/20260830_0024_audit_chain_checkpoint.py
+1 -1
View File
@@ -581,7 +581,7 @@ def _remove_postgres_audit_boundary(connection: Connection) -> None:
"drop trigger if exists trg_modelforge_audit_head_owner on public.audit_chain_heads; "
"drop trigger if exists trg_modelforge_audit_head_truncate_owner "
"on public.audit_chain_heads; "
"drop function if exists modelforge_audit.append_event_v2("
"drop function if exists modelforge_audit.append_event_v2("
"uuid, timestamptz, text, text, text, text, text, text, text, jsonb, "
"bigint, bigint, text, text, bigint, bigint, text); "
"drop function if exists modelforge_audit.enforce_owner_mutation(); "
backend/alembic/versions/20260907_0025_configuration.py
+46
View File
@@ -0,0 +1,46 @@
"""Durable operator configuration and revision history.
Revision ID: 20260907_0025
Revises: 20260830_0024
"""
import sqlalchemy as sa
from alembic import op
revision = "20260907_0025"
down_revision = "20260830_0024"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"configuration_state",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("revision", sa.Integer(), nullable=False),
sa.Column("overrides", sa.JSON(), nullable=False),
sa.CheckConstraint("id = 1"),
sa.CheckConstraint("revision >= 0"),
)
op.create_table(
"configuration_revisions",
sa.Column("revision", sa.Integer(), primary_key=True),
sa.Column("previous_revision", sa.Integer(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("actor", sa.String(255), nullable=False),
sa.Column("correlation_id", sa.String(255), nullable=False),
sa.Column("overrides", sa.JSON(), nullable=False),
sa.Column("preview_digest", sa.String(64), nullable=False),
)
if op.get_bind().dialect.name == "postgresql":
op.execute("GRANT SELECT, INSERT, UPDATE ON configuration_state TO modelforge_runtime")
op.execute("REVOKE UPDATE, DELETE, TRUNCATE ON configuration_revisions FROM modelforge_runtime")
op.execute("GRANT SELECT, INSERT ON configuration_revisions TO modelforge_runtime")
def downgrade() -> None:
if op.get_bind().execute(sa.text("SELECT count(*) FROM configuration_revisions")).scalar_one():
raise RuntimeError("Configuration history exists; recover the pre-migration backup instead.")
op.drop_table("configuration_revisions")
op.drop_table("configuration_state")
backend/alembic/versions/20260908_0026_project_onboarding.py
+51
View File
@@ -0,0 +1,51 @@
"""Project onboarding and bounded credential overlap.
Revision ID: 20260908_0026
Revises: 20260907_0025
"""
import sqlalchemy as sa
from alembic import op
revision = "20260908_0026"
down_revision = "20260907_0025"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"project_onboarding",
sa.Column("project_id", sa.Uuid(), sa.ForeignKey("projects.id", ondelete="RESTRICT"), primary_key=True),
sa.Column("owner", sa.String(128), nullable=False),
sa.Column("environment", sa.String(32), nullable=False),
sa.Column("labels", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.add_column("service_credentials", sa.Column("superseded_at", sa.DateTime(timezone=True)))
op.drop_index("uq_active_service_client_credential", table_name="service_credentials")
op.create_index(
"uq_active_service_client_credential", "service_credentials", ["service_client_id"],
unique=True,
postgresql_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"),
sqlite_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"),
)
if op.get_bind().dialect.name == "postgresql":
op.execute("GRANT SELECT, INSERT ON project_onboarding TO modelforge_runtime")
op.execute("REVOKE UPDATE, DELETE, TRUNCATE ON project_onboarding FROM modelforge_runtime")
def downgrade() -> None:
connection = op.get_bind()
if connection.execute(sa.text("SELECT count(*) FROM project_onboarding")).scalar_one() or connection.execute(
sa.text("SELECT count(*) FROM service_credentials WHERE superseded_at IS NOT NULL")
).scalar_one():
raise RuntimeError("Onboarding or rotation history exists; recover the pre-migration backup instead.")
op.drop_index("uq_active_service_client_credential", table_name="service_credentials")
op.create_index(
"uq_active_service_client_credential", "service_credentials", ["service_client_id"],
unique=True, postgresql_where=sa.text("revoked_at IS NULL"), sqlite_where=sa.text("revoked_at IS NULL"),
)
op.drop_column("service_credentials", "superseded_at")
op.drop_table("project_onboarding")
backend/alembic/versions/20260908_0027_managed_operator_identities.py
+60
View File
@@ -0,0 +1,60 @@
"""Managed personal identities with recovery-cleared credential admission.
Revision ID: 20260908_0027
Revises: 20260908_0026
"""
import sqlalchemy as sa
from alembic import op
revision = "20260908_0027"
down_revision = "20260908_0026"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table("managed_operator_state",
sa.Column("singleton_id", sa.Integer(), primary_key=True),
sa.Column("revision", sa.Integer(), nullable=False),
sa.CheckConstraint("singleton_id = 1", name="ck_managed_operator_singleton"),
sa.CheckConstraint("revision >= 0", name="ck_managed_operator_revision"))
op.execute("INSERT INTO managed_operator_state (singleton_id, revision) VALUES (1, 0)")
op.create_table("managed_operator_identities",
sa.Column("slug", sa.String(63), primary_key=True),
sa.Column("role", sa.String(16), nullable=False),
sa.Column("enabled", sa.Boolean(), nullable=False),
sa.Column("version", sa.Integer(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("role IN ('viewer','operator','admin')", name="ck_managed_operator_role"),
sa.CheckConstraint("version >= 1", name="ck_managed_operator_version"))
op.create_table("managed_operator_credentials",
sa.Column("id", sa.Uuid(), primary_key=True),
sa.Column("identity_slug", sa.String(63), sa.ForeignKey("managed_operator_identities.slug", ondelete="RESTRICT"), nullable=False),
sa.Column("token_sha256", sa.String(64), nullable=False, unique=True),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("revoked_at", sa.DateTime(timezone=True)),
sa.Column("superseded_at", sa.DateTime(timezone=True)),
sa.Column("last_used_at", sa.DateTime(timezone=True)),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False))
op.create_index("ix_managed_operator_credentials_identity_slug", "managed_operator_credentials", ["identity_slug"])
op.create_index("uq_managed_operator_current", "managed_operator_credentials", ["identity_slug"], unique=True,
postgresql_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"),
sqlite_where=sa.text("revoked_at IS NULL AND superseded_at IS NULL"))
op.create_table("managed_operator_credential_activations",
sa.Column("credential_id", sa.Uuid(), sa.ForeignKey("managed_operator_credentials.id", ondelete="RESTRICT"), primary_key=True),
sa.Column("activated_at", sa.DateTime(timezone=True), nullable=False))
if op.get_bind().dialect.name == "postgresql":
op.execute("GRANT SELECT, INSERT, UPDATE ON managed_operator_state, managed_operator_identities, managed_operator_credentials TO modelforge_runtime")
op.execute("GRANT SELECT, INSERT, DELETE ON managed_operator_credential_activations TO modelforge_runtime")
op.execute("REVOKE DELETE, TRUNCATE ON managed_operator_state, managed_operator_identities, managed_operator_credentials FROM modelforge_runtime")
def downgrade() -> None:
connection = op.get_bind()
if connection.execute(sa.text("SELECT revision FROM managed_operator_state WHERE singleton_id = 1")).scalar_one() != 0 or connection.execute(sa.text("SELECT count(*) FROM managed_operator_identities")).scalar_one():
raise RuntimeError("Managed identity history exists; recover the pre-migration backup instead.")
op.drop_table("managed_operator_credential_activations")
op.drop_table("managed_operator_credentials")
op.drop_table("managed_operator_identities")
op.drop_table("managed_operator_state")
backend/alembic/versions/20260908_0028_platform_secret_slots.py
+56
View File
@@ -0,0 +1,56 @@
"""Encrypted HF control-plane secret history and recovery-cleared admission.
Revision ID: 20260908_0028
Revises: 20260908_0027
"""
import sqlalchemy as sa
from alembic import op
revision = "20260908_0028"
down_revision = "20260908_0027"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table("platform_secret_state",
sa.Column("singleton_id", sa.Integer(), primary_key=True),
sa.Column("revision", sa.Integer(), nullable=False),
sa.Column("initialized", sa.Boolean(), nullable=False),
sa.CheckConstraint("singleton_id = 1", name="ck_secret_state_singleton"),
sa.CheckConstraint("revision >= 0", name="ck_secret_state_revision"))
op.execute("INSERT INTO platform_secret_state (singleton_id, revision, initialized) VALUES (1, 0, false)")
op.create_table("platform_secret_versions",
sa.Column("version", sa.Integer(), primary_key=True, autoincrement=False),
sa.Column("key_id", sa.String(64), nullable=False),
sa.Column("secret_ciphertext", sa.Text(), nullable=False),
sa.Column("secret_nonce", sa.String(32), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("revoked_at", sa.DateTime(timezone=True)),
sa.CheckConstraint("version > 0", name="ck_secret_version_positive"))
op.create_table("platform_secret_tests",
sa.Column("version", sa.Integer(), sa.ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), primary_key=True),
sa.Column("passed", sa.Boolean(), nullable=False),
sa.Column("tested_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("code", sa.String(64), nullable=False))
op.create_table("platform_secret_activations",
sa.Column("singleton_id", sa.Integer(), primary_key=True),
sa.Column("version", sa.Integer(), sa.ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), nullable=False),
sa.Column("activated_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("singleton_id = 1", name="ck_secret_activation_singleton"))
if op.get_bind().dialect.name == "postgresql":
op.execute("REVOKE ALL ON platform_secret_state, platform_secret_versions, platform_secret_tests, platform_secret_activations FROM modelforge_runtime")
op.execute("GRANT SELECT, UPDATE ON platform_secret_state TO modelforge_runtime")
op.execute("GRANT SELECT, INSERT ON platform_secret_versions TO modelforge_runtime")
op.execute("GRANT UPDATE (revoked_at) ON platform_secret_versions TO modelforge_runtime")
op.execute("GRANT SELECT, INSERT, UPDATE, DELETE ON platform_secret_tests, platform_secret_activations TO modelforge_runtime")
def downgrade() -> None:
connection = op.get_bind()
state = connection.execute(sa.text("SELECT revision, initialized FROM platform_secret_state WHERE singleton_id=1")).one()
if state.revision != 0 or state.initialized or connection.scalar(sa.text("SELECT count(*) FROM platform_secret_versions")):
raise RuntimeError("Secret slot history exists; recover the pre-migration backup instead.")
for table in ("platform_secret_activations", "platform_secret_tests", "platform_secret_versions", "platform_secret_state"):
op.drop_table(table)
backend/alembic/versions/20260909_0029_external_creative_coordination.py
+57
View File
@@ -0,0 +1,57 @@
"""External creative GPU coordination.
Revision ID: 20260909_0029
Revises: 20260908_0028
"""
import sqlalchemy as sa
from alembic import op
revision = "20260909_0029"
down_revision = "20260908_0028"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"external_creative_allocations",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("service_client_id", sa.Uuid(), nullable=False),
sa.Column("project_id", sa.Uuid(), nullable=False),
sa.Column("accelerator_id", sa.Uuid()),
sa.Column("request_id", sa.String(128), nullable=False),
sa.Column("idempotency_key", sa.String(128), nullable=False),
sa.Column("request_sha256", sa.String(64), nullable=False),
sa.Column("workload", sa.String(16), nullable=False),
sa.Column("resource_ref", sa.String(255), nullable=False),
sa.Column("provider_resource_id", sa.String(128), nullable=False),
sa.Column("required_vram_bytes", sa.BigInteger(), nullable=False),
sa.Column("priority", sa.String(32), nullable=False),
sa.Column("ttl_seconds", sa.Integer(), nullable=False),
sa.Column("state", sa.String(32), nullable=False),
sa.Column("decision_code", sa.String(64), nullable=False),
sa.Column("decision_evidence", sa.JSON(), nullable=False),
sa.Column("required_actions", sa.JSON(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("acquired_at", sa.DateTime(timezone=True)),
sa.Column("heartbeat_at", sa.DateTime(timezone=True)),
sa.Column("expires_at", sa.DateTime(timezone=True)),
sa.Column("uncertain_at", sa.DateTime(timezone=True)),
sa.Column("released_at", sa.DateTime(timezone=True)),
sa.CheckConstraint("required_vram_bytes > 0", name="ck_external_creative_vram_positive"),
sa.CheckConstraint("ttl_seconds >= 30 AND ttl_seconds <= 3600", name="ck_external_creative_ttl"),
sa.ForeignKeyConstraint(["service_client_id"], ["service_clients.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["accelerator_id"], ["accelerators.id"], ondelete="RESTRICT"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("service_client_id", "idempotency_key", name="uq_external_creative_idempotency"),
)
for column in ("service_client_id", "project_id", "accelerator_id", "request_id", "state", "expires_at"):
op.create_index(f"ix_external_creative_allocations_{column}", "external_creative_allocations", [column])
def downgrade() -> None:
op.drop_table("external_creative_allocations")
backend/alembic/versions/20260909_0030_model_update_inbox.py
+43
View File
@@ -0,0 +1,43 @@
"""Explicit selected-model upstream watches and deduplicated revision notices."""
import sqlalchemy as sa
from alembic import op
revision = "20260909_0030"
down_revision = "20260909_0029"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table("model_update_watches",
sa.Column("id", sa.Uuid(), primary_key=True),
sa.Column("model_id", sa.Uuid(), sa.ForeignKey("models.id", ondelete="RESTRICT"), nullable=False),
sa.Column("baseline_revision_id", sa.Uuid(), sa.ForeignKey("model_revisions.id", ondelete="RESTRICT"), nullable=False),
sa.Column("baseline_commit_sha", sa.String(64), nullable=False),
sa.Column("upstream_ref", sa.String(255), nullable=False),
sa.Column("last_checked_at", sa.DateTime(timezone=True)),
sa.Column("last_error_code", sa.String(64)),
sa.Column("check_until", sa.DateTime(timezone=True)),
sa.Column("check_token", sa.Uuid()),
sa.UniqueConstraint("model_id", "upstream_ref", name="uq_model_watch_ref"))
op.create_table("model_update_notices",
sa.Column("id", sa.Uuid(), primary_key=True),
sa.Column("watch_id", sa.Uuid(), sa.ForeignKey("model_update_watches.id", ondelete="RESTRICT"), nullable=False),
sa.Column("model_id", sa.Uuid(), sa.ForeignKey("models.id", ondelete="RESTRICT"), nullable=False),
sa.Column("baseline_commit_sha", sa.String(64), nullable=False),
sa.Column("candidate_revision_id", sa.Uuid(), sa.ForeignKey("model_revisions.id", ondelete="RESTRICT"), nullable=False),
sa.Column("candidate_commit_sha", sa.String(64), nullable=False),
sa.Column("status", sa.String(16), nullable=False),
sa.Column("discovered_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.UniqueConstraint("watch_id", "candidate_commit_sha", name="uq_model_notice_commit"),
sa.CheckConstraint("status IN ('new', 'dismissed')", name="ck_model_notice_status"))
if op.get_bind().dialect.name == "postgresql":
op.execute("GRANT SELECT, INSERT, UPDATE ON model_update_watches, model_update_notices TO modelforge_runtime")
def downgrade() -> None:
if op.get_bind().scalar(sa.text("SELECT count(*) FROM model_update_watches")):
raise RuntimeError("Model update history exists; restore the pre-migration backup instead.")
op.drop_table("model_update_notices")
op.drop_table("model_update_watches")
backend/alembic/versions/20260909_0031_external_creative_governance.py
+114
View File
@@ -0,0 +1,114 @@
"""Govern external creative resources, measurements, and action attestations.
Revision ID: 20260909_0031
Revises: 20260909_0030
"""
import sqlalchemy as sa
from alembic import op
revision = "20260909_0031"
down_revision = "20260909_0030"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"external_creative_resources",
sa.Column("id", sa.Uuid(), primary_key=True),
sa.Column(
"project_id",
sa.Uuid(),
sa.ForeignKey("projects.id", ondelete="RESTRICT"),
nullable=False,
),
sa.Column("resource_ref", sa.String(255), nullable=False),
sa.Column("workload", sa.String(16), nullable=False),
sa.Column("provider_resource_id", sa.String(128), nullable=False),
sa.Column("approved", sa.Boolean(), nullable=False, server_default=sa.false()),
sa.Column("configured_vram_bytes", sa.BigInteger(), nullable=False),
sa.Column("measured_peak_vram_bytes", sa.BigInteger()),
sa.Column("measurement_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column(
"updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
),
sa.UniqueConstraint(
"project_id",
"resource_ref",
"workload",
"provider_resource_id",
name="uq_external_creative_resource_binding",
),
sa.CheckConstraint("configured_vram_bytes > 0", name="ck_external_resource_vram_positive"),
)
op.create_index(
"ix_external_creative_resources_project",
"external_creative_resources",
["project_id", "approved"],
)
op.create_table(
"external_creative_action_attestations",
sa.Column("id", sa.Uuid(), primary_key=True),
sa.Column(
"allocation_id",
sa.Uuid(),
sa.ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"),
nullable=False,
),
sa.Column("action", sa.String(64), nullable=False),
sa.Column("before_vram_bytes", sa.BigInteger(), nullable=False),
sa.Column("after_vram_bytes", sa.BigInteger(), nullable=False),
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("validated", sa.Boolean(), nullable=False),
sa.Column(
"created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
),
sa.UniqueConstraint(
"allocation_id", "action", name="uq_external_creative_action_attestation"
),
sa.CheckConstraint(
"before_vram_bytes >= 0 AND after_vram_bytes >= 0",
name="ck_external_action_vram_nonnegative",
),
)
op.create_table(
"external_creative_profile_observations",
sa.Column("id", sa.Uuid(), primary_key=True),
sa.Column(
"allocation_id",
sa.Uuid(),
sa.ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"),
nullable=False,
),
sa.Column("resource_ref", sa.String(255), nullable=False),
sa.Column("provider_resource_id", sa.String(128), nullable=False),
sa.Column("workload", sa.String(16), nullable=False),
sa.Column("observed_peak_vram_bytes", sa.BigInteger(), nullable=False),
sa.Column("reserved_vram_bytes", sa.BigInteger(), nullable=False),
sa.Column("duration_ms", sa.BigInteger(), nullable=False),
sa.Column("outcome", sa.String(16), nullable=False),
sa.Column(
"created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
),
sa.UniqueConstraint(
"allocation_id", name="uq_external_creative_profile_observation_allocation"
),
sa.CheckConstraint(
"observed_peak_vram_bytes > 0 AND reserved_vram_bytes > 0",
name="ck_external_profile_vram_positive",
),
sa.CheckConstraint(
"duration_ms >= 0 AND duration_ms <= 604800000", name="ck_external_profile_duration"
),
)
def downgrade() -> None:
op.drop_table("external_creative_profile_observations")
op.drop_table("external_creative_action_attestations")
op.drop_index(
"ix_external_creative_resources_project", table_name="external_creative_resources"
)
op.drop_table("external_creative_resources")
backend/alembic/versions/20260909_0032_audio_creative_workloads.py
+44
View File
@@ -0,0 +1,44 @@
"""Allow explicit audio and film creative workload names.
Revision ID: 20260909_0032
Revises: 20260909_0031
"""
import sqlalchemy as sa
from alembic import op
revision = "20260909_0032"
down_revision = "20260909_0031"
branch_labels = None
depends_on = None
def upgrade() -> None:
for table in (
"external_creative_allocations",
"external_creative_resources",
"external_creative_profile_observations",
):
with op.batch_alter_table(table) as batch:
batch.alter_column(
"workload",
existing_type=sa.String(16),
type_=sa.String(32),
existing_nullable=False,
)
def downgrade() -> None:
for table in (
"external_creative_profile_observations",
"external_creative_resources",
"external_creative_allocations",
):
with op.batch_alter_table(table) as batch:
batch.alter_column(
"workload",
existing_type=sa.String(32),
type_=sa.String(16),
existing_nullable=False,
)
backend/alembic/versions/20260909_0033_deployed_hub_convergence.py
+154
View File
@@ -0,0 +1,154 @@
"""Persist one explicitly delegated current/candidate comparison per project binding."""
import sqlalchemy as sa
from alembic import op
revision = "20260909_0033"
down_revision = "20260909_0032"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"model_update_watches",
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
)
op.add_column(
"model_update_watches",
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
)
op.create_table(
"selected_model_comparisons",
sa.Column("id", sa.Uuid(), primary_key=True),
sa.Column(
"project_binding_id",
sa.Uuid(),
sa.ForeignKey("project_bindings.id", ondelete="RESTRICT"),
nullable=False,
),
sa.Column(
"consumer_client_id",
sa.Uuid(),
sa.ForeignKey("service_clients.id", ondelete="RESTRICT"),
nullable=False,
),
sa.Column(
"suite_revision_id",
sa.Uuid(),
sa.ForeignKey("evaluation_suite_revisions.id", ondelete="RESTRICT"),
nullable=False,
),
sa.Column(
"current_deployment_id",
sa.Uuid(),
sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT"),
nullable=False,
),
sa.Column(
"candidate_deployment_id",
sa.Uuid(),
sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT"),
nullable=False,
),
sa.Column(
"candidate_route_id",
sa.Uuid(),
sa.ForeignKey("capability_experiment_routes.id", ondelete="RESTRICT"),
nullable=False,
),
sa.Column("idempotency_key", sa.String(128), nullable=False),
sa.Column("request_digest", sa.String(64), nullable=False),
sa.Column("actor_id", sa.String(128), nullable=False),
sa.Column("pins", sa.JSON(), nullable=False),
sa.Column("status", sa.String(32), nullable=False),
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
sa.Column("current_completed_cases", sa.Integer(), nullable=False, server_default="0"),
sa.Column("candidate_completed_cases", sa.Integer(), nullable=False, server_default="0"),
sa.Column("expected_cases", sa.Integer(), nullable=False),
sa.Column("claim_token_hash", sa.String(64)),
sa.Column("claim_expires_at", sa.DateTime(timezone=True)),
sa.Column("deadline_at", sa.DateTime(timezone=True)),
sa.Column("claimed_at", sa.DateTime(timezone=True)),
sa.Column("cancel_requested_at", sa.DateTime(timezone=True)),
sa.Column("finished_at", sa.DateTime(timezone=True)),
sa.Column("error_code", sa.String(64)),
sa.Column("result_digest", sa.String(64)),
sa.Column(
"baseline_run_id", sa.Uuid(), sa.ForeignKey("evaluation_runs.id", ondelete="RESTRICT")
),
sa.Column(
"candidate_run_id", sa.Uuid(), sa.ForeignKey("evaluation_runs.id", ondelete="RESTRICT")
),
sa.Column(
"evaluation_comparison_id",
sa.Uuid(),
sa.ForeignKey("evaluation_comparisons.id", ondelete="RESTRICT"),
),
sa.Column(
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
),
sa.Column(
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
),
sa.UniqueConstraint(
"project_binding_id", "idempotency_key", name="uq_selected_comparison_key"
),
sa.CheckConstraint(
"status IN ('pending', 'running', 'cancel_requested', 'completed', 'failed', 'cancelled', 'interrupted')",
name="ck_selected_comparison_status",
),
sa.CheckConstraint("expected_cases BETWEEN 1 AND 100", name="ck_selected_comparison_cases"),
sa.CheckConstraint(
"current_completed_cases BETWEEN 0 AND expected_cases AND candidate_completed_cases BETWEEN 0 AND expected_cases",
name="ck_selected_comparison_progress",
),
sa.CheckConstraint("version > 0", name="ck_selected_comparison_version"),
sa.CheckConstraint(
"current_deployment_id <> candidate_deployment_id",
name="ck_selected_comparison_distinct_targets",
),
)
op.create_index(
"uq_selected_comparison_active_binding",
"selected_model_comparisons",
["project_binding_id"],
unique=True,
postgresql_where=sa.text("status IN ('pending', 'running', 'cancel_requested')"),
sqlite_where=sa.text("status IN ('pending', 'running', 'cancel_requested')"),
)
if op.get_bind().dialect.name == "postgresql":
op.execute(
"REVOKE DELETE, TRUNCATE ON model_update_watches, model_update_notices FROM modelforge_runtime"
)
op.execute(
"GRANT SELECT, INSERT, UPDATE ON model_update_watches, model_update_notices TO modelforge_runtime"
)
op.execute("REVOKE DELETE, TRUNCATE ON selected_model_comparisons FROM modelforge_runtime")
op.execute(
"GRANT SELECT, INSERT, UPDATE ON selected_model_comparisons TO modelforge_runtime"
)
def downgrade() -> None:
# Earlier deployed migrations have destructive downgrades. Never permit this
# release to cross back into that graph while authoritative history exists.
for table in (
"selected_model_comparisons",
"model_update_watches",
"model_update_notices",
"external_creative_allocations",
"external_creative_resources",
"external_creative_action_attestations",
"external_creative_profile_observations",
):
if op.get_bind().scalar(sa.select(sa.func.count()).select_from(sa.table(table))):
raise RuntimeError(
"Control-plane history exists; restore the pre-migration backup instead."
)
op.drop_index("uq_selected_comparison_active_binding", table_name="selected_model_comparisons")
op.drop_table("selected_model_comparisons")
op.drop_column("model_update_watches", "version")
op.drop_column("model_update_watches", "enabled")
# Deliberately retain the safer runtime privilege restrictions on empty old tables.
backend/alembic/versions/20260910_0034_approved_serving_transition.py
+28
View File
@@ -0,0 +1,28 @@
"""Fence approved stable serving transitions and rollback against ABA changes."""
import sqlalchemy as sa
from alembic import op
revision = "20260910_0034"
down_revision = "20260909_0033"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("capability_contracts", sa.Column(
"serving_generation", sa.BigInteger(), nullable=False, server_default="0"))
with op.batch_alter_table("capability_contracts") as batch:
batch.create_check_constraint("ck_capability_serving_generation", "serving_generation >= 0")
def downgrade() -> None:
contracts = sa.table("capability_contracts", sa.column("serving_generation"))
if op.get_bind().scalar(sa.select(sa.func.count()).select_from(contracts).where(
contracts.c.serving_generation != 0
)):
raise RuntimeError("Serving route history exists; restore the pre-migration backup instead.")
with op.batch_alter_table("capability_contracts") as batch:
batch.drop_constraint("ck_capability_serving_generation", type_="check")
batch.drop_column("serving_generation")
backend/alembic/versions/20260911_0035_user_model_change_cases.py
+95
View File
@@ -0,0 +1,95 @@
"""Add the user-facing model replacement case without duplicating lifecycle truth."""
import sqlalchemy as sa
from alembic import op
revision = "20260911_0035"
down_revision = "20260910_0034"
branch_labels = None
depends_on = None
STATUSES = (
"searching",
"found",
"source_checked",
"downloaded",
"ready_to_test",
"tested",
"comparison_available",
"migration_required",
"ready_to_activate",
"active",
"checked",
"rolled_back",
"rejected",
)
def upgrade() -> None:
op.create_table(
"model_change_cases",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("project_binding_id", sa.Uuid(), nullable=False),
sa.Column("current_deployment_id", sa.Uuid(), nullable=False),
sa.Column("candidate_model_id", sa.Uuid(), nullable=True),
sa.Column("candidate_revision_id", sa.Uuid(), nullable=True),
sa.Column("candidate_artifact_set_id", sa.Uuid(), nullable=True),
sa.Column("candidate_deployment_id", sa.Uuid(), nullable=True),
sa.Column("discovery_assessment_id", sa.Uuid(), nullable=True),
sa.Column("download_plan_id", sa.Uuid(), nullable=True),
sa.Column("artifact_job_id", sa.Uuid(), nullable=True),
sa.Column("selected_comparison_id", sa.Uuid(), nullable=True),
sa.Column("migration_plan_id", sa.Uuid(), nullable=True),
sa.Column("lifecycle_plan_id", sa.Uuid(), nullable=True),
sa.Column("status", sa.String(length=32), nullable=False, server_default="searching"),
sa.Column("status_reason", sa.Text(), nullable=False, server_default=""),
sa.Column("version", sa.Integer(), nullable=False, server_default="1"),
sa.Column("created_by", sa.String(length=255), nullable=False),
sa.Column("updated_by", sa.String(length=255), nullable=False),
sa.Column("context", sa.JSON(), nullable=False),
sa.Column("activated_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("checked_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("closed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.ForeignKeyConstraint(["project_binding_id"], ["project_bindings.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["current_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["candidate_model_id"], ["models.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["candidate_revision_id"], ["model_revisions.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["candidate_artifact_set_id"], ["artifact_sets.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["candidate_deployment_id"], ["capability_deployments.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["discovery_assessment_id"], ["discovery_candidate_assessments.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["download_plan_id"], ["download_plans.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["artifact_job_id"], ["artifact_jobs.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["selected_comparison_id"], ["selected_model_comparisons.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["migration_plan_id"], ["migration_plans.id"], ondelete="RESTRICT"),
sa.ForeignKeyConstraint(["lifecycle_plan_id"], ["lifecycle_promotion_plans.id"], ondelete="RESTRICT"),
sa.PrimaryKeyConstraint("id"),
sa.CheckConstraint("version >= 1", name="ck_model_change_case_version"),
sa.CheckConstraint(
"status IN (" + ",".join(repr(value) for value in STATUSES) + ")",
name="ck_model_change_case_status",
),
)
op.create_index("ix_model_change_cases_binding", "model_change_cases", ["project_binding_id"])
op.create_index("ix_model_change_cases_status", "model_change_cases", ["status"])
op.create_index(
"uq_model_change_cases_open_binding",
"model_change_cases",
["project_binding_id"],
unique=True,
postgresql_where=sa.text("closed_at IS NULL"),
sqlite_where=sa.text("closed_at IS NULL"),
)
def downgrade() -> None:
changes = sa.table("model_change_cases", sa.column("id"))
if op.get_bind().scalar(sa.select(sa.func.count()).select_from(changes)):
raise RuntimeError("Model change cases exist; restore the pre-migration backup instead.")
op.drop_index("uq_model_change_cases_open_binding", table_name="model_change_cases")
op.drop_index("ix_model_change_cases_status", table_name="model_change_cases")
op.drop_index("ix_model_change_cases_binding", table_name="model_change_cases")
op.drop_table("model_change_cases")
backend/alembic/versions/20260923_0036_portfolio_repository_reconciliation.py
+122
View File
@@ -0,0 +1,122 @@
"""Persist secret-free repository observations and snapshot idempotency."""
import sqlalchemy as sa
from alembic import op
revision = "20260923_0036"
down_revision = "20260911_0035"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"portfolio_reconciliation_runs",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("source_system", sa.String(length=16), nullable=False),
sa.Column("snapshot_id", sa.String(length=128), nullable=False),
sa.Column("payload_digest", sa.String(length=64), nullable=False),
sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("complete", sa.Boolean(), nullable=False),
sa.Column("stale_snapshot", sa.Boolean(), nullable=False),
sa.Column("projects_created", sa.Integer(), nullable=False),
sa.Column("repositories_created", sa.Integer(), nullable=False),
sa.Column("repositories_updated", sa.Integer(), nullable=False),
sa.Column("stale_observations", sa.Integer(), nullable=False),
sa.Column("repositories_marked_stale", sa.Integer(), nullable=False),
sa.Column("runtime_observations_updated", sa.Integer(), nullable=False),
sa.Column("head_changes_review_required", sa.Integer(), nullable=False),
sa.Column("head_changes_accepted", sa.Integer(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("source_system", "snapshot_id", name="uq_portfolio_snapshot_identity"),
sa.CheckConstraint("length(payload_digest) = 64", name="ck_portfolio_snapshot_digest_length"),
sa.CheckConstraint(
"projects_created >= 0 AND repositories_created >= 0 "
"AND repositories_updated >= 0 AND stale_observations >= 0 "
"AND repositories_marked_stale >= 0 AND runtime_observations_updated >= 0 "
"AND head_changes_review_required >= 0 AND head_changes_accepted >= 0",
name="ck_portfolio_snapshot_counts_nonnegative",
),
)
op.create_index(
"ix_portfolio_snapshot_source_observed",
"portfolio_reconciliation_runs",
["source_system", "complete", "observed_at"],
)
op.create_table(
"portfolio_repository_observations",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("project_id", sa.Uuid(), nullable=True),
sa.Column("project_key", sa.String(length=128), nullable=True),
sa.Column("source_system", sa.String(length=16), nullable=False),
sa.Column("external_repository_id", sa.String(length=128), nullable=False),
sa.Column("slug", sa.String(length=255), nullable=False),
sa.Column("default_branch", sa.String(length=255), nullable=False),
sa.Column("head_sha", sa.String(length=40), nullable=False),
sa.Column("classification", sa.String(length=32), nullable=False),
sa.Column("code_evidence", sa.JSON(), nullable=False),
sa.Column("runtime_evidence", sa.JSON(), nullable=False),
sa.Column("source_observed_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("source_snapshot_id", sa.String(length=128), nullable=False),
sa.Column("source_snapshot_digest", sa.String(length=64), nullable=False),
sa.Column("pending_head_sha", sa.String(length=40), nullable=True),
sa.Column("pending_default_branch", sa.String(length=255), nullable=True),
sa.Column("pending_project_key", sa.String(length=128), nullable=True),
sa.Column("pending_classification", sa.String(length=32), nullable=True),
sa.Column("pending_code_evidence", sa.JSON(), nullable=True),
sa.Column("pending_observed_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("pending_snapshot_id", sa.String(length=128), nullable=True),
sa.Column("pending_snapshot_digest", sa.String(length=64), nullable=True),
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("stale_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint(
"source_system", "external_repository_id", name="uq_portfolio_repo_external_id"
),
sa.UniqueConstraint(
"source_system", "project_key", name="uq_portfolio_source_project_key"
),
sa.UniqueConstraint("project_id", name="uq_portfolio_repository_project_id"),
sa.CheckConstraint("length(head_sha) = 40", name="ck_portfolio_repo_head_sha_length"),
sa.CheckConstraint(
"pending_head_sha IS NULL OR length(pending_head_sha) = 40",
name="ck_portfolio_repo_pending_sha_length",
),
sa.CheckConstraint(
"classification IN ('local_model', 'hosted_model', 'gateway', 'mixed', "
"'model_management', 'runtime_observer', 'non_ai', 'unknown')",
name="ck_portfolio_repo_classification",
),
)
op.create_index(
"ix_portfolio_repository_observations_project_id",
"portfolio_repository_observations",
["project_id"],
)
def downgrade() -> None:
bind = op.get_bind()
runs = sa.table("portfolio_reconciliation_runs", sa.column("id"))
repositories = sa.table("portfolio_repository_observations", sa.column("id"))
if bind.scalar(sa.select(sa.func.count()).select_from(runs)) or bind.scalar(
sa.select(sa.func.count()).select_from(repositories)
):
raise RuntimeError(
"portfolio reconciliation history exists; restore the pre-migration backup instead."
)
op.drop_index(
"ix_portfolio_repository_observations_project_id",
table_name="portfolio_repository_observations",
)
op.drop_table("portfolio_repository_observations")
op.drop_index(
"ix_portfolio_snapshot_source_observed", table_name="portfolio_reconciliation_runs"
)
op.drop_table("portfolio_reconciliation_runs")
backend/alembic/versions/20260923_0037_alert_notification_outbox.py
+99
View File
@@ -0,0 +1,99 @@
"""Add durable, single-route operational alert notification outbox."""
import sqlalchemy as sa
from alembic import op
revision = "20260923_0037"
down_revision = "20260923_0036"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"alert_notification_deliveries",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("alert_history_event_id", sa.Uuid(), nullable=True),
sa.Column("route_key", sa.String(length=64), nullable=False),
sa.Column("idempotency_key", sa.String(length=160), nullable=False),
sa.Column("event_kind", sa.String(length=16), nullable=False),
sa.Column("severity", sa.String(length=16), nullable=True),
sa.Column("payload", sa.JSON(), nullable=False),
sa.Column("status", sa.String(length=24), nullable=False),
sa.Column("attempt_count", sa.Integer(), nullable=False),
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("delivered_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("claim_token_hash", sa.String(length=64), nullable=True),
sa.Column("last_http_status", sa.Integer(), nullable=True),
sa.Column("last_error_code", sa.String(length=64), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.CheckConstraint(
"event_kind IN ('FIRING', 'RESOLVED', 'TEST')",
name="ck_alert_notification_event_kind",
),
sa.CheckConstraint(
"status IN ('PENDING', 'DISPATCHING', 'RETRY_WAIT', 'DELIVERED', 'FAILED', 'HELD_FOR_REVIEW')",
name="ck_alert_notification_status",
),
sa.CheckConstraint("attempt_count >= 0", name="ck_alert_notification_attempt_count"),
sa.ForeignKeyConstraint(
["alert_history_event_id"], ["alert_history_events.id"], ondelete="RESTRICT"
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("idempotency_key", name="uq_alert_notification_idempotency"),
)
op.create_index(
"ix_alert_notification_deliveries_alert_history_event_id",
"alert_notification_deliveries",
["alert_history_event_id"],
)
op.create_index(
"ix_alert_notification_deliveries_event_kind",
"alert_notification_deliveries",
["event_kind"],
)
op.create_index(
"ix_alert_notification_deliveries_severity",
"alert_notification_deliveries",
["severity"],
)
op.create_index(
"ix_alert_notification_deliveries_status",
"alert_notification_deliveries",
["status"],
)
op.create_index(
"ix_alert_notification_deliveries_next_attempt_at",
"alert_notification_deliveries",
["next_attempt_at"],
)
op.create_index(
"ix_alert_notification_deliveries_lease_expires_at",
"alert_notification_deliveries",
["lease_expires_at"],
)
op.create_index(
"ix_alert_notification_dispatch_due",
"alert_notification_deliveries",
["status", "next_attempt_at", "created_at"],
)
def downgrade() -> None:
bind = op.get_bind()
deliveries = sa.table("alert_notification_deliveries", sa.column("id"))
if bind.scalar(sa.select(sa.func.count()).select_from(deliveries)):
raise RuntimeError(
"alert notification history exists; restore the pre-migration backup instead."
)
op.drop_index("ix_alert_notification_dispatch_due", table_name="alert_notification_deliveries")
op.drop_index("ix_alert_notification_deliveries_lease_expires_at", table_name="alert_notification_deliveries")
op.drop_index("ix_alert_notification_deliveries_next_attempt_at", table_name="alert_notification_deliveries")
op.drop_index("ix_alert_notification_deliveries_status", table_name="alert_notification_deliveries")
op.drop_index("ix_alert_notification_deliveries_severity", table_name="alert_notification_deliveries")
op.drop_index("ix_alert_notification_deliveries_event_kind", table_name="alert_notification_deliveries")
op.drop_index("ix_alert_notification_deliveries_alert_history_event_id", table_name="alert_notification_deliveries")
op.drop_table("alert_notification_deliveries")
backend/alembic/versions/20260924_0038_lab_execution_runs.py
+104
View File
@@ -0,0 +1,104 @@
"""Persist actor-scoped Model Lab execution identities."""
import sqlalchemy as sa
from alembic import op
revision = "20260924_0038"
down_revision = "20260923_0037"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"lab_execution_runs",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("actor_subject", sa.String(length=255), nullable=False),
sa.Column("idempotency_key", sa.String(length=128), nullable=False),
sa.Column("request_fingerprint", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=24), server_default="accepted", nullable=False),
sa.Column("capability", sa.String(length=128), server_default="rag.embedding@1", nullable=False),
sa.Column("target_snapshot", sa.JSON(), nullable=False),
sa.Column("input_sha256", sa.String(length=64), nullable=False),
sa.Column("input_count", sa.Integer(), nullable=False),
sa.Column("input_encoded_bytes", sa.Integer(), nullable=False),
sa.Column("payload_reference", sa.String(length=128), nullable=True),
sa.Column("input_payload_expires_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("gateway_request_id", sa.Uuid(), nullable=False),
sa.Column("timeout_seconds", sa.Integer(), server_default="60", nullable=False),
sa.Column("queue_timeout_seconds", sa.Integer(), server_default="10", nullable=False),
sa.Column("retention_seconds", sa.Integer(), server_default="300", nullable=False),
sa.Column("output_metadata", sa.JSON(), nullable=True),
sa.Column("worker_observations", sa.JSON(), nullable=True),
sa.Column("failure_code", sa.String(length=64), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("dispatch_started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("cancellation_requested_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("cancellation_acknowledged_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
sa.CheckConstraint(
"status IN ('accepted', 'preparing', 'queued', 'running', 'cancel_requested', "
"'completed', 'failed', 'cancelled', 'interrupted')",
name="ck_lab_execution_run_status",
),
sa.CheckConstraint("input_count BETWEEN 1 AND 8", name="ck_lab_execution_run_input_count"),
sa.CheckConstraint("timeout_seconds BETWEEN 5 AND 300", name="ck_lab_execution_run_timeout"),
sa.CheckConstraint("queue_timeout_seconds BETWEEN 1 AND 60", name="ck_lab_execution_run_queue_timeout"),
sa.CheckConstraint("retention_seconds BETWEEN 60 AND 3600", name="ck_lab_execution_run_retention"),
sa.ForeignKeyConstraint(
["gateway_request_id"], ["gateway_requests.id"], ondelete="RESTRICT"
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("actor_subject", "idempotency_key", name="uq_lab_run_actor_idempotency"),
sa.UniqueConstraint("gateway_request_id", name="uq_lab_execution_run_gateway_request"),
)
op.create_index("ix_lab_execution_runs_actor_subject", "lab_execution_runs", ["actor_subject"])
op.create_index("ix_lab_execution_runs_status", "lab_execution_runs", ["status"])
op.create_index("ix_lab_execution_runs_gateway_request_id", "lab_execution_runs", ["gateway_request_id"])
op.create_index("ix_lab_execution_runs_actor_status_created", "lab_execution_runs", ["actor_subject", "status", "created_at"])
op.add_column(
"serving_jobs",
sa.Column("lab_execution_run_id", sa.Uuid(), nullable=True),
)
op.create_foreign_key(
"fk_serving_jobs_lab_execution_run_id_lab_execution_runs",
"serving_jobs",
"lab_execution_runs",
["lab_execution_run_id"],
["id"],
ondelete="SET NULL",
)
op.create_index(
"ix_serving_jobs_lab_execution_run_id", "serving_jobs", ["lab_execution_run_id"]
)
if op.get_bind().dialect.name == "postgresql":
op.execute(
"REVOKE DELETE, TRUNCATE ON lab_execution_runs FROM modelforge_runtime"
)
op.execute(
"GRANT SELECT, INSERT, UPDATE ON lab_execution_runs TO modelforge_runtime"
)
def downgrade() -> None:
bind = op.get_bind()
if bind.scalar(sa.select(sa.func.count()).select_from(sa.table("lab_execution_runs", sa.column("id")))):
raise RuntimeError(
"LAB run history exists; restore the pre-migration backup instead of deleting it."
)
op.drop_index("ix_serving_jobs_lab_execution_run_id", table_name="serving_jobs")
op.drop_constraint(
"fk_serving_jobs_lab_execution_run_id_lab_execution_runs",
"serving_jobs",
type_="foreignkey",
)
op.drop_column("serving_jobs", "lab_execution_run_id")
op.drop_index("ix_lab_execution_runs_actor_status_created", table_name="lab_execution_runs")
op.drop_index("ix_lab_execution_runs_gateway_request_id", table_name="lab_execution_runs")
op.drop_index("ix_lab_execution_runs_status", table_name="lab_execution_runs")
op.drop_index("ix_lab_execution_runs_actor_subject", table_name="lab_execution_runs")
op.drop_table("lab_execution_runs")
backend/alembic/versions/20260924_0039_operator_benchmark_runs.py
+164
View File
@@ -0,0 +1,164 @@
"""Bind persisted benchmark evidence to authenticated capability runs."""
import sqlalchemy as sa
from alembic import op
revision = "20260924_0039"
down_revision = "20260924_0038"
branch_labels = None
depends_on = None
def upgrade() -> None:
if op.get_bind().dialect.name == "sqlite":
# SQLite cannot ALTER TABLE to add the FK/check/unique constraints. Alembic's batch
# recreation preserves historical benchmark rows and their indexes.
with op.batch_alter_table("benchmark_runs", recreate="always") as batch:
batch.alter_column("deployment_id", existing_type=sa.Uuid(), nullable=True)
batch.add_column(sa.Column("capability_deployment_id", sa.Uuid(), nullable=True))
batch.add_column(sa.Column("target_snapshot", sa.JSON(), nullable=True))
batch.add_column(sa.Column("actor_subject", sa.String(255), nullable=True))
batch.add_column(sa.Column("idempotency_key", sa.String(128), nullable=True))
batch.add_column(sa.Column("request_fingerprint", sa.String(64), nullable=True))
batch.add_column(sa.Column("failure_code", sa.String(64), nullable=True))
batch.add_column(sa.Column("cancellation_requested_at", sa.DateTime(timezone=True)))
batch.add_column(
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.func.now(),
nullable=False,
)
)
batch.add_column(
sa.Column("generation", sa.Integer(), server_default="1", nullable=False)
)
batch.create_foreign_key(
"fk_benchmark_runs_capability_deployment",
"capability_deployments",
["capability_deployment_id"],
["id"],
ondelete="RESTRICT",
)
batch.create_check_constraint(
"ck_benchmark_runs_operator_identity",
"(actor_subject IS NULL AND idempotency_key IS NULL AND "
"request_fingerprint IS NULL AND capability_deployment_id IS NULL) OR "
"(actor_subject IS NOT NULL AND idempotency_key IS NOT NULL AND "
"request_fingerprint IS NOT NULL AND capability_deployment_id IS NOT NULL)",
)
batch.create_unique_constraint(
"uq_benchmark_runs_actor_idempotency", ["actor_subject", "idempotency_key"]
)
else:
op.alter_column("benchmark_runs", "deployment_id", existing_type=sa.Uuid(), nullable=True)
op.add_column(
"benchmark_runs", sa.Column("capability_deployment_id", sa.Uuid(), nullable=True)
)
op.add_column("benchmark_runs", sa.Column("target_snapshot", sa.JSON(), nullable=True))
op.add_column("benchmark_runs", sa.Column("actor_subject", sa.String(255), nullable=True))
op.add_column("benchmark_runs", sa.Column("idempotency_key", sa.String(128), nullable=True))
op.add_column(
"benchmark_runs", sa.Column("request_fingerprint", sa.String(64), nullable=True)
)
op.add_column("benchmark_runs", sa.Column("failure_code", sa.String(64), nullable=True))
op.add_column(
"benchmark_runs", sa.Column("cancellation_requested_at", sa.DateTime(timezone=True))
)
op.add_column(
"benchmark_runs",
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.func.now(),
nullable=False,
),
)
op.add_column(
"benchmark_runs",
sa.Column("generation", sa.Integer(), server_default="1", nullable=False),
)
op.create_foreign_key(
"fk_benchmark_runs_capability_deployment",
"benchmark_runs",
"capability_deployments",
["capability_deployment_id"],
["id"],
ondelete="RESTRICT",
)
op.create_check_constraint(
"ck_benchmark_runs_operator_identity",
"benchmark_runs",
"(actor_subject IS NULL AND idempotency_key IS NULL AND "
"request_fingerprint IS NULL AND capability_deployment_id IS NULL) OR "
"(actor_subject IS NOT NULL AND idempotency_key IS NOT NULL AND "
"request_fingerprint IS NOT NULL AND capability_deployment_id IS NOT NULL)",
)
op.create_unique_constraint(
"uq_benchmark_runs_actor_idempotency",
"benchmark_runs",
["actor_subject", "idempotency_key"],
)
op.create_index("ix_benchmark_runs_actor_subject", "benchmark_runs", ["actor_subject"])
op.create_index(
"ix_benchmark_runs_capability_deployment_completed",
"benchmark_runs",
["capability_deployment_id", "completed_at"],
)
if op.get_bind().dialect.name == "postgresql":
op.execute("REVOKE DELETE, TRUNCATE ON benchmark_runs FROM modelforge_runtime")
op.execute("GRANT SELECT, INSERT, UPDATE ON benchmark_runs TO modelforge_runtime")
def downgrade() -> None:
bind = op.get_bind()
benchmark_runs = sa.table("benchmark_runs", sa.column("actor_subject", sa.String()))
operator_history = bind.scalar(
sa.select(sa.func.count())
.select_from(benchmark_runs)
.where(benchmark_runs.c.actor_subject.is_not(None))
)
if operator_history:
raise RuntimeError(
"Operator benchmark history exists; restore the pre-migration backup instead of deleting it."
)
op.drop_index("ix_benchmark_runs_capability_deployment_completed", table_name="benchmark_runs")
op.drop_index("ix_benchmark_runs_actor_subject", table_name="benchmark_runs")
if op.get_bind().dialect.name == "sqlite":
with op.batch_alter_table("benchmark_runs", recreate="always") as batch:
batch.drop_constraint("uq_benchmark_runs_actor_idempotency", type_="unique")
batch.drop_constraint("ck_benchmark_runs_operator_identity", type_="check")
batch.drop_constraint("fk_benchmark_runs_capability_deployment", type_="foreignkey")
for column in (
"generation",
"created_at",
"cancellation_requested_at",
"request_fingerprint",
"failure_code",
"idempotency_key",
"actor_subject",
"target_snapshot",
"capability_deployment_id",
):
batch.drop_column(column)
batch.alter_column("deployment_id", existing_type=sa.Uuid(), nullable=False)
else:
op.drop_constraint("uq_benchmark_runs_actor_idempotency", "benchmark_runs", type_="unique")
op.drop_constraint("ck_benchmark_runs_operator_identity", "benchmark_runs", type_="check")
op.drop_constraint(
"fk_benchmark_runs_capability_deployment", "benchmark_runs", type_="foreignkey"
)
for column in (
"generation",
"created_at",
"cancellation_requested_at",
"request_fingerprint",
"failure_code",
"idempotency_key",
"actor_subject",
"target_snapshot",
"capability_deployment_id",
):
op.drop_column("benchmark_runs", column)
op.alter_column("benchmark_runs", "deployment_id", existing_type=sa.Uuid(), nullable=False)
backend/pyproject.toml
+6 -3
View File
@@ -1,10 +1,12 @@
[build-system]
requires = ["hatchling"]
requires = [
"hatchling==1.32.4",
]
build-backend = "hatchling.build"
[project]
name = "modelforge-api"
version = "1.2.2"
version = "1.3.0-rc.1"
description = "ITWorx ModelForge control-plane API"
license = "AGPL-3.0-or-later"
requires-python = ">=3.12"
@@ -19,6 +21,7 @@ dependencies = [
"cryptography==50.0.1",
"redis==6.4.0",
"httpx==0.28.1",
"msgpack==1.2.2",
"huggingface-hub==1.29.0",
"nvidia-ml-py==13.610.43",
"psutil==7.2.2",
@@ -28,7 +31,7 @@ dependencies = [
[project.optional-dependencies]
dev = [
"pytest==8.4.2",
"pytest==9.0.3",
"pytest-asyncio==1.4.0",
"ruff==0.16.5",
"mypy==1.20.2",
backend/src/modelforge_api/api/authorization.py
+93 -17
View File
@@ -1,13 +1,16 @@
from __future__ import annotations
import hashlib
import hmac
import re
from dataclasses import dataclass
from datetime import UTC, datetime
from enum import StrEnum
from typing import Annotated
from fastapi import Depends, Header, HTTPException, Request
from modelforge_api.domain.operator_identity import parse_operator_identities
from modelforge_api.settings import Settings, get_settings
@@ -35,6 +38,7 @@ class AccessBoundary(StrEnum):
_PUBLIC_REQUESTS = {
("POST", "/api/v1/operator/login"),
("GET", "/"),
("GET", "/api/v1/health/live"),
("GET", "/api/v1/health/ready"),
@@ -56,14 +60,18 @@ _NODE_REQUESTS = {
("GET", "/api/v1/agent/serving-jobs/next"),
("POST", "/api/v1/agent/serving-jobs/{job_id}/complete"),
("POST", "/api/v1/agent/serving-jobs/{job_id}/fail"),
("POST", "/api/v1/agent/serving-jobs/{job_id}/cancellation"),
("POST", "/api/v1/agent/serving-jobs/{job_id}/cancelled"),
("POST", "/api/v1/agent/serving-state"),
}
_NODE_PARAMETERIZED_REQUESTS = (
("POST", re.compile(r"^/api/v1/agent/artifact-jobs/[^/]+/(?:progress|complete|fail)$")),
("POST", re.compile(r"^/api/v1/agent/runtime-probes/[^/]+/(?:progress|complete|fail)$")),
("POST", re.compile(r"^/api/v1/agent/serving-jobs/[^/]+/(?:complete|fail)$")),
("POST", re.compile(r"^/api/v1/agent/serving-jobs/[^/]+/(?:complete|fail|cancellation|cancelled)$")),
)
_CAPABILITY_REQUIREMENTS = {
("POST", "/api/v1/capabilities/creative.resource@1/allocations"): "creative.resource@1",
("POST", "/api/v1/capabilities/creative.resource@1/profiles/observations"): "creative.resource@1",
("POST", "/api/v1/capabilities/rag.embedding@1/invoke"): "rag.embedding@1",
("POST", "/api/v1/capabilities/rag.reranking@1/invoke"): "rag.reranking@1",
("POST", "/api/v1/capabilities/document.ocr@1/invoke"): "document.ocr@1",
@@ -72,8 +80,19 @@ _CAPABILITY_REQUIREMENTS = {
("POST", "/api/v1/capability-experiments/{route_key}/invoke"): "rag.embedding@1",
("POST", "/v1/embeddings"): "rag.embedding@1",
}
for _capability in (
"rag.embedding@1", "rag.reranking@1", "document.ocr@1",
"vision.embedding@1", "speech.transcription@1",
):
_CAPABILITY_REQUIREMENTS[("GET", f"/api/v1/capabilities/{_capability}/check")] = _capability
_CAPABILITY_CLIENT_REQUESTS = set(_CAPABILITY_REQUIREMENTS)
_CAPABILITY_REQUIREMENTS[("GET", "/api/v1/selected-model-comparisons/assigned")] = "rag.embedding@1"
_CAPABILITY_CLIENT_REQUESTS.add(("GET", "/api/v1/selected-model-comparisons/assigned"))
_CAPABILITY_EXPERIMENT_REQUEST = re.compile(r"^/api/v1/capability-experiments/[^/]+/invoke$")
_CREATIVE_ALLOCATION_REQUEST = re.compile(r"^/api/v1/capabilities/creative\.resource@1/allocations/[^/]+(?:/(?:attest|heartbeat|release|uncertain))?$")
_COMPARISON_CONSUMER_REQUEST = re.compile(
r"^/api/v1/selected-model-comparisons/[^/]+/(claim|progress|result|finish)$"
)
def access_boundary_for_request(
@@ -98,7 +117,12 @@ def access_boundary_for_request(
):
return AccessBoundary.NODE
if (normalized_method, path) in _CAPABILITY_CLIENT_REQUESTS or (
normalized_method == "POST" and _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
normalized_method in {"GET", "POST"} and _CREATIVE_ALLOCATION_REQUEST.fullmatch(path)
) or (
normalized_method == "POST" and (
_CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
or _COMPARISON_CONSUMER_REQUEST.fullmatch(path)
)
):
return AccessBoundary.CAPABILITY_CLIENT
return AccessBoundary.CONTROL_PLANE
@@ -111,7 +135,12 @@ def required_capability_for_request(method: str, path: str) -> str | None:
requirement = _CAPABILITY_REQUIREMENTS.get((normalized_method, path))
if requirement is not None:
return requirement
if normalized_method == "POST" and _CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path):
if normalized_method in {"GET", "POST"} and _CREATIVE_ALLOCATION_REQUEST.fullmatch(path):
return "creative.resource@1"
if normalized_method == "POST" and (
_CAPABILITY_EXPERIMENT_REQUEST.fullmatch(path)
or _COMPARISON_CONSUMER_REQUEST.fullmatch(path)
):
return "rag.embedding@1"
return None
@@ -145,28 +174,70 @@ class Principal:
subject: str
role: PrincipalRole
authentication_method: str
credential_id: str | None = None
expires_at: datetime | None = None
def permits(self, required: PrincipalRole) -> bool:
return _ROLE_RANK[self.role] >= _ROLE_RANK[required]
def authenticate_operator_token(settings: Settings, token: str | None) -> Principal:
"""Authenticate the backward-compatible operator key as an admin principal.
The legacy key is the sole human credential in this release. OIDC/session authentication can
add another principal producer later without changing route authorization policy.
"""
"""Authenticate a named expiring human key or the legacy bootstrap administrator."""
configured = settings.operator_api_key
if configured is None or not configured.get_secret_value():
document = settings.operator_identities
identities = parse_operator_identities(document.get_secret_value()) if (
document is not None and document.get_secret_value()
) else ()
if not identities and (configured is None or not configured.get_secret_value()):
raise HTTPException(status_code=503, detail="operator API authentication is not configured")
if token is None or not hmac.compare_digest(token, configured.get_secret_value()):
if token is None or len(token) > 512 or token.startswith(("mfopm_", "mfsvc_", "mfnode_", "mfenroll_")):
raise HTTPException(status_code=401, detail="invalid operator credential")
return Principal(
subject="legacy-operator",
role=PrincipalRole.ADMIN,
authentication_method="legacy_admin_token",
)
if token.startswith("mfop_"):
if not re.fullmatch(r"mfop_[A-Za-z0-9_-]{43}", token):
raise HTTPException(status_code=401, detail="invalid operator credential")
digest = hashlib.sha256(token.encode()).hexdigest()
for identity in identities:
if hmac.compare_digest(digest, identity.token_sha256):
if identity.expires_at <= datetime.now(UTC):
raise HTTPException(status_code=401, detail="invalid operator credential")
return Principal(
subject=identity.subject,
role=PrincipalRole(identity.role),
authentication_method="named_operator_key",
credential_id=identity.credential_id,
expires_at=identity.expires_at,
)
# The named namespace never falls back to the legacy administrator.
raise HTTPException(status_code=401, detail="invalid operator credential")
if configured is not None and hmac.compare_digest(
token.encode("utf-8"), configured.get_secret_value().encode("utf-8")
):
return Principal(
subject="legacy-operator",
role=PrincipalRole.ADMIN,
authentication_method="legacy_admin_token",
)
if configured is not None and token.startswith("mfpw_"):
from modelforge_api.services.operator_password import verify_session
password_hash = settings.operator_password_hash
if password_hash is not None and verify_session(
token, configured.get_secret_value(), password_hash.get_secret_value()
):
return Principal(
"password-operator", PrincipalRole.ADMIN, "password_session",
expires_at=datetime.fromtimestamp(int(token[5:].split(".")[1]), UTC),
)
raise HTTPException(status_code=401, detail="invalid operator credential")
def operator_token_from_headers(admin_token: str | None, authorization: str | None) -> str | None:
if admin_token is not None:
return admin_token
if authorization is not None and authorization.startswith("Bearer mfpw_"):
return authorization.removeprefix("Bearer ")
return None
def authenticate_operator_principal(
@@ -179,7 +250,9 @@ def authenticate_operator_principal(
principal = getattr(request.state, "principal", None)
if isinstance(principal, Principal):
return principal
return authenticate_operator_token(settings, token)
return authenticate_operator_token(
settings, operator_token_from_headers(token, request.headers.get("authorization"))
)
AuthenticatedPrincipal = Annotated[Principal, Depends(authenticate_operator_principal)]
@@ -187,7 +260,10 @@ AuthenticatedPrincipal = Annotated[Principal, Depends(authenticate_operator_prin
def _require_role(principal: Principal, required: PrincipalRole) -> Principal:
if not principal.permits(required):
raise HTTPException(status_code=403, detail="operator role is not authorized")
raise HTTPException(
status_code=403,
detail=f"This action requires the {required.value} role; your role is {principal.role.value}.",
)
return principal
backend/src/modelforge_api/api/request_limits.py
+8 -1
View File
@@ -79,7 +79,14 @@ class RequestBodyLimitMiddleware:
"details": details or {},
}
},
headers={"X-Correlation-ID": str(correlation_id)},
headers={
"X-Correlation-ID": str(correlation_id),
**(
{"Cache-Control": "no-store"}
if scope.get("path") == "/api/v1/operator/login"
else {}
),
},
)
await response(scope, receive, send)
backend/src/modelforge_api/api/routes/acquisition.py
+3 -4
View File
@@ -18,8 +18,8 @@ from modelforge_api.domain.acquisition import (
UpstreamRefreshRequest,
UpstreamSnapshotResponse,
)
from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider
from modelforge_api.services.acquisition import AcquisitionService
from modelforge_api.services.secret_slots import SecretAwareHuggingFaceProvider
from modelforge_api.settings import Settings, get_settings
router = APIRouter(
@@ -33,9 +33,8 @@ def get_acquisition_service(
session: Annotated[Session, Depends(get_session)],
settings: Annotated[Settings, Depends(get_settings)],
) -> AcquisitionService:
token = settings.hf_token.get_secret_value() if settings.hf_token else None
provider = OfficialHuggingFaceProvider(token=token, timeout=settings.hf_timeout_seconds)
return AcquisitionService(session, settings, provider)
provider = SecretAwareHuggingFaceProvider(session, settings)
return AcquisitionService(session, settings, provider, credential_configured=provider.credential_configured)
Service = Annotated[AcquisitionService, Depends(get_acquisition_service)]
backend/src/modelforge_api/api/routes/agent.py
+37 -3
View File
@@ -37,6 +37,12 @@ from modelforge_api.domain.node_decommission import (
NodeDecommissionPreview,
NodeDecommissionResult,
)
from modelforge_api.domain.node_lifecycle import (
NodeLifecycleChange,
NodeLifecyclePreview,
NodeLifecyclePreviewRequest,
NodeLifecycleResult,
)
from modelforge_api.domain.runtime import (
AgentRuntimeProbeComplete,
AgentRuntimeProbeControl,
@@ -46,11 +52,11 @@ from modelforge_api.domain.runtime import (
RuntimeProbeResponse,
)
from modelforge_api.persistence.models import ComputeNode, NodeCredential
from modelforge_api.providers.huggingface import OfficialHuggingFaceProvider
from modelforge_api.services.acquisition import AcquisitionService
from modelforge_api.services.node_agent import NodeAgentService, NodeAuthenticationEvidence
from modelforge_api.services.node_decommission import NodeDecommissionService
from modelforge_api.services.runtime import RuntimeService
from modelforge_api.services.secret_slots import SecretAwareHuggingFaceProvider
from modelforge_api.settings import Settings, get_settings
router = APIRouter(tags=["node-agent"])
@@ -119,6 +125,7 @@ def list_enrollments(service: Service, _admin: Admin) -> list[EnrollmentTokenSum
expires_at=item.expires_at,
used_at=item.used_at,
revoked_at=item.revoked_at,
enrolled_node_id=str(item.enrolled_node_id) if item.enrolled_node_id is not None else None,
)
for item in service.repository.enrollments()
]
@@ -141,6 +148,32 @@ def update_node(
return ActionResponse()
@router.post(
"/api/v1/admin/hardware/nodes/{node_id}/lifecycle/preview",
response_model=NodeLifecyclePreview,
)
def preview_node_lifecycle(
node_id: uuid.UUID,
request: NodeLifecyclePreviewRequest,
service: DecommissionService,
_admin: Admin,
) -> NodeLifecyclePreview:
return service.preview_enabled_change(node_id, request)
@router.post(
"/api/v1/admin/hardware/nodes/{node_id}/lifecycle",
response_model=NodeLifecycleResult,
)
def change_node_lifecycle(
node_id: uuid.UUID,
request: NodeLifecycleChange,
service: DecommissionService,
_admin: Admin,
) -> NodeLifecycleResult:
return service.apply_enabled_change(node_id, request)
@router.post(
"/api/v1/admin/hardware/nodes/{node_id}/decommission/preview",
response_model=NodeDecommissionPreview,
@@ -213,13 +246,14 @@ def get_agent_acquisition_service(
session: Annotated[Session, Depends(get_session)],
settings: Annotated[Settings, Depends(get_settings)],
) -> AcquisitionService:
token = settings.hf_token.get_secret_value() if settings.hf_token else None
provider = SecretAwareHuggingFaceProvider(session, settings)
return AcquisitionService(
session,
settings,
OfficialHuggingFaceProvider(token=token, timeout=settings.hf_timeout_seconds),
provider,
actor_type="node_agent",
actor_id="authenticated-node",
credential_configured=provider.credential_configured,
)
backend/src/modelforge_api/api/routes/audit_timeline.py
+37
View File
@@ -0,0 +1,37 @@
"""Admin-only, read-only audit timeline API."""
from __future__ import annotations
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Admin
from modelforge_api.db import get_session
from modelforge_api.domain.audit_timeline import AuditTimelineRequest, AuditTimelineResponse
from modelforge_api.services.audit_timeline import (
AuditTimelineCursorError,
AuditTimelineService,
)
router = APIRouter(prefix="/api/v1/admin/audit", tags=["audit"])
def get_service(session: Annotated[Session, Depends(get_session)]) -> AuditTimelineService:
return AuditTimelineService(session)
Service = Annotated[AuditTimelineService, Depends(get_service)]
@router.get("/events", response_model=AuditTimelineResponse)
def audit_timeline(
query: Annotated[AuditTimelineRequest, Query()],
service: Service,
_admin: Admin,
) -> AuditTimelineResponse:
try:
return service.list_events(query)
except AuditTimelineCursorError as error:
raise HTTPException(status_code=422, detail=str(error)) from None
backend/src/modelforge_api/api/routes/catalog.py
+328 -109
View File
@@ -1,9 +1,13 @@
from collections import defaultdict
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy import select
from sqlalchemy.orm import Session
from sqlalchemy import select, tuple_
from sqlalchemy.orm import Session, aliased, selectinload
from modelforge_api.api.authorization import require_viewer
from modelforge_api.db import get_session
from modelforge_api.domain.contracts import ProjectBindingManifest
from modelforge_api.domain.schemas import (
CapabilityContractResponse,
CapabilityEstateResponse,
@@ -40,22 +44,31 @@ def list_capabilities(
registry: ManifestRegistry = Depends(get_manifest_registry),
session: Session = Depends(get_session),
) -> list[CapabilityContractResponse]:
responses = []
for contract in registry.capabilities():
stable = session.execute(
select(CapabilityDeployment)
manifests = registry.capabilities()
identities = {(item.capability, item.version) for item in manifests}
stable_by_contract: dict[tuple[str, int], CapabilityDeployment] = {}
if identities:
stable_rows = session.execute(
select(Capability.key, CapabilityContract.version, CapabilityDeployment)
.join(
CapabilityContract,
CapabilityContract.id == CapabilityDeployment.capability_contract_id,
)
.join(Capability, Capability.id == CapabilityContract.capability_id)
.where(
Capability.key == contract.capability,
CapabilityContract.version == contract.version,
tuple_(Capability.key, CapabilityContract.version).in_(identities),
CapabilityDeployment.status == "stable",
CapabilityDeployment.production.is_(True),
)
).scalar_one_or_none()
)
for key, version, deployment in stable_rows:
identity = (key, version)
if identity in stable_by_contract:
raise ValueError(f"multiple stable deployments for {key}@{version}")
stable_by_contract[identity] = deployment
responses = []
for contract in manifests:
stable = stable_by_contract.get((contract.capability, contract.version))
responses.append(
CapabilityContractResponse(
key=contract.capability,
@@ -82,45 +95,121 @@ def capability_estate(
registry: ManifestRegistry = Depends(get_manifest_registry),
session: Session = Depends(get_session),
) -> list[CapabilityEstateResponse]:
responses: list[CapabilityEstateResponse] = []
for manifest in registry.capabilities():
contract = session.scalar(
select(CapabilityContract)
manifests = registry.capabilities()
contracts_by_identity: dict[tuple[str, int], CapabilityContract] = {}
identities = [(item.capability, item.version) for item in manifests]
if identities:
for contract, capability_key in session.execute(
select(CapabilityContract, Capability.key)
.join(Capability, Capability.id == CapabilityContract.capability_id)
.where(Capability.key == manifest.capability, CapabilityContract.version == manifest.version)
.where(tuple_(Capability.key, CapabilityContract.version).in_(identities))
):
contracts_by_identity[capability_key, contract.version] = contract
contract_ids = [contract.id for contract in contracts_by_identity.values()]
deployments_by_contract: dict[UUID, CapabilityDeployment] = {}
if contract_ids:
deployment_candidate = aliased(CapabilityDeployment)
selected_deployment_id = (
select(deployment_candidate.id)
.where(
deployment_candidate.capability_contract_id == CapabilityContract.id
)
.order_by(
deployment_candidate.production.desc(),
deployment_candidate.created_at.desc(),
)
.limit(1)
.correlate(CapabilityContract)
.scalar_subquery()
)
deployment = None
model = None
revision = None
profile = None
envelope = None
latest_evaluation = None
if contract:
deployment = session.scalar(
select(CapabilityDeployment)
.where(CapabilityDeployment.capability_contract_id == contract.id)
.order_by(
CapabilityDeployment.production.desc(),
CapabilityDeployment.created_at.desc(),
)
.limit(1)
for selected_deployment in session.scalars(
select(CapabilityDeployment)
.join(
CapabilityContract,
CapabilityContract.id == CapabilityDeployment.capability_contract_id,
)
if deployment:
artifact_set = session.get(ArtifactSet, deployment.artifact_set_id)
revision = session.get(ModelRevision, artifact_set.revision_id) if artifact_set else None
model = session.get(Model, revision.model_id) if revision else None
profile = session.get(RuntimeProfile, deployment.runtime_profile_id)
envelope = session.scalar(
select(CapabilityResourceEnvelope).where(
CapabilityResourceEnvelope.capability_deployment_id == deployment.id
)
.where(
CapabilityContract.id.in_(contract_ids),
CapabilityDeployment.id == selected_deployment_id,
)
latest_evaluation = session.scalar(
select(CapabilityEvaluationRun)
.where(CapabilityEvaluationRun.capability_deployment_id == deployment.id)
.order_by(CapabilityEvaluationRun.created_at.desc())
.limit(1)
):
deployments_by_contract[selected_deployment.capability_contract_id] = selected_deployment
deployments = list(deployments_by_contract.values())
deployment_ids = [item.id for item in deployments]
artifact_set_ids = list({item.artifact_set_id for item in deployments})
artifact_sets = (
session.scalars(select(ArtifactSet).where(ArtifactSet.id.in_(artifact_set_ids))).all()
if artifact_set_ids
else []
)
artifact_sets_by_id = {item.id: item for item in artifact_sets}
revision_ids = list({item.revision_id for item in artifact_sets})
revisions = (
session.scalars(select(ModelRevision).where(ModelRevision.id.in_(revision_ids))).all()
if revision_ids
else []
)
revisions_by_id = {item.id: item for item in revisions}
model_ids = list({item.model_id for item in revisions})
models = (
session.scalars(select(Model).where(Model.id.in_(model_ids))).all() if model_ids else []
)
models_by_id = {item.id: item for item in models}
profile_ids = list({item.runtime_profile_id for item in deployments})
profiles = (
session.scalars(select(RuntimeProfile).where(RuntimeProfile.id.in_(profile_ids))).all()
if profile_ids
else []
)
profiles_by_id = {item.id: item for item in profiles}
envelopes = (
session.scalars(
select(CapabilityResourceEnvelope).where(
CapabilityResourceEnvelope.capability_deployment_id.in_(deployment_ids)
)
).all()
if deployment_ids
else []
)
envelopes_by_deployment = {item.capability_deployment_id: item for item in envelopes}
latest_evaluations: dict[UUID, CapabilityEvaluationRun] = {}
if deployment_ids:
evaluation_candidate = aliased(CapabilityEvaluationRun)
latest_evaluation_id = (
select(evaluation_candidate.id)
.where(
evaluation_candidate.capability_deployment_id == CapabilityDeployment.id
)
.order_by(evaluation_candidate.created_at.desc())
.limit(1)
.correlate(CapabilityDeployment)
.scalar_subquery()
)
for evaluation in session.scalars(
select(CapabilityEvaluationRun)
.join(
CapabilityDeployment,
CapabilityDeployment.id == CapabilityEvaluationRun.capability_deployment_id,
)
.where(
CapabilityDeployment.id.in_(deployment_ids),
CapabilityEvaluationRun.id == latest_evaluation_id,
)
):
latest_evaluations[evaluation.capability_deployment_id] = evaluation
responses: list[CapabilityEstateResponse] = []
for manifest in manifests:
contract = contracts_by_identity.get((manifest.capability, manifest.version))
deployment = deployments_by_contract.get(contract.id) if contract else None
artifact_set = artifact_sets_by_id.get(deployment.artifact_set_id) if deployment else None
revision = revisions_by_id.get(artifact_set.revision_id) if artifact_set else None
model = models_by_id.get(revision.model_id) if revision else None
profile = profiles_by_id.get(deployment.runtime_profile_id) if deployment else None
envelope = envelopes_by_deployment.get(deployment.id) if deployment else None
latest_evaluation = latest_evaluations.get(deployment.id) if deployment else None
operational_state = (
deployment.status
if deployment
@@ -157,67 +246,142 @@ def capability_estate(
def model_installation_rationale(
session: Session = Depends(get_session),
) -> list[ModelInstallationRationaleResponse]:
responses: list[ModelInstallationRationaleResponse] = []
for model in session.scalars(select(Model).order_by(Model.display_name)).all():
revisions = session.scalars(
select(ModelRevision).where(ModelRevision.model_id == model.id)
).all()
revision_ids = [item.id for item in revisions]
sets = (
session.scalars(select(ArtifactSet).where(ArtifactSet.revision_id.in_(revision_ids))).all()
if revision_ids
else []
models = session.scalars(select(Model).order_by(Model.display_name)).all()
installed_sets_by_model: dict[UUID, list[ArtifactSet]] = defaultdict(list)
if models:
for artifact_set, model_id in session.execute(
select(ArtifactSet, ModelRevision.model_id)
.join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
.join(Model, Model.id == ModelRevision.model_id)
.where(ArtifactSet.availability == "local")
):
installed_sets_by_model[model_id].append(artifact_set)
installed_sets = [
artifact_set
for model in models
for artifact_set in installed_sets_by_model.get(model.id, [])
]
deployments_by_artifact_set: dict[UUID, list[CapabilityDeployment]] = defaultdict(list)
if installed_sets:
for deployment in session.scalars(
select(CapabilityDeployment)
.join(ArtifactSet, ArtifactSet.id == CapabilityDeployment.artifact_set_id)
.join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
.join(Model, Model.id == ModelRevision.model_id)
.where(ArtifactSet.availability == "local")
):
deployments_by_artifact_set[deployment.artifact_set_id].append(deployment)
contract_and_capability_by_deployment: dict[
UUID, tuple[CapabilityContract, Capability]
] = {}
if installed_sets:
for deployment, contract, capability in session.execute(
select(CapabilityDeployment, CapabilityContract, Capability)
.join(ArtifactSet, ArtifactSet.id == CapabilityDeployment.artifact_set_id)
.join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
.join(Model, Model.id == ModelRevision.model_id)
.join(
CapabilityContract,
CapabilityContract.id == CapabilityDeployment.capability_contract_id,
)
.join(Capability, Capability.id == CapabilityContract.capability_id)
.where(ArtifactSet.availability == "local")
):
contract_and_capability_by_deployment[deployment.id] = (contract, capability)
eligible_deployments = (
select(
CapabilityDeployment.id.label("deployment_id"),
CapabilityDeployment.capability_contract_id.label("contract_id"),
)
installed_sets = [item for item in sets if item.availability == "local"]
.join(ArtifactSet, ArtifactSet.id == CapabilityDeployment.artifact_set_id)
.join(ModelRevision, ModelRevision.id == ArtifactSet.revision_id)
.join(Model, Model.id == ModelRevision.model_id)
.join(
CapabilityContract,
CapabilityContract.id == CapabilityDeployment.capability_contract_id,
)
.join(Capability, Capability.id == CapabilityContract.capability_id)
.where(ArtifactSet.availability == "local")
.subquery()
)
eligible_contract_ids = select(eligible_deployments.c.contract_id).distinct()
eligible_deployment_ids = select(eligible_deployments.c.deployment_id)
project_consumers: dict[UUID, list[str]] = defaultdict(list)
if installed_sets:
for contract_id, project_key in session.execute(
select(ProjectBinding.capability_contract_id, Project.key)
.join(Project, Project.id == ProjectBinding.project_id)
.where(
ProjectBinding.capability_contract_id.in_(eligible_contract_ids),
ProjectBinding.deprecated_at.is_(None),
)
.distinct()
):
project_consumers[contract_id].append(project_key)
active_project_consumers: dict[UUID, list[str]] = defaultdict(list)
if installed_sets:
for contract_id, project_key in session.execute(
select(ProjectBinding.capability_contract_id, Project.key)
.join(Project, Project.id == ProjectBinding.project_id)
.join(ServiceClient, ServiceClient.project_binding_id == ProjectBinding.id)
.where(
ProjectBinding.capability_contract_id.in_(eligible_contract_ids),
ServiceClient.status == "active",
)
.distinct()
):
active_project_consumers[contract_id].append(project_key)
fit_evidence_ids: dict[UUID, list[UUID]] = defaultdict(list)
if installed_sets:
for contract_id, evidence_id in session.execute(
select(ProjectBinding.capability_contract_id, ProjectFitEvidence.id)
.join(ProjectBinding, ProjectBinding.id == ProjectFitEvidence.project_binding_id)
.where(ProjectBinding.capability_contract_id.in_(eligible_contract_ids))
):
fit_evidence_ids[contract_id].append(evidence_id)
evaluation_ids: dict[UUID, list[UUID]] = defaultdict(list)
if installed_sets:
for deployment_id, evaluation_id in session.execute(
select(
CapabilityEvaluationRun.capability_deployment_id,
CapabilityEvaluationRun.id,
).where(
CapabilityEvaluationRun.capability_deployment_id.in_(eligible_deployment_ids)
)
):
evaluation_ids[deployment_id].append(evaluation_id)
residency_by_deployment = (
{
item.capability_deployment_id: item
for item in session.scalars(
select(ResidencyAllocation).where(
ResidencyAllocation.capability_deployment_id.in_(eligible_deployment_ids)
)
)
}
if installed_sets
else {}
)
responses: list[ModelInstallationRationaleResponse] = []
for model in models:
model_installed_sets = installed_sets_by_model.get(model.id, [])
dependencies: list[InstallationDependencyResponse] = []
for artifact_set in installed_sets:
deployments = session.scalars(
select(CapabilityDeployment).where(
CapabilityDeployment.artifact_set_id == artifact_set.id
)
).all()
for deployment in deployments:
contract = session.get(CapabilityContract, deployment.capability_contract_id)
capability = session.get(Capability, contract.capability_id) if contract else None
if not contract or not capability:
for artifact_set in model_installed_sets:
for deployment in deployments_by_artifact_set.get(artifact_set.id, []):
relation = contract_and_capability_by_deployment.get(deployment.id)
if not relation:
continue
projects = session.scalars(
select(Project)
.join(ProjectBinding, ProjectBinding.project_id == Project.id)
.where(
ProjectBinding.capability_contract_id == contract.id,
ProjectBinding.deprecated_at.is_(None),
)
.distinct()
).all()
active_projects = session.scalars(
select(Project)
.join(ProjectBinding, ProjectBinding.project_id == Project.id)
.join(ServiceClient, ServiceClient.project_binding_id == ProjectBinding.id)
.where(
ProjectBinding.capability_contract_id == contract.id,
ServiceClient.status == "active",
)
.distinct()
).all()
project_fit_ids = session.scalars(
select(ProjectFitEvidence.id)
.join(
ProjectBinding,
ProjectBinding.id == ProjectFitEvidence.project_binding_id,
)
.where(ProjectBinding.capability_contract_id == contract.id)
).all()
evaluations = session.scalars(
select(CapabilityEvaluationRun).where(
CapabilityEvaluationRun.capability_deployment_id == deployment.id
)
).all()
residency = session.scalar(
select(ResidencyAllocation).where(
ResidencyAllocation.capability_deployment_id == deployment.id
)
)
contract, capability = relation
residency = residency_by_deployment.get(deployment.id)
dependencies.append(
InstallationDependencyResponse(
capability=capability.key,
@@ -225,10 +389,10 @@ def model_installation_rationale(
deployment_id=deployment.id,
channel=deployment.channel,
production=deployment.production,
project_consumers=[item.key for item in projects],
active_project_consumers=[item.key for item in active_projects],
project_fit_evidence_ids=list(project_fit_ids),
evaluation_run_ids=[item.id for item in evaluations],
project_consumers=project_consumers.get(contract.id, []),
active_project_consumers=active_project_consumers.get(contract.id, []),
project_fit_evidence_ids=fit_evidence_ids.get(contract.id, []),
evaluation_run_ids=evaluation_ids.get(deployment.id, []),
last_used_at=residency.last_used_at if residency else None,
)
)
@@ -242,10 +406,10 @@ def model_installation_rationale(
model_id=model.id,
display_name=model.display_name,
upstream_source=model.upstream_source,
installed=bool(installed_sets),
installed_bytes=sum(item.total_size_bytes for item in installed_sets),
installed=bool(model_installed_sets),
installed_bytes=sum(item.total_size_bytes for item in model_installed_sets),
dependencies=dependencies,
can_delete=bool(installed_sets) and not blockers,
can_delete=bool(model_installed_sets) and not blockers,
deletion_blockers=blockers,
)
)
@@ -255,8 +419,9 @@ def model_installation_rationale(
@router.get("/projects", response_model=list[ProjectResponse])
def list_projects(
registry: ManifestRegistry = Depends(get_manifest_registry),
session: Session = Depends(get_session),
) -> list[ProjectResponse]:
return [
declared = [
ProjectResponse(
**project.project.model_dump(),
notes=project.notes,
@@ -271,3 +436,57 @@ def list_projects(
)
for project in registry.projects()
]
responses = {project.id: project for project in declared}
projects = session.scalars(
select(Project).options(selectinload(Project.bindings)).order_by(Project.key)
).all()
contract_ids = {
binding.capability_contract_id
for project in projects
for binding in project.bindings
if binding.deprecated_at is None
}
contracts = (
{
item.id: item
for item in session.scalars(
select(CapabilityContract).where(CapabilityContract.id.in_(contract_ids))
)
}
if contract_ids
else {}
)
capability_ids = {item.capability_id for item in contracts.values()}
capabilities = (
{
item.id: item
for item in session.scalars(
select(Capability).where(Capability.id.in_(capability_ids))
)
}
if capability_ids
else {}
)
for item in projects:
bindings = []
for binding in item.bindings:
if binding.deprecated_at:
continue
contract = contracts.get(binding.capability_contract_id)
capability = capabilities.get(contract.capability_id) if contract else None
if not contract or not capability:
continue
bindings.append(ProjectBindingResponse(
capability=capability.key, contract_version=contract.version,
binding=ProjectBindingManifest.model_validate({
"contract_version": contract.version, "channel": binding.channel,
"priority": binding.priority, "optional": binding.optional,
"fallback": binding.fallback_policy, "migration_support": binding.migration_support,
"slo": binding.slo, "benchmark_suites": binding.benchmark_requirements,
}),
))
responses[item.key] = ProjectResponse(
id=item.key, name=item.name, description=item.description, bindings=bindings,
notes=responses[item.key].notes if item.key in responses else [],
)
return sorted(responses.values(), key=lambda item: item.id)
backend/src/modelforge_api/api/routes/configuration.py
+154
View File
@@ -0,0 +1,154 @@
from typing import Annotated
from fastapi import APIRouter, Depends, Path, Query, Request
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Admin, Operator, Viewer, require_viewer
from modelforge_api.db import get_session
from modelforge_api.domain.configuration import (
ConfigurationApplied,
ConfigurationApply,
ConfigurationCatalogResponse,
ConfigurationChange,
ConfigurationExport,
ConfigurationImport,
ConfigurationPreview,
ConfigurationProposal,
ConfigurationRevisionResponse,
ConfigurationRollback,
)
from modelforge_api.domain.secret_slots import (
SecretSlotCatalog,
SecretSlotMetadata,
SecretSlotRevoke,
SecretSlotSet,
SecretSlotVersionRequest,
)
from modelforge_api.services.audit import AuditContext
from modelforge_api.services.configuration import build_configuration_catalog
from modelforge_api.services.configuration_changes import (
ConfigurationService,
resolved_configuration,
)
from modelforge_api.services.secret_slots import SecretSlotService
from modelforge_api.settings import Settings, get_settings
router = APIRouter(
prefix="/api/v1/configuration",
tags=["configuration"],
dependencies=[Depends(require_viewer)],
)
Database = Annotated[Session, Depends(get_session)]
RuntimeSettings = Annotated[Settings, Depends(get_settings)]
def get_configuration_service(
session: Database,
settings: RuntimeSettings,
principal: Viewer,
request: Request,
) -> ConfigurationService:
return ConfigurationService(
session,
settings,
AuditContext(
actor_type="operator",
actor_id=principal.subject,
correlation_id=request.state.correlation_id,
),
)
Service = Annotated[ConfigurationService, Depends(get_configuration_service)]
@router.get("/settings", response_model=ConfigurationCatalogResponse)
def configuration_settings(
session: Database,
settings: RuntimeSettings,
) -> ConfigurationCatalogResponse:
revision, overrides, effective = resolved_configuration(session, settings)
return build_configuration_catalog(
effective,
revision=revision,
overrides=overrides,
)
@router.post("/preview", response_model=ConfigurationPreview)
def configuration_preview(
body: ConfigurationChange, service: Service, _principal: Operator
) -> ConfigurationPreview:
return service.preview(body)
@router.post("/apply", response_model=ConfigurationApplied)
def configuration_apply(
body: ConfigurationApply, service: Service, _principal: Admin
) -> ConfigurationApplied:
return service.apply(body)
@router.get("/history", response_model=list[ConfigurationRevisionResponse])
def configuration_history(
service: Service, limit: Annotated[int, Query(ge=1, le=100)] = 30
) -> list[ConfigurationRevisionResponse]:
return service.history(limit)
@router.post("/revisions/{revision}/preview", response_model=ConfigurationProposal)
def configuration_rollback_preview(
revision: Annotated[int, Path(ge=0)],
body: ConfigurationRollback,
service: Service,
_principal: Operator,
) -> ConfigurationProposal:
change = service.rollback_change(revision, body.expected_revision)
return ConfigurationProposal(change=change, preview=service.preview(change))
@router.get("/export", response_model=ConfigurationExport)
def configuration_export(service: Service) -> ConfigurationExport:
return service.export()
@router.post("/import/preview", response_model=ConfigurationProposal)
def configuration_import_preview(
body: ConfigurationImport,
service: Service,
_principal: Operator,
) -> ConfigurationProposal:
change = service.import_change(body)
return ConfigurationProposal(change=change, preview=service.preview(change))
def secret_slot_service(session: Database, settings: RuntimeSettings, principal: Viewer, request: Request) -> SecretSlotService:
return SecretSlotService(session, settings, principal, request.state.correlation_id)
Secrets = Annotated[SecretSlotService, Depends(secret_slot_service)]
@router.get("/secret-slots", response_model=SecretSlotCatalog)
def secret_slots(service: Secrets) -> SecretSlotCatalog:
return service.catalog()
@router.post("/secret-slots/huggingface_control_plane/set", response_model=SecretSlotMetadata)
def secret_slot_set(body: SecretSlotSet, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
return service.set(body)
@router.post("/secret-slots/huggingface_control_plane/test", response_model=SecretSlotMetadata)
def secret_slot_test(body: SecretSlotVersionRequest, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
return service.test(body)
@router.post("/secret-slots/huggingface_control_plane/activate", response_model=SecretSlotMetadata)
def secret_slot_activate(body: SecretSlotVersionRequest, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
return service.activate(body)
@router.post("/secret-slots/huggingface_control_plane/revoke", response_model=SecretSlotMetadata)
def secret_slot_revoke(body: SecretSlotRevoke, service: Secrets, _admin: Admin) -> SecretSlotMetadata:
return service.revoke(body)
backend/src/modelforge_api/api/routes/external_creative.py
+189
View File
@@ -0,0 +1,189 @@
from __future__ import annotations
import uuid
from typing import Annotated
from fastapi import APIRouter, Depends, Header, Request, status
from modelforge_api.api.authorization import Operator
from modelforge_api.api.routes.serving import Service, _authenticate_capability_client
from modelforge_api.domain.external_creative import (
ExternalAllocationAction,
ExternalAllocationCreate,
ExternalAllocationList,
ExternalAllocationReconcile,
ExternalAllocationResponse,
ExternalProfileObservation,
ExternalResourceApproval,
ExternalResourceList,
ExternalResourceResponse,
)
from modelforge_api.persistence.models import ServiceClient
from modelforge_api.services.external_creative import ExternalCreativeService
router = APIRouter(tags=["creative-resource-coordination"])
def creative_client(
request: Request,
service: Service,
authorization: Annotated[str | None, Header(alias="Authorization")] = None,
) -> ServiceClient:
return _authenticate_capability_client(request, service, authorization, "creative.resource@1")
Client = Annotated[ServiceClient, Depends(creative_client)]
@router.post(
"/api/v1/capabilities/creative.resource@1/allocations",
response_model=ExternalAllocationResponse,
status_code=status.HTTP_201_CREATED,
)
def acquire(
body: ExternalAllocationCreate, service: Service, client: Client
) -> ExternalAllocationResponse:
return ExternalCreativeService(service.session, service).acquire(body, client)
@router.get(
"/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}",
response_model=ExternalAllocationResponse,
)
def allocation(
allocation_id: uuid.UUID, service: Service, client: Client
) -> ExternalAllocationResponse:
return ExternalCreativeService(service.session, service).get(allocation_id, client)
def _action(
allocation_id: uuid.UUID,
body: ExternalAllocationAction,
action: str,
service: Service,
client: ServiceClient,
) -> ExternalAllocationResponse:
return ExternalCreativeService(service.session, service).action(
allocation_id,
body.request_id,
action,
client,
body.observed_peak_vram_bytes,
body.attestation,
)
@router.post(
"/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/heartbeat",
response_model=ExternalAllocationResponse,
)
def heartbeat(
allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
) -> ExternalAllocationResponse:
return _action(allocation_id, body, "heartbeat", service, client)
@router.post(
"/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/attest",
response_model=ExternalAllocationResponse,
)
def attest(
allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
) -> ExternalAllocationResponse:
return _action(allocation_id, body, "attest", service, client)
@router.post(
"/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/release",
response_model=ExternalAllocationResponse,
)
def release(
allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
) -> ExternalAllocationResponse:
return _action(allocation_id, body, "release", service, client)
@router.post(
"/api/v1/capabilities/creative.resource@1/allocations/{allocation_id}/uncertain",
response_model=ExternalAllocationResponse,
)
def uncertain(
allocation_id: uuid.UUID, body: ExternalAllocationAction, service: Service, client: Client
) -> ExternalAllocationResponse:
return _action(allocation_id, body, "uncertain", service, client)
@router.post(
"/api/v1/capabilities/creative.resource@1/profiles/observations",
response_model=ExternalResourceResponse,
)
def observe_profile(
body: ExternalProfileObservation, service: Service, client: Client
) -> ExternalResourceResponse:
return ExternalCreativeService(service.session, service).observe(body, client)
@router.put(
"/api/v1/operator/creative-resource/resources",
response_model=ExternalResourceResponse,
)
def configure_resource(
body: ExternalResourceApproval, service: Service, operator: Operator
) -> ExternalResourceResponse:
return ExternalCreativeService(service.session, service).upsert_resource(body, operator.subject)
@router.get(
"/api/v1/operator/creative-resource/resources",
response_model=ExternalResourceList,
)
def operator_resources(
service: Service,
_operator: Operator,
project_id: uuid.UUID | None = None,
limit: int = 50,
offset: int = 0,
) -> ExternalResourceList:
return ExternalCreativeService(service.session, service).list_resources(
project_id,
min(max(limit, 1), 200),
max(offset, 0),
)
@router.get(
"/api/v1/operator/creative-resource/allocations",
response_model=ExternalAllocationList,
)
def operator_allocations(
service: Service,
_operator: Operator,
state: str | None = None,
limit: int = 50,
offset: int = 0,
) -> ExternalAllocationList:
if state not in {None, "granted", "queued", "blocked", "uncertain", "released"}:
from fastapi import HTTPException
raise HTTPException(status_code=422, detail="unsupported allocation state")
return ExternalCreativeService(service.session, service).list_allocations(
state, min(max(limit, 1), 200), max(offset, 0)
)
@router.post(
"/api/v1/operator/creative-resource/allocations/{allocation_id}/reconcile",
response_model=ExternalAllocationResponse,
)
def reconcile_allocation(
allocation_id: uuid.UUID,
body: ExternalAllocationReconcile,
service: Service,
operator: Operator,
) -> ExternalAllocationResponse:
return ExternalCreativeService(service.session, service).reconcile(
allocation_id,
body.action,
body.reason,
operator.subject,
)
backend/src/modelforge_api/api/routes/identity.py
+79
View File
@@ -0,0 +1,79 @@
from datetime import UTC, datetime
from typing import Annotated, Literal
from fastapi import APIRouter, Depends, Request
from pydantic import BaseModel
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Admin, PrincipalRole, Viewer
from modelforge_api.db import get_session
from modelforge_api.domain.managed_identity import (
ManagedIdentityChange,
ManagedIdentityCreate,
ManagedIdentityIssued,
ManagedIdentityList,
ManagedIdentityResult,
ManagedIdentityRevoke,
ManagedIdentityRotate,
)
from modelforge_api.services.operator_identities import ManagedIdentityService
from modelforge_api.settings import Settings, get_settings
router = APIRouter(prefix="/api/v1/operator", tags=["operator-identity"])
class CurrentOperator(BaseModel):
subject: str
role: PrincipalRole
authentication_method: str
credential_id: str | None
credential_expires_at: datetime | None
server_time: datetime
console_lock_after_seconds: Literal[900] = 900
@router.get("/identity", response_model=CurrentOperator)
def current_operator(principal: Viewer) -> CurrentOperator:
return CurrentOperator(
subject=principal.subject,
role=principal.role,
authentication_method=principal.authentication_method,
credential_id=principal.credential_id,
credential_expires_at=principal.expires_at,
server_time=datetime.now(UTC),
)
def managed_identity_service(
session: Annotated[Session, Depends(get_session)], principal: Admin, request: Request,
) -> ManagedIdentityService:
return ManagedIdentityService(session, principal, request.state.correlation_id)
Service = Annotated[ManagedIdentityService, Depends(managed_identity_service)]
@router.get("/managed-identities", response_model=ManagedIdentityList)
def managed_identities(service: Service, _admin: Admin,
settings: Annotated[Settings, Depends(get_settings)]) -> ManagedIdentityList:
return service.list(settings)
@router.post("/managed-identities", response_model=ManagedIdentityIssued, status_code=201)
def create_managed_identity(body: ManagedIdentityCreate, service: Service, _admin: Admin) -> ManagedIdentityIssued:
return service.create(body)
@router.post("/managed-identities/{slug}/rotate", response_model=ManagedIdentityIssued)
def rotate_managed_identity(slug: str, body: ManagedIdentityRotate, service: Service, _admin: Admin) -> ManagedIdentityIssued:
return service.rotate(slug, body)
@router.patch("/managed-identities/{slug}", response_model=ManagedIdentityResult)
def change_managed_identity(slug: str, body: ManagedIdentityChange, service: Service, _admin: Admin) -> ManagedIdentityResult:
return service.change(slug, body)
@router.post("/managed-identities/{slug}/revoke", response_model=ManagedIdentityResult)
def revoke_managed_identity(slug: str, body: ManagedIdentityRevoke, service: Service, _admin: Admin) -> ManagedIdentityResult:
return service.revoke(slug, body)
backend/src/modelforge_api/api/routes/lab.py
+162
View File
@@ -0,0 +1,162 @@
import uuid
from typing import Annotated
from fastapi import APIRouter, Depends, Header, Response, status
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Operator, require_operator
from modelforge_api.api.routes.serving import Service as ServingDependency
from modelforge_api.db import get_session
from modelforge_api.domain.benchmarking import (
BenchmarkRunResponse,
BenchmarkRunStartRequest,
BenchmarkSuiteResponse,
)
from modelforge_api.domain.lab import (
LabExecutionRunResponse,
LabExecutionStartRequest,
LabPreflightRequest,
LabPreflightResponse,
)
from modelforge_api.services.benchmark_runs import BenchmarkRunService
from modelforge_api.services.lab import LabPreflightService
from modelforge_api.services.lab_runs import LabRunService
from modelforge_api.settings import Settings, get_settings
router = APIRouter(
prefix="/api/v1/lab", tags=["model-lab"], dependencies=[Depends(require_operator)]
)
def get_lab_service(
session: Annotated[Session, Depends(get_session)],
settings: Annotated[Settings, Depends(get_settings)],
) -> LabPreflightService:
return LabPreflightService(session, settings)
def get_lab_run_service(serving: ServingDependency) -> LabRunService:
return LabRunService(serving)
def get_benchmark_run_service(serving: ServingDependency) -> BenchmarkRunService:
return BenchmarkRunService(serving)
LabRunServiceDependency = Annotated[LabRunService, Depends(get_lab_run_service)]
BenchmarkRunServiceDependency = Annotated[BenchmarkRunService, Depends(get_benchmark_run_service)]
@router.post("/preflight", response_model=LabPreflightResponse)
def preflight(
request: LabPreflightRequest,
service: Annotated[LabPreflightService, Depends(get_lab_service)],
) -> LabPreflightResponse:
"""Inspect a bounded LAB request without storing input or starting execution."""
return service.preflight(request)
@router.post("/runs", response_model=LabExecutionRunResponse, status_code=status.HTTP_202_ACCEPTED)
def start_run(
request: LabExecutionStartRequest,
response: Response,
service: LabRunServiceDependency,
operator: Operator,
idempotency_key: Annotated[
str,
Header(
alias="Idempotency-Key", min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$"
),
],
) -> LabExecutionRunResponse:
response.headers["Cache-Control"] = "no-store"
return service.start(request, actor_subject=operator.subject, idempotency_key=idempotency_key)
@router.get("/runs/{run_id}", response_model=LabExecutionRunResponse)
def get_run(
run_id: uuid.UUID,
response: Response,
service: LabRunServiceDependency,
operator: Operator,
) -> LabExecutionRunResponse:
response.headers["Cache-Control"] = "no-store"
return service.get(run_id, actor_subject=operator.subject)
@router.post("/runs/{run_id}/cancel", response_model=LabExecutionRunResponse)
def cancel_run(
run_id: uuid.UUID,
response: Response,
service: LabRunServiceDependency,
operator: Operator,
) -> LabExecutionRunResponse:
response.headers["Cache-Control"] = "no-store"
return service.cancel(run_id, actor_subject=operator.subject)
@router.get("/benchmark-suites", response_model=list[BenchmarkSuiteResponse])
def benchmark_suites(
response: Response,
service: BenchmarkRunServiceDependency,
) -> list[BenchmarkSuiteResponse]:
response.headers["Cache-Control"] = "no-store"
return service.suites()
@router.get("/benchmark-runs", response_model=list[BenchmarkRunResponse])
def benchmark_history(
response: Response,
service: BenchmarkRunServiceDependency,
operator: Operator,
limit: int = 50,
cursor: str | None = None,
) -> list[BenchmarkRunResponse]:
response.headers["Cache-Control"] = "no-store"
page = service.list_page(actor_subject=operator.subject, limit=limit, cursor=cursor)
response.headers["X-Next-Cursor"] = page.next_cursor or ""
response.headers["Access-Control-Expose-Headers"] = "X-Next-Cursor"
return page.runs
@router.post(
"/benchmark-runs",
response_model=BenchmarkRunResponse,
status_code=status.HTTP_202_ACCEPTED,
)
def start_benchmark_run(
request: BenchmarkRunStartRequest,
response: Response,
service: BenchmarkRunServiceDependency,
operator: Operator,
idempotency_key: Annotated[
str,
Header(
alias="Idempotency-Key", min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$"
),
],
) -> BenchmarkRunResponse:
response.headers["Cache-Control"] = "no-store"
return service.start(request, actor_subject=operator.subject, idempotency_key=idempotency_key)
@router.get("/benchmark-runs/{run_id}", response_model=BenchmarkRunResponse)
def get_benchmark_run(
run_id: uuid.UUID,
response: Response,
service: BenchmarkRunServiceDependency,
operator: Operator,
) -> BenchmarkRunResponse:
response.headers["Cache-Control"] = "no-store"
return service.get(run_id, actor_subject=operator.subject)
@router.post("/benchmark-runs/{run_id}/cancel", response_model=BenchmarkRunResponse)
def cancel_benchmark_run(
run_id: uuid.UUID,
response: Response,
service: BenchmarkRunServiceDependency,
operator: Operator,
) -> BenchmarkRunResponse:
response.headers["Cache-Control"] = "no-store"
return service.cancel(run_id, actor_subject=operator.subject)
backend/src/modelforge_api/api/routes/lifecycle.py
+8 -3
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import uuid
from typing import Annotated
from fastapi import APIRouter, Depends, Query, status
from fastapi import APIRouter, Depends, Query, Request, status
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Admin, require_viewer
@@ -28,6 +28,7 @@ from modelforge_api.domain.lifecycle_contracts import (
RetentionPolicyCreate,
RetentionPolicyResponse,
)
from modelforge_api.services.audit import AuditContext
from modelforge_api.services.lifecycle import LifecycleService
router = APIRouter(
@@ -171,9 +172,13 @@ def operation(
@router.post("/operations/{operation_id}/canary", response_model=LifecycleOperationResponse)
def observe_canary(
operation_id: uuid.UUID, request: CanaryObservation, service: Service, _admin: Admin
operation_id: uuid.UUID, request: CanaryObservation, service: Service, _admin: Admin,
http_request: Request,
) -> LifecycleOperationResponse:
return service.observe_canary(operation_id, request)
return service.observe_canary(operation_id, request, audit_context=AuditContext(
actor_type="operator", actor_id=_admin.subject,
correlation_id=http_request.state.correlation_id,
))
@router.post("/operations/{operation_id}/commit", response_model=LifecycleOperationResponse)
backend/src/modelforge_api/api/routes/migrations.py
+6 -2
View File
@@ -60,8 +60,12 @@ def create_validation_policy(
@router.get("/plans", response_model=list[MigrationPlanResponse])
def plans(service: Service, _admin: Admin) -> list[MigrationPlanResponse]:
return service.plans()
def plans(
service: Service,
_admin: Admin,
limit: Annotated[int, Query(ge=1, le=200)] = 200,
) -> list[MigrationPlanResponse]:
return service.plans(limit=limit)
@router.post("/plans", response_model=MigrationPlanResponse, status_code=status.HTTP_201_CREATED)
backend/src/modelforge_api/api/routes/model_updates.py
+80
View File
@@ -0,0 +1,80 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Principal, require_operator, require_viewer
from modelforge_api.api.routes.acquisition import get_acquisition_service
from modelforge_api.db import get_session
from modelforge_api.domain.model_updates import (
ModelUpdateMonitorResponse,
ModelUpdateNoticeResponse,
ModelWatchCreate,
ModelWatchResponse,
ModelWatchUpdate,
)
from modelforge_api.services.acquisition import AcquisitionService
from modelforge_api.services.audit import AuditContext
from modelforge_api.services.model_updates import ModelUpdateService
from modelforge_api.settings import Settings, get_settings
router = APIRouter(prefix="/api/v1/model-updates", tags=["model-updates"],
dependencies=[Depends(require_viewer)])
def get_model_update_service(
request: Request,
session: Annotated[Session, Depends(get_session)],
acquisition: Annotated[AcquisitionService, Depends(get_acquisition_service)],
principal: Annotated[Principal, Depends(require_viewer)],
) -> ModelUpdateService:
context = AuditContext("operator", principal.subject,
getattr(request.state, "correlation_id", None))
return ModelUpdateService(session, acquisition, context)
Service = Annotated[ModelUpdateService, Depends(get_model_update_service)]
Limit = Annotated[int, Query(ge=1, le=100)]
Offset = Annotated[int, Query(ge=0, le=10000)]
@router.get("/monitor", response_model=ModelUpdateMonitorResponse)
def monitor(settings: Annotated[Settings, Depends(get_settings)]) -> ModelUpdateMonitorResponse:
return ModelUpdateMonitorResponse(
enabled=settings.env == "production" and settings.model_update_monitor_enabled,
check_interval_seconds=settings.model_update_check_interval_seconds,
)
@router.get("/watches", response_model=list[ModelWatchResponse])
def watches(service: Service, limit: Limit = 50, offset: Offset = 0) -> list[ModelWatchResponse]:
return service.watches(limit, offset)
@router.post("/watches", response_model=ModelWatchResponse, dependencies=[Depends(require_operator)])
def create(request: ModelWatchCreate, service: Service) -> ModelWatchResponse:
return service.create(request)
@router.post("/watches/{watch_id}/check", response_model=ModelWatchResponse,
dependencies=[Depends(require_operator)])
def check(watch_id: UUID, service: Service) -> ModelWatchResponse:
return service.check(watch_id)
@router.patch("/watches/{watch_id}", response_model=ModelWatchResponse,
dependencies=[Depends(require_operator)])
def change(watch_id: UUID, request: ModelWatchUpdate, service: Service) -> ModelWatchResponse:
return service.change(watch_id, request)
@router.get("/notices", response_model=list[ModelUpdateNoticeResponse])
def notices(service: Service, limit: Limit = 50, offset: Offset = 0) -> list[ModelUpdateNoticeResponse]:
return service.notices(limit, offset)
@router.post("/notices/{notice_id}/dismiss", response_model=ModelUpdateNoticeResponse,
dependencies=[Depends(require_operator)])
def dismiss(notice_id: UUID, service: Service) -> ModelUpdateNoticeResponse:
return service.dismiss(notice_id)
backend/src/modelforge_api/api/routes/observability.py
+80 -5
View File
@@ -1,16 +1,21 @@
from __future__ import annotations
import uuid
from typing import Annotated, Any
from typing import Annotated, Any, Literal
from fastapi import APIRouter, Depends, Query, Response, status
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response, status
from sqlalchemy import select
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Admin, require_viewer
from modelforge_api.db import get_session
from modelforge_api.domain.alert_notifications import (
NotificationDeliveryResponse,
NotificationTestResponse,
)
from modelforge_api.domain.observability import (
ActiveSchedulerLeaseInventoryResponse,
AlertAction,
AlertHistoryResponse,
AlertResponse,
@@ -28,12 +33,17 @@ from modelforge_api.domain.observability import (
TrendResponse,
metrics,
)
from modelforge_api.domain.self_check import SelfCheckReport
from modelforge_api.persistence.models import (
AlertHistoryEvent,
IncidentTimelineEvent,
OperationalIncident,
)
from modelforge_api.services.alert_notifications import AlertNotificationService
from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
from modelforge_api.services.observability import ObservabilityService
from modelforge_api.services.self_check import run_self_check
from modelforge_api.settings import Settings, get_settings
router = APIRouter(tags=["operations"], dependencies=[Depends(require_viewer)])
@@ -63,6 +73,47 @@ def overview(service: Service, _admin: Admin) -> OperationsOverview:
return service.overview()
@router.get("/api/v1/admin/operations/self-check", response_model=SelfCheckReport)
def self_check(
_admin: Admin,
session: Annotated[Session, Depends(get_session)],
settings: Annotated[Settings, Depends(get_settings)],
registry: Annotated[ManifestRegistry, Depends(get_manifest_registry)],
) -> SelfCheckReport:
"""Run passive diagnostics only; no repair, write probe, or external action is performed."""
return run_self_check(session, settings, registry)
@router.get(
"/api/v1/admin/operations/notification-deliveries",
response_model=list[NotificationDeliveryResponse],
)
def notification_deliveries(
session: Annotated[Session, Depends(get_session)],
settings: Annotated[Settings, Depends(get_settings)],
_admin: Admin,
limit: int = Query(default=100, ge=1, le=500),
) -> list[NotificationDeliveryResponse]:
return AlertNotificationService(session, settings).deliveries(limit)
@router.post(
"/api/v1/admin/operations/notifications/test",
response_model=NotificationTestResponse,
status_code=status.HTTP_202_ACCEPTED,
)
def queue_notification_test(
session: Annotated[Session, Depends(get_session)],
settings: Annotated[Settings, Depends(get_settings)],
principal: Admin,
request: Request,
) -> NotificationTestResponse:
return AlertNotificationService(session, settings).queue_test(
principal, getattr(request.state, "correlation_id", None)
)
@router.get(
"/api/v1/admin/operations/slis", response_model=list[SLIDefinitionResponse]
)
@@ -185,6 +236,20 @@ def capacity(
return service.capacity(limit)
@router.get(
"/api/v1/admin/operations/active-leases",
response_model=ActiveSchedulerLeaseInventoryResponse,
)
def active_scheduler_leases(
service: Service,
_admin: Admin,
limit: int = Query(default=100, ge=1, le=500),
sort_by: Literal["expires_at", "created_at", "priority"] = Query(default="expires_at"),
order: Literal["asc", "desc"] = Query(default="asc"),
) -> ActiveSchedulerLeaseInventoryResponse:
return service.active_scheduler_leases(limit=limit, sort_by=sort_by, order=order)
@router.post(
"/api/v1/admin/operations/capacity/collect", response_model=list[CapacitySnapshotResponse]
)
@@ -208,10 +273,20 @@ def apply_retention(service: Service, _admin: Admin) -> dict[str, int]:
@router.get("/api/v1/admin/operations/incidents", response_model=list[IncidentResponse])
def incidents(service: Service, _admin: Admin) -> list[IncidentResponse]:
rows = service.session.scalars(
select(OperationalIncident).order_by(OperationalIncident.last_seen_at.desc())
def incidents(
service: Service,
_admin: Admin,
limit: int | None = Query(default=None, ge=1, le=500),
offset: int = Query(default=0, ge=0),
) -> list[IncidentResponse]:
if limit is None and offset:
raise HTTPException(status_code=422, detail="offset requires limit")
statement = select(OperationalIncident).order_by(
OperationalIncident.last_seen_at.desc(), OperationalIncident.id
)
if limit is not None:
statement = statement.offset(offset).limit(limit)
rows = service.session.scalars(statement)
return [IncidentResponse.model_validate(item) for item in rows]
backend/src/modelforge_api/api/routes/operator_login.py
+80
View File
@@ -0,0 +1,80 @@
from __future__ import annotations
import hashlib
from datetime import UTC, datetime
from typing import Annotated
import redis
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from pydantic import BaseModel, ConfigDict, Field, SecretStr
from modelforge_api.services.operator_password import (
SESSION_SECONDS,
issue_session,
verify_password,
)
from modelforge_api.settings import Settings, get_settings
router = APIRouter(prefix="/api/v1/operator", tags=["operator"])
# One atomic operation across all API processes; keys expire even for failed attempts.
_RATE_SCRIPT = """
local global = redis.call('INCR', KEYS[1])
if global == 1 then redis.call('EXPIRE', KEYS[1], 60) end
if global > 30 then return 0 end
local peer = redis.call('INCR', KEYS[2])
if peer == 1 then redis.call('EXPIRE', KEYS[2], 60) end
if peer > 5 then return 0 end
return 1
"""
def limit_login(request: Request, settings: Settings) -> None:
# Use only the direct socket peer. Forwarded headers do not identify trusted clients.
peer = request.client.host if request.client else "unknown"
peer_digest = hashlib.sha256(peer.encode()).hexdigest()
try:
with redis.Redis.from_url(
settings.redis_url, socket_connect_timeout=2, socket_timeout=2
) as client:
accepted = client.eval(
_RATE_SCRIPT, 2, "modelforge:login:global", f"modelforge:login:peer:{peer_digest}"
)
except (redis.RedisError, OSError, ValueError):
raise HTTPException(503, "Password login is temporarily unavailable") from None
if accepted != 1:
raise HTTPException(429, "Too many login attempts; try again later")
class PasswordLogin(BaseModel):
model_config = ConfigDict(extra="forbid")
password: SecretStr = Field(min_length=1, max_length=1024)
class PasswordSession(BaseModel):
access_token: str
expires_at: datetime
expires_in: int = SESSION_SECONDS
token_type: str = "bearer" # noqa: S105 - protocol token type, not a secret
@router.post("/login", response_model=PasswordSession)
def login(
payload: PasswordLogin,
request: Request,
response: Response,
settings: Annotated[Settings, Depends(get_settings)],
) -> PasswordSession:
# The pre-body middleware applies the shared limiter, including malformed requests.
configured = settings.operator_password_hash
bootstrap = settings.operator_api_key
if configured is None or bootstrap is None or len(bootstrap.get_secret_value()) < 32:
raise HTTPException(503, "Password login is not configured")
try:
valid = verify_password(payload.password.get_secret_value(), configured.get_secret_value())
except (ValueError, UnicodeError):
raise HTTPException(503, "Password login is not configured") from None
if not valid:
raise HTTPException(401, "Invalid password")
token, expires = issue_session(bootstrap.get_secret_value(), configured.get_secret_value())
response.headers["Cache-Control"] = "no-store"
return PasswordSession(access_token=token, expires_at=datetime.fromtimestamp(expires, UTC))
backend/src/modelforge_api/api/routes/portfolio.py
+61
View File
@@ -0,0 +1,61 @@
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Request
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Operator, Viewer, require_viewer
from modelforge_api.db import get_session
from modelforge_api.domain.portfolio import (
PortfolioInventoryResponse,
PortfolioReconcileRequest,
PortfolioReconcileResponse,
)
from modelforge_api.services.audit import AuditContext
from modelforge_api.services.portfolio_registry import (
PortfolioRegistryError,
PortfolioRegistryService,
)
router = APIRouter(
prefix="/api/v1/portfolio",
tags=["portfolio"],
dependencies=[Depends(require_viewer)],
)
def get_portfolio_registry_service(
session: Annotated[Session, Depends(get_session)],
) -> PortfolioRegistryService:
return PortfolioRegistryService(session)
Service = Annotated[PortfolioRegistryService, Depends(get_portfolio_registry_service)]
@router.get("", response_model=PortfolioInventoryResponse)
def portfolio_inventory(service: Service, _viewer: Viewer) -> PortfolioInventoryResponse:
return service.inventory()
@router.post(
"/reconcile",
response_model=PortfolioReconcileResponse,
)
def reconcile_portfolio(
body: PortfolioReconcileRequest,
service: Service,
operator: Operator,
request: Request,
) -> PortfolioReconcileResponse:
context = AuditContext(
actor_type="operator",
actor_id=operator.subject,
correlation_id=getattr(request.state, "correlation_id", None),
)
try:
return service.reconcile(body, context)
except PortfolioRegistryError as exc:
raise HTTPException(
status_code=exc.status_code,
detail={"code": exc.code, "message": exc.message},
) from exc
backend/src/modelforge_api/api/routes/projects.py
+172
View File
@@ -0,0 +1,172 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from modelforge_api.api.authorization import Operator, Viewer
from modelforge_api.api.routes.serving import Service as Serving
from modelforge_api.domain.model_changes import (
ModelChangeCandidateLink,
ModelChangeCreate,
ModelChangeEvidenceLink,
ModelChangeList,
ModelChangeReject,
ModelChangeResponse,
)
from modelforge_api.domain.projects import (
ProjectCreate,
ProjectRequirementCreate,
ProjectWorkspaceResponse,
)
from modelforge_api.services.audit import AuditContext, AuditWriter
from modelforge_api.services.model_changes import ModelChangeError, ModelChangeService
from modelforge_api.services.projects import ProjectService
router = APIRouter(prefix="/api/v1/project-workspace", tags=["project-onboarding"])
def get_project_service(serving: Serving, principal: Viewer, request: Request) -> ProjectService:
serving.audit = AuditWriter(serving.session, context=AuditContext(
"operator", principal.subject, request.state.correlation_id,
))
return ProjectService(serving)
def get_model_change_service(
serving: Serving, principal: Viewer, request: Request
) -> ModelChangeService:
return ModelChangeService(
serving.session,
AuditContext("operator", principal.subject, request.state.correlation_id),
)
Service = Annotated[ProjectService, Depends(get_project_service)]
ChangeService = Annotated[ModelChangeService, Depends(get_model_change_service)]
def _change_error(exc: ModelChangeError) -> HTTPException:
return HTTPException(status_code=exc.status_code, detail=exc.message)
@router.get("", response_model=list[ProjectWorkspaceResponse])
def projects(
service: Service,
_viewer: Viewer,
limit: int | None = Query(default=None, ge=1, le=200),
after_key: str | None = Query(default=None, min_length=1, max_length=128),
) -> list[ProjectWorkspaceResponse]:
if after_key is not None and limit is None:
raise HTTPException(status_code=422, detail="after_key requires limit")
return service.list(limit=limit, after_key=after_key)
@router.post("", response_model=ProjectWorkspaceResponse, status_code=201)
def create_project(body: ProjectCreate, service: Service, _operator: Operator) -> ProjectWorkspaceResponse:
return service.create(body)
@router.get("/changes", response_model=ModelChangeList)
def model_changes(
service: ChangeService,
_viewer: Viewer,
project_key: str | None = None,
status: str | None = None,
project_binding_id: UUID | None = None,
open_only: bool = False,
limit: int = Query(default=100, ge=1, le=200),
offset: int = Query(default=0, ge=0),
) -> ModelChangeList:
try:
return service.list(
project_key=project_key,
status=status,
project_binding_id=project_binding_id,
open_only=open_only,
limit=limit,
offset=offset,
)
except ModelChangeError as exc:
raise _change_error(exc) from exc
@router.post("/changes", response_model=ModelChangeResponse, status_code=201)
def create_model_change(
body: ModelChangeCreate, service: ChangeService, _operator: Operator
) -> ModelChangeResponse:
try:
return service.create(body)
except ModelChangeError as exc:
raise _change_error(exc) from exc
@router.get("/changes/{case_id}", response_model=ModelChangeResponse)
def model_change(case_id: UUID, service: ChangeService, _viewer: Viewer) -> ModelChangeResponse:
try:
return service.get(case_id)
except ModelChangeError as exc:
raise _change_error(exc) from exc
@router.put("/changes/{case_id}/candidate", response_model=ModelChangeResponse)
def link_model_change_candidate(
case_id: UUID,
body: ModelChangeCandidateLink,
service: ChangeService,
_operator: Operator,
) -> ModelChangeResponse:
try:
return service.link_candidate(case_id, body)
except ModelChangeError as exc:
raise _change_error(exc) from exc
@router.put("/changes/{case_id}/evidence", response_model=ModelChangeResponse)
def link_model_change_evidence(
case_id: UUID,
body: ModelChangeEvidenceLink,
service: ChangeService,
_operator: Operator,
) -> ModelChangeResponse:
try:
return service.link_evidence(case_id, body)
except ModelChangeError as exc:
raise _change_error(exc) from exc
@router.post("/changes/{case_id}/refresh", response_model=ModelChangeResponse)
def refresh_model_change(
case_id: UUID,
service: ChangeService,
_operator: Operator,
expected_version: int = Query(ge=1),
) -> ModelChangeResponse:
try:
return service.refresh(case_id, expected_version)
except ModelChangeError as exc:
raise _change_error(exc) from exc
@router.post("/changes/{case_id}/reject", response_model=ModelChangeResponse)
def reject_model_change(
case_id: UUID,
body: ModelChangeReject,
service: ChangeService,
_operator: Operator,
) -> ModelChangeResponse:
try:
return service.reject(case_id, body)
except ModelChangeError as exc:
raise _change_error(exc) from exc
@router.get("/{key}", response_model=ProjectWorkspaceResponse)
def project(key: str, service: Service, _viewer: Viewer) -> ProjectWorkspaceResponse:
return service.detail(key)
@router.post("/{key}/requirements", response_model=ProjectWorkspaceResponse, status_code=201)
def bind_project(
key: str, body: ProjectRequirementCreate, service: Service, _operator: Operator,
) -> ProjectWorkspaceResponse:
return service.bind(key, body)
backend/src/modelforge_api/api/routes/recovery.py
+6
View File
@@ -26,6 +26,7 @@ from modelforge_api.domain.recovery import (
RecoveryDashboard,
RecoveryPolicyCreate,
RecoveryPolicyResponse,
RecoveryRetentionPreview,
RestoreAdvanceRequest,
RestoreOperationEventResponse,
RestoreOperationResponse,
@@ -116,6 +117,11 @@ def apply_retention(service: Service, _admin: Admin) -> dict[str, int]:
return service.apply_retention()
@router.get("/retention/preview", response_model=RecoveryRetentionPreview)
def preview_retention(service: Service, _admin: Admin) -> RecoveryRetentionPreview:
return service.preview_retention()
@router.get("/restore-plans", response_model=list[RestorePlanResponse])
def restore_plans(
service: Service, _admin: Admin, limit: int = Query(default=100, ge=1, le=1000)
backend/src/modelforge_api/api/routes/selected_model_comparison.py
+106
View File
@@ -0,0 +1,106 @@
"""Separate operator planning from designated project-consumer execution reports."""
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, Query, Request, Response, status
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Principal, require_operator, require_viewer
from modelforge_api.api.routes.serving import (
Service as Serving,
)
from modelforge_api.api.routes.serving import (
authenticate_rag_embedding_client,
)
from modelforge_api.db import get_session
from modelforge_api.domain.selected_model_comparison import (
ComparisonCancel,
ComparisonClaim,
ComparisonCreate,
ComparisonFinish,
ComparisonList,
ComparisonProgress,
ComparisonResponse,
ComparisonResult,
)
from modelforge_api.persistence.models import ServiceClient
from modelforge_api.services.audit import AuditContext
from modelforge_api.services.selected_model_comparison import SelectedModelComparisonService
router = APIRouter(prefix="/api/v1/selected-model-comparisons", tags=["selected-model-comparisons"])
Client = Annotated[ServiceClient, Depends(authenticate_rag_embedding_client)]
Limit = Annotated[int, Query(ge=1, le=100)]
Offset = Annotated[int, Query(ge=0, le=10000)]
def operator_service(
request: Request,
session: Annotated[Session, Depends(get_session)],
principal: Annotated[Principal, Depends(require_viewer)],
) -> SelectedModelComparisonService:
return SelectedModelComparisonService(session, audit_context=AuditContext(
"operator", principal.subject, getattr(request.state, "correlation_id", None),
))
def consumer_service(request: Request, response: Response, serving: Serving, client: Client) -> SelectedModelComparisonService:
response.headers["Cache-Control"] = "no-store"
return SelectedModelComparisonService(serving.session, audit_context=AuditContext(
"service_client", str(client.id), getattr(request.state, "correlation_id", None),
))
OperatorService = Annotated[SelectedModelComparisonService, Depends(operator_service)]
ConsumerService = Annotated[SelectedModelComparisonService, Depends(consumer_service)]
@router.get("", response_model=ComparisonList, dependencies=[Depends(require_viewer)])
def history(service: OperatorService, project_binding_id: UUID | None = None,
limit: Limit = 50, offset: Offset = 0) -> ComparisonList:
return service.list(project_binding_id=project_binding_id, limit=limit, offset=offset)
@router.post("", response_model=ComparisonResponse, status_code=status.HTTP_201_CREATED,
dependencies=[Depends(require_operator)])
def create(body: ComparisonCreate, service: OperatorService) -> ComparisonResponse:
return service.create(body)
@router.get("/assigned", response_model=ComparisonList)
def assigned(service: ConsumerService, client: Client, limit: Limit = 50) -> ComparisonList:
return service.assigned(client, limit=limit)
@router.get("/{comparison_id}", response_model=ComparisonResponse, dependencies=[Depends(require_viewer)])
def detail(comparison_id: UUID, service: OperatorService) -> ComparisonResponse:
return service.get(comparison_id)
@router.post("/{comparison_id}/cancel", response_model=ComparisonResponse,
dependencies=[Depends(require_operator)])
def cancel(comparison_id: UUID, body: ComparisonCancel, service: OperatorService) -> ComparisonResponse:
return service.cancel(comparison_id, body)
@router.post("/{comparison_id}/claim", response_model=ComparisonClaim)
def claim(comparison_id: UUID, service: ConsumerService, client: Client) -> ComparisonClaim:
return service.claim(comparison_id, client)
@router.post("/{comparison_id}/progress", response_model=ComparisonResponse)
def progress(comparison_id: UUID, body: ComparisonProgress, service: ConsumerService,
client: Client) -> ComparisonResponse:
return service.progress(comparison_id, body, client)
@router.post("/{comparison_id}/result", response_model=ComparisonResponse)
def result(comparison_id: UUID, body: ComparisonResult, service: ConsumerService,
client: Client) -> ComparisonResponse:
return service.result(comparison_id, body, client)
@router.post("/{comparison_id}/finish", response_model=ComparisonResponse)
def finish(comparison_id: UUID, body: ComparisonFinish, service: ConsumerService,
client: Client) -> ComparisonResponse:
return service.finish(comparison_id, body, client)
backend/src/modelforge_api/api/routes/selected_replacement.py
+20
View File
@@ -0,0 +1,20 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, Response
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import require_viewer
from modelforge_api.db import get_session
from modelforge_api.domain.selected_replacement import ReplacementPreview
from modelforge_api.services.selected_replacement import SelectedReplacementService
router = APIRouter(prefix="/api/v1/selected-model-comparisons", tags=["selected-model-comparisons"])
@router.get("/{comparison_id}/replacement-preview", response_model=ReplacementPreview,
dependencies=[Depends(require_viewer)])
def preview(comparison_id: UUID, response: Response,
session: Annotated[Session, Depends(get_session)]) -> ReplacementPreview:
response.headers["Cache-Control"] = "no-store"
return SelectedReplacementService(session).preview(comparison_id)
backend/src/modelforge_api/api/routes/serving.py
+153 -2
View File
@@ -7,10 +7,14 @@ from typing import Annotated
from fastapi import APIRouter, Depends, Header, Query, Request, Response, status
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Admin, require_viewer
from modelforge_api.api.authorization import Admin, require_viewer, required_capability_for_request
from modelforge_api.api.routes.agent import AgentIdentity
from modelforge_api.db import get_session
from modelforge_api.domain.projects import ProjectConnectivityResponse
from modelforge_api.domain.serving import (
AgentServingJobCancellationAck,
AgentServingJobCancellationCheck,
AgentServingJobCancellationStatus,
AgentServingJobComplete,
AgentServingJobFailure,
AgentServingJobLease,
@@ -48,6 +52,9 @@ from modelforge_api.domain.serving import (
ServiceClientCreate,
ServiceClientCreated,
ServiceClientResponse,
ServiceCredentialResponse,
ServiceCredentialRevoke,
ServiceCredentialRotate,
SpeechTranscriptionInvokeRequest,
SpeechTranscriptionInvokeResponse,
StableEmbeddingInvokeResponse,
@@ -55,6 +62,7 @@ from modelforge_api.domain.serving import (
VisionEmbeddingInvokeResponse,
)
from modelforge_api.persistence.models import ServiceClient
from modelforge_api.services.audit import AuditContext, AuditWriter
from modelforge_api.services.manifest_registry import ManifestRegistry, get_manifest_registry
from modelforge_api.services.serving import (
CapabilityAuthenticationEvidence,
@@ -146,6 +154,60 @@ SpeechTranscriptionClient = Annotated[
]
def authenticate_check_client(
request: Request,
service: Service,
authorization: Annotated[str | None, Header(alias="Authorization")] = None,
) -> ServiceClient:
capability = required_capability_for_request(request.method, request.url.path)
if capability is None:
raise ServingError(403, "CAPABILITY_NOT_AUTHORIZED", "capability is not registered")
return _authenticate_capability_client(request, service, authorization, capability)
@router.get(
"/api/v1/capabilities/rag.embedding@1/check", response_model=ProjectConnectivityResponse
)
@router.get(
"/api/v1/capabilities/rag.reranking@1/check", response_model=ProjectConnectivityResponse
)
@router.get("/api/v1/capabilities/document.ocr@1/check", response_model=ProjectConnectivityResponse)
@router.get(
"/api/v1/capabilities/vision.embedding@1/check", response_model=ProjectConnectivityResponse
)
@router.get(
"/api/v1/capabilities/speech.transcription@1/check", response_model=ProjectConnectivityResponse
)
def check_project_connectivity(
request: Request,
response: Response,
service: Service,
client: Annotated[ServiceClient, Depends(authenticate_check_client)],
) -> ProjectConnectivityResponse:
response.headers["Cache-Control"] = "no-store"
capability = required_capability_for_request(request.method, request.url.path)
if capability is None:
raise ServingError(403, "CAPABILITY_NOT_AUTHORIZED", "capability is not registered")
service.audit = AuditWriter(
service.session,
context=AuditContext(
"service_client",
str(client.id),
request.state.correlation_id,
),
)
service.audit.write(
"PROJECT_CONNECTIVITY_CHECKED", "service_client", str(client.id), {"capability": capability}
)
service.session.commit()
return ProjectConnectivityResponse(
client_id=client.id,
project_key=service.client_response(client).project_key,
capability=capability,
correlation_id=request.state.correlation_id,
)
@router.post(
"/api/v1/admin/deployment-candidates/{candidate_id}/production-approvals",
response_model=ProductionApprovalResponse,
@@ -254,11 +316,74 @@ def create_service_client(
service: Service,
_admin: Admin,
response: Response,
http_request: Request,
) -> ServiceClientCreated:
response.headers["Cache-Control"] = "no-store"
service.audit = AuditWriter(
service.session,
context=AuditContext(
"operator",
_admin.subject,
http_request.state.correlation_id,
),
)
return service.create_client(request)
@router.get(
"/api/v1/admin/service-clients/{client_id}/credentials",
response_model=list[ServiceCredentialResponse],
)
def service_credential_history(
client_id: uuid.UUID, service: Service, _admin: Admin
) -> list[ServiceCredentialResponse]:
return service.credential_history(client_id)
@router.post(
"/api/v1/admin/service-clients/{client_id}/credentials/rotate",
response_model=ServiceClientCreated,
)
def rotate_project_credential(
client_id: uuid.UUID,
body: ServiceCredentialRotate,
service: Service,
principal: Admin,
request: Request,
) -> ServiceClientCreated:
service.audit = AuditWriter(
service.session,
context=AuditContext(
"operator",
principal.subject,
request.state.correlation_id,
),
)
return service.rotate_client_credential(client_id, body)
@router.post(
"/api/v1/admin/service-clients/{client_id}/credentials/revoke",
response_model=ServiceClientResponse,
)
def revoke_project_credential(
client_id: uuid.UUID,
body: ServiceCredentialRevoke,
service: Service,
principal: Admin,
request: Request,
) -> ServiceClientResponse:
service.audit = AuditWriter(
service.session,
context=AuditContext(
"operator",
principal.subject,
request.state.correlation_id,
),
)
return service.revoke_client_credential(client_id, body)
@router.get("/api/v1/admin/service-clients", response_model=list[ServiceClientResponse])
def service_clients(service: Service, _admin: Admin) -> list[ServiceClientResponse]:
return service.clients()
@@ -311,7 +436,7 @@ def record_project_fit_evidence(
dependencies=[Depends(require_viewer)],
)
def scheduler_overview(service: Service) -> list[SchedulerBudgetResponse]:
return service.scheduler_overview()
return service.scheduler_overview(commit=False, update_pressure=False)
@router.get(
@@ -569,6 +694,32 @@ def fail_serving_job(
return {"status": "accepted"}
@router.post(
"/api/v1/agent/serving-jobs/{job_id}/cancellation",
response_model=AgentServingJobCancellationStatus,
)
def check_serving_job_cancellation(
job_id: uuid.UUID,
request: AgentServingJobCancellationCheck,
service: Service,
identity: AgentIdentity,
) -> AgentServingJobCancellationStatus:
_credential, node = identity
return service.check_lab_cancellation(job_id, node, request)
@router.post("/api/v1/agent/serving-jobs/{job_id}/cancelled")
def acknowledge_serving_job_cancellation(
job_id: uuid.UUID,
request: AgentServingJobCancellationAck,
service: Service,
identity: AgentIdentity,
) -> dict[str, str]:
_credential, node = identity
service.acknowledge_lab_cancellation(job_id, node, request)
return {"status": "accepted"}
@router.post(
"/api/v1/agent/serving-state",
response_model=AgentServingStateAck,
backend/src/modelforge_api/api/routes/update_preflight.py
+36
View File
@@ -0,0 +1,36 @@
from typing import Annotated
from fastapi import APIRouter, Depends
from modelforge_api.api.authorization import Admin
from modelforge_api.domain.release import build_identity
from modelforge_api.domain.update_preflight import UpdatePreflight, update_preflight
from modelforge_api.settings import Settings, get_settings
router = APIRouter(prefix="/api/v1/admin/updates", tags=["update-preflight"])
@router.get(
"/preflight",
response_model=UpdatePreflight,
)
def read_update_preflight(
_admin: Admin,
settings: Annotated[Settings, Depends(get_settings)],
) -> UpdatePreflight:
"""Report local update evidence; schema is the app build target, not the applied DB revision."""
identity = build_identity(
source_commit=settings.build_commit,
built_at=settings.build_timestamp,
image_digest=settings.build_image_digest,
)
return update_preflight(
version=identity.version,
channel=identity.channel,
schema_revision=identity.schema_revision,
agent_protocol_version=identity.agent_protocol_version,
source_commit=identity.source_commit,
built_at=identity.built_at,
image_digest=identity.image_digest,
)
backend/src/modelforge_api/domain/agent_protocol.py
+1
View File
@@ -123,6 +123,7 @@ class EnrollmentTokenSummary(AgentProtocolModel):
expires_at: datetime
used_at: datetime | None
revoked_at: datetime | None
enrolled_node_id: str | None = None
class NodeManagementUpdate(AgentProtocolModel):
backend/src/modelforge_api/domain/alert_notifications.py
+49
View File
@@ -0,0 +1,49 @@
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from uuid import UUID
from pydantic import BaseModel, ConfigDict
class NotificationStatus(StrEnum):
PENDING = "PENDING"
DISPATCHING = "DISPATCHING"
RETRY_WAIT = "RETRY_WAIT"
DELIVERED = "DELIVERED"
FAILED = "FAILED"
HELD_FOR_REVIEW = "HELD_FOR_REVIEW"
class NotificationKind(StrEnum):
FIRING = "FIRING"
RESOLVED = "RESOLVED"
TEST = "TEST"
class NotificationDeliveryResponse(BaseModel):
"""Safe operator-facing delivery metadata. Payload and destination are never returned."""
model_config = ConfigDict(extra="forbid", from_attributes=True)
id: UUID
alert_history_event_id: UUID | None
route_key: str
event_kind: NotificationKind
severity: str | None
status: NotificationStatus
attempt_count: int
next_attempt_at: datetime | None
last_attempt_at: datetime | None
delivered_at: datetime | None
last_http_status: int | None
last_error_code: str | None
created_at: datetime
class NotificationTestResponse(BaseModel):
model_config = ConfigDict(extra="forbid")
delivery: NotificationDeliveryResponse
message: str
backend/src/modelforge_api/domain/audit_timeline.py
+69
View File
@@ -0,0 +1,69 @@
"""Typed, bounded query and response models for the operator audit timeline."""
from __future__ import annotations
from datetime import timedelta
from uuid import UUID
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, model_validator
AUDIT_TIMELINE_MAX_PAGE_SIZE = 100
AUDIT_TIMELINE_MAX_TIME_RANGE = timedelta(days=366)
class AuditTimelineRequest(BaseModel):
"""Validated filters for one bounded page of audit events.
Text filters are case-insensitive substring matches. A time filter must specify both ends,
and its window is limited to one year plus one leap day.
"""
model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
principal: str | None = Field(default=None, min_length=1, max_length=255)
action: str | None = Field(default=None, min_length=1, max_length=128)
resource: str | None = Field(default=None, min_length=1, max_length=128)
outcome: str | None = Field(default=None, min_length=1, max_length=32)
occurred_after: AwareDatetime | None = None
occurred_before: AwareDatetime | None = None
correlation_id: str | None = Field(default=None, min_length=1, max_length=64)
limit: int = Field(default=50, ge=1, le=AUDIT_TIMELINE_MAX_PAGE_SIZE)
cursor: str | None = Field(default=None, min_length=1, max_length=256)
@model_validator(mode="after")
def validate_time_window(self) -> AuditTimelineRequest:
if (self.occurred_after is None) != (self.occurred_before is None):
raise ValueError("occurred_after and occurred_before must be supplied together")
if self.occurred_after is not None and self.occurred_before is not None:
if self.occurred_after >= self.occurred_before:
raise ValueError("occurred_after must be earlier than occurred_before")
if self.occurred_before - self.occurred_after > AUDIT_TIMELINE_MAX_TIME_RANGE:
raise ValueError("audit time range exceeds the 366-day maximum")
return self
class AuditTimelineEvent(BaseModel):
"""Safe event fields returned to the operator console."""
model_config = ConfigDict(frozen=True)
id: UUID
sequence: int = Field(ge=1)
occurred_at: AwareDatetime
correlation_id: str
actor_type: str
actor_id: str
action: str
resource_type: str
resource_id: str | None
outcome: str
class AuditTimelineResponse(BaseModel):
"""One page of audit events and an opaque keyset cursor for the next page."""
model_config = ConfigDict(frozen=True)
events: list[AuditTimelineEvent]
next_cursor: str | None
limit: int = Field(ge=1, le=AUDIT_TIMELINE_MAX_PAGE_SIZE)
backend/src/modelforge_api/domain/benchmarking.py
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
import uuid
from datetime import datetime
from typing import Any, Literal
from pydantic import Field
from modelforge_api.domain.contracts import BenchmarkRunContract, StrictModel
from modelforge_api.domain.lab import LabTarget
class BenchmarkSuiteResponse(StrictModel):
id: uuid.UUID
key: str
revision: str
capability: Literal["rag.embedding@1"]
dataset_revision: str
definition: dict[str, Any]
immutable_at: datetime
class BenchmarkRunStartRequest(StrictModel):
capability: Literal["rag.embedding@1"]
suite_id: uuid.UUID
target: LabTarget
deployment_id: uuid.UUID
experiment_route_id: uuid.UUID
BenchmarkRunStatus = Literal[
"accepted",
"running",
"cancel_requested",
"completed",
"failed",
"cancelled",
"interrupted",
]
class BenchmarkRunProgress(StrictModel):
completed_invocations: int = Field(ge=0)
total_invocations: int = Field(ge=1)
active_case: str | None = None
class BenchmarkRunResponse(StrictModel):
id: uuid.UUID
capability: Literal["rag.embedding@1"]
suite_id: uuid.UUID
capability_deployment_id: uuid.UUID
status: BenchmarkRunStatus
contract: BenchmarkRunContract
target_snapshot: dict[str, Any]
progress: BenchmarkRunProgress
results: dict[str, Any]
failure_code: str | None = None
cancellation_requested_at: datetime | None = None
created_at: datetime
completed_at: datetime | None = None
class BenchmarkRunHistoryPage(StrictModel):
"""A bounded page of actor-scoped benchmark history."""
runs: list[BenchmarkRunResponse]
next_cursor: str | None
limit: int = Field(ge=1, le=100)
backend/src/modelforge_api/domain/configuration.py
+133
View File
@@ -0,0 +1,133 @@
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from modelforge_api.domain.configuration_reference import Sensitivity
class ConfigurationSource(StrEnum):
"""Source that supplied the effective value in Pydantic's unchanged priority order."""
PYDANTIC_DEFAULT = "pydantic_default"
INITIALIZATION = "initialization"
PROCESS_ENVIRONMENT = "process_environment"
DOTENV = "dotenv"
SECRETS_FILE = "secrets_file"
RUNTIME_UPDATE = "runtime_update"
OPERATOR_OVERRIDE = "operator_override"
class ConfigurationApplicationMode(StrEnum):
"""How a setting can take effect in the current product contract."""
DYNAMIC = "dynamic"
RESTART_REQUIRED = "restart_required"
class ConfigurationScope(StrEnum):
CONTROL_PLANE = "control_plane"
class ConfigurationSettingResponse(BaseModel):
key: str
environment_variable: str
description: str
sensitivity: Sensitivity
scope: ConfigurationScope = ConfigurationScope.CONTROL_PLANE
required_in_production: bool
application_mode: ConfigurationApplicationMode
editable: bool = False
configured: bool
source: ConfigurationSource
value: str | int | float | bool | None = None
display_name: str = ""
category: str = "process"
data_type: str = "string"
default_value: str | int | float | bool | None = None
validation_rules: dict[str, str | int | float] = Field(default_factory=dict)
consumer: str | None = None
rollback: str = "Restore deployment configuration and restart the component."
class ConfigurationCatalogResponse(BaseModel):
environment: str
read_only: bool = True
settings: list[ConfigurationSettingResponse] = Field(default_factory=list)
revision: int = 0
overrides: dict[str, int | float] = Field(default_factory=dict)
class ConfigurationChange(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
expected_revision: int = Field(ge=0)
# Null removes the operator override and reveals the deployment-resolved value.
changes: dict[str, int | float | None] = Field(min_length=1, max_length=14)
class ConfigurationApply(ConfigurationChange):
preview_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
confirm: Literal[True]
class ConfigurationDifference(BaseModel):
key: str
before: int | float
after: int | float
source_before: ConfigurationSource
source_after: ConfigurationSource
class ConfigurationPreview(BaseModel):
expected_revision: int
preview_digest: str
differences: list[ConfigurationDifference]
affected_components: list[str]
impact: str
rollback: str
overrides: dict[str, int | float]
class ConfigurationApplied(BaseModel):
revision: int
status: Literal["accepted_for_new_requests"] = "accepted_for_new_requests"
acknowledged_by: str
preview_digest: str
class ConfigurationRevisionResponse(BaseModel):
revision: int
previous_revision: int
created_at: datetime
actor: str
correlation_id: str
overrides: dict[str, int | float]
preview_digest: str
class ConfigurationExport(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
format: str
product: Literal["ModelForge"] = "ModelForge"
source_version: str
overrides: dict[str, int | float] = Field(max_length=14)
secret_slots: list[str] = Field(max_length=20)
class ConfigurationImport(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
expected_revision: int = Field(ge=0)
document: ConfigurationExport
class ConfigurationProposal(BaseModel):
change: ConfigurationChange
preview: ConfigurationPreview
class ConfigurationRollback(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
expected_revision: int = Field(ge=0)
backend/src/modelforge_api/domain/configuration_policy.py
+14
View File
@@ -0,0 +1,14 @@
"""The closed set of settings with a proven per-request serving consumer."""
SERVING_OVERRIDE_KEYS = frozenset(
{
"gateway_max_batch_size",
"gateway_max_input_characters",
"gateway_request_timeout_seconds",
"gateway_queue_timeout_seconds",
"scheduler_telemetry_stale_seconds",
}
)
CONFIGURATION_FORMAT = "modelforge.serving-configuration.v1"
CONFIGURATION_CONSUMER = "control_plane.serving"
backend/src/modelforge_api/domain/configuration_reference.py
+40 -5
View File
@@ -73,14 +73,30 @@ SETTING_DOCS: dict[str, SettingDoc] = {
"A wildcard is refused in production because requests are credentialed."
),
# ---------------------------------------------------------------- credentials
"operator_identities": _secret(
"Optional JSON array of named human identities with role, credential ID, SHA-256 key "
"digest and timezone-aware expiry. Provision with scripts/operator_identity.py. "
"Replace this deployment setting and recreate the API to rotate or revoke access."
),
"operator_password_hash": _secret(
"Optional salted scrypt verifier generated by scripts/operator_password.py. Enables "
"password login with 15-minute sessions; requires the separate strong operator API key. "
"Changing either value invalidates password sessions."
),
"operator_api_key": _secret(
"Operator API key guarding every admin route. Generate at least 32 random characters; "
"ModelForge never mints one for you.",
required=True,
"Legacy bootstrap administrator key. Generate at least 32 random characters. "
"Production requires this key or an unexpired named administrator in operator_identities."
),
"hf_token": _secret(
"Optional Hugging Face token, used only for acquiring gated repositories. "
"It is never passed to a runtime and never leaves the control plane."
"Optional deployment Hugging Face token for control-plane metadata. Compose may also "
"provision it independently to node-agent downloads; API secret activation does not update nodes."
),
"secret_slot_root_key": _secret(
"External base64 32-byte AES-256 key for the managed Hugging Face control-plane slot. "
"Keep independent of backup and operator keys; retain it externally for historical restore."
),
"secret_slot_root_key_id": _p(
"External key identifier bound to encrypted HF slot versions. Replacing a key does not rewrap history."
),
"backup_encryption_key": _secret(
"Base64 AES-256 key for backup encryption. Without it no backup can be produced, and "
@@ -160,6 +176,25 @@ SETTING_DOCS: dict[str, SettingDoc] = {
),
"observability_monitor_enabled": _p("Run SLO and alert evaluation in this process."),
"observability_poll_interval_seconds": _p("Interval between observability evaluation passes."),
"alert_notifications_enabled": _p("Queue and dispatch the deployment-managed operational alert webhook."),
"alert_webhook_url": _secret(
"Deployment-managed HTTPS notification destination. Its exact hostname must also appear "
"in alert_webhook_allowed_hosts; the URL is never returned by the API."
),
"alert_webhook_allowed_hosts": _p(
"Comma-separated exact destination host allowlist for the alert webhook. Wildcards and "
"redirects are not accepted."
),
"alert_webhook_min_severity": _p(
"Minimum alert severity sent to the single webhook route: INFO, WARNING or CRITICAL."
),
"alert_notification_timeout_seconds": _p("Total timeout for one webhook delivery."),
"alert_notification_max_attempts": _p("Maximum webhook attempts before a delivery fails permanently."),
"alert_notification_batch_size": _p("Maximum outbox rows claimed by one notification dispatch pass."),
"alert_notification_lease_seconds": _p("Lease duration before an interrupted delivery can be reclaimed."),
"alert_notification_retry_base_seconds": _p("Base delay for bounded exponential webhook retries."),
"model_update_monitor_enabled": _p("In production, check explicitly followed model references for changed revisions. Metadata only; never downloads, tests or promotes a model."),
"model_update_check_interval_seconds": _p("Minimum interval between automatic checks of each followed model reference; default one day. Each API process checks at most one due reference per minute."),
"recovery_reconciliation_enabled": _p("Reconcile interrupted backups and restores at startup."),
# ---------------------------------------------------------------- recovery
"backup_pg_dump_path": _p("pg_dump executable. Must match the server major version."),
backend/src/modelforge_api/domain/contracts.py
+2 -2
View File
@@ -341,12 +341,12 @@ class BenchmarkEnvironmentFingerprint(StrictModel):
launch_arguments: dict[str, Any]
cuda_version: str | None
driver_version: str | None
accelerator_name: str
accelerator_name: str | None
accelerator_uuid: str | None
context_length: int | None
concurrency: int = Field(ge=1)
seed: int | None
operating_system: str
operating_system: str | None
python_version: str | None = None
@property
backend/src/modelforge_api/domain/dispatch_context.py
+53
View File
@@ -0,0 +1,53 @@
"""Internal trusted scheduler identity; this is not an authorization or public input contract."""
from __future__ import annotations
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, model_validator
DispatchPriority = Literal["production", "interactive", "background", "benchmark", "lab"]
class DispatchContext(BaseModel):
"""Identifiers only, constructed after gateway authentication or operator admission.
An operator context does not grant permission to run a candidate. The eventual caller must
independently admit the exact target and persist its operation before using this context.
"""
model_config = ConfigDict(frozen=True, extra="forbid", strict=True)
actor_kind: Literal["service_client", "operator"]
actor_id: str = Field(min_length=1, max_length=128)
lease_owner: str = Field(min_length=1, max_length=255)
workload_priority: DispatchPriority
service_client_id: UUID | None = None
operation_id: UUID | None = None
@model_validator(mode="after")
def coherent_identity(self) -> DispatchContext:
if self.actor_kind == "service_client":
if (self.service_client_id is None or self.actor_id != str(self.service_client_id)
or self.operation_id is not None or self.workload_priority == "lab"):
raise ValueError("service dispatch requires its exact client identity and configured priority")
elif (self.service_client_id is not None or self.operation_id is None
or self.workload_priority != "lab" or self.lease_owner != f"operator:{self.actor_id}"):
raise ValueError("operator dispatch requires a persisted operation and fixed lab priority")
return self
@classmethod
def for_service_client(cls, client_id: UUID, name: str, priority: str) -> DispatchContext:
"""Priority is read from the authenticated persisted client, never the request body."""
return cls.model_validate({"actor_kind": "service_client", "actor_id": str(client_id),
"lease_owner": name, "service_client_id": client_id, "workload_priority": priority})
@classmethod
def for_operator(cls, subject: str, operation_id: UUID) -> DispatchContext:
return cls(actor_kind="operator", actor_id=subject, lease_owner=f"operator:{subject}",
workload_priority="lab", operation_id=operation_id)
def provenance(self) -> dict[str, str | None]:
return {"actor_kind": self.actor_kind, "actor_id": self.actor_id,
"service_client_id": str(self.service_client_id) if self.service_client_id else None,
"operation_id": str(self.operation_id) if self.operation_id else None,
"workload_priority": self.workload_priority}
backend/src/modelforge_api/domain/external_creative.py
+139
View File
@@ -0,0 +1,139 @@
from __future__ import annotations
import uuid
from datetime import datetime
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
CreativeWorkload = Literal[
"image",
"video",
"chat",
"speech_generation",
"voice_clone",
"music_generation",
"sound_generation",
"foley_generation",
"transcription",
"audio_enhancement",
"lip_sync",
]
CreativeAllocationState = Literal["granted", "queued", "blocked", "uncertain", "released"]
CreativeRequiredAction = Literal["release_idle_comfy", "unload_configured_ollama"]
class ExternalAllocationCreate(BaseModel):
model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
request_id: str = Field(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$")
idempotency_key: str = Field(
min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
)
installation_id: str = Field(
min_length=1, max_length=120, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
)
workload: CreativeWorkload
resource_ref: str = Field(
min_length=1, max_length=255, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$"
)
provider_resource_id: str = Field(
min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
)
required_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
priority: Literal["production", "interactive", "background", "benchmark"] = "interactive"
ttl_seconds: int = Field(default=600, ge=30, le=3600)
use_measured_profile: bool = True
class ExternalAllocationResponse(BaseModel):
allocation_id: uuid.UUID
project_id: uuid.UUID
request_id: str
installation_id: str
workload: CreativeWorkload
resource_ref: str
provider_resource_id: str
required_vram_bytes: int
priority: str
state: CreativeAllocationState
decision_code: str
required_actions: list[CreativeRequiredAction]
accelerator_id: uuid.UUID | None
acquired_at: datetime | None
heartbeat_at: datetime | None
expires_at: datetime | None
uncertain_at: datetime | None
released_at: datetime | None
heartbeat_interval_seconds: int = 30
decision_evidence: dict[str, Any]
queue_position: int | None = None
class ExternalAllocationAction(BaseModel):
model_config = ConfigDict(extra="forbid")
request_id: str = Field(min_length=1, max_length=128)
observed_peak_vram_bytes: int | None = Field(default=None, gt=0, le=256 * 1024**3)
attestation: ExternalActionAttestation | None = None
class ExternalResourceApproval(BaseModel):
model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
project_id: uuid.UUID
resource_ref: str = Field(
min_length=1, max_length=255, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:@/-]*$"
)
workload: CreativeWorkload
provider_resource_id: str = Field(
min_length=1, max_length=128, pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$"
)
approved: bool = True
configured_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
class ExternalResourceResponse(ExternalResourceApproval):
resource_id: uuid.UUID
measured_peak_vram_bytes: int | None = None
measurement_count: int = 0
updated_at: datetime
class ExternalResourceList(BaseModel):
items: list[ExternalResourceResponse]
total: int
class ExternalActionAttestation(BaseModel):
model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
action: Literal["release_idle_comfy", "unload_configured_ollama"]
observed_before: int = Field(ge=0, le=256 * 1024**3)
observed_after: int = Field(ge=0, le=256 * 1024**3)
completed_at: datetime
class ExternalActionAttestationResponse(ExternalActionAttestation):
attestation_id: uuid.UUID
allocation_id: uuid.UUID
validated: bool
class ExternalAllocationReconcile(BaseModel):
model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
action: Literal["confirm_released", "keep_uncertain"]
reason: str = Field(min_length=3, max_length=500)
class ExternalAllocationList(BaseModel):
items: list[ExternalAllocationResponse]
total: int
class ExternalProfileObservation(BaseModel):
model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
allocation_id: uuid.UUID
resource_ref: str = Field(min_length=1, max_length=255)
provider_resource_id: str = Field(min_length=1, max_length=128)
workload: CreativeWorkload
observed_peak_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
reserved_vram_bytes: int = Field(gt=0, le=256 * 1024**3)
duration_ms: int = Field(ge=0, le=7 * 24 * 3600 * 1000)
outcome: Literal["succeeded", "failed", "cancelled", "uncertain"]
backend/src/modelforge_api/domain/lab.py
+145
View File
@@ -0,0 +1,145 @@
from __future__ import annotations
import uuid
from datetime import datetime
from typing import Annotated, Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
from modelforge_api.domain.serving import (
EmbeddingInvokeRequest,
OCRInvokeRequest,
RerankingInvokeRequest,
SpeechTranscriptionInvokeRequest,
VisionEmbeddingInvokeRequest,
)
class LabModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class LabTarget(LabModel):
deployment_candidate_id: uuid.UUID
artifact_set_id: uuid.UUID
runtime_profile_id: uuid.UUID
compute_node_id: uuid.UUID
execution_approval_id: uuid.UUID
class LabEmbeddingInput(LabModel):
capability: Literal["rag.embedding"]
payload: EmbeddingInvokeRequest
class LabRerankingInput(LabModel):
capability: Literal["rag.reranking"]
payload: RerankingInvokeRequest
class LabOCRInput(LabModel):
capability: Literal["document.ocr"]
payload: OCRInvokeRequest
class LabVisionInput(LabModel):
capability: Literal["vision.embedding"]
payload: VisionEmbeddingInvokeRequest
class LabSpeechInput(LabModel):
capability: Literal["speech.transcription"]
payload: SpeechTranscriptionInvokeRequest
LabInput = Annotated[
LabEmbeddingInput | LabRerankingInput | LabOCRInput | LabVisionInput | LabSpeechInput,
Field(discriminator="capability"),
]
class LabLimits(LabModel):
timeout_seconds: int = Field(default=60, ge=5, le=300, strict=True)
queue_timeout_seconds: int = Field(default=10, ge=1, le=60, strict=True)
retention_seconds: int = Field(default=300, ge=60, le=3600, strict=True)
max_output_bytes: int = Field(default=65536, ge=1, le=1048576, strict=True)
concurrency: Literal[1] = 1
@model_validator(mode="after")
def bounded_queue(self) -> LabLimits:
if self.queue_timeout_seconds > self.timeout_seconds:
raise ValueError("queue timeout cannot exceed total timeout")
return self
class LabPreflightRequest(LabModel):
target: LabTarget
input: LabInput
contract_version: Literal[1] = 1
limits: LabLimits = Field(default_factory=LabLimits)
class LabExecutionStartRequest(LabPreflightRequest):
"""An explicitly selected embedding experiment; body input is never echoed in status."""
deployment_id: uuid.UUID
experiment_route_id: uuid.UUID
class LabInputSummary(LabModel):
sha256: str
item_count: int
encoded_bytes: int
class LabBinding(LabTarget):
compatibility_assessment_id: uuid.UUID
runtime_probe_id: uuid.UUID
capability: str
contract_version: Literal[1] = 1
runtime_profile_fingerprint: str
assessment_fingerprint: str
approval_evidence_fingerprint: str
probe_environment_fingerprint: str
accelerator_id: uuid.UUID | None = None
class LabPreflightResponse(LabModel):
input_summary: LabInputSummary
binding: LabBinding | None = None
limits: LabLimits
blockers: list[str]
checks_passed: bool
execution_started: Literal[False] = False
permission_granted: Literal[False] = False
LabExecutionStatus = Literal[
"accepted",
"preparing",
"queued",
"running",
"cancel_requested",
"completed",
"failed",
"cancelled",
"interrupted",
]
class LabExecutionRunResponse(LabModel):
id: uuid.UUID
capability: Literal["rag.embedding@1"] = "rag.embedding@1"
status: LabExecutionStatus
target_snapshot: dict[str, object]
input_summary: LabInputSummary
output_metadata: dict[str, object] | None = None
worker_observations: dict[str, object] | None = None
failure_code: str | None = None
input_payload_expires_at: datetime
retention_seconds: int = Field(ge=60, le=3600)
created_at: datetime
started_at: datetime | None = None
cancellation_requested_at: datetime | None = None
cancellation_acknowledged_at: datetime | None = None
finished_at: datetime | None = None
backend/src/modelforge_api/domain/lifecycle_contracts.py
+24
View File
@@ -133,7 +133,31 @@ def assert_deployment_transition(
raise ValueError(f"lifecycle transition {current} -> {target} is not allowed")
class ServingTransitionSelection(LifecycleModel):
"""Explicit selection; the server derives the complete immutable serving contract."""
current_deployment_id: uuid.UUID
candidate_deployment_id: uuid.UUID
production_approval_id: uuid.UUID
expected_route_generation: int = Field(ge=0)
comparison_ids: dict[uuid.UUID, uuid.UUID] = Field(min_length=1, max_length=1000)
class ServingTransitionContract(LifecycleModel):
version: Literal[1] = 1
production_validation: Literal["SATISFIED"] = "SATISFIED"
selection: ServingTransitionSelection
capability_contract_id: uuid.UUID
current: dict[str, Any]
candidate: dict[str, Any]
experiment_routes: dict[str, dict[str, Any]]
bindings: dict[str, dict[str, Any]]
comparisons: dict[str, dict[str, Any]]
production_approval_fingerprint: str
class LifecycleEvidenceBundle(LifecycleModel):
serving_transition: ServingTransitionSelection | None = None
artifact_set_id: uuid.UUID | None = None
model_revision_id: uuid.UUID | None = None
runtime_profile_id: uuid.UUID | None = None
backend/src/modelforge_api/domain/managed_identity.py
+89
View File
@@ -0,0 +1,89 @@
"""Bounded, secret-safe contracts for the managed personal-key provider."""
from datetime import datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
Role = Literal["viewer", "operator", "admin"]
class ManagedIdentityRequest(BaseModel):
model_config = ConfigDict(extra="forbid", hide_input_in_errors=True, strict=True)
reason: str = Field(min_length=8, max_length=500)
@field_validator("reason")
@classmethod
def safe_reason(cls, value: str) -> str:
value = value.strip()
if len(value) < 8 or any(prefix in value for prefix in ("mfopm_", "mfop_", "mfsvc_", "mfnode_", "mfenroll_")):
raise ValueError("Use a non-secret explanation of at least eight characters")
return value
class ManagedIdentityCreate(ManagedIdentityRequest):
slug: str = Field(pattern=r"^[a-z][a-z0-9-]{0,62}$")
role: Role
expires_in_days: int = Field(ge=1, le=365)
expected_revision: int = Field(ge=0)
class ManagedIdentityRotate(ManagedIdentityRequest):
expected_version: int = Field(ge=1)
expires_in_days: int = Field(ge=1, le=365)
overlap_seconds: int = Field(ge=0, le=900)
class ManagedIdentityChange(ManagedIdentityRequest):
expected_version: int = Field(ge=1)
role: Role
enabled: bool
class ManagedIdentityRevoke(ManagedIdentityRequest):
expected_version: int = Field(ge=1)
confirm_subject: str = Field(max_length=71)
class ManagedCredentialMetadata(BaseModel):
id: str
expires_at: datetime
revoked_at: datetime | None
superseded_at: datetime | None
last_used_at: datetime | None
activated: bool
usable: bool
class ManagedIdentityMetadata(BaseModel):
subject: str
slug: str
source: Literal["managed"] = "managed"
role: Role
enabled: bool
version: int
created_at: datetime
credentials: list[ManagedCredentialMetadata]
class DeploymentIdentityMetadata(BaseModel):
subject: str
role: Role
credential_id: str
expires_at: datetime
source: Literal["deployment"] = "deployment"
class ManagedIdentityList(BaseModel):
revision: int
initialized: bool
identities: list[ManagedIdentityMetadata]
deployment_identities: list[DeploymentIdentityMetadata]
class ManagedIdentityResult(BaseModel):
revision: int
identity: ManagedIdentityMetadata
class ManagedIdentityIssued(ManagedIdentityResult):
one_time_key: str = Field(repr=False)
backend/src/modelforge_api/domain/model_changes.py
+146
View File
@@ -0,0 +1,146 @@
from __future__ import annotations
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, model_validator
ModelChangeStatus = Literal[
"searching",
"found",
"source_checked",
"downloaded",
"ready_to_test",
"tested",
"comparison_available",
"migration_required",
"ready_to_activate",
"active",
"checked",
"rolled_back",
"rejected",
]
class Contract(BaseModel):
model_config = ConfigDict(extra="forbid")
class ModelChangeCreate(Contract):
project_binding_id: UUID
reason: str = Field(default="Model verbeteren", min_length=1, max_length=500)
class ModelChangeCandidateLink(Contract):
expected_version: int = Field(ge=1)
candidate_model_id: UUID | None = None
candidate_revision_id: UUID | None = None
candidate_artifact_set_id: UUID | None = None
candidate_deployment_id: UUID | None = None
discovery_assessment_id: UUID | None = None
download_plan_id: UUID | None = None
artifact_job_id: UUID | None = None
@model_validator(mode="after")
def at_least_one_identity(self) -> ModelChangeCandidateLink:
if not any((
self.candidate_model_id,
self.candidate_revision_id,
self.candidate_artifact_set_id,
self.candidate_deployment_id,
self.discovery_assessment_id,
self.download_plan_id,
self.artifact_job_id,
)):
raise ValueError("at least one candidate or acquisition identity is required")
return self
class ModelChangeEvidenceLink(Contract):
expected_version: int = Field(ge=1)
selected_comparison_id: UUID | None = None
migration_plan_id: UUID | None = None
lifecycle_plan_id: UUID | None = None
@model_validator(mode="after")
def at_least_one_evidence(self) -> ModelChangeEvidenceLink:
if not any((self.selected_comparison_id, self.migration_plan_id, self.lifecycle_plan_id)):
raise ValueError("at least one evidence reference is required")
return self
class ModelChangeReject(Contract):
expected_version: int = Field(ge=1)
reason: str = Field(min_length=3, max_length=1000)
class ChangeModelIdentity(Contract):
model_id: UUID | None = None
display_name: str | None = None
revision_id: UUID | None = None
resolved_commit_sha: str | None = None
artifact_set_id: UUID | None = None
deployment_id: UUID | None = None
class ChangeAcquisition(Contract):
discovery_assessment_id: UUID | None = None
security_gate: str = "unknown"
license_gate: str = "unknown"
hardware_fit: str = "unknown"
storage_bytes: int | None = None
download_plan_id: UUID | None = None
artifact_job_id: UUID | None = None
download_status: str | None = None
class ChangeImpact(Contract):
affected_projects: list[str] = Field(default_factory=list)
affected_project_count: int = 0
application_change_required: bool = False
migration_required: bool = False
migration_kind: str = "none"
downtime_expected: bool = False
rollback_available: bool = False
blockers: list[str] = Field(default_factory=list)
class ChangeProgressStep(Contract):
key: str
label: str
state: Literal["done", "current", "upcoming", "blocked"]
class ModelChangeResponse(Contract):
id: UUID
project_binding_id: UUID
project_id: UUID
project_key: str
project_name: str
capability: str
status: ModelChangeStatus
stored_status: ModelChangeStatus
status_reason: str
version: int
current: ChangeModelIdentity
candidate: ChangeModelIdentity | None = None
acquisition: ChangeAcquisition
selected_comparison_id: UUID | None = None
migration_plan_id: UUID | None = None
lifecycle_plan_id: UUID | None = None
impact: ChangeImpact
next_action: str
progress: list[ChangeProgressStep]
created_by: str
updated_by: str
activated_at: datetime | None = None
checked_at: datetime | None = None
closed_at: datetime | None = None
created_at: datetime
updated_at: datetime
class ModelChangeList(Contract):
items: list[ModelChangeResponse]
total: int
backend/src/modelforge_api/domain/model_updates.py
+51
View File
@@ -0,0 +1,51 @@
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
class UpdateModel(BaseModel):
model_config = ConfigDict(extra="forbid", from_attributes=True)
class ModelWatchCreate(UpdateModel):
model_id: UUID
baseline_revision_id: UUID
upstream_ref: str = Field(default="main", min_length=1, max_length=255, pattern=r"^[A-Za-z0-9][A-Za-z0-9._/-]*$")
class ModelWatchResponse(UpdateModel):
id: UUID
model_id: UUID
baseline_revision_id: UUID
baseline_commit_sha: str
upstream_ref: str
last_checked_at: datetime | None
last_error_code: str | None
enabled: bool
version: int
class ModelWatchUpdate(UpdateModel):
expected_version: int = Field(ge=1)
expected_baseline_revision_id: UUID
baseline_revision_id: UUID
enabled: bool
reason: str = Field(min_length=1, max_length=500, pattern=r"\S")
class ModelUpdateMonitorResponse(UpdateModel):
enabled: bool
check_interval_seconds: int
class ModelUpdateNoticeResponse(UpdateModel):
id: UUID
watch_id: UUID
model_id: UUID
baseline_commit_sha: str
candidate_revision_id: UUID
candidate_commit_sha: str
status: Literal["new", "dismissed"]
discovered_at: datetime
backend/src/modelforge_api/domain/node_lifecycle.py
+54
View File
@@ -0,0 +1,54 @@
from __future__ import annotations
import uuid
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
class NodeLifecycleModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class NodeLifecyclePreviewRequest(NodeLifecycleModel):
target_enabled: bool
class NodeLifecycleChange(NodeLifecycleModel):
target_enabled: bool
expected_generation: int = Field(ge=1)
dependency_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
operator: str = Field(min_length=1, max_length=255)
reason: str = Field(min_length=10, max_length=2000)
confirmation: str = Field(min_length=1, max_length=255)
class NodeLifecycleBlocker(NodeLifecycleModel):
code: str
message: str
record_type: str
count: int = Field(ge=1)
resource_ids: list[str] = Field(default_factory=list)
class NodeLifecyclePreview(NodeLifecycleModel):
node_id: uuid.UUID
persisted_identity: str
hostname: str
display_name: str
current_enabled: bool
current_liveness: str
target_enabled: bool
node_generation: int = Field(ge=1)
dependency_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
can_apply: bool
blockers: list[NodeLifecycleBlocker]
agent_evidence_after_enable: Literal["unchanged", "awaiting_fresh_authenticated_heartbeat"]
class NodeLifecycleResult(NodeLifecycleModel):
node_id: uuid.UUID
enabled: bool
generation: int = Field(ge=1)
liveness: Literal["offline", "disabled"]
agent_evidence_state: Literal["unchanged", "awaiting_fresh_authenticated_heartbeat"]
backend/src/modelforge_api/domain/observability.py
+28
View File
@@ -363,6 +363,34 @@ class CapacitySnapshotResponse(OperationalModel):
freshness_seconds: float
class ActiveSchedulerLeaseResponse(OperationalModel):
lease_id: uuid.UUID
project_id: uuid.UUID | None
node_id: uuid.UUID | None
accelerator_id: uuid.UUID
deployment_id: uuid.UUID
lease_state: Literal["PENDING", "GRANTED", "ACTIVE", "RELEASING"]
workload_status: Literal["QUEUED", "RUNNING", "UNKNOWN"]
priority: str
lease_type: Literal[
"RESIDENCY_RESERVATION", "REQUEST_EXECUTION", "LOAD_TRANSITION", "UNKNOWN"
]
reserved_vram_bytes: int
materialized_vram_bytes: int
created_at: datetime
acquired_at: datetime | None
heartbeat_at: datetime | None
expires_at: datetime
class ActiveSchedulerLeaseInventoryResponse(OperationalModel):
observed_at: datetime
completeness: Literal["COMPLETE", "TRUNCATED"]
limit: int
returned_count: int
leases: list[ActiveSchedulerLeaseResponse]
class TrendResponse(OperationalModel):
subject: str
sample_count: int
backend/src/modelforge_api/domain/operator_identity.py
+57
View File
@@ -0,0 +1,57 @@
"""Deployment-owned, hash-only human identities; independent of machine credentials."""
from __future__ import annotations
import json
from functools import lru_cache
from typing import Literal
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
class OperatorIdentity(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, hide_input_in_errors=True)
credential_id: str = Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$")
subject: str = Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._@-]{0,127}$")
role: Literal["viewer", "operator", "admin"]
token_sha256: str = Field(pattern=r"^[0-9a-f]{64}$", repr=False)
expires_at: AwareDatetime
def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
result: dict[str, object] = {}
for key, value in pairs:
if key in result:
raise ValueError("Duplicate identity field")
result[key] = value
return result
@lru_cache(maxsize=8)
def parse_operator_identities(document: str) -> tuple[OperatorIdentity, ...]:
"""Bound and validate once per deployment snapshot; never echo rejected input."""
try:
if len(document) > 65_536:
raise ValueError("Identity document too large")
payload = json.loads(document, object_pairs_hook=_unique_object)
if not isinstance(payload, list) or not 1 <= len(payload) <= 100:
raise ValueError("Identity list required")
identities = TypeAdapter(tuple[OperatorIdentity, ...]).validate_python(payload)
identifiers: set[str] = set()
digests: set[str] = set()
roles: dict[str, str] = {}
for identity in identities:
if (
identity.subject == "legacy-operator"
or identity.credential_id in identifiers
or identity.token_sha256 in digests
or roles.get(identity.subject, identity.role) != identity.role
):
raise ValueError("Ambiguous identity configuration")
identifiers.add(identity.credential_id)
digests.add(identity.token_sha256)
roles[identity.subject] = identity.role
return identities
except (ValueError, TypeError, ValidationError, RecursionError) as exc:
raise ValueError("Invalid operator identities; use the documented hash-only format") from exc
backend/src/modelforge_api/domain/portfolio.py
+197
View File
@@ -0,0 +1,197 @@
"""Typed, secret-free contracts for repository inventory reconciliation."""
from __future__ import annotations
import re
from datetime import UTC, datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
RepositoryClassification = Literal[
"local_model",
"hosted_model",
"gateway",
"mixed",
"model_management",
"runtime_observer",
"non_ai",
"unknown",
]
PROJECT_LINK_CLASSIFICATIONS = frozenset({"local_model", "gateway", "mixed"})
class PortfolioCodeEvidence(StrictModel):
"""A bounded source-code signal; source text and arbitrary notes are never accepted."""
path: str = Field(min_length=1, max_length=512)
signal: Literal[
"provider_dependency",
"inference_callsite",
"model_configuration",
"capability_client",
"runtime_declaration",
"model_management",
"runtime_observer",
]
capabilities: list[str] = Field(default_factory=list, max_length=8)
@field_validator("path")
@classmethod
def validate_repository_relative_path(cls, value: str) -> str:
if (
value.startswith(("/", "\\"))
or "\\" in value
or ":" in value
or any(ord(character) < 32 for character in value)
or any(part in {"", ".", ".."} for part in value.split("/"))
):
raise ValueError("path must be a normalized repository-relative file path")
return value
@field_validator("capabilities")
@classmethod
def validate_capabilities(cls, values: list[str]) -> list[str]:
pattern = re.compile(r"^[a-z][a-z0-9_-]*(?:\.[a-z][a-z0-9_-]*)+(?:@[1-9][0-9]*)?$")
if any(not pattern.fullmatch(value) for value in values):
raise ValueError("capabilities must be safe capability identifiers")
if len(values) != len(set(values)):
raise ValueError("capabilities must be unique")
return values
class PortfolioRuntimeEvidence(StrictModel):
"""A runtime observation separate from source-code classification."""
component: str = Field(pattern=r"^[a-z][a-z0-9-]{0,63}$")
state: Literal["running", "stopped", "unknown"]
observed_at: datetime
observation_source: Literal["container_inventory", "health_probe"]
git_revision: str | None = Field(default=None, pattern=r"^(?:[0-9a-f]{7,40})$")
image_digest: str | None = Field(default=None, pattern=r"^sha256:[0-9a-f]{64}$")
@field_validator("observed_at")
@classmethod
def require_timezone(cls, value: datetime) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError("runtime observation time must include a timezone")
return value.astimezone(UTC)
class PortfolioRepositoryInput(StrictModel):
external_repository_id: str = Field(pattern=r"^[a-z0-9][a-z0-9._:-]{0,127}$")
project_key: str | None = Field(default=None, pattern=r"^[a-z][a-z0-9-]{0,127}$")
slug: str = Field(min_length=3, max_length=255)
default_branch: str = Field(min_length=1, max_length=255)
head_sha: str = Field(pattern=r"^[0-9a-f]{40}$")
classification: RepositoryClassification
accept_head_update: bool = False
code_evidence: list[PortfolioCodeEvidence] = Field(default_factory=list, max_length=128)
runtime_evidence: list[PortfolioRuntimeEvidence] = Field(default_factory=list, max_length=64)
@field_validator("slug")
@classmethod
def validate_slug(cls, value: str) -> str:
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", value):
raise ValueError("slug must be an owner/repository identifier, not a URL")
return value
@field_validator("default_branch")
@classmethod
def validate_branch(cls, value: str) -> str:
if (
not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,254}", value)
or ".." in value
or "//" in value
or value.endswith(("/", ".", ".lock"))
):
raise ValueError("default branch must be a safe Git branch name")
return value
@model_validator(mode="after")
def validate_runtime_components(self) -> PortfolioRepositoryInput:
if self.classification in PROJECT_LINK_CLASSIFICATIONS and self.project_key is None:
raise ValueError("project_key is required for a local model consumer or provider")
components = [item.component for item in self.runtime_evidence]
if len(components) != len(set(components)):
raise ValueError("runtime components must be unique within a repository observation")
return self
class PortfolioReconcileRequest(StrictModel):
source_system: Literal["gitea"] = "gitea"
snapshot_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
observed_at: datetime
complete: bool = False
repositories: list[PortfolioRepositoryInput] = Field(max_length=256)
@field_validator("observed_at")
@classmethod
def require_timezone(cls, value: datetime) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError("snapshot observation time must include a timezone")
return value.astimezone(UTC)
@model_validator(mode="after")
def validate_repository_identity_uniqueness(self) -> PortfolioReconcileRequest:
if not self.repositories and not self.complete:
raise ValueError("an empty inventory is valid only as an explicit complete snapshot")
identities = [item.external_repository_id for item in self.repositories]
if len(identities) != len(set(identities)):
raise ValueError("a snapshot may contain each external repository only once")
return self
class PortfolioRepositoryView(StrictModel):
id: str
project_key: str | None
project_name: str | None
project_id: str | None
source_system: str
external_repository_id: str
slug: str
default_branch: str
head_sha: str
classification: RepositoryClassification
code_evidence: list[PortfolioCodeEvidence]
runtime_evidence: list[PortfolioRuntimeEvidence]
observed_at: datetime
snapshot_id: str
snapshot_digest: str
pending_head_sha: str | None
pending_default_branch: str | None
pending_project_key: str | None
pending_classification: RepositoryClassification | None
pending_code_evidence: list[PortfolioCodeEvidence]
pending_observed_at: datetime | None
pending_snapshot_id: str | None
pending_snapshot_digest: str | None
last_seen_at: datetime
stale_at: datetime | None
class PortfolioInventoryResponse(StrictModel):
repositories: list[PortfolioRepositoryView]
class PortfolioReconcileResponse(StrictModel):
source_system: str
snapshot_id: str
snapshot_digest: str
observed_at: datetime
complete: bool
replayed: bool = False
stale_snapshot: bool = False
projects_created: int = 0
repositories_created: int = 0
repositories_updated: int = 0
stale_observations: int = 0
repositories_marked_stale: int = 0
runtime_observations_updated: int = 0
head_changes_review_required: int = 0
head_changes_accepted: int = 0
backend/src/modelforge_api/domain/projects.py
+87
View File
@@ -0,0 +1,87 @@
from __future__ import annotations
import uuid
from datetime import datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from modelforge_api.domain.serving import ServiceClientResponse
class ProjectCreate(BaseModel):
model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
key: str = Field(pattern=r"^[a-z][a-z0-9-]{0,127}$")
name: str = Field(min_length=1, max_length=255)
description: str = Field(min_length=1, max_length=2000)
owner: str = Field(min_length=1, max_length=128)
environment: Literal["lab", "shadow", "evaluation", "production"] = "lab"
labels: list[str] = Field(default_factory=list, max_length=20)
class ProjectConnectivityResponse(BaseModel):
client_id: uuid.UUID
project_key: str | None
capability: str
connected: Literal[True] = True
correlation_id: str
class ProjectRequirementCreate(BaseModel):
model_config = ConfigDict(extra="forbid")
capability: str = Field(pattern=r"^[a-z][a-z0-9.]*(?:@[1-9][0-9]*)$", max_length=128)
migration_support: Literal["none", "reindex", "schema"] = "none"
priority: Literal["interactive", "background", "benchmark"] = "interactive"
class ProjectServingModelResponse(BaseModel):
"""Exact model identity resolved from the stable deployment used by a project requirement."""
deployment_id: uuid.UUID
artifact_set_id: uuid.UUID
model_id: uuid.UUID
model_key: str
display_name: str
upstream_source: str
revision_id: uuid.UUID
upstream_revision: str
resolved_commit_sha: str
runtime_profile_id: uuid.UUID
compute_node_id: uuid.UUID
channel: str
production: bool
artifact_availability: str
class ProjectRequirementResponse(BaseModel):
id: uuid.UUID
capability: str
channel: str
priority: str
migration_support: str
deprecated: bool
deployment_id: uuid.UUID | None
deployment_health: str
serving_model: ProjectServingModelResponse | None = None
credential_ready: bool
connectivity_checked_at: datetime | None
capability_checked_at: datetime | None
capability_check_request_id: str | None
state: str
blockers: list[str]
class ProjectWorkspaceResponse(BaseModel):
id: uuid.UUID
key: str
name: str
description: str
active: bool
owner: str | None
environment: str | None
labels: list[str]
source: Literal["operator", "manifest"]
production_eligible: bool | None
state: str
requirements: list[ProjectRequirementResponse]
clients: list[ServiceClientResponse]
backend/src/modelforge_api/domain/recovery.py
+26
View File
@@ -292,6 +292,32 @@ class BackupSetResponse(RecoveryModel):
entries: list[BackupManifestEntryResponse] = Field(default_factory=list)
class RecoveryRetentionDecision(RecoveryModel):
backup_id: str
current_state: BackupState
decision: Literal["RETAIN", "EXPIRE"]
reason: Literal[
"ACTIVE_RESTORE_IN_PROGRESS",
"LEGAL_HOLD",
"MILESTONE",
"MINIMUM_VERIFIED_BACKUPS",
"RETENTION_NOT_REACHED",
"RETENTION_ELIGIBLE",
]
expires_at: datetime | None
class RecoveryRetentionPreview(RecoveryModel):
observed_at: datetime
policy_key: str
policy_revision: int
minimum_verified_backups: int
active_restore_in_progress: bool
retained_backup_ids: list[str]
expiring_backup_ids: list[str]
decisions: list[RecoveryRetentionDecision]
class RestorePlanCreate(RecoveryModel):
backup_set_id: uuid.UUID
mode: RestoreMode
backend/src/modelforge_api/domain/release.py
+69 -7
View File
@@ -14,6 +14,7 @@ support is a far better outcome than one that starts and writes rows the next ve
from __future__ import annotations
import re
from dataclasses import dataclass
from datetime import UTC, datetime
from enum import StrEnum
@@ -23,7 +24,7 @@ from pathlib import Path
#: Schema revisions this application version can run against, oldest first. The application refuses
#: to serve against anything outside this list rather than guessing.
SUPPORTED_SCHEMA_REVISIONS: tuple[str, ...] = ("20260830_0024",)
SUPPORTED_SCHEMA_REVISIONS: tuple[str, ...] = ("20260924_0039",)
#: The revision a clean installation and a completed upgrade must both arrive at.
TARGET_SCHEMA_REVISION = SUPPORTED_SCHEMA_REVISIONS[-1]
@@ -35,6 +36,21 @@ SUPPORTED_UPGRADE_SOURCE_SCHEMA_REVISIONS: tuple[str, ...] = (
"20260827_0021",
"20260828_0022",
"20260830_0023",
"20260830_0024",
"20260907_0025",
"20260908_0026",
"20260908_0027",
"20260908_0028",
"20260909_0029",
"20260909_0030",
"20260909_0031",
"20260909_0032",
"20260909_0033",
"20260910_0034",
"20260911_0035",
"20260923_0036",
"20260923_0037",
"20260924_0038",
TARGET_SCHEMA_REVISION,
)
@@ -47,14 +63,36 @@ SUPPORTED_AGENT_PROTOCOL_VERSIONS: tuple[int, ...] = (1,)
#: The protocol version this control plane itself speaks.
CURRENT_AGENT_PROTOCOL_VERSION = SUPPORTED_AGENT_PROTOCOL_VERSIONS[-1]
#: Release channels. v1 ships one; the enum exists so adding another is a typed change.
RELEASE_CHANNEL = "stable"
#: Minimum PostgreSQL major version. The schema uses partial unique indexes and generated columns
#: that older majors either lack or plan differently.
MINIMUM_POSTGRES_MAJOR = 16
_PEP440_NUMBERED_RC = re.compile(
r"^(?P<major>0|[1-9][0-9]*)\.(?P<minor>0|[1-9][0-9]*)\."
r"(?P<patch>0|[1-9][0-9]*)rc(?P<number>0|[1-9][0-9]*)$"
)
def _canonicalize_packaged_version(value: str) -> str:
"""Convert the one PEP 440 form produced for a numbered SemVer RC back to SemVer.
Build backends correctly normalize ``1.3.0-rc.1`` to ``1.3.0rc1`` in installed
distribution metadata. The release contract, image labels, and API intentionally expose the
reviewed SemVer spelling, so installed images must restore it rather than reporting a second
version representation. All other metadata is left untouched and remains subject to the
strict semantic-version validation below.
"""
match = _PEP440_NUMBERED_RC.fullmatch(value)
if match is None:
return value
return (
f"{match.group('major')}.{match.group('minor')}.{match.group('patch')}"
f"-rc.{match.group('number')}"
)
def _read_version_file() -> str:
"""Read the repository's VERSION file, falling back to the packaged distribution metadata."""
@@ -68,7 +106,7 @@ def _read_version_file() -> str:
from importlib.metadata import PackageNotFoundError, version
try:
return version("modelforge-api")
return _canonicalize_packaged_version(version("modelforge-api"))
except PackageNotFoundError: # pragma: no cover - only in a broken install
return "0.0.0"
@@ -89,9 +127,13 @@ class SemanticVersion:
@classmethod
def parse(cls, value: str) -> SemanticVersion:
core, _, prerelease = value.partition("-")
core, separator, prerelease = value.partition("-")
parts = core.split(".")
if len(parts) != 3 or not all(part.isdigit() for part in parts):
if (
len(parts) != 3
or not all(part.isdigit() for part in parts)
or (separator and not prerelease)
):
raise ValueError(f"{value!r} is not a MAJOR.MINOR.PATCH version")
major, minor, patch = (int(part) for part in parts)
return cls(major, minor, patch, prerelease or None)
@@ -105,6 +147,26 @@ class SemanticVersion:
return self.prerelease is not None
def release_channel_for(version: str) -> str:
"""Derive the reported channel from the single product-version source.
A release candidate must never identify itself as ``stable`` merely because it shares the
executable with the eventual stable release. Unknown prereleases stay visibly non-stable too,
rather than acquiring the RC label without an explicit ``rc.<number>`` version.
"""
prerelease = SemanticVersion.parse(version).prerelease
if prerelease is None:
return "stable"
if prerelease.startswith("rc.") and prerelease.removeprefix("rc.").isdigit():
return "rc"
return "prerelease"
#: The channel is derived from VERSION rather than maintained as a second, drift-prone constant.
RELEASE_CHANNEL = release_channel_for(PRODUCT_VERSION)
# --------------------------------------------------------------------------- compatibility
backend/src/modelforge_api/domain/secret_slots.py
+79
View File
@@ -0,0 +1,79 @@
"""Contracts for the sole supported retrievable platform secret."""
from datetime import datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, SecretStr, field_validator
SLOT = "huggingface_control_plane"
class SecretSlotRequest(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True, hide_input_in_errors=True)
expected_revision: int = Field(ge=0)
reason: str = Field(min_length=8, max_length=500)
@field_validator("reason")
@classmethod
def safe_reason(cls, value: str) -> str:
value = value.strip()
if len(value) < 8 or any(prefix in value for prefix in ("hf_", "mfop", "mfsvc_", "mfnode_", "mfenroll_")):
raise ValueError("Provide a non-secret explanation of at least eight characters")
return value
class SecretSlotSet(SecretSlotRequest):
value: SecretStr = Field(min_length=1, max_length=4096)
@field_validator("value")
@classmethod
def valid_token(cls, value: SecretStr) -> SecretStr:
raw = value.get_secret_value()
if not raw.startswith("hf_") or not raw.isascii() or not all(c.isalnum() or c in "_-" for c in raw) or len(raw) < 8:
raise ValueError("Provide a valid Hugging Face token without whitespace")
return value
class SecretSlotVersionRequest(SecretSlotRequest):
version: int = Field(ge=1)
class SecretSlotRevoke(SecretSlotRequest):
confirm_slot: Literal["huggingface_control_plane"]
class SecretVersionMetadata(BaseModel):
version: int
key_id: str
created_at: datetime
revoked_at: datetime | None
test_status: Literal["untested", "passed", "failed"]
tested_at: datetime | None
test_code: str | None
class SecretSlotMetadata(BaseModel):
slot: Literal["huggingface_control_plane"] = "huggingface_control_plane"
revision: int
source: Literal["deployment", "managed"]
state: Literal["deployment", "pending", "active", "revoked", "quarantined"]
configured: bool
consumer: Literal["huggingface_control_plane_metadata"] = "huggingface_control_plane_metadata"
application_mode: Literal["new_requests"] = "new_requests"
root_key_available: bool
candidate_version: int | None
active_version: int | None
versions: list[SecretVersionMetadata]
node_credentials_managed: Literal[False] = False
class DeploymentSecretMetadata(BaseModel):
key: str
configured: bool
consumer: str
application_mode: Literal["restart_required"] = "restart_required"
editable: Literal[False] = False
class SecretSlotCatalog(BaseModel):
slot: SecretSlotMetadata
deployment_secrets: list[DeploymentSecretMetadata]
backend/src/modelforge_api/domain/selected_model_comparison.py
+106
View File
@@ -0,0 +1,106 @@
"""Bounded operator intent and designated-consumer evidence contracts."""
from __future__ import annotations
from datetime import datetime
from typing import Any, Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, SecretStr
from modelforge_api.domain.evaluation import EvaluationCaseDefinitionResponse, EvaluationRunComplete
class ComparisonModel(BaseModel):
model_config = ConfigDict(extra="forbid", from_attributes=True)
class ComparisonCreate(ComparisonModel):
project_binding_id: UUID
consumer_client_id: UUID
suite_revision_id: UUID
current_deployment_id: UUID
candidate_deployment_id: UUID
candidate_route_id: UUID
current_route_id: UUID | None = None
corpus_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
retrieval_config_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
current_index_ref: str = Field(min_length=1, max_length=512)
candidate_index_ref: str = Field(min_length=1, max_length=512)
idempotency_key: str = Field(min_length=1, max_length=128, pattern=r"^[A-Za-z0-9._:-]+$")
class ComparisonEvaluationSummary(ComparisonModel):
comparability: str
eligibility: str
metric_deltas: dict[str, float]
critical_regressions: int
class ComparisonResponse(ComparisonModel):
id: UUID
project_binding_id: UUID
consumer_client_id: UUID
suite_revision_id: UUID
current_deployment_id: UUID
candidate_deployment_id: UUID
candidate_route_id: UUID
actor_id: str
pins: dict[str, Any]
pins_digest: str
status: str
version: int
expected_cases: int
current_completed_cases: int
candidate_completed_cases: int
claimed_at: datetime | None
deadline_at: datetime | None
cancel_requested_at: datetime | None
finished_at: datetime | None
error_code: str | None
baseline_run_id: UUID | None
candidate_run_id: UUID | None
evaluation_comparison_id: UUID | None
created_at: datetime
updated_at: datetime
evaluation_summary: ComparisonEvaluationSummary | None = None
production_promotion_performed: Literal[False] = False
class ComparisonList(ComparisonModel):
items: list[ComparisonResponse]
total: int
class ComparisonCancel(ComparisonModel):
expected_version: int = Field(ge=1)
class ComparisonClaim(ComparisonModel):
comparison: ComparisonResponse
claim_token: str
claim_expires_at: datetime
cases: list[EvaluationCaseDefinitionResponse]
class ComparisonClaimRequest(ComparisonModel):
claim_token: SecretStr = Field(min_length=32, max_length=128)
class ComparisonProgress(ComparisonClaimRequest):
current_completed_cases: int = Field(ge=0, le=100)
candidate_completed_cases: int = Field(ge=0, le=100)
class ComparisonResult(ComparisonClaimRequest):
pins_digest: str = Field(pattern=r"^[0-9a-f]{64}$")
consumer_revision_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
current: EvaluationRunComplete
candidate: EvaluationRunComplete
current_gateway_requests: dict[UUID, UUID] = Field(max_length=100)
candidate_gateway_requests: dict[UUID, UUID] = Field(max_length=100)
class ComparisonFinish(ComparisonClaimRequest):
outcome: Literal["failed", "cancelled"]
error_code: str | None = Field(default=None, max_length=64, pattern=r"^[A-Z][A-Z0-9_]*$")
backend/src/modelforge_api/domain/selected_replacement.py
+49
View File
@@ -0,0 +1,49 @@
"""Read-only, advisory inputs to the existing approved lifecycle workflow."""
from datetime import datetime
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, ConfigDict
from modelforge_api.domain.lifecycle_contracts import ServingTransitionSelection
class PreviewModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class ReplacementBlocker(PreviewModel):
code: str
message: str
project_binding_id: UUID | None = None
class ReplacementBinding(PreviewModel):
project_binding_id: UUID
project_id: UUID
project_name: str
comparison_id: UUID | None
comparison_status: str | None
eligible: bool
class ReplacementApproval(PreviewModel):
id: UUID
approved_by: str
approved_at: datetime
evidence_fingerprint: str
class ReplacementPreview(PreviewModel):
comparison_id: UUID
capability_contract_id: UUID | None
current_deployment_id: UUID
candidate_deployment_id: UUID
route_generation: int | None
same_embedding_space: bool
affected_bindings: list[ReplacementBinding]
production_approvals: list[ReplacementApproval]
blockers: list[ReplacementBlocker]
selection: ServingTransitionSelection | None
production_validation_required: Literal["SATISFIED"] = "SATISFIED"
production_promotion_performed: Literal[False] = False
backend/src/modelforge_api/domain/self_check.py
+72
View File
@@ -0,0 +1,72 @@
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from pydantic import BaseModel, Field, JsonValue
class SelfCheckStatus(StrEnum):
PASS = "pass" # noqa: S105 - diagnostic status, not a credential
WARN = "warn"
FAIL = "fail"
UNKNOWN = "unknown"
class SelfCheckKey(StrEnum):
API = "api"
DATABASE = "database"
REDIS = "redis"
MANIFESTS = "manifests"
STORAGE = "storage"
NODES = "nodes"
SCHEDULER = "scheduler"
DEPLOYMENTS = "deployments"
BACKUP = "backup"
CREDENTIALS = "credentials"
CONFIGURATION = "configuration"
RELEASE = "release"
class SelfCheckActionCode(StrEnum):
NONE_REQUIRED = "none_required"
REVIEW_API_IDENTITY = "review_api_identity"
CHECK_DATABASE = "check_database"
CHECK_REDIS = "check_redis"
REVIEW_MANIFESTS = "review_manifests"
REVIEW_STORAGE = "review_storage"
REVIEW_NODE_FRESHNESS = "review_node_freshness"
REVIEW_SCHEDULER_INPUTS = "review_scheduler_inputs"
REVIEW_DEPLOYMENTS = "review_deployments"
VERIFY_BACKUP_PAYLOAD = "verify_backup_payload"
REVIEW_CREDENTIALS = "review_credentials"
COMPLETE_CONFIGURATION = "complete_configuration"
VERIFY_RELEASE_IDENTITY = "verify_release_identity"
class SelfCheckOverall(StrEnum):
HEALTHY = "healthy"
DEGRADED = "degraded"
UNHEALTHY = "unhealthy"
class SelfCheckNextAction(BaseModel):
code: SelfCheckActionCode
summary: str
class SelfCheckResult(BaseModel):
key: SelfCheckKey
label: str
status: SelfCheckStatus
summary: str
detail: str | None = None
evidence: dict[str, JsonValue] = Field(default_factory=dict)
next_action: SelfCheckNextAction
class SelfCheckReport(BaseModel):
observed_at: datetime
overall: SelfCheckOverall
read_only: bool = True
checks: list[SelfCheckResult] = Field(default_factory=list)
backend/src/modelforge_api/domain/serving.py
+49 -12
View File
@@ -8,7 +8,7 @@ import uuid
from datetime import datetime
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, field_validator, model_validator
CAPABILITY_VERSION_KEY = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+@[1-9][0-9]*$")
ServingOperation = Literal["load", "invoke", "health", "drain", "unload"]
@@ -119,6 +119,7 @@ class ResidencyResponse(ServingModel):
class CapabilityDeploymentResponse(ServingModel):
id: uuid.UUID
route_generation: int = Field(ge=0)
capability: str
contract_version: int
deployment_candidate_id: uuid.UUID
@@ -159,7 +160,8 @@ class ServiceClientCreate(ServingModel):
project_key: str | None = Field(default=None, pattern=r"^[a-z][a-z0-9-]*$")
integration_environment: Literal["production", "shadow", "evaluation", "lab"] | None = None
purpose: str | None = Field(default=None, min_length=3, max_length=500)
credential_expires_at: datetime | None = None
credential_expires_at: AwareDatetime | None = None
project_binding_id: uuid.UUID | None = None
@field_validator("allowed_capabilities")
@classmethod
@@ -176,9 +178,13 @@ class ServiceClientCreate(ServingModel):
if any(value is not None for value in project_fields) and not all(
value is not None for value in project_fields
):
raise ValueError("project_key, integration_environment and purpose are required together")
raise ValueError(
"project_key, integration_environment and purpose are required together"
)
if self.project_key is not None and len(self.allowed_capabilities) != 1:
raise ValueError("a project service identity must bind exactly one capability")
if self.project_binding_id is not None and self.project_key is None:
raise ValueError("an explicit binding requires a project identity")
return self
@@ -195,6 +201,7 @@ class ServiceClientResponse(ServingModel):
integration_environment: str | None
purpose: str | None
credential_prefix: str | None
credential_id: uuid.UUID | None = None
credential_created_at: datetime | None
credential_expires_at: datetime | None
credential_revoked_at: datetime | None
@@ -206,20 +213,35 @@ class ServiceClientCreated(ServiceClientResponse):
credential: str
class ServiceCredentialRotate(ServingModel):
expected_credential_id: uuid.UUID
overlap_seconds: int = Field(default=0, strict=True, ge=0, le=86400)
expires_at: AwareDatetime
class ServiceCredentialRevoke(ServingModel):
expected_credential_id: uuid.UUID
confirmation: str = Field(min_length=1, max_length=255)
class ServiceCredentialResponse(ServingModel):
id: uuid.UUID
secret_prefix: str
created_at: datetime
expires_at: datetime | None
superseded_at: datetime | None
revoked_at: datetime | None
last_used_at: datetime | None
class ProjectFitEvidenceCreate(ServingModel):
project_key: str = Field(pattern=r"^[a-z][a-z0-9-]*$")
capability: str = Field(pattern=r"^[a-z][a-z0-9.]*@[1-9][0-9]*$")
environment: Literal["production", "shadow", "evaluation", "lab"]
recommendation: Literal[
"PROMOTION_ELIGIBLE", "KEEP_LAB", "BLOCKED", "REQUIRES_MORE_EVIDENCE"
]
evidence_class: Literal[
"OWNER_PHOTO", "PUBLIC_PHYSICAL_CAPTURE", "CATALOG_REFERENCE"
]
recommendation: Literal["PROMOTION_ELIGIBLE", "KEEP_LAB", "BLOCKED", "REQUIRES_MORE_EVIDENCE"]
evidence_class: Literal["OWNER_PHOTO", "PUBLIC_PHYSICAL_CAPTURE", "CATALOG_REFERENCE"]
engineering_integration: Literal["PASS", "INCOMPLETE", "BLOCKED"]
production_validation: Literal[
"DEFERRED_EXTERNAL_VALIDATION", "REQUIRED", "SATISFIED"
]
production_validation: Literal["DEFERRED_EXTERNAL_VALIDATION", "REQUIRED", "SATISFIED"]
production_action: Literal["NONE"] = "NONE"
case_count: int = Field(ge=0, le=1_000_000)
metric_values: dict[str, float | int | str | bool | None]
@@ -693,6 +715,7 @@ class AgentServingJobLease(ServingModel):
top_n: int | None = Field(default=None, ge=1, le=10)
modality_payload: dict[str, Any] | None = None
request_id: uuid.UUID | None = None
lab_run_id: uuid.UUID | None = None
@model_validator(mode="after")
def validate_capability_payload(self) -> AgentServingJobLease:
@@ -736,6 +759,20 @@ class AgentServingJobFailure(ServingModel):
details: dict[str, Any] = Field(default_factory=dict)
class AgentServingJobCancellationCheck(ServingModel):
lease_token: str = Field(min_length=32, max_length=512)
class AgentServingJobCancellationStatus(ServingModel):
cancel_requested: bool
class AgentServingJobCancellationAck(ServingModel):
lease_token: str = Field(min_length=32, max_length=512)
worker_instance_id: str = Field(min_length=1, max_length=255)
state: Literal["cold", "warm", "busy", "loading", "draining", "unloading", "failed"]
class AgentResidentState(ServingModel):
deployment_id: uuid.UUID
artifact_set_id: uuid.UUID
backend/src/modelforge_api/domain/update_preflight.py
+115
View File
@@ -0,0 +1,115 @@
"""Read-only evidence contract for planning a ModelForge release update."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Literal
from pydantic import BaseModel, Field
EvidenceStatus = Literal["KNOWN", "UNKNOWN"]
class UpdateEvidence(BaseModel):
status: EvidenceStatus
value: str | bool | None = None
reason: str | None = None
class CurrentRelease(BaseModel):
version: str
channel: str
schema_revision: str = Field(
description="Schema target advertised by this application build; not the applied database revision."
)
agent_protocol_version: int
class ApprovedRelease(BaseModel):
status: Literal["UNKNOWN"] = "UNKNOWN"
version: str | None = None
notes: UpdateEvidence
schema_revision: UpdateEvidence
class BuildProvenance(BaseModel):
status: Literal["AVAILABLE", "UNKNOWN"]
source_commit: str | None = None
built_at: str | None = None
image_digest: str | None = None
reason: str | None = None
class UpdateImpact(BaseModel):
schema_impact: UpdateEvidence = Field(alias="schema")
breaking: UpdateEvidence
class BackupRequirement(BaseModel):
required: UpdateEvidence
class UpdateExecution(BaseModel):
available: Literal[False] = False
control: Literal["HUMAN_CONTROLLED"] = "HUMAN_CONTROLLED"
class UpdatePreflight(BaseModel):
observed_at: datetime
status: Literal["INCOMPLETE"] = "INCOMPLETE"
read_only: Literal[True] = True
current: CurrentRelease
approved: ApprovedRelease
impact: UpdateImpact
backup: BackupRequirement
provenance: dict[Literal["current", "approved"], BuildProvenance]
execution: UpdateExecution
def update_preflight(
*,
version: str,
channel: str,
schema_revision: str,
agent_protocol_version: int,
source_commit: str | None,
built_at: str | None,
image_digest: str | None,
) -> UpdatePreflight:
"""Build a plan view from local release identity without probing or mutating other systems."""
provenance_complete = all((source_commit, built_at, image_digest))
provenance_status: Literal["AVAILABLE", "UNKNOWN"] = "AVAILABLE" if provenance_complete else "UNKNOWN"
unknown_reason = "No approved update metadata is configured; this value is not inferred."
return UpdatePreflight(
observed_at=datetime.now(UTC),
current=CurrentRelease(
version=version,
channel=channel,
schema_revision=schema_revision,
agent_protocol_version=agent_protocol_version,
),
approved=ApprovedRelease(
notes=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
schema_revision=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
),
impact=UpdateImpact(
schema=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
breaking=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
),
backup=BackupRequirement(
required=UpdateEvidence(status="UNKNOWN", reason=unknown_reason),
),
provenance={
"current": BuildProvenance(
status=provenance_status,
source_commit=source_commit,
built_at=built_at,
image_digest=image_digest,
reason=None if provenance_complete else "One or more build provenance fields are unavailable.",
),
"approved": BuildProvenance(status="UNKNOWN", reason=unknown_reason),
},
execution=UpdateExecution(),
)
backend/src/modelforge_api/main.py
+151 -9
View File
@@ -18,8 +18,10 @@ from sqlalchemy.orm import Session
from modelforge_api import __version__
from modelforge_api.api.authorization import (
AccessBoundary,
Principal,
access_boundary_for_request,
authenticate_operator_token,
operator_token_from_headers,
request_body_limit_bytes,
required_capability_for_request,
)
@@ -27,21 +29,35 @@ from modelforge_api.api.request_limits import RequestBodyLimitMiddleware
from modelforge_api.api.routes import (
acquisition,
agent,
audit_timeline,
catalog,
configuration,
evaluation,
external_creative,
hardware,
health,
identity,
lab,
lifecycle,
migrations,
model_updates,
observability,
operator_login,
portfolio,
projects,
recovery,
registry,
runtime,
selected_model_comparison,
selected_replacement,
serving,
update_preflight,
)
from modelforge_api.db import engine, get_session
from modelforge_api.domain.observability import metrics, route_class
from modelforge_api.domain.release import build_identity
from modelforge_api.services.alert_notifications import AlertNotificationError
from modelforge_api.services.configuration_changes import ConfigurationError
from modelforge_api.services.evaluation import EvaluationError
from modelforge_api.services.hardware_polling import HardwarePollingService
from modelforge_api.services.lifecycle import LifecycleError, LifecycleService
@@ -50,6 +66,7 @@ from modelforge_api.services.migration_engine import (
MigrationEngineError,
MigrationEngineService,
)
from modelforge_api.services.model_update_polling import ModelUpdatePollingService
from modelforge_api.services.node_agent import (
AgentProtocolError,
NodeAgentService,
@@ -59,9 +76,14 @@ from modelforge_api.services.node_decommission import NodeDecommissionError
from modelforge_api.services.node_liveness import NodeLivenessPollingService
from modelforge_api.services.observability import ObservabilityError, ObservabilityService
from modelforge_api.services.observability_polling import ObservabilityPollingService
from modelforge_api.services.operator_identities import (
ManagedIdentityError,
authenticate_managed_operator,
)
from modelforge_api.services.project_registry import sync_project_registry
from modelforge_api.services.recovery import RecoveryError, RecoveryService
from modelforge_api.services.registry import RegistryError, seed_candidate_registry
from modelforge_api.services.secret_slots import SecretSlotError
from modelforge_api.services.serving import (
CapabilityAuthenticationEvidence,
ServingError,
@@ -151,6 +173,18 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
"project_bindings_waiting_for_contracts",
bindings=sync_result.unavailable_contracts,
)
if sync_result.unavailable_bindings:
logger.warning(
"project_bindings_unavailable_for_declared_stability",
bindings=[
{
"project": item.project_key,
"capability": f"{item.capability}@{item.contract_version}",
"declared_stability": item.declared_stability,
}
for item in sync_result.unavailable_bindings
],
)
poller = HardwarePollingService(settings)
hardware_task = (
asyncio.create_task(poller.run()) if settings.hardware_refresh_on_startup else None
@@ -166,15 +200,24 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
else None
)
observability_poller = ObservabilityPollingService(settings, engine)
if settings.observability_monitor_enabled:
if settings.observability_monitor_enabled or settings.alert_notifications_enabled:
with Session(engine) as session:
ObservabilityService(session, settings).ensure_defaults()
observability_task = (
asyncio.create_task(observability_poller.run())
if settings.observability_monitor_enabled
if settings.observability_monitor_enabled or settings.alert_notifications_enabled
else None
)
model_update_poller = ModelUpdatePollingService(settings, engine)
model_update_task = (
asyncio.create_task(model_update_poller.run())
if settings.env == "production" and settings.model_update_monitor_enabled
else None
)
yield
if model_update_task:
model_update_poller.stop()
await model_update_task
if observability_task:
observability_poller.stop()
await observability_task
@@ -199,19 +242,51 @@ app = FastAPI(
redoc_url="/redoc" if interactive_api_enabled else None,
openapi_url="/openapi.json" if interactive_api_enabled else None,
)
app.include_router(operator_login.router)
app.include_router(health.router)
app.include_router(lifecycle.router)
app.include_router(migrations.router)
app.include_router(observability.router)
app.include_router(recovery.router)
app.include_router(agent.router)
app.include_router(audit_timeline.router)
app.include_router(hardware.router)
app.include_router(catalog.router)
app.include_router(configuration.router)
app.include_router(identity.router)
app.include_router(portfolio.router)
app.include_router(projects.router)
app.include_router(registry.router)
app.include_router(acquisition.router)
app.include_router(model_updates.router)
app.include_router(selected_model_comparison.router)
app.include_router(selected_replacement.router)
app.include_router(runtime.router)
app.include_router(lab.router)
app.include_router(serving.router)
app.include_router(external_creative.router)
app.include_router(evaluation.router)
app.include_router(update_preflight.router)
@app.exception_handler(ConfigurationError)
async def configuration_error(request: Request, exc: ConfigurationError) -> JSONResponse:
return JSONResponse(status_code=exc.status_code, content={"error": {
"code": exc.code, "message": exc.message,
"correlation_id": getattr(request.state, "correlation_id", "unknown"), "details": {},
}})
@app.exception_handler(ManagedIdentityError)
async def managed_identity_error(request: Request, exc: ManagedIdentityError) -> JSONResponse:
return _authentication_error(exc.status_code, exc.code, exc.message,
getattr(request.state, "correlation_id", "unknown"))
@app.exception_handler(SecretSlotError)
async def secret_slot_error(request: Request, exc: SecretSlotError) -> JSONResponse:
return _authentication_error(exc.status_code, exc.code, exc.message,
getattr(request.state, "correlation_id", "unknown"))
@app.exception_handler(LifecycleError)
@@ -277,6 +352,24 @@ async def observability_error(request: Request, exc: ObservabilityError) -> JSON
)
@app.exception_handler(AlertNotificationError)
async def alert_notification_error(
request: Request, exc: AlertNotificationError
) -> JSONResponse:
correlation_id = getattr(request.state, "correlation_id", "unknown")
return JSONResponse(
status_code=exc.status_code,
content={
"error": {
"code": exc.code,
"message": exc.message,
"correlation_id": correlation_id,
"details": {},
}
},
)
@contextmanager
def _authentication_session(application: FastAPI) -> Iterator[Session]:
"""Use a test/application session override when present, otherwise a short owned session."""
@@ -316,6 +409,7 @@ def _authenticate_node_before_body(
service.session.commit()
return evidence
with _authentication_session(application) as session:
service = NodeAgentService(session, request_settings)
_identity, evidence = service.authenticate_with_evidence(authorization)
@@ -323,6 +417,13 @@ def _authenticate_node_before_body(
return evidence
def _authenticate_managed_before_body(application: FastAPI, token: str) -> Principal:
with _authentication_session(application) as session:
principal = authenticate_managed_operator(session, token)
session.commit()
return principal
def _authenticate_capability_before_body(
application: FastAPI,
request_settings: Settings,
@@ -389,12 +490,31 @@ async def correlation_middleware(
boundary,
)
response: Response | None = None
if request.method == "POST" and request.url.path == "/api/v1/operator/login":
request.state.request_body_limit_bytes = 8192
try:
await run_in_threadpool(operator_login.limit_login, request, request_settings)
except HTTPException as exc:
response = _authentication_error(
exc.status_code, f"http_{exc.status_code}", str(exc.detail), correlation_id
)
if boundary is AccessBoundary.CONTROL_PLANE:
try:
request.state.principal = authenticate_operator_token(
request_settings,
operator_token = operator_token_from_headers(
request.headers.get("x-modelforge-admin-token"),
request.headers.get("authorization"),
)
if operator_token is not None and operator_token.startswith("mfopm_"):
request.state.principal = await run_in_threadpool(
_authenticate_managed_before_body, request.app, operator_token,
)
else:
request.state.principal = authenticate_operator_token(request_settings, operator_token)
except ManagedIdentityError as exc:
response = _authentication_error(exc.status_code, exc.code, exc.message, correlation_id)
except SQLAlchemyError:
response = _authentication_error(503, "MANAGED_AUTHENTICATION_UNAVAILABLE",
"managed authentication backend is unavailable", correlation_id)
except HTTPException as exc:
response = _authentication_error(
exc.status_code,
@@ -461,15 +581,15 @@ async def correlation_middleware(
response = await call_next(request)
duration_seconds = time.perf_counter() - started
response.headers["x-correlation-id"] = correlation_id
if boundary is AccessBoundary.CONTROL_PLANE or request.url.path == "/api/v1/operator/login":
response.headers["Cache-Control"] = "no-store"
body_error_status_code = getattr(
request.state,
"request_body_error_status_code",
None,
)
effective_status_code = (
body_error_status_code
if isinstance(body_error_status_code, int)
else response.status_code
body_error_status_code if isinstance(body_error_status_code, int) else response.status_code
)
labels = {
"method": request.method,
@@ -536,6 +656,20 @@ def _renderable(value: object) -> object:
@app.exception_handler(RequestValidationError)
async def validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
correlation_id = getattr(request.state, "correlation_id", "unknown")
errors = exc.errors()
if request.url.path.startswith("/api/v1/selected-model-comparisons"):
# Claim secrets, dataset content and arbitrary discriminator locations never echo.
errors = [{"type": error["type"], "msg": "Invalid comparison request field"} for error in errors]
elif request.url.path.startswith("/api/v1/lab/"):
# Discriminator messages and locations can contain caller-controlled payload data.
errors = [{"type": error["type"], "msg": "Invalid lab request field"} for error in errors]
elif request.url.path.startswith("/api/v1/portfolio"):
# Portfolio snapshots are intentionally secret-free; never echo body values or paths.
errors = [{"type": error["type"], "msg": "Invalid portfolio request field"} for error in errors]
if request.url.path == "/api/v1/operator/login":
return _authentication_error(
422, "request_validation_failed", "Invalid login request", correlation_id
)
return JSONResponse(
status_code=422,
content={
@@ -545,8 +679,16 @@ async def validation_error(request: Request, exc: RequestValidationError) -> JSO
"correlation_id": correlation_id,
"details": {
"errors": [
{key: _renderable(value) for key, value in error.items() if key != "ctx"}
for error in exc.errors()
{key: _renderable(value) for key, value in error.items()
if key != "ctx" and not (
(request.url.path.startswith("/api/v1/configuration/")
or request.url.path.startswith("/api/v1/operator/managed-identities"))
and key in {"input", "loc"}
) and not (
request.url.path.startswith("/api/v1/portfolio")
and key in {"input", "loc"}
)}
for error in errors
]
},
}
backend/src/modelforge_api/persistence/__init__.py
+2 -1
View File
@@ -1,3 +1,4 @@
from .model_change import ModelChangeCase
from .models import Base
__all__ = ["Base"]
__all__ = ["Base", "ModelChangeCase"]
backend/src/modelforge_api/persistence/hardware_repository.py
+4 -1
View File
@@ -40,7 +40,10 @@ class HardwareRepository:
def node_for_update(self, node_id: uuid.UUID) -> ComputeNode | None:
return self.session.scalar(
select(ComputeNode).where(ComputeNode.id == node_id).with_for_update()
select(ComputeNode)
.where(ComputeNode.id == node_id)
.with_for_update()
.execution_options(populate_existing=True)
)
def accelerators(self, node_id: uuid.UUID | None = None) -> list[Accelerator]:
backend/src/modelforge_api/persistence/model_change.py
+94
View File
@@ -0,0 +1,94 @@
from __future__ import annotations
import uuid
from datetime import datetime
from typing import Any
import sqlalchemy as sa
from sqlalchemy.orm import Mapped, mapped_column
from modelforge_api.persistence.models import Base, IdMixin, TimestampMixin
MODEL_CHANGE_STATUSES = (
"searching",
"found",
"source_checked",
"downloaded",
"ready_to_test",
"tested",
"comparison_available",
"migration_required",
"ready_to_activate",
"active",
"checked",
"rolled_back",
"rejected",
)
class ModelChangeCase(Base, IdMixin, TimestampMixin):
"""User-facing workflow shell that only references authoritative ModelForge evidence."""
__tablename__ = "model_change_cases"
__table_args__ = (
sa.CheckConstraint("version >= 1", name="ck_model_change_case_version"),
sa.CheckConstraint(
"status IN (" + ",".join(repr(value) for value in MODEL_CHANGE_STATUSES) + ")",
name="ck_model_change_case_status",
),
sa.Index("ix_model_change_cases_binding", "project_binding_id"),
sa.Index("ix_model_change_cases_status", "status"),
sa.Index(
"uq_model_change_cases_open_binding",
"project_binding_id",
unique=True,
postgresql_where=sa.text("closed_at IS NULL"),
sqlite_where=sa.text("closed_at IS NULL"),
),
)
project_binding_id: Mapped[uuid.UUID] = mapped_column(
sa.ForeignKey("project_bindings.id", ondelete="RESTRICT"), nullable=False
)
current_deployment_id: Mapped[uuid.UUID] = mapped_column(
sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT"), nullable=False
)
candidate_model_id: Mapped[uuid.UUID | None] = mapped_column(
sa.ForeignKey("models.id", ondelete="RESTRICT")
)
candidate_revision_id: Mapped[uuid.UUID | None] = mapped_column(
sa.ForeignKey("model_revisions.id", ondelete="RESTRICT")
)
candidate_artifact_set_id: Mapped[uuid.UUID | None] = mapped_column(
sa.ForeignKey("artifact_sets.id", ondelete="RESTRICT")
)
candidate_deployment_id: Mapped[uuid.UUID | None] = mapped_column(
sa.ForeignKey("capability_deployments.id", ondelete="RESTRICT")
)
discovery_assessment_id: Mapped[uuid.UUID | None] = mapped_column(
sa.ForeignKey("discovery_candidate_assessments.id", ondelete="RESTRICT")
)
download_plan_id: Mapped[uuid.UUID | None] = mapped_column(
sa.ForeignKey("download_plans.id", ondelete="RESTRICT")
)
artifact_job_id: Mapped[uuid.UUID | None] = mapped_column(
sa.ForeignKey("artifact_jobs.id", ondelete="RESTRICT")
)
selected_comparison_id: Mapped[uuid.UUID | None] = mapped_column(
sa.ForeignKey("selected_model_comparisons.id", ondelete="RESTRICT")
)
migration_plan_id: Mapped[uuid.UUID | None] = mapped_column(
sa.ForeignKey("migration_plans.id", ondelete="RESTRICT")
)
lifecycle_plan_id: Mapped[uuid.UUID | None] = mapped_column(
sa.ForeignKey("lifecycle_promotion_plans.id", ondelete="RESTRICT")
)
status: Mapped[str] = mapped_column(sa.String(32), nullable=False, default="searching")
status_reason: Mapped[str] = mapped_column(sa.Text, nullable=False, default="")
version: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=1)
created_by: Mapped[str] = mapped_column(sa.String(255), nullable=False)
updated_by: Mapped[str] = mapped_column(sa.String(255), nullable=False)
context: Mapped[dict[str, Any]] = mapped_column(sa.JSON, nullable=False, default=dict)
activated_at: Mapped[datetime | None] = mapped_column(sa.DateTime(timezone=True))
checked_at: Mapped[datetime | None] = mapped_column(sa.DateTime(timezone=True))
closed_at: Mapped[datetime | None] = mapped_column(sa.DateTime(timezone=True))
backend/src/modelforge_api/persistence/models.py
+511 -5
View File
@@ -58,6 +58,123 @@ class Base(DeclarativeBase):
pass
class SecretSlotState(Base):
__tablename__ = "platform_secret_state"
__table_args__ = (CheckConstraint("singleton_id = 1", name="ck_secret_state_singleton"), CheckConstraint("revision >= 0", name="ck_secret_state_revision"))
singleton_id: Mapped[int] = mapped_column(Integer, primary_key=True)
revision: Mapped[int] = mapped_column(Integer, nullable=False)
initialized: Mapped[bool] = mapped_column(Boolean, nullable=False)
@event.listens_for(SecretSlotState.__table__, "after_create")
def _seed_secret_slot_state(_table: Any, connection: Connection, **_kwargs: Any) -> None:
connection.execute(insert(SecretSlotState).values(singleton_id=1, revision=0, initialized=False))
class SecretSlotVersion(Base):
__tablename__ = "platform_secret_versions"
__table_args__ = (CheckConstraint("version > 0", name="ck_secret_version_positive"),)
version: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=False)
key_id: Mapped[str] = mapped_column(String(64), nullable=False)
secret_ciphertext: Mapped[str] = mapped_column(Text, nullable=False)
secret_nonce: Mapped[str] = mapped_column(String(32), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class SecretSlotTest(Base):
__tablename__ = "platform_secret_tests"
version: Mapped[int] = mapped_column(ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), primary_key=True)
passed: Mapped[bool] = mapped_column(Boolean, nullable=False)
tested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
code: Mapped[str] = mapped_column(String(64), nullable=False)
class SecretSlotActivation(Base):
__tablename__ = "platform_secret_activations"
__table_args__ = (CheckConstraint("singleton_id = 1", name="ck_secret_activation_singleton"),)
singleton_id: Mapped[int] = mapped_column(Integer, primary_key=True)
version: Mapped[int] = mapped_column(ForeignKey("platform_secret_versions.version", ondelete="RESTRICT"), nullable=False)
activated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class ManagedOperatorState(Base):
__tablename__ = "managed_operator_state"
__table_args__ = (
CheckConstraint("singleton_id = 1", name="ck_managed_operator_singleton"),
CheckConstraint("revision >= 0", name="ck_managed_operator_revision"),
)
singleton_id: Mapped[int] = mapped_column(Integer, primary_key=True)
revision: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
@event.listens_for(ManagedOperatorState.__table__, "after_create")
def _seed_managed_operator_state(_table: Any, connection: Connection, **_kwargs: Any) -> None:
connection.execute(insert(ManagedOperatorState).values(singleton_id=1, revision=0))
class ManagedOperatorIdentity(Base):
__tablename__ = "managed_operator_identities"
__table_args__ = (
CheckConstraint("role IN ('viewer','operator','admin')", name="ck_managed_operator_role"),
CheckConstraint("version >= 1", name="ck_managed_operator_version"),
)
slug: Mapped[str] = mapped_column(String(63), primary_key=True)
role: Mapped[str] = mapped_column(String(16), nullable=False)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class ManagedOperatorCredential(Base):
__tablename__ = "managed_operator_credentials"
__table_args__ = (
Index("uq_managed_operator_current", "identity_slug", unique=True,
postgresql_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL"),
sqlite_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL")),
)
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=uuid.uuid4)
identity_slug: Mapped[str] = mapped_column(
ForeignKey("managed_operator_identities.slug", ondelete="RESTRICT"), index=True,
)
token_sha256: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
superseded_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class ManagedOperatorCredentialActivation(Base):
"""Rebuildable admission evidence, deliberately cleared by supported recovery."""
__tablename__ = "managed_operator_credential_activations"
credential_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("managed_operator_credentials.id", ondelete="RESTRICT"), primary_key=True,
)
activated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class ConfigurationState(Base):
__tablename__ = "configuration_state"
__table_args__ = (CheckConstraint("id = 1"), CheckConstraint("revision >= 0"))
id: Mapped[int] = mapped_column(Integer, primary_key=True)
revision: Mapped[int] = mapped_column(Integer, nullable=False)
overrides: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
class ConfigurationRevision(Base):
__tablename__ = "configuration_revisions"
revision: Mapped[int] = mapped_column(Integer, primary_key=True)
previous_revision: Mapped[int] = mapped_column(Integer, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
actor: Mapped[str] = mapped_column(String(255), nullable=False)
correlation_id: Mapped[str] = mapped_column(String(255), nullable=False)
overrides: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
preview_digest: Mapped[str] = mapped_column(String(64), nullable=False)
_PROTECTED_AUDIT_TABLES = frozenset({"audit_events", "audit_chain_heads"})
@@ -974,12 +1091,14 @@ class CapabilityContract(Base, IdMixin, TimestampMixin):
__tablename__ = "capability_contracts"
__table_args__ = (
UniqueConstraint("capability_id", "version", name="uq_capability_contract_version"),
CheckConstraint("serving_generation >= 0", name="ck_capability_serving_generation"),
)
capability_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("capabilities.id", ondelete="RESTRICT"), index=True
)
version: Mapped[int] = mapped_column(Integer)
serving_generation: Mapped[int] = mapped_column(BigInteger, default=0, server_default="0")
input_schema: Mapped[dict[str, Any]] = mapped_column(JSON)
output_schema: Mapped[dict[str, Any]] = mapped_column(JSON)
contract: Mapped[dict[str, Any]] = mapped_column(JSON)
@@ -1000,6 +1119,87 @@ class Project(Base, IdMixin, TimestampMixin):
bindings: Mapped[list[ProjectBinding]] = relationship(back_populates="project")
class PortfolioRepositoryObservation(Base, IdMixin, TimestampMixin):
"""Latest source inventory state, optionally linked to a ModelForge project identity."""
__tablename__ = "portfolio_repository_observations"
__table_args__ = (
UniqueConstraint(
"source_system", "external_repository_id", name="uq_portfolio_repo_external_id"
),
UniqueConstraint("source_system", "project_key", name="uq_portfolio_source_project_key"),
UniqueConstraint("project_id", name="uq_portfolio_repository_project_id"),
CheckConstraint("length(head_sha) = 40", name="ck_portfolio_repo_head_sha_length"),
CheckConstraint(
"pending_head_sha IS NULL OR length(pending_head_sha) = 40",
name="ck_portfolio_repo_pending_sha_length",
),
CheckConstraint(
"classification IN ('local_model', 'hosted_model', 'gateway', 'mixed', "
"'model_management', 'runtime_observer', 'non_ai', 'unknown')",
name="ck_portfolio_repo_classification",
),
)
project_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("projects.id", ondelete="RESTRICT"), index=True, nullable=True
)
project_key: Mapped[str | None] = mapped_column(String(128))
source_system: Mapped[str] = mapped_column(String(16))
external_repository_id: Mapped[str] = mapped_column(String(128))
slug: Mapped[str] = mapped_column(String(255))
default_branch: Mapped[str] = mapped_column(String(255))
head_sha: Mapped[str] = mapped_column(String(40))
classification: Mapped[str] = mapped_column(String(32))
code_evidence: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list)
runtime_evidence: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list)
source_observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
source_snapshot_id: Mapped[str] = mapped_column(String(128))
source_snapshot_digest: Mapped[str] = mapped_column(String(64))
pending_head_sha: Mapped[str | None] = mapped_column(String(40))
pending_default_branch: Mapped[str | None] = mapped_column(String(255))
pending_project_key: Mapped[str | None] = mapped_column(String(128))
pending_classification: Mapped[str | None] = mapped_column(String(32))
pending_code_evidence: Mapped[list[dict[str, Any]] | None] = mapped_column(JSON)
pending_observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
pending_snapshot_id: Mapped[str | None] = mapped_column(String(128))
pending_snapshot_digest: Mapped[str | None] = mapped_column(String(64))
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
stale_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class PortfolioReconciliationRun(Base, IdMixin, TimestampMixin):
"""Idempotency ledger for exact replay and conflict detection of source snapshots."""
__tablename__ = "portfolio_reconciliation_runs"
__table_args__ = (
UniqueConstraint("source_system", "snapshot_id", name="uq_portfolio_snapshot_identity"),
CheckConstraint("length(payload_digest) = 64", name="ck_portfolio_snapshot_digest_length"),
CheckConstraint(
"projects_created >= 0 AND repositories_created >= 0 "
"AND repositories_updated >= 0 AND stale_observations >= 0 "
"AND repositories_marked_stale >= 0 AND runtime_observations_updated >= 0 "
"AND head_changes_review_required >= 0 AND head_changes_accepted >= 0",
name="ck_portfolio_snapshot_counts_nonnegative",
),
)
source_system: Mapped[str] = mapped_column(String(16))
snapshot_id: Mapped[str] = mapped_column(String(128))
payload_digest: Mapped[str] = mapped_column(String(64))
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
complete: Mapped[bool] = mapped_column(Boolean, default=False)
stale_snapshot: Mapped[bool] = mapped_column(Boolean, default=False)
projects_created: Mapped[int] = mapped_column(Integer, default=0)
repositories_created: Mapped[int] = mapped_column(Integer, default=0)
repositories_updated: Mapped[int] = mapped_column(Integer, default=0)
stale_observations: Mapped[int] = mapped_column(Integer, default=0)
repositories_marked_stale: Mapped[int] = mapped_column(Integer, default=0)
runtime_observations_updated: Mapped[int] = mapped_column(Integer, default=0)
head_changes_review_required: Mapped[int] = mapped_column(Integer, default=0)
head_changes_accepted: Mapped[int] = mapped_column(Integer, default=0)
class ProjectBinding(Base, IdMixin, TimestampMixin):
__tablename__ = "project_bindings"
__table_args__ = (
@@ -1026,6 +1226,18 @@ class ProjectBinding(Base, IdMixin, TimestampMixin):
project: Mapped[Project] = relationship(back_populates="bindings")
class ProjectOnboarding(Base):
__tablename__ = "project_onboarding"
project_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("projects.id", ondelete="RESTRICT"), primary_key=True
)
owner: Mapped[str] = mapped_column(String(128))
environment: Mapped[str] = mapped_column(String(32))
labels: Mapped[list[str]] = mapped_column(JSON, default=list)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class ComputeNode(Base, IdMixin, TimestampMixin):
__tablename__ = "compute_nodes"
@@ -1335,9 +1547,28 @@ class BenchmarkSuite(Base, IdMixin, TimestampMixin):
class BenchmarkRun(Base, IdMixin):
__tablename__ = "benchmark_runs"
deployment_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("deployments.id", ondelete="RESTRICT"), index=True
__table_args__ = (
UniqueConstraint(
"actor_subject", "idempotency_key", name="uq_benchmark_runs_actor_idempotency"
),
CheckConstraint(
"(actor_subject IS NULL AND idempotency_key IS NULL AND "
"request_fingerprint IS NULL AND capability_deployment_id IS NULL) OR "
"(actor_subject IS NOT NULL AND idempotency_key IS NOT NULL AND "
"request_fingerprint IS NOT NULL AND capability_deployment_id IS NOT NULL)",
name="ck_benchmark_runs_operator_identity",
),
)
deployment_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("deployments.id", ondelete="RESTRICT"), index=True, nullable=True
)
capability_deployment_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("capability_deployments.id", ondelete="RESTRICT")
)
target_snapshot: Mapped[dict[str, Any] | None] = mapped_column(JSON)
actor_subject: Mapped[str | None] = mapped_column(String(255))
idempotency_key: Mapped[str | None] = mapped_column(String(128))
request_fingerprint: Mapped[str | None] = mapped_column(String(64))
suite_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("benchmark_suites.id", ondelete="RESTRICT"), index=True
)
@@ -1354,10 +1585,16 @@ class BenchmarkRun(Base, IdMixin):
seed: Mapped[int | None] = mapped_column(Integer)
results: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
status: Mapped[str] = mapped_column(String(32))
failure_code: Mapped[str | None] = mapped_column(String(64))
comparable: Mapped[bool] = mapped_column(Boolean, default=True)
stale_reason: Mapped[str | None] = mapped_column(Text)
cancellation_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
generation: Mapped[int] = mapped_column(Integer, default=1, server_default="1", nullable=False)
__mapper_args__ = {"version_id_col": generation} # noqa: RUF012
class Experiment(Base, IdMixin, TimestampMixin):
@@ -1621,8 +1858,8 @@ class ServiceCredential(Base, IdMixin):
"uq_active_service_client_credential",
"service_client_id",
unique=True,
postgresql_where=sa_text("revoked_at IS NULL"),
sqlite_where=sa_text("revoked_at IS NULL"),
postgresql_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL"),
sqlite_where=sa_text("revoked_at IS NULL AND superseded_at IS NULL"),
),
)
@@ -1631,12 +1868,115 @@ class ServiceCredential(Base, IdMixin):
)
secret_hash: Mapped[str] = mapped_column(String(64), unique=True)
secret_prefix: Mapped[str] = mapped_column(String(16))
superseded_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class ExternalCreativeAllocation(Base, IdMixin):
"""A service-client-owned reservation for a creative workload run outside ModelForge."""
__tablename__ = "external_creative_allocations"
__table_args__ = (
UniqueConstraint(
"service_client_id", "idempotency_key", name="uq_external_creative_idempotency"
),
CheckConstraint("required_vram_bytes > 0", name="ck_external_creative_vram_positive"),
CheckConstraint("ttl_seconds >= 30 AND ttl_seconds <= 3600", name="ck_external_creative_ttl"),
)
service_client_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("service_clients.id", ondelete="RESTRICT"), index=True
)
project_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("projects.id", ondelete="RESTRICT"), index=True
)
accelerator_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("accelerators.id", ondelete="RESTRICT"), index=True
)
request_id: Mapped[str] = mapped_column(String(128), index=True)
idempotency_key: Mapped[str] = mapped_column(String(128))
request_sha256: Mapped[str] = mapped_column(String(64))
workload: Mapped[str] = mapped_column(String(32))
resource_ref: Mapped[str] = mapped_column(String(255))
provider_resource_id: Mapped[str] = mapped_column(String(128))
required_vram_bytes: Mapped[int] = mapped_column(BigInteger)
priority: Mapped[str] = mapped_column(String(32))
ttl_seconds: Mapped[int] = mapped_column(Integer)
state: Mapped[str] = mapped_column(String(32), index=True)
decision_code: Mapped[str] = mapped_column(String(64))
decision_evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
required_actions: Mapped[list[str]] = mapped_column(JSON, default=list)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
acquired_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
uncertain_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
released_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class ExternalCreativeResource(Base, IdMixin):
__tablename__ = "external_creative_resources"
__table_args__ = (
UniqueConstraint("project_id", "resource_ref", "workload", "provider_resource_id",
name="uq_external_creative_resource_binding"),
CheckConstraint("configured_vram_bytes > 0", name="ck_external_resource_vram_positive"),
Index("ix_external_creative_resources_project", "project_id", "approved"),
)
project_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("projects.id", ondelete="RESTRICT"))
resource_ref: Mapped[str] = mapped_column(String(255))
workload: Mapped[str] = mapped_column(String(32))
provider_resource_id: Mapped[str] = mapped_column(String(128))
approved: Mapped[bool] = mapped_column(Boolean, server_default="false")
configured_vram_bytes: Mapped[int] = mapped_column(BigInteger)
measured_peak_vram_bytes: Mapped[int | None] = mapped_column(BigInteger)
measurement_count: Mapped[int] = mapped_column(Integer, server_default="0")
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class ExternalCreativeActionAttestation(Base, IdMixin):
__tablename__ = "external_creative_action_attestations"
__table_args__ = (
UniqueConstraint("allocation_id", "action", name="uq_external_creative_action_attestation"),
CheckConstraint("before_vram_bytes >= 0 AND after_vram_bytes >= 0",
name="ck_external_action_vram_nonnegative"),
)
allocation_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"))
action: Mapped[str] = mapped_column(String(64))
before_vram_bytes: Mapped[int] = mapped_column(BigInteger)
after_vram_bytes: Mapped[int] = mapped_column(BigInteger)
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
validated: Mapped[bool] = mapped_column(Boolean)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class ExternalCreativeProfileObservation(Base, IdMixin):
__tablename__ = "external_creative_profile_observations"
__table_args__ = (
UniqueConstraint("allocation_id", name="uq_external_creative_profile_observation_allocation"),
CheckConstraint("observed_peak_vram_bytes > 0 AND reserved_vram_bytes > 0",
name="ck_external_profile_vram_positive"),
CheckConstraint("duration_ms >= 0 AND duration_ms <= 604800000",
name="ck_external_profile_duration"),
)
allocation_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("external_creative_allocations.id", ondelete="RESTRICT"))
resource_ref: Mapped[str] = mapped_column(String(255))
provider_resource_id: Mapped[str] = mapped_column(String(128))
workload: Mapped[str] = mapped_column(String(32))
observed_peak_vram_bytes: Mapped[int] = mapped_column(BigInteger)
reserved_vram_bytes: Mapped[int] = mapped_column(BigInteger)
duration_ms: Mapped[int] = mapped_column(BigInteger)
outcome: Mapped[str] = mapped_column(String(16))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class ResidencyAllocation(Base, IdMixin, TimestampMixin):
__tablename__ = "residency_allocations"
__table_args__ = (UniqueConstraint("capability_deployment_id", name="uq_residency_deployment"),)
@@ -1739,6 +2079,9 @@ class ServingJob(Base, IdMixin, TimestampMixin):
gateway_request_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("gateway_requests.id", ondelete="SET NULL"), index=True
)
lab_execution_run_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("lab_execution_runs.id", ondelete="SET NULL"), index=True
)
operation: Mapped[str] = mapped_column(String(32), index=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
priority: Mapped[str] = mapped_column(String(32), default="production")
@@ -1755,6 +2098,53 @@ class ServingJob(Base, IdMixin, TimestampMixin):
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class LabExecutionRun(Base, IdMixin, TimestampMixin):
__tablename__ = "lab_execution_runs"
__table_args__ = (
UniqueConstraint("actor_subject", "idempotency_key", name="uq_lab_run_actor_idempotency"),
CheckConstraint(
"status IN ('accepted', 'preparing', 'queued', 'running', 'cancel_requested', "
"'completed', 'failed', 'cancelled', 'interrupted')",
name="ck_lab_execution_run_status",
),
CheckConstraint("input_count BETWEEN 1 AND 8", name="ck_lab_execution_run_input_count"),
CheckConstraint("timeout_seconds BETWEEN 5 AND 300", name="ck_lab_execution_run_timeout"),
CheckConstraint(
"queue_timeout_seconds BETWEEN 1 AND 60", name="ck_lab_execution_run_queue_timeout"
),
CheckConstraint(
"retention_seconds BETWEEN 60 AND 3600", name="ck_lab_execution_run_retention"
),
)
actor_subject: Mapped[str] = mapped_column(String(255), index=True)
idempotency_key: Mapped[str] = mapped_column(String(128))
request_fingerprint: Mapped[str] = mapped_column(String(64))
status: Mapped[str] = mapped_column(String(24), default="accepted", index=True)
capability: Mapped[str] = mapped_column(String(128), default="rag.embedding@1")
target_snapshot: Mapped[dict[str, Any]] = mapped_column(JSON)
input_sha256: Mapped[str] = mapped_column(String(64))
input_count: Mapped[int] = mapped_column(Integer)
input_encoded_bytes: Mapped[int] = mapped_column(Integer)
payload_reference: Mapped[str | None] = mapped_column(String(128))
input_payload_expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
gateway_request_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("gateway_requests.id", ondelete="RESTRICT"), unique=True, index=True
)
timeout_seconds: Mapped[int] = mapped_column(Integer, default=60)
queue_timeout_seconds: Mapped[int] = mapped_column(Integer, default=10)
retention_seconds: Mapped[int] = mapped_column(Integer, default=300)
output_metadata: Mapped[dict[str, Any] | None] = mapped_column(JSON)
worker_observations: Mapped[dict[str, Any] | None] = mapped_column(JSON)
failure_code: Mapped[str | None] = mapped_column(String(64))
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
dispatch_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
cancellation_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
cancellation_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class SchedulerPolicyRevision(Base, IdMixin):
__tablename__ = "scheduler_policy_revisions"
@@ -2439,6 +2829,12 @@ Index("ix_migrations_binding_status", Migration.project_binding_id, Migration.st
Index(
"ix_benchmark_runs_deployment_completed", BenchmarkRun.deployment_id, BenchmarkRun.completed_at
)
Index("ix_benchmark_runs_actor_subject", BenchmarkRun.actor_subject)
Index(
"ix_benchmark_runs_capability_deployment_completed",
BenchmarkRun.capability_deployment_id,
BenchmarkRun.completed_at,
)
Index(
"ix_capability_deployments_contract_channel_status",
CapabilityDeployment.capability_contract_id,
@@ -3085,7 +3481,7 @@ def _append_canonical_audit_event(
"""
if not isinstance(details, dict):
raise ValueError("canonical audit details must be an object")
raise TypeError("canonical audit details must be an object")
if expected_event_count != expected_last_sequence:
raise ValueError("canonical audit append requires a coherent predecessor count")
if expected_hash_format != AUDIT_CURRENT_HASH_FORMAT:
@@ -3518,6 +3914,43 @@ class AlertHistoryEvent(Base, IdMixin):
)
class AlertNotificationDelivery(Base, IdMixin):
__tablename__ = "alert_notification_deliveries"
__table_args__ = (
UniqueConstraint("idempotency_key", name="uq_alert_notification_idempotency"),
CheckConstraint(
"event_kind IN ('FIRING', 'RESOLVED', 'TEST')",
name="ck_alert_notification_event_kind",
),
CheckConstraint(
"status IN ('PENDING', 'DISPATCHING', 'RETRY_WAIT', 'DELIVERED', 'FAILED', 'HELD_FOR_REVIEW')",
name="ck_alert_notification_status",
),
CheckConstraint("attempt_count >= 0", name="ck_alert_notification_attempt_count"),
)
alert_history_event_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("alert_history_events.id", ondelete="RESTRICT"), index=True
)
route_key: Mapped[str] = mapped_column(String(64), default="default")
idempotency_key: Mapped[str] = mapped_column(String(160), nullable=False)
event_kind: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
severity: Mapped[str | None] = mapped_column(String(16), index=True)
payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
status: Mapped[str] = mapped_column(String(24), nullable=False, index=True)
attempt_count: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
delivered_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
claim_token_hash: Mapped[str | None] = mapped_column(String(64))
last_http_status: Mapped[int | None] = mapped_column(Integer)
last_error_code: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), nullable=False
)
class MaintenanceWindow(Base, IdMixin):
__tablename__ = "maintenance_windows"
@@ -4289,3 +4722,76 @@ def _prevent_immutable_field_changes(_mapper: Any, _connection: Any, target: Bas
for _immutable_type in IMMUTABLE_FIELDS:
event.listen(_immutable_type, "before_update", _prevent_immutable_field_changes)
class ModelUpdateWatch(Base, IdMixin):
__tablename__ = "model_update_watches"
__table_args__ = (UniqueConstraint("model_id", "upstream_ref", name="uq_model_watch_ref"),)
model_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("models.id", ondelete="RESTRICT"))
baseline_revision_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("model_revisions.id", ondelete="RESTRICT"))
baseline_commit_sha: Mapped[str] = mapped_column(String(64))
upstream_ref: Mapped[str] = mapped_column(String(255))
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
version: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_error_code: Mapped[str | None] = mapped_column(String(64))
check_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
check_token: Mapped[uuid.UUID | None] = mapped_column(Uuid)
class ModelUpdateNotice(Base, IdMixin):
__tablename__ = "model_update_notices"
__table_args__ = (
UniqueConstraint("watch_id", "candidate_commit_sha", name="uq_model_notice_commit"),
CheckConstraint("status IN ('new', 'dismissed')", name="ck_model_notice_status"),
)
watch_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("model_update_watches.id", ondelete="RESTRICT"))
model_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("models.id", ondelete="RESTRICT"))
baseline_commit_sha: Mapped[str] = mapped_column(String(64))
candidate_revision_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("model_revisions.id", ondelete="RESTRICT"))
candidate_commit_sha: Mapped[str] = mapped_column(String(64))
status: Mapped[str] = mapped_column(String(16), default="new")
discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class SelectedModelComparison(Base, IdMixin, TimestampMixin):
"""One explicitly delegated current/candidate comparison; no implicit replay."""
__tablename__ = "selected_model_comparisons"
__table_args__ = (
UniqueConstraint("project_binding_id", "idempotency_key", name="uq_selected_comparison_key"),
Index("uq_selected_comparison_active_binding", "project_binding_id", unique=True,
postgresql_where=sa_text("status IN ('pending', 'running', 'cancel_requested')"),
sqlite_where=sa_text("status IN ('pending', 'running', 'cancel_requested')")),
CheckConstraint("status IN ('pending', 'running', 'cancel_requested', 'completed', 'failed', 'cancelled', 'interrupted')", name="ck_selected_comparison_status"),
CheckConstraint("expected_cases BETWEEN 1 AND 100", name="ck_selected_comparison_cases"),
CheckConstraint("current_completed_cases BETWEEN 0 AND expected_cases AND candidate_completed_cases BETWEEN 0 AND expected_cases", name="ck_selected_comparison_progress"),
CheckConstraint("version > 0", name="ck_selected_comparison_version"),
CheckConstraint("current_deployment_id <> candidate_deployment_id", name="ck_selected_comparison_distinct_targets"),
)
project_binding_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("project_bindings.id", ondelete="RESTRICT"))
consumer_client_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("service_clients.id", ondelete="RESTRICT"))
suite_revision_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("evaluation_suite_revisions.id", ondelete="RESTRICT"))
current_deployment_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("capability_deployments.id", ondelete="RESTRICT"))
candidate_deployment_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("capability_deployments.id", ondelete="RESTRICT"))
candidate_route_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("capability_experiment_routes.id", ondelete="RESTRICT"))
idempotency_key: Mapped[str] = mapped_column(String(128))
request_digest: Mapped[str] = mapped_column(String(64))
actor_id: Mapped[str] = mapped_column(String(128))
pins: Mapped[dict[str, Any]] = mapped_column(JSON)
status: Mapped[str] = mapped_column(String(32), default="pending")
version: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
current_completed_cases: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
candidate_completed_cases: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
expected_cases: Mapped[int] = mapped_column(Integer)
claim_token_hash: Mapped[str | None] = mapped_column(String(64))
claim_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
deadline_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
claimed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
cancel_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
error_code: Mapped[str | None] = mapped_column(String(64))
result_digest: Mapped[str | None] = mapped_column(String(64))
baseline_run_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("evaluation_runs.id", ondelete="RESTRICT"))
candidate_run_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("evaluation_runs.id", ondelete="RESTRICT"))
evaluation_comparison_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("evaluation_comparisons.id", ondelete="RESTRICT"))
backend/src/modelforge_api/persistence/registry_repository.py
+16
View File
@@ -13,6 +13,8 @@ from modelforge_api.persistence.models import (
Model,
ModelArtifact,
ModelRevision,
ModelUpdateNotice,
ModelUpdateWatch,
RuntimeProfile,
StorageRoot,
)
@@ -171,6 +173,20 @@ class RegistryRepository:
def dependencies(self, resource_type: str, resource_id: uuid.UUID) -> list[dict[str, str]]:
dependencies: list[dict[str, str]] = []
if resource_type in {"model", "model_revision"}:
for entity, relation, column in (
(ModelUpdateWatch, "update_tracking_baseline",
ModelUpdateWatch.model_id if resource_type == "model"
else ModelUpdateWatch.baseline_revision_id),
(ModelUpdateNotice, "update_candidate_history",
ModelUpdateNotice.model_id if resource_type == "model"
else ModelUpdateNotice.candidate_revision_id),
):
dependencies.extend(
{"resource_type": "model_update_watch" if entity is ModelUpdateWatch
else "model_update_notice", "resource_id": str(item), "relation": relation}
for item in self.session.scalars(select(entity.id).where(column == resource_id))
)
if resource_type == "model":
ids = self.session.scalars(
select(ModelRevision.id).where(ModelRevision.model_id == resource_id)
backend/src/modelforge_api/persistence/runtime_engine.py
-1
View File
@@ -20,4 +20,3 @@ def register_application_engine(engine: Engine) -> Engine:
def is_application_connection(connection: Connection) -> bool:
return connection.engine in _APPLICATION_ENGINES
backend/src/modelforge_api/persistence/serving_repository.py
+13 -1
View File
@@ -53,6 +53,15 @@ class ServingRepository:
self.session.flush()
return entity
def lock_contract_route(self, contract_id: uuid.UUID) -> CapabilityContract | None:
"""Serialize stable-route writers; caller retains this lock through commit."""
return self.session.scalar(
select(CapabilityContract)
.where(CapabilityContract.id == contract_id)
.with_for_update()
.execution_options(populate_existing=True)
)
def capability(self, key: str) -> Capability | None:
return self.session.scalar(select(Capability).where(Capability.key == key))
@@ -251,6 +260,7 @@ class ServingRepository:
select(ServiceCredential).where(
ServiceCredential.service_client_id == client_id,
ServiceCredential.revoked_at.is_(None),
ServiceCredential.superseded_at.is_(None),
)
)
@@ -342,7 +352,9 @@ class ServingRepository:
ServingJob.compute_node_id == node_id,
or_(
ServingJob.status == "queued",
(ServingJob.status == "running") & (ServingJob.lease_expires_at < now),
(ServingJob.status == "running")
& ServingJob.lab_execution_run_id.is_(None)
& (ServingJob.lease_expires_at < now),
),
)
.order_by(priority_order, ServingJob.created_at)
backend/src/modelforge_api/providers/huggingface.py
+5 -3
View File
@@ -4,7 +4,7 @@ from collections.abc import Iterable
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import PurePosixPath
from typing import Any, Protocol
from typing import Any, Literal, Protocol
import httpx
from huggingface_hub import HfApi
@@ -116,8 +116,10 @@ class OfficialHuggingFaceProvider:
"""Narrow adapter over the official huggingface_hub client; no HTML scraping."""
def __init__(self, *, token: str | None = None, timeout: float = 30) -> None:
self.api = HfApi(token=token)
self.token = token
# Explicit False disables ambient HF_TOKEN/cache credentials. The official public
# provider endpoint is fixed; HF_ENDPOINT must not redirect credentials elsewhere.
self.api = HfApi(endpoint="https://huggingface.co", token=token or False)
self.token: str | Literal[False] = token or False
self.timeout = timeout
@staticmethod
backend/src/modelforge_api/services/acquisition.py
+9 -1
View File
@@ -4,6 +4,7 @@ import hashlib
import json
import secrets
import uuid
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -126,13 +127,20 @@ class AcquisitionService:
*,
actor_type: str = "operator",
actor_id: str = "local-operator",
credential_configured: bool | Callable[[], bool] | None = None,
) -> None:
self.session = session
self.settings = settings
self.provider = provider
self._credential_configured = bool(settings.hf_token) if credential_configured is None else credential_configured
self.repo = AcquisitionRepository(session)
self.audit = AuditWriter(session, actor_type, actor_id)
@property
def credential_configured(self) -> bool:
value = self._credential_configured
return value() if callable(value) else value
def _commit(self) -> None:
self.session.commit()
@@ -386,7 +394,7 @@ class AcquisitionService:
revision = self.session.get(ModelRevision, artifact_set.revision_id)
if not snapshot or not revision:
raise AcquisitionError("artifact set provenance is incomplete")
if snapshot.access_state != "public" and not self.settings.hf_token:
if snapshot.access_state != "public" and not self.credential_configured:
raise AcquisitionError(
f"repository access is {snapshot.access_state}; Hugging Face authentication is required"
)
backend/src/modelforge_api/services/alert_notifications.py
+317
View File
@@ -0,0 +1,317 @@
from __future__ import annotations
import hashlib
import secrets
from datetime import UTC, datetime, timedelta
from typing import Any
from urllib.parse import urlsplit
import httpx
import structlog
from sqlalchemy import or_, select, update
from sqlalchemy.orm import Session
from modelforge_api.api.authorization import Principal
from modelforge_api.domain.alert_notifications import (
NotificationDeliveryResponse,
NotificationKind,
NotificationStatus,
NotificationTestResponse,
)
from modelforge_api.persistence.models import (
AlertHistoryEvent,
AlertNotificationDelivery,
OperationalAlert,
)
from modelforge_api.services.audit import AuditContext, AuditWriter
from modelforge_api.settings import Settings
logger = structlog.get_logger()
_SEVERITY_RANK = {"INFO": 0, "WARNING": 1, "CRITICAL": 2}
_ACTIVE_STATES = ("PENDING", "DISPATCHING", "RETRY_WAIT")
class AlertNotificationError(RuntimeError):
def __init__(self, status_code: int, code: str, message: str) -> None:
super().__init__(message)
self.status_code = status_code
self.code = code
self.message = message
def _now() -> datetime:
return datetime.now(UTC)
def _route_ready(settings: Settings) -> bool:
if not settings.alert_notifications_enabled or settings.alert_webhook_url is None:
return False
try:
parsed = urlsplit(settings.alert_webhook_url.get_secret_value())
hostname = (parsed.hostname or "").casefold().rstrip(".")
allowed = set(settings.alert_webhook_allowed_hosts.split(","))
return (
parsed.scheme.casefold() == "https"
and hostname in allowed
and parsed.username is None
and parsed.password is None
and not parsed.fragment
and parsed.port in (None, 443)
)
except ValueError:
return False
def _payload(alert: OperationalAlert, event: AlertHistoryEvent) -> dict[str, Any]:
return {
"schema_version": 1,
"event_id": str(event.id),
"event": event.to_state,
"alert_id": str(alert.id),
"alert_type": alert.alert_type[:64],
"severity": alert.severity,
"subject_type": alert.subject_type[:64],
"subject_ref": alert.subject_ref[:255],
"summary": alert.summary[:500],
"occurred_at": event.occurred_at.isoformat(),
}
def enqueue_alert_transition(
session: Session,
settings: Settings,
alert: OperationalAlert,
event: AlertHistoryEvent,
) -> None:
"""Add one route delivery in the caller's alert/history transaction."""
kind = event.to_state
if (
not settings.alert_notifications_enabled
or kind not in {NotificationKind.FIRING.value, NotificationKind.RESOLVED.value}
or _SEVERITY_RANK.get(alert.severity, -1)
< _SEVERITY_RANK[settings.alert_webhook_min_severity]
):
return
session.add(
AlertNotificationDelivery(
alert_history_event_id=event.id,
route_key="default",
idempotency_key=f"alert:{event.id}:default",
event_kind=kind,
severity=alert.severity,
payload=_payload(alert, event),
status=NotificationStatus.PENDING.value,
attempt_count=0,
next_attempt_at=None,
)
)
class AlertNotificationService:
def __init__(self, session: Session, settings: Settings) -> None:
self.session = session
self.settings = settings
def queue_test(self, principal: Principal, correlation_id: str | None) -> NotificationTestResponse:
if not _route_ready(self.settings):
raise AlertNotificationError(
503,
"NOTIFICATION_ROUTE_UNAVAILABLE",
"The deployment-managed alert notification route is unavailable",
)
event_id = secrets.token_hex(16)
row = AlertNotificationDelivery(
alert_history_event_id=None,
route_key="default",
idempotency_key=f"test:{event_id}:default",
event_kind=NotificationKind.TEST.value,
severity=None,
payload={
"schema_version": 1,
"event_id": event_id,
"event": "TEST",
"summary": "ModelForge operational alert webhook test",
"occurred_at": _now().isoformat(),
},
status=NotificationStatus.PENDING.value,
attempt_count=0,
next_attempt_at=None,
)
self.session.add(row)
self.session.flush()
AuditWriter(
self.session,
context=AuditContext("operator", principal.subject, correlation_id),
).write(
"ALERT_NOTIFICATION_TEST_QUEUED",
"alert_notification_delivery",
str(row.id),
{"route_key": "default", "delivery_id": str(row.id)},
)
self.session.commit()
return NotificationTestResponse(
delivery=NotificationDeliveryResponse.model_validate(row),
message="Test notification queued for asynchronous delivery",
)
def deliveries(self, limit: int) -> list[NotificationDeliveryResponse]:
rows = self.session.scalars(
select(AlertNotificationDelivery)
.order_by(AlertNotificationDelivery.created_at.desc(), AlertNotificationDelivery.id.desc())
.limit(limit)
)
return [NotificationDeliveryResponse.model_validate(row) for row in rows]
def _claim(
session: Session, settings: Settings, now: datetime
) -> list[tuple[Any, str, str, dict[str, Any], int]]:
due = or_(
AlertNotificationDelivery.status == NotificationStatus.PENDING.value,
(
(AlertNotificationDelivery.status == NotificationStatus.RETRY_WAIT.value)
& (AlertNotificationDelivery.next_attempt_at <= now)
),
(
(AlertNotificationDelivery.status == NotificationStatus.DISPATCHING.value)
& (AlertNotificationDelivery.lease_expires_at <= now)
),
)
rows = list(
session.scalars(
select(AlertNotificationDelivery)
.where(due)
.order_by(AlertNotificationDelivery.created_at, AlertNotificationDelivery.id)
.limit(settings.alert_notification_batch_size)
.with_for_update(skip_locked=True)
)
)
claims: list[tuple[Any, str, str, dict[str, Any], int]] = []
for row in rows:
token = secrets.token_urlsafe(32)
row.status = NotificationStatus.DISPATCHING.value
row.attempt_count += 1
row.last_attempt_at = now
row.lease_expires_at = now + timedelta(seconds=settings.alert_notification_lease_seconds)
row.claim_token_hash = hashlib.sha256(token.encode()).hexdigest()
claims.append((row.id, row.idempotency_key, token, dict(row.payload), row.attempt_count))
session.commit()
return claims
def _retry_delay(settings: Settings, attempt: int) -> int:
multiplier = 1 << min(max(attempt - 1, 0), 10)
return min(settings.alert_notification_retry_base_seconds * multiplier, 3600)
def _safe_result(
status_code: int | None,
error_code: str | None,
*,
settings: Settings,
attempt: int,
) -> tuple[str, datetime | None, int | None, str | None]:
if status_code is not None and 200 <= status_code < 300:
return NotificationStatus.DELIVERED.value, None, status_code, None
retryable = (
(status_code is None and error_code != "DESTINATION_REJECTED")
or status_code in {408, 425, 429}
or (status_code is not None and status_code >= 500)
)
safe_code = error_code or (
"REMOTE_RETRYABLE" if retryable else "REMOTE_REJECTED"
)
if status_code is not None and 300 <= status_code < 400:
retryable = False
safe_code = "REDIRECT_REJECTED"
if retryable and attempt < settings.alert_notification_max_attempts:
due = _now() + timedelta(seconds=_retry_delay(settings, attempt))
return NotificationStatus.RETRY_WAIT.value, due, status_code, safe_code
return NotificationStatus.FAILED.value, None, status_code, safe_code
def _post_once(
settings: Settings,
payload: dict[str, Any],
idempotency_key: str,
transport: httpx.BaseTransport | None,
) -> tuple[int | None, str | None]:
webhook = settings.alert_webhook_url
if not _route_ready(settings) or webhook is None:
return None, "DESTINATION_REJECTED"
url = webhook.get_secret_value()
try:
with httpx.Client(
timeout=httpx.Timeout(settings.alert_notification_timeout_seconds),
follow_redirects=False,
trust_env=False,
transport=transport,
) as client, client.stream(
"POST",
url,
json=payload,
headers={
"Accept": "application/json",
"Content-Type": "application/json",
"Idempotency-Key": idempotency_key,
},
) as response:
return response.status_code, None
except httpx.TimeoutException:
return None, "DELIVERY_TIMEOUT"
except httpx.HTTPError:
return None, "DELIVERY_TRANSPORT_ERROR"
except (ValueError, OSError, RuntimeError):
return None, "DELIVERY_TRANSPORT_ERROR"
def dispatch_pending(
session: Session,
settings: Settings,
*,
now: datetime | None = None,
transport: httpx.BaseTransport | None = None,
) -> int:
"""Claim and deliver one bounded batch; network calls happen outside the claim transaction."""
if not settings.alert_notifications_enabled:
return 0
claims = _claim(session, settings, now or _now())
for row_id, idempotency_key, token, payload, attempt in claims:
event_id = str(payload.get("event_id", ""))[:64]
status_code, error_code = _post_once(
settings, payload, idempotency_key, transport
)
status, next_attempt, status_code, safe_code = _safe_result(
status_code, error_code, settings=settings, attempt=attempt
)
token_hash = hashlib.sha256(token.encode()).hexdigest()
values: dict[str, Any] = {
"status": status,
"next_attempt_at": next_attempt,
"lease_expires_at": None,
"claim_token_hash": None,
"last_http_status": status_code,
"last_error_code": safe_code,
}
if status == NotificationStatus.DELIVERED.value:
values["delivered_at"] = _now()
session.execute(
update(AlertNotificationDelivery)
.where(
AlertNotificationDelivery.id == row_id,
AlertNotificationDelivery.status == NotificationStatus.DISPATCHING.value,
AlertNotificationDelivery.claim_token_hash == token_hash,
)
.values(**values)
)
session.commit()
logger.info(
"alert_notification_delivery_attempted",
delivery_id=str(row_id),
event_id=event_id,
status=status,
error_code=safe_code,
http_status=status_code,
attempt=attempt,
)
return len(claims)
backend/src/modelforge_api/services/audit.py
+2 -1
View File
@@ -126,7 +126,8 @@ def _decode_canonical_audit_payload(value: str) -> dict[str, Any]:
decoded = json.loads(value, object_pairs_hook=reject_duplicate_keys)
if not isinstance(decoded, dict):
raise ValueError("canonical audit payload is not an object")
# The string is valid JSON but an invalid persisted audit value, not a caller type error.
raise ValueError("canonical audit payload is not an object") # noqa: TRY004
return decoded
backend/src/modelforge_api/services/audit_timeline.py
+209
View File
@@ -0,0 +1,209 @@
"""Read-only, bounded queries over append-only audit events."""
from __future__ import annotations
import base64
import binascii
import hashlib
import json
import re
import uuid
from dataclasses import dataclass
from datetime import UTC
from sqlalchemy import and_, or_, select
from sqlalchemy.orm import Session
from modelforge_api.domain.audit_timeline import (
AuditTimelineEvent,
AuditTimelineRequest,
AuditTimelineResponse,
)
from modelforge_api.persistence.models import AuditEvent
_CURSOR_VERSION = 1
_CURSOR_MAX_LENGTH = 256
class AuditTimelineCursorError(ValueError):
"""A cursor is malformed or was created for a different filter set."""
@dataclass(frozen=True, slots=True)
class _AuditCursor:
sequence: int
event_id: uuid.UUID
def _filters_digest(request: AuditTimelineRequest) -> str:
values: dict[str, object] = {
"principal": request.principal,
"action": request.action,
"resource": request.resource,
"outcome": request.outcome,
"occurred_after": (
request.occurred_after.astimezone(UTC).isoformat()
if request.occurred_after is not None
else None
),
"occurred_before": (
request.occurred_before.astimezone(UTC).isoformat()
if request.occurred_before is not None
else None
),
"correlation_id": request.correlation_id,
}
encoded = json.dumps(values, sort_keys=True, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _encode_cursor(sequence: int, event_id: uuid.UUID, filters_digest: str) -> str:
payload = json.dumps(
{
"v": _CURSOR_VERSION,
"sequence": sequence,
"event_id": str(event_id),
"filters": filters_digest,
},
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return base64.urlsafe_b64encode(payload).rstrip(b"=").decode("ascii")
def _decode_cursor(value: str, filters_digest: str) -> _AuditCursor:
if len(value) > _CURSOR_MAX_LENGTH or not re.fullmatch(r"[A-Za-z0-9_-]+", value):
raise AuditTimelineCursorError("invalid audit timeline cursor")
padded = value + "=" * (-len(value) % 4)
try:
decoded = base64.b64decode(padded, altchars=b"-_", validate=True)
payload = json.loads(decoded)
except (binascii.Error, UnicodeDecodeError, json.JSONDecodeError, ValueError):
raise AuditTimelineCursorError("invalid audit timeline cursor") from None
if not isinstance(payload, dict) or set(payload) != {"v", "sequence", "event_id", "filters"}:
raise AuditTimelineCursorError("invalid audit timeline cursor")
sequence = payload["sequence"]
event_id_value = payload["event_id"]
cursor_version = payload["v"]
cursor_filters = payload["filters"]
if (
type(cursor_version) is not int
or cursor_version != _CURSOR_VERSION
or type(sequence) is not int
or sequence < 1
or not isinstance(event_id_value, str)
or not isinstance(cursor_filters, str)
or not re.fullmatch(r"[0-9a-f]{64}", cursor_filters)
):
raise AuditTimelineCursorError("invalid audit timeline cursor")
try:
event_id = uuid.UUID(event_id_value)
except (AttributeError, TypeError, ValueError):
raise AuditTimelineCursorError("invalid audit timeline cursor") from None
if str(event_id) != event_id_value or cursor_filters != filters_digest:
raise AuditTimelineCursorError("audit timeline cursor does not match the supplied filters")
return _AuditCursor(sequence=sequence, event_id=event_id)
class AuditTimelineService:
"""Read a bounded page without returning arbitrary event details or changing the chain."""
def __init__(self, session: Session) -> None:
self.session = session
def list_events(self, request: AuditTimelineRequest) -> AuditTimelineResponse:
filters_digest = _filters_digest(request)
cursor = _decode_cursor(request.cursor, filters_digest) if request.cursor else None
predicates = []
if request.principal:
predicates.append(
or_(
AuditEvent.actor_id.icontains(request.principal, autoescape=True),
AuditEvent.actor_type.icontains(request.principal, autoescape=True),
)
)
if request.action:
predicates.append(AuditEvent.action.icontains(request.action, autoescape=True))
if request.resource:
predicates.append(
or_(
AuditEvent.resource_type.icontains(request.resource, autoescape=True),
AuditEvent.resource_id.icontains(request.resource, autoescape=True),
)
)
if request.outcome:
predicates.append(AuditEvent.outcome.icontains(request.outcome, autoescape=True))
if request.occurred_after is not None:
predicates.append(AuditEvent.occurred_at >= request.occurred_after)
if request.occurred_before is not None:
predicates.append(AuditEvent.occurred_at <= request.occurred_before)
if request.correlation_id:
predicates.append(
AuditEvent.correlation_id.icontains(request.correlation_id, autoescape=True)
)
if cursor is not None:
predicates.append(
or_(
AuditEvent.sequence < cursor.sequence,
and_(
AuditEvent.sequence == cursor.sequence,
AuditEvent.id < cursor.event_id,
),
)
)
statement = (
select(
AuditEvent.id,
AuditEvent.sequence,
AuditEvent.occurred_at,
AuditEvent.correlation_id,
AuditEvent.actor_type,
AuditEvent.actor_id,
AuditEvent.action,
AuditEvent.resource_type,
AuditEvent.resource_id,
AuditEvent.outcome,
)
.where(*predicates)
.order_by(AuditEvent.sequence.desc(), AuditEvent.id.desc())
.limit(request.limit + 1)
)
rows = self.session.execute(statement).all()
has_more = len(rows) > request.limit
page = rows[: request.limit]
events = [
AuditTimelineEvent(
id=event_id,
sequence=sequence,
occurred_at=(
occurred_at.replace(tzinfo=UTC)
if occurred_at.tzinfo is None
else occurred_at.astimezone(UTC)
),
correlation_id=correlation_id,
actor_type=actor_type,
actor_id=actor_id,
action=action,
resource_type=resource_type,
resource_id=resource_id,
outcome=outcome,
)
for (
event_id,
sequence,
occurred_at,
correlation_id,
actor_type,
actor_id,
action,
resource_type,
resource_id,
outcome,
) in page
]
next_cursor = None
if has_more and page:
last_event_id = uuid.UUID(str(page[-1][0]))
next_cursor = _encode_cursor(page[-1][1], last_event_id, filters_digest)
return AuditTimelineResponse(events=events, next_cursor=next_cursor, limit=request.limit)
backend/src/modelforge_api/services/benchmark_runs.py
+1222
View File
File diff suppressed because it is too large. Load diff
backend/src/modelforge_api/services/configuration.py
+145
View File
@@ -0,0 +1,145 @@
from __future__ import annotations
from pathlib import Path
from urllib.parse import urlsplit, urlunsplit
from pydantic import SecretStr
from modelforge_api.domain.configuration import (
ConfigurationApplicationMode,
ConfigurationCatalogResponse,
ConfigurationSettingResponse,
ConfigurationSource,
)
from modelforge_api.domain.configuration_policy import CONFIGURATION_CONSUMER, SERVING_OVERRIDE_KEYS
from modelforge_api.domain.configuration_reference import SETTING_DOCS, Sensitivity
from modelforge_api.settings import Settings
def _environment_variable_name(setting_key: str) -> str:
return f"MODELFORGE_{setting_key.upper()}"
def _is_configured(value: object) -> bool:
if value is None:
return False
if isinstance(value, SecretStr):
return bool(value.get_secret_value())
if isinstance(value, str):
return bool(value)
return True
def _redact_url_userinfo(value: str) -> str:
"""Remove URL userinfo even when a legacy setting is not classified as SECRET."""
try:
parsed = urlsplit(value)
except ValueError:
return value
if not parsed.scheme or "@" not in parsed.netloc:
return value
# Keep useful topology without leaking a username or password. rsplit handles userinfo that
# legitimately contains percent-encoded separators while preserving the complete host/port.
safe_netloc = parsed.netloc.rsplit("@", 1)[-1]
return urlunsplit((parsed.scheme, safe_netloc, parsed.path, parsed.query, parsed.fragment))
def _safe_effective_value(
value: object, sensitivity: Sensitivity
) -> str | int | float | bool | None:
if sensitivity is Sensitivity.SECRET:
return None
if value is None:
return None
if isinstance(value, bool):
return value
if isinstance(value, str):
return _redact_url_userinfo(value)
if isinstance(value, int | float):
return value
if isinstance(value, Path):
return str(value)
# Future settings may gain structured values. A read-only catalog must not guess a wire format
# or accidentally serialise an object with surprising/revealing repr semantics.
return None
def build_configuration_catalog(
settings: Settings,
*,
revision: int = 0,
overrides: dict[str, int | float] | None = None,
) -> ConfigurationCatalogResponse:
"""Return the runtime configuration reference without exposing credential material."""
setting_fields = set(type(settings).model_fields)
documented_fields = set(SETTING_DOCS)
if setting_fields != documented_fields:
missing_docs = sorted(setting_fields - documented_fields)
unknown_docs = sorted(documented_fields - setting_fields)
raise RuntimeError(
"configuration reference is out of sync with Settings: "
f"missing_docs={missing_docs}, unknown_docs={unknown_docs}"
)
active = overrides or {}
schema = Settings.model_json_schema()["properties"]
entries: list[ConfigurationSettingResponse] = []
for key in type(settings).model_fields:
documentation = SETTING_DOCS[key]
environment_variable = _environment_variable_name(key)
value = getattr(settings, key)
field = type(settings).model_fields[key]
rules = {
name: rule
for name, rule in schema[key].items()
if name in {"minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "maxLength"}
}
editable = key in SERVING_OVERRIDE_KEYS
if key == "scheduler_safety_reserve_percentage":
rules.pop("maximum", None)
rules["exclusiveMaximum"] = 0.5
if key == "gateway_queue_timeout_seconds":
rules["less_than_setting"] = "gateway_request_timeout_seconds"
entries.append(
ConfigurationSettingResponse(
key=key,
display_name=key.replace("_", " ").capitalize(),
category=key.split("_", 1)[0],
data_type=schema[key].get("type", "optional"),
default_value=_safe_effective_value(field.default, documentation.sensitivity),
validation_rules=rules,
editable=editable,
consumer=CONFIGURATION_CONSUMER if editable else None,
rollback=(
"Preview a previous revision or remove the override, then apply."
if editable
else "Restore deployment configuration and restart the component."
),
environment_variable=environment_variable,
description=documentation.description,
sensitivity=documentation.sensitivity,
required_in_production=documentation.required_in_production,
application_mode=(
ConfigurationApplicationMode.DYNAMIC
if editable
else ConfigurationApplicationMode.RESTART_REQUIRED
),
configured=_is_configured(value),
source=(
ConfigurationSource.OPERATOR_OVERRIDE
if key in active
else ConfigurationSource(settings.setting_source(key))
),
value=_safe_effective_value(value, documentation.sensitivity),
)
)
return ConfigurationCatalogResponse(
environment=settings.env,
settings=entries,
read_only=False,
revision=revision,
overrides=active,
)
backend/src/modelforge_api/services/configuration_changes.py
+326
View File
@@ -0,0 +1,326 @@
"""Atomic, preview-bound safe overrides; all persisted payloads are numeric and allowlisted."""
from __future__ import annotations
import hashlib
import hmac
import json
import math
from datetime import UTC, datetime
from pydantic import ValidationError
from sqlalchemy import select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from modelforge_api.db import rows_affected
from modelforge_api.domain.configuration import (
ConfigurationApplied,
ConfigurationApply,
ConfigurationChange,
ConfigurationDifference,
ConfigurationExport,
ConfigurationImport,
ConfigurationPreview,
ConfigurationRevisionResponse,
ConfigurationSource,
)
from modelforge_api.domain.configuration_policy import (
CONFIGURATION_CONSUMER,
CONFIGURATION_FORMAT,
SERVING_OVERRIDE_KEYS,
)
from modelforge_api.domain.configuration_reference import SETTING_DOCS, Sensitivity
from modelforge_api.domain.release import PRODUCT_VERSION
from modelforge_api.persistence.models import ConfigurationRevision, ConfigurationState
from modelforge_api.services.audit import AuditContext, AuditWriter
from modelforge_api.settings import Settings
class ConfigurationError(Exception):
def __init__(self, code: str, message: str, status_code: int = 409) -> None:
super().__init__(message)
self.code = code
self.message = message
self.status_code = status_code
def _validate_overrides(settings: Settings, overrides: dict[str, int | float]) -> Settings:
for key, value in overrides.items():
if key not in SERVING_OVERRIDE_KEYS:
raise ConfigurationError(
"CONFIGURATION_KEY_NOT_EDITABLE",
"Only the documented serving settings can be changed here.",
422,
)
if type(value) not in (int, float) or (
isinstance(value, float) and not math.isfinite(value)
):
raise ConfigurationError("CONFIGURATION_INVALID", "A finite number is required.", 422)
if Settings.model_fields[key].annotation is int and type(value) is not int:
raise ConfigurationError("CONFIGURATION_INVALID", f"{key} requires an integer.", 422)
try:
candidate = Settings.model_validate(settings.model_dump() | overrides)
candidate._inherit_setting_sources(settings)
except ValidationError as exc:
# Validation input may contain credentials from the deployment settings. Never echo it.
raise ConfigurationError(
"CONFIGURATION_INVALID",
"A value is outside the documented validation bounds.",
422,
) from exc
if candidate.gateway_queue_timeout_seconds >= candidate.gateway_request_timeout_seconds:
raise ConfigurationError(
"CONFIGURATION_INVALID",
"Queue timeout must be lower than request timeout.",
422,
)
if candidate.scheduler_safety_reserve_percentage >= 0.5:
raise ConfigurationError(
"CONFIGURATION_INVALID",
"The GPU safety reserve must be below 50%.",
422,
)
return candidate
def configuration_state(session: Session) -> tuple[int, dict[str, int | float]]:
row = session.execute(
select(
ConfigurationState.revision,
ConfigurationState.overrides,
).where(ConfigurationState.id == 1)
).first()
if row is None:
# Empty create_all test databases use the same first-write CAS as a fresh registry.
return 0, {}
return row.revision, dict(row.overrides)
def resolve_serving_settings(session: Session, settings: Settings) -> Settings:
return resolved_configuration(session, settings)[2]
def resolved_configuration(
session: Session,
settings: Settings,
) -> tuple[int, dict[str, int | float], Settings]:
revision, overrides = configuration_state(session)
return revision, overrides, _validate_overrides(settings, overrides) if overrides else settings
class ConfigurationService:
def __init__(self, session: Session, settings: Settings, context: AuditContext) -> None:
self.session = session
self.settings = settings
self.context = context
def preview(self, request: ConfigurationChange) -> ConfigurationPreview:
revision, current = configuration_state(self.session)
if request.expected_revision != revision:
raise ConfigurationError(
"CONFIGURATION_CONFLICT", "Configuration changed; refresh and preview again."
)
desired = dict(current)
for key, value in request.changes.items():
if key not in SERVING_OVERRIDE_KEYS:
raise ConfigurationError(
"CONFIGURATION_KEY_NOT_EDITABLE",
"The requested setting is deployment-managed.",
422,
)
if value is None:
desired.pop(key, None)
else:
desired[key] = value
before = _validate_overrides(self.settings, current)
after = _validate_overrides(self.settings, desired)
differences = [
ConfigurationDifference(
key=key,
before=getattr(before, key),
after=getattr(after, key),
source_before=(
ConfigurationSource.OPERATOR_OVERRIDE
if key in current
else ConfigurationSource(self.settings.setting_source(key))
),
source_after=(
ConfigurationSource.OPERATOR_OVERRIDE
if key in desired
else ConfigurationSource(self.settings.setting_source(key))
),
)
for key in sorted(request.changes)
if getattr(before, key) != getattr(after, key) or (key in current) != (key in desired)
]
if not differences:
raise ConfigurationError(
"CONFIGURATION_NO_CHANGE", "The proposal does not change configuration.", 422
)
# Include every editable base value: deployment changes invalidate earlier previews, even
# if the operator revision has not advanced. No credential enters this digest.
baseline = {key: getattr(self.settings, key) for key in sorted(SERVING_OVERRIDE_KEYS)}
payload = {
"format": CONFIGURATION_FORMAT,
"revision": revision,
"base": baseline,
"current": current,
"desired": desired,
}
digest = hashlib.sha256(
json.dumps(
payload,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
).hexdigest()
return ConfigurationPreview(
expected_revision=revision,
preview_digest=digest,
differences=differences,
affected_components=[CONFIGURATION_CONSUMER],
overrides=desired,
impact="New serving requests use this snapshot. In-flight requests retain their settings. "
"Admission and queue changes may reject new work; no model is promoted or worker restarted.",
rollback="Preview a retained revision or remove an override, then explicitly apply it as a new revision.",
)
def apply(self, request: ConfigurationApply) -> ConfigurationApplied:
preview = self.preview(request)
if not hmac.compare_digest(request.preview_digest, preview.preview_digest):
raise ConfigurationError(
"CONFIGURATION_PREVIEW_STALE", "The proposal changed; preview it again."
)
next_revision = request.expected_revision + 1
try:
result = self.session.execute(
update(ConfigurationState)
.where(
ConfigurationState.id == 1,
ConfigurationState.revision == request.expected_revision,
)
.values(revision=next_revision, overrides=preview.overrides)
)
if rows_affected(result) != 1:
if request.expected_revision != 0:
raise ConfigurationError(
"CONFIGURATION_CONFLICT", "A concurrent configuration change won."
)
self.session.add(ConfigurationState(id=1, revision=1, overrides=preview.overrides))
self.session.flush() # Unique singleton also protects the empty-registry race.
self.session.add(
ConfigurationRevision(
revision=next_revision,
previous_revision=request.expected_revision,
created_at=datetime.now(UTC),
actor=self.context.actor_id,
correlation_id=self.context.correlation_id or "unknown",
overrides=preview.overrides,
preview_digest=preview.preview_digest,
)
)
self.session.flush()
# Exercise the exact reader used by every new ServingService before acknowledging.
resolve_serving_settings(self.session, self.settings)
AuditWriter(self.session, context=self.context).write(
"configuration.applied",
"configuration_revision",
str(next_revision),
{
"previous_revision": request.expected_revision,
"preview_digest": preview.preview_digest,
"differences": [item.model_dump() for item in preview.differences],
"acknowledged_by": CONFIGURATION_CONSUMER,
},
)
self.session.commit()
except IntegrityError as exc:
self.session.rollback()
raise ConfigurationError(
"CONFIGURATION_CONFLICT", "A concurrent configuration change won."
) from exc
except Exception:
self.session.rollback()
raise
return ConfigurationApplied(
revision=next_revision,
acknowledged_by=CONFIGURATION_CONSUMER,
preview_digest=preview.preview_digest,
)
def history(self, limit: int = 30) -> list[ConfigurationRevisionResponse]:
rows = self.session.scalars(
select(ConfigurationRevision)
.order_by(
ConfigurationRevision.revision.desc(),
)
.limit(limit)
)
return [
ConfigurationRevisionResponse.model_validate(row, from_attributes=True) for row in rows
]
def rollback_change(self, revision: int, expected_revision: int) -> ConfigurationChange:
_, current = configuration_state(self.session)
if revision == 0:
desired: dict[str, int | float] = {}
else:
row = self.session.get(ConfigurationRevision, revision)
if row is None:
raise ConfigurationError(
"CONFIGURATION_REVISION_NOT_FOUND", "Configuration revision was not found.", 404
)
desired = row.overrides
if desired == current:
raise ConfigurationError("CONFIGURATION_NO_CHANGE", "That revision is already effective.", 422)
return ConfigurationChange(
expected_revision=expected_revision,
changes={key: desired.get(key) for key in current.keys() | desired.keys()},
)
def export(self) -> ConfigurationExport:
_, overrides = configuration_state(self.session)
_validate_overrides(self.settings, overrides)
return ConfigurationExport(
format=CONFIGURATION_FORMAT,
source_version=PRODUCT_VERSION,
overrides=overrides,
secret_slots=sorted({"huggingface_control_plane"} | {
key for key, doc in SETTING_DOCS.items() if doc.sensitivity is Sensitivity.SECRET
}),
)
def import_change(self, request: ConfigurationImport) -> ConfigurationChange:
document = request.document
if document.format != CONFIGURATION_FORMAT:
raise ConfigurationError(
"CONFIGURATION_FORMAT_UNSUPPORTED", "Unsupported configuration format.", 422
)
if document.source_version.split(".")[0] != PRODUCT_VERSION.split(".")[0]:
raise ConfigurationError(
"CONFIGURATION_VERSION_UNSUPPORTED", "Incompatible product major version.", 422
)
expected_slots = {"huggingface_control_plane"} | {
key for key, doc in SETTING_DOCS.items() if doc.sensitivity is Sensitivity.SECRET
}
if (
len(document.secret_slots) != len(set(document.secret_slots))
or set(document.secret_slots) != expected_slots
):
raise ConfigurationError(
"CONFIGURATION_SLOTS_INVALID",
"Secret slot references do not match this product.",
422,
)
_validate_overrides(self.settings, document.overrides)
_, current = configuration_state(self.session)
changes = {
key: document.overrides.get(key) for key in current.keys() | document.overrides.keys()
}
if not changes:
raise ConfigurationError(
"CONFIGURATION_NO_CHANGE", "The import contains no overrides to change.", 422
)
return ConfigurationChange(expected_revision=request.expected_revision, changes=changes)
backend/src/modelforge_api/services/evaluation.py
+32 -8
View File
@@ -130,8 +130,16 @@ def _percentile(values: list[float], fraction: float) -> float:
class EvaluationService:
def __init__(self, session: Session) -> None:
def __init__(self, session: Session, *, commit: bool = True) -> None:
self.session = session
self._commit_results = commit
def _finish_result(self, row: EvaluationRun | EvaluationComparison) -> None:
if self._commit_results:
self.session.commit()
else:
self.session.flush()
self.session.refresh(row)
def create_suite(self, request: EvaluationSuiteCreate) -> EvaluationSuiteResponse:
if self.session.get(Project, request.project_id) is None:
@@ -220,6 +228,8 @@ class EvaluationService:
"project or suite revision not found",
status_code=404,
)
if not self._suite_belongs_to_project(revision.id, request.project_id):
raise EvaluationError("suite_project_mismatch", "suite revision belongs to another project")
cases = list(
self.session.scalars(
select(EvaluationCase).where(EvaluationCase.suite_revision_id == revision.id)
@@ -254,8 +264,7 @@ class EvaluationService:
started_at=datetime.now(UTC),
)
self.session.add(run)
self.session.commit()
self.session.refresh(run)
self._finish_result(run)
return EvaluationRunResponse.model_validate(run)
def complete_run(
@@ -266,6 +275,8 @@ class EvaluationService:
raise EvaluationError("run_not_found", "evaluation run not found", status_code=404)
if run.status != "running":
raise EvaluationError("run_not_active", "only a running evaluation can complete")
if not self._suite_belongs_to_project(run.suite_revision_id, run.project_id):
raise EvaluationError("suite_project_mismatch", "suite revision belongs to another project")
cases = {
case.id: case
for case in self.session.scalars(
@@ -310,10 +321,17 @@ class EvaluationService:
run.aggregate_metrics = {name: totals[name] / len(request.results) for name in METRICS}
run.status = "completed" if errors == 0 else "completed_with_errors"
run.completed_at = datetime.now(UTC)
self.session.commit()
self.session.refresh(run)
self._finish_result(run)
return EvaluationRunResponse.model_validate(run)
def _suite_belongs_to_project(self, revision_id: uuid.UUID, project_id: uuid.UUID) -> bool:
owner = self.session.scalar(
select(EvaluationSuite.project_id)
.join(EvaluationSuiteRevision, EvaluationSuiteRevision.suite_id == EvaluationSuite.id)
.where(EvaluationSuiteRevision.id == revision_id)
)
return owner == project_id
def runs(self) -> list[EvaluationRunResponse]:
return [
EvaluationRunResponse.model_validate(item)
@@ -758,6 +776,8 @@ class EvaluationService:
raise EvaluationError("run_incomplete", "both runs must be complete")
factors = {
"project": baseline.project_id == candidate.project_id,
"baseline_suite_project": self._suite_belongs_to_project(baseline.suite_revision_id, baseline.project_id),
"candidate_suite_project": self._suite_belongs_to_project(candidate.suite_revision_id, candidate.project_id),
"suite_revision": baseline.suite_revision_id == candidate.suite_revision_id,
"corpus_revision": baseline.corpus_revision == candidate.corpus_revision,
"retrieval_config": baseline.retrieval_config_digest
@@ -765,7 +785,8 @@ class EvaluationService:
}
if all(factors.values()):
comparability = "comparable"
elif factors["project"] and factors["suite_revision"]:
elif (factors["project"] and factors["suite_revision"]
and factors["baseline_suite_project"] and factors["candidate_suite_project"]):
comparability = "partially_comparable"
else:
comparability = "not_comparable"
@@ -817,8 +838,10 @@ class EvaluationService:
for name in METRICS
}
error_rate = candidate.error_count / candidate.expected_cases
baseline_error_free = baseline.status == "completed" and baseline.error_count == 0
eligible = (
comparability == "comparable"
and baseline_error_free
and error_rate <= request.maximum_error_rate
and deltas["recall_at_10"] >= request.minimum_recall_at_10_delta
and (not request.require_no_critical_regressions or critical_regressions == 0)
@@ -843,13 +866,14 @@ class EvaluationService:
promotion_eligibility=eligibility,
eligibility_evidence={
"error_rate": error_rate,
"baseline_error_count": baseline.error_count,
"baseline_error_free": baseline_error_free,
"criteria": request.model_dump(mode="json"),
"promotion_performed": False,
},
)
self.session.add(comparison)
self.session.commit()
self.session.refresh(comparison)
self._finish_result(comparison)
return EvaluationComparisonResponse.model_validate(comparison)
def comparison(self, comparison_id: uuid.UUID) -> EvaluationComparisonResponse:
backend/src/modelforge_api/services/external_creative.py
+770
View File
@@ -0,0 +1,770 @@
from __future__ import annotations
import hashlib
import json
import uuid
from datetime import UTC, datetime, timedelta
from typing import cast
import sqlalchemy as sa
from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from modelforge_api.domain.external_creative import (
CreativeAllocationState,
CreativeRequiredAction,
CreativeWorkload,
ExternalActionAttestation,
ExternalAllocationCreate,
ExternalAllocationList,
ExternalAllocationResponse,
ExternalProfileObservation,
ExternalResourceApproval,
ExternalResourceList,
ExternalResourceResponse,
)
from modelforge_api.persistence.models import (
Accelerator,
ExternalCreativeActionAttestation,
ExternalCreativeAllocation,
ExternalCreativeProfileObservation,
ExternalCreativeResource,
Project,
ProjectBinding,
ServiceClient,
)
from modelforge_api.services.audit import AuditWriter
from modelforge_api.services.serving import ServingError, ServingService
ACTIVE = {"granted", "uncertain"}
PRIORITY = {"production": 0, "interactive": 1, "background": 2, "benchmark": 3}
resources = cast(sa.Table, ExternalCreativeResource.__table__)
attestations = cast(sa.Table, ExternalCreativeActionAttestation.__table__)
observations = cast(sa.Table, ExternalCreativeProfileObservation.__table__)
def _now() -> datetime:
return datetime.now(UTC)
def _digest(value: ExternalAllocationCreate) -> str:
return hashlib.sha256(
json.dumps(value.model_dump(mode="json"), sort_keys=True, separators=(",", ":")).encode()
).hexdigest()
class ExternalCreativeService:
def __init__(self, session: Session, serving: ServingService):
self.session, self.serving = session, serving
def _lock_accelerators(self) -> None:
self.session.scalars(select(Accelerator).order_by(Accelerator.id).with_for_update()).all()
def _project(self, client: ServiceClient) -> uuid.UUID:
fresh = self.session.get(
ServiceClient, client.id, populate_existing=True, with_for_update=True
)
binding = (
self.session.get(
ProjectBinding,
fresh.project_binding_id,
populate_existing=True,
with_for_update=True,
)
if fresh and fresh.project_binding_id
else None
)
project = (
self.session.get(
Project, binding.project_id, populate_existing=True, with_for_update=True
)
if binding
else None
)
if (
fresh is None
or fresh.status != "active"
or fresh.disabled_at is not None
or "creative.resource@1" not in fresh.allowed_capabilities
or binding is None
or binding.deprecated_at is not None
or project is None
or not project.active
):
raise ServingError(
403,
"PROJECT_BINDING_REQUIRED",
"creative coordination requires an active project-bound client",
)
return binding.project_id
def _priority(self, client: ServiceClient, requested: str) -> str:
binding = self.session.get(ProjectBinding, client.project_binding_id)
allowed = max(
PRIORITY.get(client.workload_priority, 99),
PRIORITY.get(binding.priority, 99) if binding else 99,
)
if PRIORITY.get(requested, -1) < allowed:
raise ServingError(
403,
"ALLOCATION_PRIORITY_FORBIDDEN",
"requested priority exceeds client or project policy",
)
return requested
def _expire(self, row: ExternalCreativeAllocation) -> bool:
expiry = row.expires_at
if row.state == "granted" and (
expiry is None
or (expiry.astimezone(UTC) if expiry.tzinfo else expiry.replace(tzinfo=UTC)) <= _now()
):
row.state, row.uncertain_at, row.updated_at = "uncertain", _now(), _now()
row.expires_at = None
row.decision_code = "ALLOCATION_EXPIRED_UNCERTAIN"
self._audit("system", "external-creative-expiry").write(
"EXTERNAL_CREATIVE_ALLOCATION_EXPIRED",
"external_creative_allocation",
str(row.id),
{"state": row.state, "reservation_retained_bytes": row.required_vram_bytes},
)
return True
return False
def _expire_grants(self) -> None:
for row in self.session.scalars(
select(ExternalCreativeAllocation)
.where(ExternalCreativeAllocation.state == "granted")
.order_by(ExternalCreativeAllocation.id)
.with_for_update()
.execution_options(populate_existing=True)
):
self._expire(row)
self.session.flush()
def _queued_authority(self, row: ExternalCreativeAllocation) -> bool:
client = self.session.get(ServiceClient, row.service_client_id, populate_existing=True)
try:
if client is None or self._project(client) != row.project_id:
return False
self._priority(client, row.priority)
except ServingError:
return False
resource = (
self.session.execute(
select(resources)
.where(
resources.c.project_id == row.project_id,
resources.c.resource_ref == row.resource_ref,
resources.c.workload == row.workload,
resources.c.provider_resource_id == row.provider_resource_id,
)
.with_for_update()
)
.mappings()
.one_or_none()
)
if resource is None or not resource["approved"]:
return False
row.required_vram_bytes = max(
row.required_vram_bytes,
resource["configured_vram_bytes"],
resource["measured_peak_vram_bytes"] or 0,
)
return True
def _audit(
self, client_or_type: ServiceClient | str, actor_id: str | None = None
) -> AuditWriter:
if isinstance(client_or_type, ServiceClient):
return AuditWriter(self.session, "service_client", str(client_or_type.id))
return AuditWriter(self.session, client_or_type, actor_id or "unknown")
def _response(
self, row: ExternalCreativeAllocation, *, as_of: datetime | None = None
) -> ExternalAllocationResponse:
expired = False
if as_of is not None and row.state == "granted":
expiry = row.expires_at
expired = (
expiry is None
or (expiry.astimezone(UTC) if expiry.tzinfo else expiry.replace(tzinfo=UTC))
<= as_of
)
return ExternalAllocationResponse(
allocation_id=row.id,
project_id=row.project_id,
request_id=row.request_id,
installation_id=row.decision_evidence.get("installation_id", "unknown"),
workload=cast(CreativeWorkload, row.workload),
resource_ref=row.resource_ref,
provider_resource_id=row.provider_resource_id,
required_vram_bytes=row.required_vram_bytes,
priority=row.priority,
state=cast(CreativeAllocationState, "uncertain" if expired else row.state),
decision_code=("ALLOCATION_EXPIRED_UNCERTAIN" if expired else row.decision_code),
required_actions=cast(list[CreativeRequiredAction], row.required_actions or []),
accelerator_id=row.accelerator_id,
acquired_at=row.acquired_at,
heartbeat_at=row.heartbeat_at,
expires_at=None if expired else row.expires_at,
uncertain_at=row.uncertain_at,
released_at=row.released_at,
decision_evidence=row.decision_evidence or {},
queue_position=(row.decision_evidence or {}).get("queue_position"),
)
def _promote_queue(self) -> None:
self._lock_accelerators()
self._expire_grants()
queued = list(
self.session.scalars(
select(ExternalCreativeAllocation)
.where(ExternalCreativeAllocation.state == "queued")
.order_by(ExternalCreativeAllocation.created_at, ExternalCreativeAllocation.id)
.with_for_update()
)
)
queued.sort(
key=lambda item: (PRIORITY.get(item.priority, 9), item.created_at, str(item.id))
)
# Snapshot already includes all existing external reservations.
reserved: dict[uuid.UUID, int] = {}
budgets, now = self.serving.scheduler_overview(commit=False), _now()
remaining_position = 0
for item in queued:
if not self._queued_authority(item):
item.state, item.decision_code = "blocked", "QUEUED_AUTHORITY_REVOKED"
item.decision_evidence = {**item.decision_evidence, "queue_position": None}
self._audit("system", "external-creative-admission").write(
"EXTERNAL_CREATIVE_ALLOCATION_BLOCKED",
"external_creative_allocation",
str(item.id),
{"decision_code": item.decision_code},
)
continue
chosen = next(
(
budget
for budget in budgets
if budget.attribution_confidence != "UNKNOWN"
and budget.pressure_state not in {"HIGH", "CRITICAL"}
and budget.schedulable_free_vram_bytes - reserved.get(budget.accelerator_id, 0)
>= item.required_vram_bytes
),
None,
)
evidence = dict(item.decision_evidence or {})
if chosen is None:
remaining_position += 1
evidence["queue_position"] = remaining_position
item.decision_evidence = evidence
continue
item.state, item.decision_code, item.accelerator_id = (
"granted",
"CAPACITY_GRANTED_FROM_QUEUE",
chosen.accelerator_id,
)
item.acquired_at = item.heartbeat_at = item.updated_at = now
item.expires_at = now + timedelta(seconds=item.ttl_seconds)
# This grant is based on observed schedulable free VRAM. A forced
# unload would discard a reusable resident model despite admission
# already succeeding. Preparation is only meaningful when a future
# decision explicitly accounts for reclaimable provider memory.
item.required_actions = []
evidence.update(
{
"queue_position": None,
"promoted_at": now.isoformat(),
"policy_revision": chosen.policy_revision,
}
)
item.decision_evidence = evidence
self._audit("system", "external-creative-admission").write(
"EXTERNAL_CREATIVE_ALLOCATION_GRANTED",
"external_creative_allocation",
str(item.id),
{
"accelerator_id": str(chosen.accelerator_id),
"reserved_vram_bytes": item.required_vram_bytes,
},
)
reserved[chosen.accelerator_id] = (
reserved.get(chosen.accelerator_id, 0) + item.required_vram_bytes
)
def acquire(
self, request: ExternalAllocationCreate, client: ServiceClient
) -> ExternalAllocationResponse:
self._lock_accelerators()
self._expire_grants()
project_id = self._project(client)
self._priority(client, request.priority)
digest = _digest(request)
previous = self.session.scalar(
select(ExternalCreativeAllocation).where(
ExternalCreativeAllocation.service_client_id == client.id,
ExternalCreativeAllocation.idempotency_key == request.idempotency_key,
)
)
if previous:
if previous.request_sha256 != digest:
raise ServingError(
409,
"IDEMPOTENCY_CONFLICT",
"idempotency key belongs to another allocation request",
)
self.session.commit()
return self._response(previous)
binding = (
self.session.execute(
select(resources)
.where(
resources.c.project_id == project_id,
resources.c.resource_ref == request.resource_ref,
resources.c.workload == request.workload,
resources.c.provider_resource_id == request.provider_resource_id,
)
.with_for_update()
)
.mappings()
.one_or_none()
)
if binding is None or not binding["approved"]:
raise ServingError(
403,
"RESOURCE_NOT_APPROVED",
"resource and provider binding is not approved for this project",
)
profiled = binding["measured_peak_vram_bytes"] if request.use_measured_profile else None
required_vram = max(
request.required_vram_bytes, profiled or 0, binding["configured_vram_bytes"]
)
# Give already-queued work first access to newly available capacity, while
# allowing requests that fit to bypass queue entries that still cannot.
self._promote_queue()
budgets = self.serving.scheduler_overview(commit=False)
chosen = next(
(
b
for b in budgets
if b.attribution_confidence != "UNKNOWN"
and b.pressure_state not in {"HIGH", "CRITICAL"}
and b.schedulable_free_vram_bytes >= required_vram
),
None,
)
queued = list(
self.session.scalars(
select(ExternalCreativeAllocation).where(
ExternalCreativeAllocation.state == "queued"
)
)
)
queue_position = 1 + sum(
PRIORITY.get(item.priority, 9) <= PRIORITY[request.priority] for item in queued
)
state, code = ("granted", "CAPACITY_GRANTED") if chosen else ("queued", "CAPACITY_QUEUED")
observed_budget = chosen or max(
budgets, key=lambda budget: budget.schedulable_free_vram_bytes, default=None
)
actions: list[str] = []
evidence = {
"installation_id": request.installation_id,
"policy_revision": observed_budget.policy_revision
if observed_budget
else "unavailable",
"observed_at": observed_budget.observed_at.isoformat()
if observed_budget and observed_budget.observed_at
else None,
"headroom_before_bytes": (
observed_budget.schedulable_free_vram_bytes if observed_budget else 0
),
"profile_source": "measured" if profiled else "configured",
"requested_vram_bytes": request.required_vram_bytes,
"queue_position": None if chosen else queue_position,
}
now = _now()
row = ExternalCreativeAllocation(
service_client_id=client.id,
project_id=project_id,
accelerator_id=chosen.accelerator_id if chosen else None,
request_id=request.request_id,
idempotency_key=request.idempotency_key,
request_sha256=digest,
workload=request.workload,
resource_ref=request.resource_ref,
provider_resource_id=request.provider_resource_id,
required_vram_bytes=required_vram,
priority=request.priority,
ttl_seconds=request.ttl_seconds,
state=state,
decision_code=code,
decision_evidence=evidence,
required_actions=actions if chosen else [],
acquired_at=now if chosen else None,
heartbeat_at=now if chosen else None,
expires_at=now + timedelta(seconds=request.ttl_seconds) if chosen else None,
)
self.session.add(row)
try:
self.session.flush()
self._audit(client).write(
"EXTERNAL_CREATIVE_ALLOCATION_DECIDED",
"external_creative_allocation",
str(row.id),
{
"project_id": str(project_id),
"workload": row.workload,
"resource_ref": row.resource_ref,
"state": row.state,
"decision_code": row.decision_code,
"accelerator_id": str(row.accelerator_id) if row.accelerator_id else None,
},
)
self.session.commit()
except IntegrityError as exc:
self.session.rollback()
raise ServingError(
409,
"ALLOCATION_CONFLICT",
"allocation was concurrently created; retry with the same idempotency key",
) from exc
self.session.refresh(row)
return self._response(row)
def get(self, allocation_id: uuid.UUID, client: ServiceClient) -> ExternalAllocationResponse:
self._lock_accelerators()
self._project(client)
self._promote_queue()
row = self.session.get(ExternalCreativeAllocation, allocation_id)
if row is None or row.service_client_id != client.id:
raise ServingError(404, "ALLOCATION_NOT_FOUND", "allocation was not found")
self.session.commit()
return self._response(row)
def action(
self,
allocation_id: uuid.UUID,
request_id: str,
action: str,
client: ServiceClient,
observed_peak_vram_bytes: int | None = None,
attestation: ExternalActionAttestation | None = None,
) -> ExternalAllocationResponse:
self._lock_accelerators()
row = self.session.get(
ExternalCreativeAllocation,
allocation_id,
with_for_update=True,
populate_existing=True,
)
if row is None or row.service_client_id != client.id or row.request_id != request_id:
raise ServingError(404, "ALLOCATION_NOT_FOUND", "allocation was not found")
if self._expire(row) and action == "heartbeat":
self.session.commit()
raise ServingError(
409, "ALLOCATION_EXPIRED", "expired allocation retains an uncertain reservation"
)
if action == "heartbeat":
self._project(client)
if action == "attest" and attestation is None:
raise ServingError(422, "ATTESTATION_REQUIRED", "attestation is required")
now = _now()
if attestation is not None:
if attestation.action not in (row.required_actions or []):
raise ServingError(409, "ACTION_NOT_REQUIRED", "attested action was not required")
if attestation.observed_after > attestation.observed_before:
raise ServingError(
422, "ATTESTATION_INVALID", "action attestation did not release VRAM"
)
self.session.execute(
attestations.insert().values(
id=uuid.uuid4(),
allocation_id=row.id,
action=attestation.action,
before_vram_bytes=attestation.observed_before,
after_vram_bytes=attestation.observed_after,
observed_at=attestation.completed_at,
validated=True,
created_at=now,
)
)
if action == "release":
if row.state != "released":
row.state, row.released_at, row.updated_at = "released", now, now
elif action == "uncertain":
if row.state != "released":
row.state, row.uncertain_at, row.expires_at, row.updated_at = (
"uncertain",
now,
None,
now,
)
elif action == "heartbeat":
if row.state != "granted":
raise ServingError(
409, "ALLOCATION_NOT_ACTIVE", "only granted allocations can be renewed"
)
row.heartbeat_at, row.expires_at, row.updated_at = (
now,
now + timedelta(seconds=row.ttl_seconds),
now,
)
self._audit(client).write(
f"EXTERNAL_CREATIVE_ALLOCATION_{action.upper()}",
"external_creative_allocation",
str(row.id),
{"state": row.state, "workload": row.workload},
)
if action == "release":
self.session.flush()
self._promote_queue()
self.session.commit()
self.session.refresh(row)
return self._response(row)
def observe(
self, body: ExternalProfileObservation, client: ServiceClient
) -> ExternalResourceResponse:
self._lock_accelerators()
self._project(client)
row = self.session.get(
ExternalCreativeAllocation,
body.allocation_id,
with_for_update=True,
populate_existing=True,
)
if row is None or row.service_client_id != client.id:
raise ServingError(404, "ALLOCATION_NOT_FOUND", "allocation was not found")
if self._expire(row):
self.session.commit()
if (
row.acquired_at is None
or row.accelerator_id is None
or row.state not in {"granted", "released"}
or body.reserved_vram_bytes != row.required_vram_bytes
):
raise ServingError(
409,
"PROFILE_GRANT_REQUIRED",
"observation requires an actual grant and exact reserved bytes",
)
if (row.resource_ref, row.provider_resource_id, row.workload) != (
body.resource_ref,
body.provider_resource_id,
body.workload,
):
raise ServingError(
409, "PROFILE_BINDING_MISMATCH", "observation does not match its allocation"
)
record = (
self.session.execute(
select(resources)
.where(
resources.c.project_id == row.project_id,
resources.c.resource_ref == row.resource_ref,
resources.c.workload == row.workload,
resources.c.provider_resource_id == row.provider_resource_id,
)
.with_for_update()
)
.mappings()
.one()
)
previous = self.session.execute(
select(observations.c.id).where(observations.c.allocation_id == row.id)
).scalar_one_or_none()
if previous is not None:
return self._resource(record["id"])
try:
self.session.execute(
observations.insert().values(
id=uuid.uuid4(),
allocation_id=row.id,
resource_ref=body.resource_ref,
provider_resource_id=body.provider_resource_id,
workload=body.workload,
observed_peak_vram_bytes=body.observed_peak_vram_bytes,
reserved_vram_bytes=row.required_vram_bytes,
duration_ms=body.duration_ms,
outcome=body.outcome,
created_at=_now(),
)
)
except IntegrityError as exc:
self.session.rollback()
raise ServingError(
409, "PROFILE_OBSERVATION_EXISTS", "allocation already has a profile observation"
) from exc
self.session.execute(
resources.update()
.where(resources.c.id == record["id"])
.values(
measured_peak_vram_bytes=max(
record["measured_peak_vram_bytes"] or 0, body.observed_peak_vram_bytes
),
measurement_count=record["measurement_count"] + 1,
updated_at=_now(),
)
)
self._audit(client).write(
"EXTERNAL_CREATIVE_PROFILE_OBSERVED",
"external_creative_resource",
str(record["id"]),
{"outcome": body.outcome, "observed_peak_vram_bytes": body.observed_peak_vram_bytes},
)
self.session.commit()
return self._resource(record["id"])
def _resource(self, identifier: uuid.UUID) -> ExternalResourceResponse:
record = (
self.session.execute(select(resources).where(resources.c.id == identifier))
.mappings()
.one()
)
return ExternalResourceResponse(
resource_id=identifier,
**{
key: record[key]
for key in ExternalResourceResponse.model_fields
if key != "resource_id"
},
)
def upsert_resource(
self, body: ExternalResourceApproval, actor_id: str
) -> ExternalResourceResponse:
record = (
self.session.execute(
select(resources)
.where(
resources.c.project_id == body.project_id,
resources.c.resource_ref == body.resource_ref,
resources.c.workload == body.workload,
resources.c.provider_resource_id == body.provider_resource_id,
)
.with_for_update()
)
.mappings()
.one_or_none()
)
identifier, now = (record["id"] if record else uuid.uuid4()), _now()
values = {**body.model_dump(), "updated_at": now}
if record:
self.session.execute(
resources.update().where(resources.c.id == identifier).values(**values)
)
else:
self.session.execute(
resources.insert().values(
id=identifier, measured_peak_vram_bytes=None, measurement_count=0, **values
)
)
self._audit("operator", actor_id).write(
"EXTERNAL_CREATIVE_RESOURCE_CONFIGURED",
"external_creative_resource",
str(identifier),
{"project_id": str(body.project_id), "approved": body.approved},
)
self.session.commit()
return self._resource(identifier)
def list_allocations(
self, state: str | None, limit: int, offset: int
) -> ExternalAllocationList:
observed_at = _now()
expired_grant = sa.and_(
ExternalCreativeAllocation.state == "granted",
sa.or_(
ExternalCreativeAllocation.expires_at.is_(None),
ExternalCreativeAllocation.expires_at <= observed_at,
),
)
effective_state = sa.case(
(expired_grant, "uncertain"), else_=ExternalCreativeAllocation.state
)
condition = effective_state == state if state else sa.true()
with self.session.no_autoflush:
total = (
self.session.scalar(
select(func.count()).select_from(ExternalCreativeAllocation).where(condition)
)
or 0
)
rows = self.session.scalars(
select(ExternalCreativeAllocation)
.where(condition)
.order_by(ExternalCreativeAllocation.created_at.desc())
.offset(offset)
.limit(limit)
).all()
return ExternalAllocationList(
items=[self._response(row, as_of=observed_at) for row in rows], total=total
)
def list_resources(
self, project_id: uuid.UUID | None, limit: int, offset: int
) -> ExternalResourceList:
condition = resources.c.project_id == project_id if project_id else sa.true()
total = (
self.session.scalar(select(func.count()).select_from(resources).where(condition)) or 0
)
records = (
self.session.execute(
select(resources)
.where(condition)
.order_by(resources.c.updated_at.desc())
.offset(offset)
.limit(limit)
)
.mappings()
.all()
)
return ExternalResourceList(
items=[
ExternalResourceResponse(
resource_id=record["id"],
**{
key: record[key]
for key in ExternalResourceResponse.model_fields
if key != "resource_id"
},
)
for record in records
],
total=total,
)
def reconcile(
self, allocation_id: uuid.UUID, action: str, reason: str, actor_id: str
) -> ExternalAllocationResponse:
self._lock_accelerators()
row = self.session.get(
ExternalCreativeAllocation,
allocation_id,
with_for_update=True,
populate_existing=True,
)
if row is None:
raise ServingError(404, "ALLOCATION_NOT_FOUND", "allocation was not found")
self._expire(row)
if row.state != "uncertain":
raise ServingError(
409, "ALLOCATION_NOT_UNCERTAIN", "only uncertain allocations can be reconciled"
)
if action == "confirm_released":
row.state, row.released_at, row.updated_at = "released", _now(), _now()
self.session.flush()
self._promote_queue()
self._audit("operator", actor_id).write(
"EXTERNAL_CREATIVE_ALLOCATION_RECONCILED",
"external_creative_allocation",
str(row.id),
{"action": action, "reason": reason},
)
self.session.commit()
self.session.refresh(row)
return self._response(row)
backend/src/modelforge_api/services/lab.py
+208
View File
@@ -0,0 +1,208 @@
from __future__ import annotations
import hashlib
import json
from sqlalchemy.orm import Session
from modelforge_api.domain.lab import (
LabBinding,
LabEmbeddingInput,
LabInputSummary,
LabPreflightRequest,
LabPreflightResponse,
LabRerankingInput,
LabVisionInput,
)
from modelforge_api.persistence.models import DeploymentCandidate
from modelforge_api.services.manifest_registry import ManifestRegistry
from modelforge_api.services.registry import RegistryNotFound
from modelforge_api.services.runtime import RuntimeService
from modelforge_api.settings import Settings
FAMILY_MODALITIES = {
"rag.embedding": "embedding",
"rag.reranking": "reranking",
"document.ocr": "document",
"vision.embedding": "vision",
"speech.transcription": "audio",
}
class LabPreflightService:
"""Read-only snapshot validation, never an execution grant or resource reservation."""
def __init__(self, session: Session, settings: Settings) -> None:
self.session = session
self.settings = settings
self.runtime = RuntimeService(session, settings)
def preflight(self, request: LabPreflightRequest) -> LabPreflightResponse:
with self.session.no_autoflush:
return self._preflight(request)
def _preflight(self, request: LabPreflightRequest) -> LabPreflightResponse:
payload = request.input.payload.model_dump(mode="json")
encoded = json.dumps(
payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False
).encode()
count = 1
blockers: list[str] = []
if isinstance(request.input, LabEmbeddingInput):
values = request.input.payload.inputs()
count = len(values)
if count > self.settings.gateway_max_batch_size or any(
len(value) > self.settings.gateway_max_input_characters for value in values
):
blockers.append("INPUT_LIMIT_EXCEEDED")
elif isinstance(request.input, LabRerankingInput):
count = len(request.input.payload.documents)
elif isinstance(request.input, LabVisionInput):
count = len(request.input.payload.items)
if len(encoded) > self.settings.gateway_max_payload_bytes:
blockers.append("PAYLOAD_LIMIT_EXCEEDED")
summary = LabInputSummary(
sha256=hashlib.sha256(encoded).hexdigest(), item_count=count, encoded_bytes=len(encoded)
)
contract = next(
(
item
for item in ManifestRegistry(self.settings.config_root).capabilities()
if item.capability == request.input.capability
and item.version == request.contract_version
),
None,
)
if contract is None:
blockers.append("CAPABILITY_CONTRACT_UNAVAILABLE")
binding: LabBinding | None = None
target = request.target
candidate = self.session.get(DeploymentCandidate, target.deployment_candidate_id)
if candidate is None:
blockers.append("CANDIDATE_NOT_FOUND")
else:
if (
candidate.production
or candidate.channel != "lab"
or candidate.status != "lab_ready"
):
blockers.append("CANDIDATE_NOT_LAB_READY")
if (
candidate.artifact_set_id != target.artifact_set_id
or candidate.runtime_profile_id != target.runtime_profile_id
or candidate.compute_node_id != target.compute_node_id
):
blockers.append("TARGET_IDENTITY_MISMATCH")
repo = self.runtime.repo
profile = repo.profile(candidate.runtime_profile_id)
node = repo.node(candidate.compute_node_id)
assessment = repo.assessment(candidate.compatibility_assessment_id)
probe = repo.probe(candidate.runtime_probe_id)
approval = repo.approval(target.execution_approval_id)
if not profile or not node or not assessment or not probe or not approval:
blockers.append("EVIDENCE_MISSING")
else:
if (
profile.artifact_set_id != candidate.artifact_set_id
or any(
item.artifact_set_id != candidate.artifact_set_id
or item.runtime_profile_id != candidate.runtime_profile_id
or item.compute_node_id != candidate.compute_node_id
for item in (assessment, probe)
)
or probe.compatibility_assessment_id != assessment.id
or probe.execution_approval_id != approval.id
or approval.artifact_set_id != candidate.artifact_set_id
):
blockers.append("EVIDENCE_IDENTITY_MISMATCH")
if profile.modality != FAMILY_MODALITIES[request.input.capability]:
blockers.append("CAPABILITY_PROFILE_MISMATCH")
if profile.trust_remote_code or profile.network_egress:
blockers.append("RUNTIME_POLICY_UNSUPPORTED")
if (
not node.enabled
or not node.lab_eligible
or node.liveness_state != "online"
or not {
"deployment.load.v1",
"deployment.invoke.v1",
"deployment.unload.v1",
}.issubset(node.agent_capabilities)
):
blockers.append("NODE_NOT_READY")
try:
if (
assessment.status not in {"compatible", "requires_probe"}
or assessment.blockers
or self.runtime.assessment_response(assessment).stale
):
blockers.append("ASSESSMENT_STALE_OR_BLOCKED")
if (
self.runtime.approval_response(approval).stale
or approval.scope != "lab_execution"
):
blockers.append("LAB_APPROVAL_STALE")
except RegistryNotFound:
blockers.append("ARTIFACT_EVIDENCE_MISSING")
if probe.status != "completed" or probe.cancel_requested or probe.failure_code:
blockers.append("PROBE_NOT_COMPLETED")
result = probe.inference_result
capability = request.input.capability
valid_output = False
if capability in {"rag.embedding", "vision.embedding"}:
dimension = result.get("dimension")
valid_output = (
isinstance(dimension, int)
and not isinstance(dimension, bool)
and 32 <= dimension <= 8192
and result.get("finite") is True
and result.get("output_type") in (None, "float_vector")
and (
not contract
or not contract.vector
or contract.vector.dimensionality is None
or dimension == contract.vector.dimensionality
)
)
elif capability == "rag.reranking":
valid_output = (
result.get("output_type") == "ranked_scores"
and result.get("finite") is True
and probe.runtime_facts.get("adapter") == "qwen3_reranker"
)
else:
expected = "ocr_text" if capability == "document.ocr" else "transcription"
valid_output = (
result.get("output_type") == expected and result.get("output_valid") is True
)
if not valid_output:
blockers.append("PROBE_OUTPUT_INVALID")
gpu_uuid = probe.runtime_facts.get("gpu_uuid")
accelerator = next(
(item for item in repo.accelerators(node.id) if item.device_uuid == gpu_uuid),
None,
)
if accelerator is None:
blockers.append("PROBED_ACCELERATOR_MISSING")
# Only expose a coherent server-resolved binding, never echo a mismatched target.
if not {"TARGET_IDENTITY_MISMATCH", "EVIDENCE_IDENTITY_MISMATCH"}.intersection(
blockers
):
binding = LabBinding(
**target.model_dump(),
compatibility_assessment_id=assessment.id,
runtime_probe_id=probe.id,
capability=capability,
runtime_profile_fingerprint=profile.fingerprint,
assessment_fingerprint=assessment.environment_fingerprint,
approval_evidence_fingerprint=approval.evidence_fingerprint,
probe_environment_fingerprint=probe.environment_fingerprint,
accelerator_id=accelerator.id if accelerator else None,
)
return LabPreflightResponse(
input_summary=summary,
binding=binding,
limits=request.limits,
blockers=list(dict.fromkeys(blockers)),
checks_passed=not blockers,
)
backend/src/modelforge_api/services/lab_runs.py
+805
View File
@@ -0,0 +1,805 @@
from __future__ import annotations
import hashlib
import json
import re
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import UUID, uuid4
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from modelforge_api.domain.dispatch_context import DispatchContext
from modelforge_api.domain.lab import (
LabEmbeddingInput,
LabExecutionRunResponse,
LabExecutionStartRequest,
LabInputSummary,
)
from modelforge_api.persistence.models import (
CapabilityDeployment,
GatewayRequest,
LabExecutionRun,
ServingGpuLease,
ServingJob,
)
from modelforge_api.services.gpu_scheduler import PlacementVerdict
from modelforge_api.services.serving import ServingError, ServingService
_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
_ACTIVE_STATUSES = frozenset({"accepted", "preparing", "queued", "running", "cancel_requested"})
_TERMINAL_STATUSES = frozenset({"completed", "failed", "cancelled", "interrupted"})
_RESIDENT_STATES = frozenset({"warm", "idle"})
def _canonical_json(value: Any) -> bytes:
return json.dumps(
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False
).encode("utf-8")
def _sha256(value: Any) -> str:
return hashlib.sha256(_canonical_json(value)).hexdigest()
def _now() -> datetime:
return datetime.now(UTC)
def _aware(value: datetime) -> datetime:
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
class LabRunService:
"""Persist actor-scoped identities for bounded, selected embedding runs."""
def __init__(self, serving: ServingService) -> None:
self.serving = serving
self.session = serving.session
def _owned(self, run_id: UUID, actor_subject: str) -> LabExecutionRun:
run = self.session.scalar(
select(LabExecutionRun).where(
LabExecutionRun.id == run_id,
LabExecutionRun.actor_subject == actor_subject,
)
)
if run is None:
raise ServingError(404, "LAB_RUN_NOT_FOUND", "LAB run was not found")
return run
@staticmethod
def _response(run: LabExecutionRun) -> LabExecutionRunResponse:
return LabExecutionRunResponse(
id=run.id,
status=run.status, # type: ignore[arg-type]
target_snapshot=run.target_snapshot,
input_summary=LabInputSummary(
sha256=run.input_sha256,
item_count=run.input_count,
encoded_bytes=run.input_encoded_bytes,
),
output_metadata=run.output_metadata,
worker_observations=run.worker_observations,
failure_code=run.failure_code,
input_payload_expires_at=run.input_payload_expires_at,
retention_seconds=run.retention_seconds,
created_at=run.created_at,
started_at=run.started_at,
cancellation_requested_at=run.cancellation_requested_at,
cancellation_acknowledged_at=run.cancellation_acknowledged_at,
finished_at=run.finished_at,
)
def _fingerprint(self, request: LabExecutionStartRequest) -> str:
return _sha256(request.model_dump(mode="json"))
def _selected_target(
self, request: LabExecutionStartRequest
) -> tuple[CapabilityDeployment, dict[str, Any], LabEmbeddingInput]:
deployment = self.serving.repo.deployment(request.deployment_id)
if deployment is None:
raise ServingError(404, "LAB_TARGET_NOT_FOUND", "selected LAB target was not found")
target = request.target
if (
deployment.deployment_candidate_id != target.deployment_candidate_id
or deployment.artifact_set_id != target.artifact_set_id
or deployment.runtime_profile_id != target.runtime_profile_id
or deployment.compute_node_id != target.compute_node_id
or deployment.execution_approval_id != target.execution_approval_id
):
raise ServingError(
409, "LAB_TARGET_MISMATCH", "selected LAB target identity does not match"
)
if not isinstance(request.input, LabEmbeddingInput):
raise ServingError(
422, "LAB_INPUT_NOT_SUPPORTED", "operator execution supports text embeddings only"
)
selected, _route_key = self.serving._admit_operator_embedding(
request.input, deployment.id, request.experiment_route_id
)
return (
selected,
self.serving._operator_lab_target_snapshot(selected, request.experiment_route_id),
request.input,
)
def _gateway(self, run: LabExecutionRun) -> GatewayRequest | None:
return self.session.get(GatewayRequest, run.gateway_request_id)
def _set_gateway_state(
self,
run: LabExecutionRun,
status: str,
*,
failure_code: str | None = None,
) -> None:
gateway = self._gateway(run)
if gateway is None:
return
gateway.status = status
if status in {"completed", "failed", "cancelled", "interrupted"}:
gateway.finished_at = _now()
if failure_code:
gateway.failure_code = failure_code
evidence = gateway.decision_evidence or {}
lab_evidence = evidence.get("lab_run_evidence", {})
if isinstance(lab_evidence, dict):
updated = {**lab_evidence, "status": status}
if failure_code:
updated["failure_code"] = failure_code
if run.output_metadata:
updated["output"] = run.output_metadata
if run.worker_observations:
updated["worker_observations"] = run.worker_observations
gateway.decision_evidence = {**evidence, "lab_run_evidence": updated}
def _delete_payload(self, run: LabExecutionRun) -> None:
if run.payload_reference:
self.serving.payload_store.delete(run.payload_reference)
run.payload_reference = None
def _release_reservation(self, run: LabExecutionRun, *, failed: bool = False) -> None:
gateway = self._gateway(run)
if gateway is None:
return
now = _now()
request_id = gateway.request_id
lease = self.session.scalar(
select(ServingGpuLease)
.where(ServingGpuLease.request_id == request_id)
.order_by(ServingGpuLease.created_at.desc())
.limit(1)
)
if lease is None:
return
if lease and lease.state not in {"released", "expired", "failed"}:
lease.state = "failed" if failed else "released"
lease.released_at = now
deployment = (
self.serving.repo.deployment(gateway.capability_deployment_id)
if gateway.capability_deployment_id
else None
)
residency = self.serving.repo.residency(deployment.id, lock=True) if deployment else None
if residency:
residency.active_requests = max(0, residency.active_requests - 1)
if residency.state == "busy":
residency.state = "warm"
elif residency.state == "loading":
residency.state = "cold"
residency.transition_reason = "LAB_RUN_CANCELLED_BEFORE_LOAD"
residency.last_used_at = now
def _finish(
self,
run: LabExecutionRun,
status: str,
*,
failure_code: str | None = None,
) -> None:
run.status = status
run.failure_code = failure_code
run.finished_at = _now()
self._set_gateway_state(run, status, failure_code=failure_code)
self._delete_payload(run)
def start(
self,
request: LabExecutionStartRequest,
*,
actor_subject: str,
idempotency_key: str,
) -> LabExecutionRunResponse:
if not actor_subject or len(actor_subject) > 255:
raise ServingError(
401, "OPERATOR_CONTEXT_REQUIRED", "operator principal is unavailable"
)
if not _IDEMPOTENCY_KEY.fullmatch(idempotency_key):
raise ServingError(422, "LAB_IDEMPOTENCY_KEY_INVALID", "idempotency key is invalid")
fingerprint = self._fingerprint(request)
existing = self.session.scalar(
select(LabExecutionRun).where(
LabExecutionRun.actor_subject == actor_subject,
LabExecutionRun.idempotency_key == idempotency_key,
)
)
if existing is not None:
if existing.request_fingerprint != fingerprint:
raise ServingError(
409,
"LAB_IDEMPOTENCY_CONFLICT",
"idempotency key was already used for a different LAB request",
)
return self._response(existing)
deployment, target_snapshot, embedding_input = self._selected_target(request)
payload = embedding_input.payload
inputs = payload.inputs()
input_sha256 = _sha256(inputs)
encoded_input = _canonical_json(payload.model_dump(mode="json"))
run_id = uuid4()
context = DispatchContext.for_operator(actor_subject, run_id)
now = _now()
payload_reference = self.serving.payload_store.put(
{"input": inputs, "input_type": payload.input_type},
request.limits.retention_seconds,
)
gateway_request = GatewayRequest(
request_id=uuid4(),
service_client_id=None,
capability_key="rag.embedding",
capability_version=1,
capability_deployment_id=deployment.id,
compute_node_id=deployment.compute_node_id,
status="queued",
priority="lab",
input_sha256=input_sha256,
input_count=len(inputs),
decision_evidence={
"dispatch_context": context.provenance(),
"operator_route_id": str(request.experiment_route_id),
"input_persisted": False,
"input_type": payload.input_type,
"lab_run_evidence": {
"schema_version": 1,
"status": "accepted",
"capability": "rag.embedding@1",
"operation_id": str(run_id),
"target": target_snapshot,
"input": {
"sha256": input_sha256,
"item_count": len(inputs),
"content_logged": False,
},
},
},
)
self.session.add(gateway_request)
try:
self.session.flush()
run = LabExecutionRun(
id=run_id,
actor_subject=actor_subject,
idempotency_key=idempotency_key,
request_fingerprint=fingerprint,
status="accepted",
capability="rag.embedding@1",
target_snapshot=target_snapshot,
input_sha256=input_sha256,
input_count=len(inputs),
input_encoded_bytes=len(encoded_input),
payload_reference=payload_reference,
input_payload_expires_at=now + timedelta(seconds=request.limits.retention_seconds),
gateway_request_id=gateway_request.id,
timeout_seconds=request.limits.timeout_seconds,
queue_timeout_seconds=request.limits.queue_timeout_seconds,
retention_seconds=request.limits.retention_seconds,
created_at=now,
)
self.session.add(run)
self.session.commit()
except IntegrityError:
self.session.rollback()
winner = self.session.scalar(
select(LabExecutionRun).where(
LabExecutionRun.actor_subject == actor_subject,
LabExecutionRun.idempotency_key == idempotency_key,
)
)
self.serving.payload_store.delete(payload_reference)
if winner is not None and winner.request_fingerprint == fingerprint:
return self._response(winner)
if winner is not None:
raise ServingError(
409,
"LAB_IDEMPOTENCY_CONFLICT",
"idempotency key was already used for a different LAB request",
) from None
raise ServingError(409, "STATE_CONFLICT", "LAB run could not be persisted") from None
except Exception:
self.session.rollback()
self.serving.payload_store.delete(payload_reference)
raise
self.session.refresh(run)
return self._response(run)
def _active_job(self, run: LabExecutionRun) -> ServingJob | None:
return self.session.scalar(
select(ServingJob)
.where(ServingJob.lab_execution_run_id == run.id)
.order_by(ServingJob.created_at.desc())
.limit(1)
)
def _input(self, run: LabExecutionRun) -> LabEmbeddingInput | None:
if not run.payload_reference or _aware(run.input_payload_expires_at) <= _now():
return None
payload = self.serving.payload_store.get(run.payload_reference)
if not payload:
return None
values = payload.get("input")
input_type = payload.get("input_type", "raw")
if (
not isinstance(values, list)
or not values
or len(values) != run.input_count
or any(not isinstance(value, str) for value in values)
or input_type not in {"raw", "query", "document"}
or _sha256(values) != run.input_sha256
):
return None
try:
return LabEmbeddingInput.model_validate(
{
"capability": "rag.embedding",
"payload": {"input": values, "input_type": input_type},
}
)
except ValueError:
return None
def _revalidate_target(
self, run: LabExecutionRun, input_value: LabEmbeddingInput
) -> CapabilityDeployment:
deployment_id = UUID(str(run.target_snapshot["deployment_id"]))
route_id = UUID(str(run.target_snapshot["experiment_route_id"]))
deployment, _ = self.serving._admit_operator_embedding(input_value, deployment_id, route_id)
current_snapshot = self.serving._operator_lab_target_snapshot(deployment, route_id)
if current_snapshot != run.target_snapshot:
raise ServingError(
409, "LAB_TARGET_CHANGED", "selected LAB target changed after run acceptance"
)
return deployment
def _add_job(
self,
run: LabExecutionRun,
deployment: CapabilityDeployment,
gateway: GatewayRequest,
operation: str,
*,
now: datetime,
) -> ServingJob:
key = _sha256({"run_id": str(run.id), "operation": operation})
existing = self.session.scalar(select(ServingJob).where(ServingJob.idempotency_key == key))
if existing is not None:
return existing
job = ServingJob(
capability_deployment_id=deployment.id,
compute_node_id=deployment.compute_node_id,
gateway_request_id=gateway.id,
lab_execution_run_id=run.id,
operation=operation,
status="queued",
priority="lab",
idempotency_key=key,
payload_reference=(run.payload_reference if operation == "invoke" else None),
attempt_count=0,
result_summary={},
created_at=now,
updated_at=now,
)
self.session.add(job)
self.session.flush()
return job
def _queue_invoke(
self,
run: LabExecutionRun,
deployment: CapabilityDeployment,
gateway: GatewayRequest,
*,
now: datetime,
) -> bool:
if self._input(run) is None:
self._finish(run, "failed", failure_code="REQUEST_PAYLOAD_EXPIRED")
self._release_reservation(run, failed=True)
return False
if self.serving.repo.queued_job_count(deployment.id) >= deployment.max_queue_depth:
return False
if (
self.serving.repo.active_gateway_requests()
> self.serving.settings.scheduler_global_queue_limit
):
return False
job = self._add_job(run, deployment, gateway, "invoke", now=now)
lease = self.session.scalar(
select(ServingGpuLease)
.where(ServingGpuLease.request_id == gateway.request_id)
.order_by(ServingGpuLease.created_at.desc())
.limit(1)
)
if lease:
lease.serving_job_id = job.id
run.status = "queued"
gateway.status = "running"
self._set_gateway_state(run, "queued")
self.session.commit()
return True
def _admit_and_queue(self, run: LabExecutionRun, now: datetime) -> bool:
input_value = self._input(run)
if input_value is None:
self._finish(run, "failed", failure_code="REQUEST_PAYLOAD_EXPIRED")
self.session.commit()
return True
try:
deployment = self._revalidate_target(run, input_value)
except ServingError as exc:
self._finish(run, "failed", failure_code=exc.code)
self.session.commit()
return True
gateway = self._gateway(run)
if gateway is None:
self._finish(run, "failed", failure_code="LAB_EVIDENCE_MISSING")
self.session.commit()
return True
if self.serving.repo.queued_job_count(deployment.id) >= deployment.max_queue_depth:
return False
if (
self.serving.repo.active_gateway_requests()
> self.serving.settings.scheduler_global_queue_limit
):
return False
self.serving.repo.lock_accelerator(deployment.accelerator_id)
residency = self.serving.repo.residency(deployment.id, lock=True)
if residency is None:
self._finish(run, "failed", failure_code="SCHEDULER_STATE_STALE")
self.session.commit()
return True
if residency.active_requests >= deployment.max_concurrency:
return False
if residency.state not in _RESIDENT_STATES and residency.state != "cold":
if residency.state in {"loading", "busy", "unloading", "draining"}:
return False
self._finish(run, "failed", failure_code="LAB_TARGET_UNAVAILABLE")
self.session.commit()
return True
plan = self.serving._placement_plan(
deployment,
priority="lab",
deadline_remaining_ms=float(run.timeout_seconds * 1000),
request_id=gateway.request_id,
dry_run=True,
persist=False,
commit=False,
)
if plan.verdict == PlacementVerdict.QUEUE.value:
self.session.commit()
return False
if plan.verdict == PlacementVerdict.ADMIT_AFTER_EVICTION.value:
self._finish(run, "failed", failure_code="LAB_EVICTION_REQUIRED")
self.session.commit()
return True
if plan.verdict != PlacementVerdict.ADMIT.value:
code = {
PlacementVerdict.REJECT_HEALTH.value: "LAB_TARGET_UNAVAILABLE",
PlacementVerdict.REJECT_POLICY.value: "LAB_POLICY_BLOCKED",
PlacementVerdict.REJECT_CAPACITY.value: "LAB_CAPACITY_BLOCKED",
}.get(plan.verdict, "LAB_ADMISSION_BLOCKED")
self._finish(run, "failed", failure_code=code)
self.session.commit()
return True
context = DispatchContext.for_operator(run.actor_subject, run.id)
deadline = _aware(run.created_at) + timedelta(seconds=run.timeout_seconds)
lease = ServingGpuLease(
accelerator_id=deployment.accelerator_id,
capability_deployment_id=deployment.id,
request_id=gateway.request_id,
reserved_vram_bytes=plan.required_vram_bytes,
priority="lab",
state="active",
owner=context.lease_owner,
created_at=now,
acquired_at=now,
heartbeat_at=now,
expires_at=deadline,
lease_type=(
"request_execution"
if residency.state in _RESIDENT_STATES
else "residency_reservation"
),
materialized_vram_bytes=(
residency.measured_resident_vram_bytes if residency.state in _RESIDENT_STATES else 0
),
generation=residency.generation,
)
self.session.add(lease)
gateway.status = "running"
gateway.decision_evidence = {
**(gateway.decision_evidence or {}),
"dispatch_context": context.provenance(),
"placement": {
"verdict": plan.verdict,
"decision_fingerprint": plan.decision_fingerprint,
"required_vram_bytes": plan.required_vram_bytes,
"reason_codes": plan.reason_codes,
},
}
run.dispatch_started_at = now
if residency.state == "cold":
residency.state = "loading"
residency.transition_started_at = now
residency.transition_reason = "LAB_RUN_LOAD"
residency.generation += 1
operation = "load"
run.status = "preparing"
else:
residency.state = "busy"
operation = "invoke"
run.status = "queued"
residency.active_requests += 1
residency.last_used_at = now
job = self._add_job(run, deployment, gateway, operation, now=now)
self.session.flush()
lease.serving_job_id = job.id
lab_evidence = (gateway.decision_evidence or {}).get("lab_run_evidence", {})
if isinstance(lab_evidence, dict):
gateway.decision_evidence = {
**gateway.decision_evidence,
"lab_run_evidence": {**lab_evidence, "status": run.status},
}
self.session.commit()
return True
def _drive_preparing(self, run: LabExecutionRun, now: datetime) -> bool:
gateway = self._gateway(run)
deployment = (
self.serving.repo.deployment(gateway.capability_deployment_id)
if gateway and gateway.capability_deployment_id
else None
)
if gateway is None or deployment is None:
self._finish(run, "failed", failure_code="LAB_EVIDENCE_MISSING")
self.session.commit()
return True
jobs = list(
self.session.scalars(
select(ServingJob)
.where(ServingJob.lab_execution_run_id == run.id)
.order_by(ServingJob.created_at.desc())
)
)
latest = jobs[0] if jobs else None
if latest is None:
self._finish(run, "interrupted", failure_code="LAB_DISPATCH_STATE_INCOMPLETE")
self.session.commit()
return True
if latest.status in {"queued", "running"}:
return False
if latest.operation != "load" or latest.status != "completed":
self._finish(run, "failed", failure_code=latest.failure_code or "RESIDENCY_LOAD_FAILED")
self._release_reservation(run, failed=True)
self.session.commit()
return True
input_value = self._input(run)
if input_value is None:
self._finish(run, "failed", failure_code="REQUEST_PAYLOAD_EXPIRED")
self._release_reservation(run, failed=True)
self.session.commit()
return True
try:
selected = self._revalidate_target(run, input_value)
except ServingError as exc:
self._finish(run, "failed", failure_code=exc.code)
self._release_reservation(run, failed=True)
self.session.commit()
return True
if _aware(run.created_at) + timedelta(seconds=run.timeout_seconds) <= now:
self._finish(run, "failed", failure_code="REQUEST_TIMEOUT")
self._release_reservation(run, failed=True)
self.session.commit()
return True
self._queue_invoke(run, selected, gateway, now=now)
return run.status == "queued"
def _expire_run(self, run: LabExecutionRun, now: datetime) -> bool:
gateway = self._gateway(run)
if gateway is None:
self._finish(run, "failed", failure_code="LAB_EVIDENCE_MISSING")
self.session.commit()
return True
job = self._active_job(run)
if job and job.status == "running":
run.status = "cancel_requested"
run.failure_code = "REQUEST_TIMEOUT"
run.cancellation_requested_at = run.cancellation_requested_at or now
self._set_gateway_state(run, "cancel_requested", failure_code="REQUEST_TIMEOUT")
self._delete_payload(run)
self.session.commit()
return True
# Do not lock/update a queued job here: claim and completion lock job then run.
# The terminal run state fences a concurrent claim, which will mark that job
# failed after acquiring the same locks in the established order.
self._release_reservation(run, failed=True)
self._finish(run, "failed", failure_code="REQUEST_TIMEOUT")
self.session.commit()
return True
def _interrupt_stale_job(self, job_id: UUID, now: datetime) -> bool:
job = self.session.scalar(
select(ServingJob).where(ServingJob.id == job_id).with_for_update()
)
if (
job is None
or job.lab_execution_run_id is None
or job.status != "running"
or job.lease_expires_at is None
or _aware(job.lease_expires_at) > now
):
self.session.rollback()
return False
run = self.session.scalar(
select(LabExecutionRun)
.where(LabExecutionRun.id == job.lab_execution_run_id)
.with_for_update()
)
if run is None or run.status in _TERMINAL_STATUSES:
self.session.rollback()
return False
job.status = "failed"
job.failure_code = "LAB_WORKER_LEASE_STALE"
job.finished_at = now
job.lease_token_hash = None
job.lease_expires_at = None
gateway = self._gateway(run)
deployment = (
self.serving.repo.deployment(gateway.capability_deployment_id)
if gateway and gateway.capability_deployment_id
else None
)
if deployment:
deployment.health_status = "unavailable"
residency = self.serving.repo.residency(deployment.id, lock=True)
if residency:
residency.state = "failed"
residency.failure_code = "LAB_WORKER_LEASE_STALE"
residency.failure_message = None
residency.transition_reason = "LAB_WORKER_LEASE_STALE"
self._finish(run, "interrupted", failure_code="LAB_WORKER_LEASE_STALE")
# The GPU reservation is intentionally left until its own expiry. A missing
# worker heartbeat is not proof that the process has stopped using the device.
self.session.commit()
return True
def reconcile(self) -> dict[str, int]:
now = _now()
interrupted = 0
stale_job_ids = list(
self.session.scalars(
select(ServingJob.id)
.where(
ServingJob.lab_execution_run_id.is_not(None),
ServingJob.status == "running",
ServingJob.lease_expires_at <= now,
)
.order_by(ServingJob.lease_expires_at)
.limit(100)
)
)
for job_id in stale_job_ids:
interrupted += self._interrupt_stale_job(job_id, now)
driven = 0
candidate_ids = list(
self.session.scalars(
select(LabExecutionRun.id)
.where(LabExecutionRun.status.in_(("accepted", "preparing", "queued")))
.order_by(LabExecutionRun.created_at)
.limit(100)
)
)
for run_id in candidate_ids:
run = self.session.scalar(
select(LabExecutionRun)
.where(
LabExecutionRun.id == run_id,
LabExecutionRun.status.in_(("accepted", "preparing", "queued")),
)
.with_for_update(skip_locked=True)
)
if run is None:
self.session.rollback()
continue
now = _now()
if _aware(run.created_at) + timedelta(seconds=run.timeout_seconds) <= now:
driven += self._expire_run(run, now)
continue
if run.status in {"accepted", "queued"} and (
_aware(run.created_at) + timedelta(seconds=run.queue_timeout_seconds) <= now
):
self._expire_run(run, now)
driven += 1
continue
if run.status == "accepted":
driven += self._admit_and_queue(run, now)
elif run.status == "preparing":
driven += self._drive_preparing(run, now)
elif run.status == "queued":
job = self._active_job(run)
if job is None or job.status not in {"queued", "running"}:
self._finish(run, "interrupted", failure_code="LAB_DISPATCH_STATE_INCOMPLETE")
self.session.commit()
driven += 1
else:
self.session.rollback()
return {"runs_driven": driven, "stale_runs_interrupted": interrupted}
def get(self, run_id: UUID, *, actor_subject: str) -> LabExecutionRunResponse:
return self._response(self._owned(run_id, actor_subject))
def cancel(self, run_id: UUID, *, actor_subject: str) -> LabExecutionRunResponse:
# Check ownership before taking locks so a foreign id stays indistinguishable from a
# missing run. Lock order is job then run, matching claim and worker completion.
self._owned(run_id, actor_subject)
job = self.session.scalar(
select(ServingJob)
.where(
ServingJob.lab_execution_run_id == run_id,
ServingJob.status.in_(("queued", "running")),
)
.order_by(ServingJob.created_at.desc())
.limit(1)
.with_for_update()
)
run = self.session.scalar(
select(LabExecutionRun)
.where(
LabExecutionRun.id == run_id,
LabExecutionRun.actor_subject == actor_subject,
)
.with_for_update()
)
if run is None:
raise ServingError(404, "LAB_RUN_NOT_FOUND", "LAB run was not found")
if run.status in _TERMINAL_STATUSES:
self.session.commit()
return self._response(run)
now = _now()
if job is not None and job.status == "running":
# Inference is synchronous inside the worker. The run remains nonterminal until
# the worker reaches a safe boundary and acknowledges cancellation.
run.status = "cancel_requested"
run.cancellation_requested_at = run.cancellation_requested_at or now
self._delete_payload(run)
self._set_gateway_state(run, "cancel_requested")
else:
if job is not None:
job.status = "cancelled"
job.failure_code = "CANCELLED"
job.finished_at = now
job.lease_token_hash = None
job.lease_expires_at = None
job.payload_reference = None
self._release_reservation(run)
self._finish(run, "cancelled")
run.cancellation_requested_at = run.cancellation_requested_at or now
run.cancellation_acknowledged_at = None
self.session.commit()
if job is not None and job.status == "cancelled":
# The worker never claimed this job, so no worker acknowledgement is expected.
pass
self.session.refresh(run)
return self._response(run)
backend/src/modelforge_api/services/lifecycle.py
+207 -22
View File
@@ -75,6 +75,9 @@ from modelforge_api.persistence.models import (
RuntimeProbe,
RuntimeProfile,
)
from modelforge_api.services.audit import AuditContext, AuditWriter
from modelforge_api.services.lifecycle_serving import LifecycleServing
from modelforge_api.services.serving import ServingError
class LifecycleError(RuntimeError):
@@ -94,6 +97,13 @@ def _now() -> datetime:
return datetime.now(UTC)
def _serving_changed_at(operation: LifecycleOperation) -> str:
if operation.finished_at is None:
raise LifecycleError(409, "SERVING_JOURNAL_INVALID", "committed serving timestamp is missing")
value = operation.finished_at
return (value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)).isoformat()
DEFAULT_POLICIES: tuple[dict[str, Any], ...] = (
{
"key": "lab-promotion",
@@ -305,6 +315,18 @@ class LifecycleService:
"SUBJECT_TARGET_MISMATCH",
"approval target does not match lifecycle subject",
)
if request.evidence.serving_transition is not None:
selected = request.evidence.serving_transition
if request.evidence.production_validation != "SATISFIED":
raise LifecycleError(409, "SERVING_PRODUCTION_VALIDATION_REQUIRED", "actual project serving requires satisfied production validation")
if (request.environment != "PRODUCTION" or policy.scope != "CAPABILITY_PRODUCTION"
or request.target_type != "CAPABILITY_DEPLOYMENT"
or request.target_ref != str(selected.candidate_deployment_id)
or request.requested_transition != "STABLE"
or request.evidence.capability_deployment_id != selected.candidate_deployment_id
or request.evidence.production_execution_approval_id != selected.production_approval_id
or request.evidence.rollback_target_ref != str(selected.current_deployment_id)):
raise LifecycleError(422, "SERVING_APPROVAL_MISMATCH", "serving transition requires exact production approval scope and targets")
resolved = self._resolve_evidence(request.evidence.model_dump(mode="json"))
snapshot = {
"declared": request.evidence.model_dump(mode="json"),
@@ -400,7 +422,9 @@ class LifecycleService:
item.status = ApprovalStatus.REJECTED
event_type = "APPROVAL_REJECTED"
elif action == "revoke":
if item.status != ApprovalStatus.APPROVED:
if item.status != ApprovalStatus.APPROVED and not (
item.status == ApprovalStatus.STALE and item.approved_by is not None
):
raise LifecycleError(409, "APPROVAL_NOT_APPROVED", "only approval can be revoked")
item.status = ApprovalStatus.REVOKED
event_type = "APPROVAL_REVOKED"
@@ -468,13 +492,29 @@ class LifecycleService:
"APPROVAL_MIGRATION_MISMATCH",
"promotion plan migration differs from approved evidence",
)
serving_contract = approval.evidence_snapshot.get("resolved", {}).get("serving_transition")
if "serving_transition" in request.affected_identities:
raise LifecycleError(422, "SERVING_CONTRACT_SERVER_OWNED", "serving contract is derived from approved evidence")
if serving_contract is not None:
if (request.canary_strategy or request.migration_class not in {"transparent", "behavioral"}
or subject.target_type != "CAPABILITY_DEPLOYMENT" or subject.environment != "PRODUCTION"
or subject.target_ref != str(request.candidate_deployment_id)
or subject.state not in {"LAB_READY", "PROMOTION_ELIGIBLE"}
or request.desired_state != "STABLE"):
raise LifecycleError(422, "SERVING_PLAN_UNSUPPORTED", "only reviewed same-space stable replacement is supported")
contract = LifecycleServing(self.session).validate(serving_contract)
request = request.model_copy(update={
"affected_identities": {**request.affected_identities, "serving_transition": serving_contract},
"project_consumers": sorted({value["project_id"] for value in contract.bindings.values()}),
})
try:
intermediate = (
DeploymentLifecycleState.CANARY
if request.canary_strategy
else request.desired_state
)
assert_deployment_transition(DeploymentLifecycleState(subject.state), intermediate)
if serving_contract is None:
assert_deployment_transition(DeploymentLifecycleState(subject.state), intermediate)
except ValueError as exc:
raise LifecycleError(422, "INVALID_LIFECYCLE_TRANSITION", str(exc)) from exc
if request.migration_class == "requires_reindex":
@@ -635,6 +675,11 @@ class LifecycleService:
raise LifecycleError(500, "SUBJECT_REFERENCE_MISSING", "plan subject is unavailable")
if subject.version != request.expected_subject_version:
raise LifecycleError(409, "VERSION_CONFLICT", "lifecycle subject changed concurrently")
serving_contract = plan.affected_identities.get("serving_transition")
if serving_contract is not None:
LifecycleServing(self.session).validate(serving_contract)
elif subject.environment == "PRODUCTION" and subject.target_type == "CAPABILITY_DEPLOYMENT" and plan.desired_state == "STABLE":
raise LifecycleError(409, "SERVING_TRANSITION_REQUIRED", "operational stable promotion requires the approved serving adapter")
plan_version = plan.version
snapshot_payload = {
"subject_id": str(subject.id),
@@ -645,7 +690,10 @@ class LifecycleService:
"version": subject.version,
"rollback_target_ref": plan.rollback_target_ref,
"affected_identities": plan.affected_identities,
"superseded_by_ref": subject.superseded_by_ref,
}
if serving_contract is not None:
snapshot_payload["serving_transition"] = serving_contract
snapshot = LifecycleRollbackSnapshot(
promotion_plan_id=plan.id,
snapshot=snapshot_payload,
@@ -741,7 +789,8 @@ class LifecycleService:
return self._operation_response(operation)
def observe_canary(
self, operation_id: uuid.UUID, observation: CanaryObservation
self, operation_id: uuid.UUID, observation: CanaryObservation,
*, audit_context: AuditContext | None = None,
) -> LifecycleOperationResponse:
operation = self._operation(operation_id)
if operation.stage != OperationStage.VERIFYING:
@@ -751,16 +800,32 @@ class LifecycleService:
)
if not canary or canary.status != CanaryStatus.RUNNING:
raise LifecycleError(409, "CANARY_NOT_RUNNING", "canary is not running")
plan = self._plan(operation.promotion_plan_id)
subject = self._require_manual_canary_rehearsal(plan)
context = audit_context or AuditContext(
actor_type="manual_lab_rehearsal", actor_id="local-lab-rehearsal",
)
canary.request_count += observation.request_count
canary.error_count += observation.error_count
canary.latency_p95_ms = observation.latency_p95_ms
canary.result = observation.model_dump(mode="json")
canary.result = {
**observation.model_dump(mode="json"),
"provenance": {
"source": "MANUAL_LAB_REHEARSAL",
"authoritative": False,
"actor": context.actor_id,
"correlation_id": context.correlation_id,
},
}
trigger = self._canary_trigger(canary, observation)
if trigger:
canary.status = CanaryStatus.ABORTED
canary.abort_trigger = trigger
canary.finished_at = _now()
self._rollback(operation, trigger, "SYSTEM", "Automatic canary abort")
self._rollback(
operation, trigger, context.actor_id,
"Automatic abort from manual LAB rehearsal observation",
)
event_type = "CANARY_ABORTED"
elif canary.request_count >= canary.target_request_count:
canary.status = CanaryStatus.READY_FOR_PROMOTION_REVIEW
@@ -768,18 +833,14 @@ class LifecycleService:
event_type = "CANARY_READY_FOR_PROMOTION_REVIEW"
else:
event_type = "CANARY_OBSERVED"
plan = self._plan(operation.promotion_plan_id)
subject = self.session.get(LifecycleSubject, plan.subject_id)
if subject is None:
raise LifecycleError(500, "SUBJECT_REFERENCE_MISSING", "plan subject is unavailable")
self._event(
event_type,
subject.target_type,
subject.target_ref,
subject.state,
canary.status,
"lifecycle-engine",
"SYSTEM",
context.actor_id,
"MANUAL_LAB_REHEARSAL",
self._approval(plan.approval_request_id).policy_revision_id,
[str(canary.id)],
trigger or "Bounded canary observation recorded without payload content",
@@ -790,14 +851,64 @@ class LifecycleService:
},
change_id=str(operation.id),
)
AuditWriter(self.session, context=context).write(
action="lifecycle.manual_lab_canary_observed",
resource_type="lifecycle_operation",
resource_id=str(operation.id),
details={
"source": "MANUAL_LAB_REHEARSAL",
"canary_id": str(canary.id),
"status": canary.status,
"authoritative": False,
},
)
self.session.commit()
return self._operation_response(operation)
def _require_manual_canary_rehearsal(
self, plan: LifecyclePromotionPlan,
) -> LifecycleSubject:
"""No manual or legacy canary evidence can qualify an operational promotion."""
subject = self.session.get(LifecycleSubject, plan.subject_id)
approval = self.session.get(LifecycleApprovalRequest, plan.approval_request_id)
policy = self.session.get(LifecyclePolicyRevision, approval.policy_revision_id) if (
approval is not None
) else None
if (
subject is None or approval is None or policy is None
or subject.target_type != "LAB_REHEARSAL"
or subject.environment != "LAB"
or approval.environment != "LAB"
or policy.scope != "LAB_PROMOTION"
or plan.desired_state != "LAB_STABLE"
or approval.subject_id != subject.id
or approval.target_type != subject.target_type
or approval.target_ref != subject.target_ref
or approval.requested_transition != plan.desired_state
):
raise LifecycleError(
409, "MANUAL_CANARY_REHEARSAL_ONLY",
"Canary observations and commit require an explicit LAB rehearsal; "
"authoritative operational canary evidence ingestion is unavailable",
)
return subject
def commit_operation(
self, operation_id: uuid.UUID, decision: ApprovalDecision
) -> LifecycleOperationResponse:
operation = self._operation(operation_id)
plan = self._plan(operation.promotion_plan_id)
if operation.stage == OperationStage.COMMITTED:
transition = plan.affected_identities.get("serving_transition")
if transition is not None:
snapshot = self.session.scalar(select(LifecycleRollbackSnapshot).where(
LifecycleRollbackSnapshot.promotion_plan_id == plan.id))
if (snapshot is None or _hash(snapshot.snapshot) != snapshot.snapshot_digest
or snapshot.snapshot.get("serving_transition") != transition):
raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact installed serving snapshot is required")
LifecycleServing(self.session).installed(transition, _serving_changed_at(operation))
self.session.commit()
return self._operation_response(operation)
if operation.stage != OperationStage.VERIFYING:
raise LifecycleError(
409, "OPERATION_NOT_REVIEWABLE", "operation is not ready for review"
@@ -805,6 +916,8 @@ class LifecycleService:
canary = self.session.scalar(
select(LifecycleCanaryRun).where(LifecycleCanaryRun.operation_id == operation.id)
)
if canary or plan.canary_strategy:
self._require_manual_canary_rehearsal(plan)
if canary and canary.status != CanaryStatus.READY_FOR_PROMOTION_REVIEW:
raise LifecycleError(
409,
@@ -823,14 +936,26 @@ class LifecycleService:
raise LifecycleError(
409, "ROLLBACK_SNAPSHOT_MISSING", "immutable rollback snapshot is unavailable"
)
serving_contract = plan.affected_identities.get("serving_transition")
serving_changed_at = None
if serving_contract is not None:
approval = self._approval(plan.approval_request_id)
self._refresh_approval(approval)
if approval.status != ApprovalStatus.APPROVED or _hash(approval.evidence_snapshot) != approval.evidence_fingerprint:
raise LifecycleError(409, "APPROVAL_STALE", "serving approval is no longer current")
if snapshot.snapshot.get("serving_transition") != serving_contract or _hash(snapshot.snapshot) != snapshot.snapshot_digest:
raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact immutable serving snapshot is required")
serving_changed_at = _now()
LifecycleServing(self.session).switch(serving_contract, serving_changed_at.isoformat())
expected_state = (
DeploymentLifecycleState.CANARY if plan.canary_strategy else plan.current_state
)
try:
assert_deployment_transition(
DeploymentLifecycleState(expected_state),
DeploymentLifecycleState(plan.desired_state),
)
if serving_contract is None:
assert_deployment_transition(
DeploymentLifecycleState(expected_state),
DeploymentLifecycleState(plan.desired_state),
)
except ValueError as exc:
raise LifecycleError(409, "STATE_CHANGED", str(exc)) from exc
previous = str(expected_state)
@@ -862,7 +987,7 @@ class LifecycleService:
)
self.session.refresh(subject)
operation.stage = OperationStage.COMMITTED
operation.finished_at = _now()
operation.finished_at = serving_changed_at or _now()
operation.version += 1
plan.status = PromotionPlanStatus.COMMITTED
plan.version += 1
@@ -1142,6 +1267,14 @@ class LifecycleService:
def _resolve_evidence(self, declared: dict[str, Any]) -> dict[str, Any]:
facts: dict[str, Any] = {}
if declared.get("serving_transition") is not None:
from modelforge_api.domain.lifecycle_contracts import ServingTransitionSelection
if declared.get("production_validation") != "SATISFIED":
raise LifecycleError(409, "SERVING_PRODUCTION_VALIDATION_REQUIRED", "actual project serving requires satisfied production validation")
facts["serving_transition"] = LifecycleServing(self.session).prepare(
ServingTransitionSelection.model_validate(declared["serving_transition"])
).model_dump(mode="json")
for field, model in (
("artifact_set_id", ArtifactSet),
("runtime_profile_id", RuntimeProfile),
@@ -1359,8 +1492,21 @@ class LifecycleService:
change_id=str(item.id),
)
return
current_resolved = self._resolve_evidence(item.evidence_snapshot["declared"])
if _hash(current_resolved) != _hash(item.evidence_snapshot["resolved"]):
stale_details: dict[str, Any] = {}
current_resolved: dict[str, Any] | None
try:
current_resolved = self._resolve_evidence(item.evidence_snapshot["declared"])
except (LifecycleError, ServingError) as exc:
# A historical serving approval must remain inspectable after its route
# changes. Expected qualification failures invalidate authority; internal
# errors still propagate rather than being disguised as stale evidence.
if (item.evidence_snapshot["declared"].get("serving_transition") is None
or exc.status_code not in {404, 409, 422}):
raise
current_resolved = None
stale_details = {"validation_code": exc.code, "validation_reason": str(exc)}
item.blockers = sorted(set(item.blockers) | {exc.code})
if current_resolved is None or _hash(current_resolved) != _hash(item.evidence_snapshot["resolved"]):
previous = item.status
item.status = ApprovalStatus.STALE
item.stale_at = now
@@ -1375,7 +1521,7 @@ class LifecycleService:
item.policy_revision_id,
[],
"Referenced evidence changed after approval request creation",
{},
stale_details,
change_id=str(item.id),
)
@@ -1461,6 +1607,7 @@ class LifecycleService:
) -> None:
started = time.perf_counter()
plan = self._plan(operation.promotion_plan_id)
self._approval(plan.approval_request_id) # Same approval-before-route lock order as commit.
snapshot = self.session.scalar(
select(LifecycleRollbackSnapshot).where(
LifecycleRollbackSnapshot.promotion_plan_id == plan.id
@@ -1492,6 +1639,21 @@ class LifecycleService:
)
return
snapshot_version = int(snapshot.snapshot["version"])
serving_contract = plan.affected_identities.get("serving_transition")
was_committed = operation.stage == OperationStage.COMMITTED
if serving_contract is not None:
if (snapshot.snapshot.get("serving_transition") != serving_contract
or _hash(snapshot.snapshot) != snapshot.snapshot_digest):
raise LifecycleError(409, "SERVING_SNAPSHOT_INVALID", "exact serving snapshot is unavailable")
# A subject CAS failure below rolls back the route in the same transaction.
# A later route is preserved rather than guessed away.
try:
LifecycleServing(self.session).restore(serving_contract,
_serving_changed_at(operation) if was_committed else _now().isoformat(),
committed=was_committed)
except Exception:
self.session.rollback()
raise
expected_version = snapshot_version + (
2 if operation.stage == OperationStage.COMMITTED else 1
)
@@ -1511,6 +1673,9 @@ class LifecycleService:
)
)
if rows_affected(restored) != 1:
if serving_contract is not None:
self.session.rollback()
raise LifecycleError(409, "ROLLBACK_VERSION_CONFLICT", "lifecycle subject changed; serving state was preserved")
operation.stage = OperationStage.FAILED
operation.failure_code = "ROLLBACK_VERSION_CONFLICT"
operation.failure_details = {
@@ -1581,6 +1746,26 @@ class LifecycleService:
500, "RETENTION_POLICY_MISSING", "rollback retention policy missing"
)
retained_until = _now() + timedelta(days=policy.minimum_rollback_days)
transition = plan.affected_identities.get("serving_transition")
if transition is not None:
prior = transition["current"]
for kind, ref in (("ARTIFACT_SET", prior["artifact_set_id"]),
("RUNTIME_PROFILE", prior["runtime_profile_id"]),
("CAPABILITY_DEPLOYMENT", prior["id"])):
retained = self.session.scalar(select(LifecycleRetentionRecord).where(
LifecycleRetentionRecord.target_type == kind,
LifecycleRetentionRecord.target_ref == ref))
if retained is None:
self.session.add(LifecycleRetentionRecord(
policy_revision_id=policy.id, target_type=kind, target_ref=ref,
state=RetentionState.ROLLBACK_RETAINED, retained_until=retained_until,
legal_hold=False, reason="Exact prior serving identity retained for approved rollback",
created_at=_now()))
else:
previous_until = retained.retained_until
if previous_until is None or previous_until.replace(tzinfo=UTC) < retained_until:
retained.retained_until = retained_until
retained.state = RetentionState.ROLLBACK_RETAINED
existing = self.session.scalar(
select(LifecycleRetentionRecord).where(
LifecycleRetentionRecord.target_type == "ROLLBACK_TARGET",
@@ -1856,19 +2041,19 @@ class LifecycleService:
)
def _approval(self, approval_id: uuid.UUID) -> LifecycleApprovalRequest:
item = self.session.get(LifecycleApprovalRequest, approval_id)
item = self.session.get(LifecycleApprovalRequest, approval_id, with_for_update=True, populate_existing=True)
if not item:
raise LifecycleError(404, "APPROVAL_NOT_FOUND", "approval request not found")
return item
def _plan(self, plan_id: uuid.UUID) -> LifecyclePromotionPlan:
item = self.session.get(LifecyclePromotionPlan, plan_id)
item = self.session.get(LifecyclePromotionPlan, plan_id, with_for_update=True, populate_existing=True)
if not item:
raise LifecycleError(404, "PROMOTION_PLAN_NOT_FOUND", "promotion plan not found")
return item
def _operation(self, operation_id: uuid.UUID) -> LifecycleOperation:
item = self.session.get(LifecycleOperation, operation_id)
item = self.session.get(LifecycleOperation, operation_id, with_for_update=True, populate_existing=True)
if not item:
raise LifecycleError(404, "LIFECYCLE_OPERATION_NOT_FOUND", "operation not found")
return item
backend/src/modelforge_api/services/lifecycle_serving.py
+337
View File
@@ -0,0 +1,337 @@
"""Transaction-only serving adapter for approved lifecycle plans.
This adapter never commits, invokes inference, or changes a consumer index.
The lifecycle engine owns the approval, journal, transaction and rollback.
"""
from __future__ import annotations
import hashlib
import json
from datetime import UTC, datetime
from typing import Any, NoReturn
from uuid import UUID
from sqlalchemy import select, update
from sqlalchemy.orm import Session
from modelforge_api.domain.lifecycle_contracts import (
ServingTransitionContract,
ServingTransitionSelection,
)
from modelforge_api.domain.serving import ProductionApprovalCreate, SupplyChainReview
from modelforge_api.persistence.models import (
Accelerator,
ArtifactSet,
CapabilityContract,
CapabilityDeployment,
CapabilityExperimentRoute,
CapabilityResourceEnvelope,
ComputeNode,
DeploymentCandidate,
EvaluationComparison,
EvaluationRun,
ProductionExecutionApproval,
Project,
ProjectBinding,
ProjectFitEvidence,
ResidencyAllocation,
RuntimeCompatibilityAssessment,
RuntimeProbe,
SelectedModelComparison,
ServiceClient,
)
from modelforge_api.persistence.serving_repository import ServingRepository
from modelforge_api.services.audit import AuditContext
from modelforge_api.services.manifest_registry import ManifestRegistry
from modelforge_api.services.selected_model_comparison import SelectedModelComparisonService
from modelforge_api.services.serving import ServingService
from modelforge_api.settings import get_settings
def _fail(code: str, message: str) -> NoReturn:
# The parent service imports this adapter; defer its public boundary error.
from modelforge_api.services.lifecycle import LifecycleError
raise LifecycleError(409, code, message)
def _json(value: Any) -> Any:
if isinstance(value, datetime):
return value.replace(tzinfo=UTC).isoformat() if value.tzinfo is None else value.astimezone(UTC).isoformat()
return str(value) if isinstance(value, UUID) else value
def digest(value: Any) -> str:
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"),
default=_json, allow_nan=False).encode()).hexdigest()
ROUTE_FIELDS = (
"id", "capability_contract_id", "deployment_candidate_id", "production_approval_id",
"execution_approval_id", "embedding_space_id", "artifact_set_id", "runtime_profile_id",
"compute_node_id", "accelerator_id", "channel", "status", "production", "routing_weight",
"fallback_policy", "residency_policy", "keep_warm_seconds", "max_concurrency", "max_queue_depth",
"config_fingerprint", "provenance", "rollback_policy", "promoted_at", "draining_at", "deprecated_at",
)
class LifecycleServing:
def __init__(self, session: Session) -> None:
self.session = session
self.repo = ServingRepository(session)
@staticmethod
def route_snapshot(row: CapabilityDeployment) -> dict[str, Any]:
return {field: _json(getattr(row, field)) for field in ROUTE_FIELDS}
def _deployment(self, identifier: UUID) -> CapabilityDeployment:
row = self.session.get(CapabilityDeployment, identifier, with_for_update=True, populate_existing=True)
if row is None:
_fail("SERVING_TARGET_MISSING", "exact serving deployment is unavailable")
return row
def _lock_accelerators(self, selection: ServingTransitionSelection) -> None:
node_ids = list(self.session.scalars(select(CapabilityDeployment.compute_node_id).where(
CapabilityDeployment.id.in_((selection.current_deployment_id, selection.candidate_deployment_id)))))
# Hardware inventory flushes compute_nodes before updating accelerators.
# Native admission does not acquire a node lock while holding its accelerator.
self.session.scalars(select(ComputeNode).where(ComputeNode.id.in_(node_ids)).order_by(
ComputeNode.id).with_for_update().execution_options(populate_existing=True)).all()
identifiers = list(self.session.scalars(select(CapabilityDeployment.accelerator_id).where(
CapabilityDeployment.id.in_((selection.current_deployment_id, selection.candidate_deployment_id)))))
self.session.scalars(select(Accelerator).where(Accelerator.id.in_(identifiers)).order_by(
Accelerator.id).with_for_update()).all()
def _bindings(self, contract_id: UUID) -> dict[str, dict[str, Any]]:
# Lock even deprecated bindings: a concurrent reactivation changes the blast radius.
bindings = list(self.session.scalars(select(ProjectBinding).where(
ProjectBinding.capability_contract_id == contract_id
).order_by(ProjectBinding.id).with_for_update().execution_options(populate_existing=True)))
result = {}
for binding in bindings:
project = self.session.get(Project, binding.project_id, with_for_update=True, populate_existing=True)
clients = list(self.session.scalars(select(ServiceClient).where(
ServiceClient.project_binding_id == binding.id
).order_by(ServiceClient.id).with_for_update().execution_options(populate_existing=True)))
if binding.channel != "stable" or binding.deprecated_at is not None:
continue
if project is None:
_fail("SERVING_BINDING_MISSING", "binding project disappeared")
result[str(binding.id)] = {
"project_id": str(binding.project_id), "project_active": project.active,
"priority": binding.priority, "optional": binding.optional,
"fallback_policy": binding.fallback_policy, "migration_support": binding.migration_support,
"slo": binding.slo, "benchmark_requirements": binding.benchmark_requirements,
"clients": [{"id": str(client.id), "status": client.status,
"disabled_at": _json(client.disabled_at),
"capabilities": client.allowed_capabilities} for client in clients],
}
if not result or len(result) > 1000:
_fail("SERVING_BINDINGS_UNBOUNDED", "transition requires one to 1000 exact stable bindings")
return result
def _experiment_routes(self, candidate_id: UUID) -> dict[str, dict[str, Any]]:
return {str(row.id): {"route_key": row.route_key, "status": row.status,
"capability_contract_id": str(row.capability_contract_id)}
for row in self.session.scalars(select(CapabilityExperimentRoute).where(
CapabilityExperimentRoute.capability_deployment_id == candidate_id
).order_by(CapabilityExperimentRoute.id).with_for_update().execution_options(populate_existing=True))}
def prepare(self, selection: ServingTransitionSelection) -> ServingTransitionContract:
# Obtain the contract before any deployment locks to match other route writers.
candidate_ref = self.session.get(CapabilityDeployment, selection.candidate_deployment_id)
if candidate_ref is None:
_fail("SERVING_TARGET_MISSING", "candidate deployment is unavailable")
contract = self.repo.lock_contract_route(candidate_ref.capability_contract_id)
if contract is None or contract.deprecated_at is not None:
_fail("SERVING_CONTRACT_UNAVAILABLE", "capability contract is unavailable")
if contract.serving_generation != selection.expected_route_generation:
_fail("SERVING_ROUTE_CHANGED", "stable route generation changed after selection")
# Match inventory node->accelerator and admission accelerator->residency/binding.
self._lock_accelerators(selection)
current = self._deployment(selection.current_deployment_id)
candidate = self._deployment(selection.candidate_deployment_id)
if (current.id == candidate.id or current.capability_contract_id != contract.id
or not current.production or current.status != "stable" or current.channel != "stable"
or candidate.production or candidate.status != "candidate" or candidate.channel != "experiment"):
_fail("SERVING_SELECTION_INVALID", "exact stable current and nonproduction candidate are required")
if current.embedding_space_id is None or current.embedding_space_id != candidate.embedding_space_id:
_fail("SERVING_REINDEX_REQUIRED", "changed embedding identity requires trusted consumer reindex and cutover")
experiment_routes = self._experiment_routes(candidate.id)
bindings = self._bindings(contract.id)
if set(bindings) != {str(identifier) for identifier in selection.comparison_ids}:
_fail("SERVING_BINDING_EVIDENCE_REQUIRED", "every affected stable binding requires its own comparison")
if len(set(selection.comparison_ids.values())) != len(bindings):
_fail("SERVING_COMPARISON_REUSED", "one project comparison cannot authorize another binding")
comparison_service = SelectedModelComparisonService(self.session, AuditContext("system", "lifecycle-serving"))
evidence = {}
for binding_id, comparison_id in sorted(selection.comparison_ids.items(), key=lambda item: str(item[0])):
comparison = self.session.get(SelectedModelComparison, comparison_id, with_for_update=True, populate_existing=True)
latest = self.session.scalar(select(SelectedModelComparison.id).where(
SelectedModelComparison.project_binding_id == binding_id
).order_by(SelectedModelComparison.created_at.desc(), SelectedModelComparison.id.desc()).limit(1))
if (comparison is None or comparison.status != "completed" or comparison.id != latest
or comparison.project_binding_id != binding_id
or comparison.current_deployment_id != current.id or comparison.candidate_deployment_id != candidate.id
or comparison.finished_at is None or comparison.result_digest is None):
_fail("SERVING_COMPARISON_STALE", "latest completed exact current/candidate comparison is required")
comparison_service._revalidate(comparison)
evaluated = self.session.get(EvaluationComparison, comparison.evaluation_comparison_id)
if (evaluated is None or evaluated.comparability != "comparable"
or evaluated.promotion_eligibility != "eligible" or evaluated.critical_regressions
or evaluated.baseline_run_id != comparison.baseline_run_id
or evaluated.candidate_run_id != comparison.candidate_run_id):
_fail("SERVING_COMPARISON_NOT_ELIGIBLE", "project comparison does not qualify replacement")
for run_id in (comparison.baseline_run_id, comparison.candidate_run_id):
run = self.session.get(EvaluationRun, run_id)
if run is None or run.status != "completed" or run.error_count or run.completed_cases != run.expected_cases:
_fail("SERVING_RUN_NOT_COMPLETE", "both comparison legs must be complete and error-free")
fit = self.session.scalar(select(ProjectFitEvidence).where(
ProjectFitEvidence.project_binding_id == binding_id,
ProjectFitEvidence.capability_deployment_id == candidate.id,
).order_by(ProjectFitEvidence.created_at.desc(), ProjectFitEvidence.id.desc()).limit(1)
.with_for_update().execution_options(populate_existing=True))
if fit is not None and (fit.recommendation != "PROMOTION_ELIGIBLE"
or fit.production_validation != "SATISFIED" or fit.engineering_integration != "PASS"
or fit.critical_errors or fit.blockers or fit.case_count < 1):
_fail("SERVING_PROJECT_FIT_BLOCKED", "latest exact binding/candidate project-fit evidence blocks production")
evidence[str(binding_id)] = {"id": str(comparison.id), "pins": comparison.pins,
"result_digest": comparison.result_digest, "evaluation_comparison_id": str(evaluated.id),
"baseline_run_id": str(evaluated.baseline_run_id), "candidate_run_id": str(evaluated.candidate_run_id),
"project_fit": ({"id": str(fit.id), "evidence_digest": fit.evidence_digest,
"record_digest": digest({column.name: _json(getattr(fit, column.name))
for column in fit.__table__.columns})} if fit is not None else None),
"evaluation_digest": digest({column.name: _json(getattr(evaluated, column.name))
for column in evaluated.__table__.columns})}
approval = self._production_evidence(candidate, contract, selection.production_approval_id)
return ServingTransitionContract(selection=selection, capability_contract_id=contract.id,
current=self.route_snapshot(current), candidate=self.route_snapshot(candidate),
experiment_routes=experiment_routes,
bindings=bindings, comparisons=evidence, production_approval_fingerprint=approval.evidence_fingerprint)
def _production_evidence(self, candidate: CapabilityDeployment, contract: CapabilityContract,
approval_id: UUID) -> ProductionExecutionApproval:
approval = self.session.get(ProductionExecutionApproval, approval_id,
with_for_update=True, populate_existing=True)
if (approval is None or approval.status != "approved" or approval.revoked_at is not None
or approval.deployment_candidate_id != candidate.deployment_candidate_id
or approval.capability_contract_id != contract.id
or approval.artifact_set_id != candidate.artifact_set_id
or approval.runtime_profile_id != candidate.runtime_profile_id
or approval.compute_node_id != candidate.compute_node_id):
_fail("SERVING_PRODUCTION_APPROVAL_INVALID", "exact current production execution approval is required")
# Immutable identity fields are already sealed. Mutable qualification/revocation
# fields must remain locked until the containing lifecycle transaction commits.
source = self.session.get(DeploymentCandidate, candidate.deployment_candidate_id,
with_for_update=True, populate_existing=True)
if source is None:
_fail("SERVING_TARGET_MISSING", "candidate provenance disappeared")
self.session.get(RuntimeProbe, source.runtime_probe_id, with_for_update=True, populate_existing=True)
self.session.get(RuntimeCompatibilityAssessment, source.compatibility_assessment_id,
with_for_update=True, populate_existing=True)
self.session.get(ArtifactSet, candidate.artifact_set_id, with_for_update=True, populate_existing=True)
self.session.get(ComputeNode, candidate.compute_node_id, with_for_update=True, populate_existing=True)
for artifact in self.repo.set_artifacts(candidate.artifact_set_id):
self.session.refresh(artifact, with_for_update=True)
# Reuse the existing production validation engine, without its create/promote commits.
settings = get_settings()
serving = ServingService(self.session, settings, ManifestRegistry(settings.config_root), None)
facts, _, probe, _, _ = serving._approval_facts(candidate.deployment_candidate_id, contract,
ProductionApprovalCreate(capability="rag.embedding", contract_version=1,
approved_by=approval.approved_by, reason=approval.reason,
supply_chain_review=SupplyChainReview.model_validate(approval.supply_chain_evidence),
deployment_config=approval.deployment_config))
if digest(facts) != approval.evidence_fingerprint:
_fail("SERVING_PRODUCTION_APPROVAL_STALE", "production probe or supply-chain evidence changed")
envelope = self.session.scalar(select(CapabilityResourceEnvelope).where(
CapabilityResourceEnvelope.capability_deployment_id == candidate.id
).with_for_update().execution_options(populate_existing=True))
if envelope is None or envelope.stale or envelope.runtime_probe_id != probe.id:
_fail("SERVING_ENVELOPE_STALE", "candidate requires its exact current measured envelope")
residency = self.session.scalar(select(ResidencyAllocation).where(
ResidencyAllocation.capability_deployment_id == candidate.id
).with_for_update().execution_options(populate_existing=True))
if (residency is None or residency.compute_node_id != candidate.compute_node_id
or residency.accelerator_id != candidate.accelerator_id
or residency.state not in {"cold", "ready", "warm"}):
_fail("SERVING_RESIDENCY_UNAVAILABLE", "candidate has no settled schedulable runtime residency")
return approval
def validate(self, payload: dict[str, Any]) -> ServingTransitionContract:
expected = ServingTransitionContract.model_validate(payload)
actual = self.prepare(expected.selection)
if actual != expected:
_fail("SERVING_PLAN_STALE", "approved serving identities or affected bindings changed")
return expected
def _generation(self, contract_id: UUID, expected: int) -> None:
changed = self.session.execute(update(CapabilityContract).where(
CapabilityContract.id == contract_id, CapabilityContract.serving_generation == expected
).values(serving_generation=expected + 1).returning(CapabilityContract.id)).scalar_one_or_none()
if changed is None:
_fail("SERVING_ROUTE_CHANGED", "stable route generation changed concurrently")
@staticmethod
def after(contract: ServingTransitionContract, changed_at: str) -> tuple[dict[str, Any], dict[str, Any]]:
old = {**contract.current, "status": "draining", "draining_at": changed_at, "routing_weight": 0}
new = {**contract.candidate, "status": "stable", "channel": "stable", "production": True,
"production_approval_id": str(contract.selection.production_approval_id),
"promoted_at": changed_at, "routing_weight": 100}
return old, new
def switch(self, payload: dict[str, Any], changed_at: str) -> None:
contract = self.validate(payload)
self._generation(contract.capability_contract_id, contract.selection.expected_route_generation)
current = self._deployment(contract.selection.current_deployment_id)
candidate = self._deployment(contract.selection.candidate_deployment_id)
current.status, current.draining_at = "draining", datetime.fromisoformat(changed_at)
current.routing_weight = 0
self.session.flush() # Release the one-stable partial unique index before activating the next.
candidate.status, candidate.channel, candidate.production = "stable", "stable", True
candidate.production_approval_id = contract.selection.production_approval_id
candidate.routing_weight = 100
candidate.promoted_at = datetime.fromisoformat(changed_at)
self.session.flush()
def installed(self, payload: dict[str, Any], changed_at: str, *, committed: bool = True) -> tuple[ServingTransitionContract, CapabilityDeployment, CapabilityDeployment]:
"""Verify exact persisted result for retry/rollback, without switching anything."""
contract = ServingTransitionContract.model_validate(payload)
route = self.repo.lock_contract_route(contract.capability_contract_id)
expected_generation = contract.selection.expected_route_generation + int(committed)
if route is None or route.serving_generation != expected_generation:
_fail("SERVING_ROLLBACK_CONFLICT", "later stable route generation must be preserved")
self._lock_accelerators(contract.selection)
current = self._deployment(contract.selection.current_deployment_id)
candidate = self._deployment(contract.selection.candidate_deployment_id)
expected = self.after(contract, changed_at) if committed else (contract.current, contract.candidate)
if ((self.route_snapshot(current), self.route_snapshot(candidate)) != expected
or self._bindings(contract.capability_contract_id) != contract.bindings
or self._experiment_routes(candidate.id) != contract.experiment_routes):
_fail("SERVING_ROLLBACK_CONFLICT", "serving state or affected bindings changed after the snapshot")
return contract, current, candidate
def restore(self, payload: dict[str, Any], changed_at: str, *, committed: bool) -> None:
contract, current, candidate = self.installed(payload, changed_at, committed=committed)
if not committed:
return
route = self.session.get(CapabilityContract, contract.capability_contract_id)
if route is None or current.production_approval_id is None:
_fail("SERVING_ROLLBACK_APPROVAL_MISSING", "prior serving authority is unavailable")
self._production_evidence(current, route, current.production_approval_id)
expected_generation = contract.selection.expected_route_generation + 1
self._generation(contract.capability_contract_id, expected_generation)
candidate.status = str(contract.candidate["status"])
candidate.channel = str(contract.candidate["channel"])
candidate.production = bool(contract.candidate["production"])
candidate.routing_weight = int(contract.candidate["routing_weight"])
candidate.production_approval_id = (UUID(contract.candidate["production_approval_id"])
if contract.candidate["production_approval_id"] else None)
candidate.promoted_at = (datetime.fromisoformat(contract.candidate["promoted_at"])
if contract.candidate["promoted_at"] else None)
self.session.flush()
current.status = str(contract.current["status"])
current.routing_weight = int(contract.current["routing_weight"])
current.draining_at = (datetime.fromisoformat(contract.current["draining_at"])
if contract.current["draining_at"] else None)
self.session.flush()
backend/src/modelforge_api/services/manifest_registry.py
+4
View File
@@ -73,6 +73,10 @@ class ManifestRegistry:
ProjectManifest.model_validate(self._read_yaml(path))
for path in sorted((self.config_root / "projects").glob("*.yaml"))
]
project_ids = [item.project.id for item in manifests]
if len(project_ids) != len(set(project_ids)):
raise ValueError("duplicate project manifest identity")
capabilities = self.capabilities()
by_identity = {(item.capability, item.version): item for item in capabilities}
capability_versions = set(by_identity)
backend/src/modelforge_api/services/migration_engine.py
+4 -2
View File
@@ -459,11 +459,13 @@ class MigrationEngineService:
}
return set()
def plans(self) -> list[MigrationPlanResponse]:
def plans(self, limit: int = 200) -> list[MigrationPlanResponse]:
return [
self._plan_response(item)
for item in self.session.scalars(
select(MigrationPlan).order_by(MigrationPlan.created_at.desc())
select(MigrationPlan)
.order_by(MigrationPlan.created_at.desc(), MigrationPlan.id.desc())
.limit(limit)
)
]
backend/src/modelforge_api/services/model_changes.py
+813
View File
@@ -0,0 +1,813 @@
from __future__ import annotations
from datetime import UTC, datetime
from typing import Literal, cast
from uuid import UUID
from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from modelforge_api.domain.model_changes import (
ChangeAcquisition,
ChangeImpact,
ChangeModelIdentity,
ChangeProgressStep,
ModelChangeCandidateLink,
ModelChangeCreate,
ModelChangeEvidenceLink,
ModelChangeList,
ModelChangeReject,
ModelChangeResponse,
ModelChangeStatus,
)
from modelforge_api.domain.selected_replacement import ReplacementPreview
from modelforge_api.persistence.model_change import ModelChangeCase
from modelforge_api.persistence.models import (
ArtifactJob,
ArtifactSet,
Capability,
CapabilityContract,
CapabilityDeployment,
DiscoveryCandidateAssessment,
DownloadPlan,
GatewayRequest,
LifecyclePromotionPlan,
MigrationPlan,
Model,
ModelRevision,
Project,
ProjectBinding,
SelectedModelComparison,
ServiceClient,
)
from modelforge_api.services.audit import AuditContext, AuditWriter
from modelforge_api.services.evaluation import EvaluationError
from modelforge_api.services.lifecycle import LifecycleError
from modelforge_api.services.selected_replacement import SelectedReplacementService
from modelforge_api.services.serving import ServingError
ProgressState = Literal["done", "current", "upcoming", "blocked"]
class ModelChangeError(ValueError):
def __init__(self, code: str, message: str, status_code: int = 409) -> None:
super().__init__(message)
self.code = code
self.message = message
self.status_code = status_code
def _now() -> datetime:
return datetime.now(UTC)
def _gate(value: object) -> str:
if not isinstance(value, dict):
return "unknown"
text = " ".join(
str(value.get(key, "")) for key in ("status", "state", "verdict", "result")
).lower()
if any(word in text for word in ("block", "reject", "deny", "fail", "incompat")):
return "blocked"
if any(
word in text for word in ("approved", "clear", "verified", "compatible", "pass", "safe")
):
return "clear"
return "review"
class ModelChangeService:
"""User workflow over authoritative ModelForge evidence; never executes lifecycle work itself."""
def __init__(self, session: Session, audit_context: AuditContext) -> None:
self.session = session
self.audit_context = audit_context
def _error(self, code: str, message: str, status: int = 409) -> ModelChangeError:
return ModelChangeError(code, message, status)
def _audit(
self,
row: ModelChangeCase,
action: str,
details: dict[str, object] | None = None,
) -> None:
AuditWriter(self.session, context=self.audit_context).write(
action,
"model_change_case",
str(row.id),
{
"project_binding_id": str(row.project_binding_id),
"current_deployment_id": str(row.current_deployment_id),
"candidate_deployment_id": (
str(row.candidate_deployment_id) if row.candidate_deployment_id else None
),
"status": row.status,
"version": row.version,
**(details or {}),
},
)
def _row(self, identifier: UUID, *, lock: bool = False) -> ModelChangeCase:
query = select(ModelChangeCase).where(ModelChangeCase.id == identifier)
if lock:
query = query.with_for_update().execution_options(populate_existing=True)
row = self.session.scalar(query)
if row is None:
raise self._error("MODEL_CHANGE_NOT_FOUND", "Model change case was not found", 404)
return row
def _binding_context(
self, binding_id: UUID
) -> tuple[ProjectBinding, Project, CapabilityContract, Capability]:
binding = self.session.get(ProjectBinding, binding_id)
project = self.session.get(Project, binding.project_id) if binding else None
contract = (
self.session.get(CapabilityContract, binding.capability_contract_id) if binding else None
)
capability = self.session.get(Capability, contract.capability_id) if contract else None
if (
binding is None
or binding.deprecated_at is not None
or binding.channel != "stable"
or project is None
or not project.active
or contract is None
or contract.deprecated_at is not None
or capability is None
):
raise self._error(
"MODEL_CHANGE_BINDING_INVALID",
"The selected project task is not an active stable binding",
422,
)
return binding, project, contract, capability
def _stable_deployment(self, contract_id: UUID) -> CapabilityDeployment | None:
return self.session.scalar(
select(CapabilityDeployment)
.where(
CapabilityDeployment.capability_contract_id == contract_id,
CapabilityDeployment.production.is_(True),
CapabilityDeployment.status == "stable",
CapabilityDeployment.deprecated_at.is_(None),
)
.order_by(
CapabilityDeployment.promoted_at.desc(),
CapabilityDeployment.created_at.desc(),
)
.limit(1)
)
def _identity(
self,
*,
model_id: UUID | None = None,
revision_id: UUID | None = None,
artifact_set_id: UUID | None = None,
deployment_id: UUID | None = None,
) -> ChangeModelIdentity | None:
deployment = self.session.get(CapabilityDeployment, deployment_id) if deployment_id else None
if deployment is not None:
artifact_set_id = deployment.artifact_set_id
artifact = self.session.get(ArtifactSet, artifact_set_id) if artifact_set_id else None
if artifact is not None:
revision_id = artifact.revision_id
revision = self.session.get(ModelRevision, revision_id) if revision_id else None
if revision is not None:
model_id = revision.model_id
model = self.session.get(Model, model_id) if model_id else None
if not any((deployment, artifact, revision, model)):
return None
return ChangeModelIdentity(
model_id=model.id if model else model_id,
display_name=model.display_name if model else None,
revision_id=revision.id if revision else revision_id,
resolved_commit_sha=revision.resolved_commit_sha if revision else None,
artifact_set_id=artifact.id if artifact else artifact_set_id,
deployment_id=deployment.id if deployment else deployment_id,
)
def _current_identity(self, row: ModelChangeCase) -> ChangeModelIdentity:
identity = self._identity(deployment_id=row.current_deployment_id)
if identity is None:
raise self._error(
"MODEL_CHANGE_CURRENT_IDENTITY_MISSING",
"The exact model identity behind the current deployment is incomplete",
)
return identity
def _candidate_identity(self, row: ModelChangeCase) -> ChangeModelIdentity | None:
return self._identity(
model_id=row.candidate_model_id,
revision_id=row.candidate_revision_id,
artifact_set_id=row.candidate_artifact_set_id,
deployment_id=row.candidate_deployment_id,
)
def _acquisition(self, row: ModelChangeCase) -> ChangeAcquisition:
assessment = (
self.session.get(DiscoveryCandidateAssessment, row.discovery_assessment_id)
if row.discovery_assessment_id
else None
)
plan = self.session.get(DownloadPlan, row.download_plan_id) if row.download_plan_id else None
job = self.session.get(ArtifactJob, row.artifact_job_id) if row.artifact_job_id else None
storage_bytes = plan.total_size_bytes if plan else None
if storage_bytes is None and row.candidate_artifact_set_id:
artifact = self.session.get(ArtifactSet, row.candidate_artifact_set_id)
storage_bytes = artifact.total_size_bytes if artifact else None
fit = "unknown"
if assessment is not None and isinstance(assessment.gpu_fit, dict):
fit = str(
assessment.gpu_fit.get("fit")
or assessment.gpu_fit.get("status")
or assessment.gpu_fit.get("verdict")
or "unknown"
)
return ChangeAcquisition(
discovery_assessment_id=assessment.id if assessment else None,
security_gate=_gate(assessment.security_state) if assessment else "unknown",
license_gate=_gate(assessment.license_state) if assessment else "unknown",
hardware_fit=fit,
storage_bytes=storage_bytes,
download_plan_id=plan.id if plan else None,
artifact_job_id=job.id if job else None,
download_status=job.status if job else (plan.status if plan else None),
)
def _successful_project_request_after(
self, binding_id: UUID, deployment_id: UUID, since: datetime | None
) -> bool:
clients = list(
self.session.scalars(
select(ServiceClient.id).where(
ServiceClient.project_binding_id == binding_id,
ServiceClient.status == "active",
ServiceClient.disabled_at.is_(None),
)
)
)
if not clients:
return False
query = select(GatewayRequest.id).where(
GatewayRequest.service_client_id.in_(clients),
GatewayRequest.capability_deployment_id == deployment_id,
GatewayRequest.status == "completed",
GatewayRequest.finished_at.is_not(None),
)
if since is not None:
query = query.where(GatewayRequest.finished_at >= since)
return (
self.session.scalar(query.order_by(GatewayRequest.finished_at.desc()).limit(1))
is not None
)
def _preview(self, row: ModelChangeCase) -> ReplacementPreview | None:
if row.selected_comparison_id is None:
return None
try:
return SelectedReplacementService(self.session).preview(row.selected_comparison_id)
except (EvaluationError, LifecycleError, ServingError):
# The detailed subsystem remains authoritative. A stale preview simply keeps this
# user projection from claiming readiness until the linked evidence is refreshed.
return None
def _derived_status(self, row: ModelChangeCase) -> ModelChangeStatus:
if row.closed_at is not None and row.status in {"rolled_back", "rejected"}:
return cast(ModelChangeStatus, row.status)
binding, _project, contract, _capability = self._binding_context(row.project_binding_id)
lifecycle = (
self.session.get(LifecyclePromotionPlan, row.lifecycle_plan_id)
if row.lifecycle_plan_id
else None
)
if lifecycle is not None and lifecycle.status == "ROLLED_BACK":
return "rolled_back"
stable = self._stable_deployment(contract.id)
if row.candidate_deployment_id and stable and stable.id == row.candidate_deployment_id:
since = stable.promoted_at or row.activated_at or row.updated_at
if self._successful_project_request_after(binding.id, stable.id, since):
return "checked"
return "active"
if lifecycle is not None and lifecycle.status == "COMMITTED":
return "active"
if lifecycle is not None and lifecycle.status == "APPROVED":
return "ready_to_activate"
comparison = (
self.session.get(SelectedModelComparison, row.selected_comparison_id)
if row.selected_comparison_id
else None
)
if comparison is not None:
if comparison.status == "completed":
preview = self._preview(row)
if preview is not None and not preview.same_embedding_space:
return "migration_required"
if preview is not None and preview.selection is not None:
return "ready_to_activate"
return "comparison_available"
if comparison.status in {"pending", "running", "cancel_requested"}:
return "tested"
if row.candidate_deployment_id:
return "ready_to_test"
job = self.session.get(ArtifactJob, row.artifact_job_id) if row.artifact_job_id else None
if job is not None and job.status == "completed":
return "downloaded"
plan = self.session.get(DownloadPlan, row.download_plan_id) if row.download_plan_id else None
if plan is not None and plan.status == "completed":
return "downloaded"
assessment = (
self.session.get(DiscoveryCandidateAssessment, row.discovery_assessment_id)
if row.discovery_assessment_id
else None
)
if assessment is not None and assessment.status.lower() not in {
"blocked",
"rejected",
"failed",
}:
return "source_checked"
if any((row.candidate_model_id, row.candidate_revision_id, row.candidate_artifact_set_id)):
return "found"
return "searching"
def _impact(self, row: ModelChangeCase) -> ChangeImpact:
_binding, project, _contract, _capability = self._binding_context(row.project_binding_id)
affected = [project.name]
blockers: list[str] = []
migration_required = False
migration_kind = "none"
rollback_available = bool(row.candidate_deployment_id)
preview = self._preview(row)
if preview is not None:
affected = list(dict.fromkeys(item.project_name for item in preview.affected_bindings)) or affected
blockers = list(dict.fromkeys(item.message for item in preview.blockers))
migration_required = not preview.same_embedding_space
migration_kind = "reindex" if migration_required else "none"
rollback_available = preview.current_deployment_id is not None
migration = (
self.session.get(MigrationPlan, row.migration_plan_id) if row.migration_plan_id else None
)
if migration is not None:
migration_kind = str(migration.migration_class).lower()
migration_required = migration_kind not in {"transparent", "none"}
lifecycle = (
self.session.get(LifecyclePromotionPlan, row.lifecycle_plan_id)
if row.lifecycle_plan_id
else None
)
downtime = False
application_change = False
if lifecycle is not None:
details = lifecycle.impact_analysis if isinstance(lifecycle.impact_analysis, dict) else {}
downtime = bool(details.get("downtime_expected", details.get("downtime", False)))
application_change = str(lifecycle.migration_class).lower() == "schema_breaking"
rollback_available = bool(lifecycle.rollback_target_ref)
if lifecycle.project_consumers:
affected = list(dict.fromkeys([*affected, *lifecycle.project_consumers]))
return ChangeImpact(
affected_projects=affected,
affected_project_count=len(affected),
application_change_required=application_change,
migration_required=migration_required,
migration_kind=migration_kind,
downtime_expected=downtime,
rollback_available=rollback_available,
blockers=blockers,
)
def _next_action(self, status: ModelChangeStatus) -> str:
return {
"searching": "Kandidaat zoeken",
"found": "Bron controleren",
"source_checked": "Downloaden voor test",
"downloaded": "Klaarzetten om te testen",
"ready_to_test": "Test uitvoeren",
"tested": "Vergelijk met huidig model",
"comparison_available": "Voorbereiden om te wisselen",
"migration_required": "Herindexering of migratie voorbereiden",
"ready_to_activate": "Veilig activeren",
"active": "Gezondheid controleren",
"checked": "Geen actie nodig",
"rolled_back": "Afgerond — teruggerold",
"rejected": "Afgerond — afgewezen",
}[status]
def _progress(self, status: ModelChangeStatus) -> list[ChangeProgressStep]:
labels = [
("candidate", "Kandidaat"),
("source", "Broncontrole"),
("download", "Download"),
("test", "Test"),
("compare", "Vergelijking"),
("impact", "Impact & migratie"),
("activate", "Activeren"),
("verify", "Controle"),
]
rank = {
"searching": 0,
"found": 0,
"source_checked": 1,
"downloaded": 2,
"ready_to_test": 3,
"tested": 4,
"comparison_available": 5,
"migration_required": 5,
"ready_to_activate": 6,
"active": 7,
"checked": 8,
"rolled_back": 8,
"rejected": 8,
}[status]
steps: list[ChangeProgressStep] = []
for index, (key, label) in enumerate(labels):
state: ProgressState = (
"done"
if index < rank
else "current"
if index == rank and rank < len(labels)
else "upcoming"
)
if status == "migration_required" and key == "impact":
state = "blocked"
steps.append(ChangeProgressStep(key=key, label=label, state=state))
return steps
def _response(self, row: ModelChangeCase) -> ModelChangeResponse:
binding, project, contract, capability = self._binding_context(row.project_binding_id)
status = self._derived_status(row)
return ModelChangeResponse(
id=row.id,
project_binding_id=binding.id,
project_id=project.id,
project_key=project.key,
project_name=project.name,
capability=f"{capability.key}@{contract.version}",
status=status,
stored_status=cast(ModelChangeStatus, row.status),
status_reason=row.status_reason,
version=row.version,
current=self._current_identity(row),
candidate=self._candidate_identity(row),
acquisition=self._acquisition(row),
selected_comparison_id=row.selected_comparison_id,
migration_plan_id=row.migration_plan_id,
lifecycle_plan_id=row.lifecycle_plan_id,
impact=self._impact(row),
next_action=self._next_action(status),
progress=self._progress(status),
created_by=row.created_by,
updated_by=row.updated_by,
activated_at=row.activated_at,
checked_at=row.checked_at,
closed_at=row.closed_at,
created_at=row.created_at,
updated_at=row.updated_at,
)
def _remember_derived_status(self, row: ModelChangeCase) -> None:
status = self._derived_status(row)
if status != row.status:
row.status = status
row.status_reason = self._next_action(status)
def create(self, request: ModelChangeCreate) -> ModelChangeResponse:
binding, project, contract, capability = self._binding_context(request.project_binding_id)
self.session.scalar(
select(ProjectBinding).where(ProjectBinding.id == binding.id).with_for_update()
)
existing = self.session.scalar(
select(ModelChangeCase).where(
ModelChangeCase.project_binding_id == binding.id,
ModelChangeCase.closed_at.is_(None),
)
)
if existing is not None:
return self._response(existing)
current = self._stable_deployment(contract.id)
if current is None:
raise self._error(
"MODEL_CHANGE_CURRENT_UNAVAILABLE",
"No healthy stable model deployment is available for this project task",
422,
)
row = ModelChangeCase(
project_binding_id=binding.id,
current_deployment_id=current.id,
status="searching",
status_reason=request.reason,
version=1,
created_by=self.audit_context.actor_id,
updated_by=self.audit_context.actor_id,
context={
"project_id": str(project.id),
"project_key": project.key,
"capability": f"{capability.key}@{contract.version}",
"reason": request.reason,
},
)
self.session.add(row)
try:
self.session.flush()
self._audit(row, "MODEL_CHANGE_CASE_CREATED")
self.session.commit()
except IntegrityError as exc:
self.session.rollback()
raise self._error(
"MODEL_CHANGE_ALREADY_OPEN",
"Another model change is already open for this project task",
) from exc
self.session.refresh(row)
return self._response(row)
def get(self, identifier: UUID) -> ModelChangeResponse:
return self._response(self._row(identifier))
def list(
self,
*,
project_key: str | None = None,
status: str | None = None,
project_binding_id: UUID | None = None,
open_only: bool = False,
limit: int = 100,
offset: int = 0,
) -> ModelChangeList:
if not 1 <= limit <= 200:
raise self._error("MODEL_CHANGE_PAGE_INVALID", "Limit must be between 1 and 200", 422)
if offset < 0:
raise self._error("MODEL_CHANGE_OFFSET_INVALID", "Offset must be zero or greater", 422)
query = select(ModelChangeCase)
if project_key:
query = (
query.join(ProjectBinding, ProjectBinding.id == ModelChangeCase.project_binding_id)
.join(Project, Project.id == ProjectBinding.project_id)
.where(Project.key == project_key)
)
if status:
query = query.where(ModelChangeCase.status == status)
if project_binding_id is not None:
query = query.where(ModelChangeCase.project_binding_id == project_binding_id)
if open_only:
query = query.where(ModelChangeCase.closed_at.is_(None))
total = self.session.scalar(select(func.count()).select_from(query.subquery())) or 0
rows = self.session.scalars(
query.order_by(ModelChangeCase.updated_at.desc(), ModelChangeCase.id.asc())
.offset(offset)
.limit(limit)
).all()
return ModelChangeList(items=[self._response(row) for row in rows], total=total)
def link_candidate(
self, identifier: UUID, request: ModelChangeCandidateLink
) -> ModelChangeResponse:
row = self._row(identifier, lock=True)
if row.closed_at is not None:
raise self._error("MODEL_CHANGE_CLOSED", "This model change is already closed")
if row.version != request.expected_version:
raise self._error("MODEL_CHANGE_VERSION_CONFLICT", "Refresh the change before editing it")
_binding, _project, contract, _capability = self._binding_context(row.project_binding_id)
deployment = (
self.session.get(CapabilityDeployment, request.candidate_deployment_id)
if request.candidate_deployment_id
else None
)
if deployment is not None and (
deployment.capability_contract_id != contract.id
or deployment.id == row.current_deployment_id
or deployment.production
or deployment.channel not in {"experiment", "lab", "candidate"}
or deployment.status not in {"candidate", "approved", "lab_ready"}
or deployment.deprecated_at is not None
):
raise self._error(
"MODEL_CHANGE_CANDIDATE_DEPLOYMENT_INVALID",
"The candidate deployment is not a safe non-production target for this project task",
422,
)
artifact_set_id = request.candidate_artifact_set_id or (
deployment.artifact_set_id if deployment else None
)
plan = self.session.get(DownloadPlan, request.download_plan_id) if request.download_plan_id else None
if plan is not None:
if artifact_set_id is not None and plan.artifact_set_id != artifact_set_id:
raise self._error(
"MODEL_CHANGE_DOWNLOAD_MISMATCH", "Download plan belongs to another candidate", 422
)
artifact_set_id = plan.artifact_set_id
job = self.session.get(ArtifactJob, request.artifact_job_id) if request.artifact_job_id else None
if job is not None:
if plan is None:
plan = self.session.get(DownloadPlan, job.plan_id)
if plan is None or job.plan_id != plan.id or (
artifact_set_id is not None and plan.artifact_set_id != artifact_set_id
):
raise self._error(
"MODEL_CHANGE_JOB_MISMATCH", "Download job belongs to another candidate", 422
)
artifact_set_id = plan.artifact_set_id
artifact = self.session.get(ArtifactSet, artifact_set_id) if artifact_set_id else None
if artifact_set_id and artifact is None:
raise self._error("MODEL_CHANGE_ARTIFACT_NOT_FOUND", "Candidate artifact set was not found", 404)
revision_id = request.candidate_revision_id or (artifact.revision_id if artifact else None)
revision = self.session.get(ModelRevision, revision_id) if revision_id else None
if revision_id and revision is None:
raise self._error("MODEL_CHANGE_REVISION_NOT_FOUND", "Candidate version was not found", 404)
model_id = request.candidate_model_id or (revision.model_id if revision else None)
model = self.session.get(Model, model_id) if model_id else None
if model_id and model is None:
raise self._error("MODEL_CHANGE_MODEL_NOT_FOUND", "Candidate model was not found", 404)
if revision and model_id and revision.model_id != model_id:
raise self._error(
"MODEL_CHANGE_IDENTITY_MISMATCH", "Candidate version belongs to another model", 422
)
if artifact and revision_id and artifact.revision_id != revision_id:
raise self._error(
"MODEL_CHANGE_IDENTITY_MISMATCH", "Candidate files belong to another version", 422
)
assessment = (
self.session.get(DiscoveryCandidateAssessment, request.discovery_assessment_id)
if request.discovery_assessment_id
else None
)
if request.discovery_assessment_id and assessment is None:
raise self._error(
"MODEL_CHANGE_ASSESSMENT_NOT_FOUND", "Candidate source review was not found", 404
)
if assessment is not None:
if assessment.model_id is not None and model_id is not None and assessment.model_id != model_id:
raise self._error(
"MODEL_CHANGE_ASSESSMENT_MISMATCH", "Source review belongs to another model", 422
)
if revision is not None and assessment.resolved_commit_sha != revision.resolved_commit_sha:
raise self._error(
"MODEL_CHANGE_ASSESSMENT_MISMATCH",
"Source review belongs to another exact version",
422,
)
row.candidate_model_id = model_id
row.candidate_revision_id = revision_id
row.candidate_artifact_set_id = artifact_set_id
row.candidate_deployment_id = (
deployment.id if deployment else request.candidate_deployment_id or row.candidate_deployment_id
)
row.discovery_assessment_id = (
assessment.id if assessment else request.discovery_assessment_id or row.discovery_assessment_id
)
row.download_plan_id = plan.id if plan else request.download_plan_id or row.download_plan_id
row.artifact_job_id = job.id if job else request.artifact_job_id or row.artifact_job_id
row.version += 1
row.updated_by = self.audit_context.actor_id
self._remember_derived_status(row)
self._audit(row, "MODEL_CHANGE_CANDIDATE_LINKED")
self.session.commit()
self.session.refresh(row)
return self._response(row)
def link_evidence(
self, identifier: UUID, request: ModelChangeEvidenceLink
) -> ModelChangeResponse:
row = self._row(identifier, lock=True)
if row.closed_at is not None:
raise self._error("MODEL_CHANGE_CLOSED", "This model change is already closed")
if row.version != request.expected_version:
raise self._error("MODEL_CHANGE_VERSION_CONFLICT", "Refresh the change before editing it")
binding, project, _contract, _capability = self._binding_context(row.project_binding_id)
comparison = (
self.session.get(SelectedModelComparison, request.selected_comparison_id)
if request.selected_comparison_id
else None
)
if request.selected_comparison_id:
if comparison is None:
raise self._error("MODEL_CHANGE_COMPARISON_NOT_FOUND", "Comparison was not found", 404)
if (
comparison.project_binding_id != binding.id
or comparison.current_deployment_id != row.current_deployment_id
or row.candidate_deployment_id is None
or comparison.candidate_deployment_id != row.candidate_deployment_id
):
raise self._error(
"MODEL_CHANGE_COMPARISON_MISMATCH",
"Comparison does not belong to this project and exact current/candidate pair",
422,
)
migration = (
self.session.get(MigrationPlan, request.migration_plan_id)
if request.migration_plan_id
else None
)
if request.migration_plan_id:
if migration is None:
raise self._error("MODEL_CHANGE_MIGRATION_NOT_FOUND", "Migration plan was not found", 404)
if migration.project_binding_id != binding.id or migration.project_id != project.id:
raise self._error(
"MODEL_CHANGE_MIGRATION_MISMATCH", "Migration plan belongs to another project task", 422
)
if row.candidate_deployment_id:
candidate = self.session.get(CapabilityDeployment, row.candidate_deployment_id)
if (
candidate
and candidate.embedding_space_id
and migration.target_space_id != candidate.embedding_space_id
):
raise self._error(
"MODEL_CHANGE_MIGRATION_MISMATCH",
"Migration target does not match the candidate embedding space",
422,
)
lifecycle = (
self.session.get(LifecyclePromotionPlan, request.lifecycle_plan_id)
if request.lifecycle_plan_id
else None
)
if request.lifecycle_plan_id:
if lifecycle is None:
raise self._error("MODEL_CHANGE_LIFECYCLE_NOT_FOUND", "Activation plan was not found", 404)
if (
row.candidate_deployment_id is None
or lifecycle.candidate_deployment_id != row.candidate_deployment_id
):
raise self._error(
"MODEL_CHANGE_LIFECYCLE_MISMATCH", "Activation plan belongs to another candidate", 422
)
if (
str(project.id) not in {str(value) for value in lifecycle.project_consumers}
and project.name not in lifecycle.project_consumers
):
impacted = lifecycle.impact_analysis if isinstance(lifecycle.impact_analysis, dict) else {}
project_ids = impacted.get("project_ids", [])
project_refs = (
{str(value) for value in project_ids} if isinstance(project_ids, list) else set()
)
if project_refs and str(project.id) not in project_refs:
raise self._error(
"MODEL_CHANGE_LIFECYCLE_MISMATCH",
"Activation plan does not affect this project",
422,
)
if comparison is not None:
row.selected_comparison_id = comparison.id
if migration is not None:
row.migration_plan_id = migration.id
if lifecycle is not None:
row.lifecycle_plan_id = lifecycle.id
row.version += 1
row.updated_by = self.audit_context.actor_id
self._remember_derived_status(row)
self._audit(row, "MODEL_CHANGE_EVIDENCE_LINKED")
self.session.commit()
self.session.refresh(row)
return self._response(row)
def refresh(self, identifier: UUID, expected_version: int) -> ModelChangeResponse:
row = self._row(identifier, lock=True)
if row.version != expected_version:
raise self._error("MODEL_CHANGE_VERSION_CONFLICT", "Refresh the change before updating status")
before = row.status
derived = self._derived_status(row)
if derived != row.status:
row.status = derived
row.status_reason = self._next_action(derived)
row.version += 1
row.updated_by = self.audit_context.actor_id
if derived in {"active", "checked"} and row.activated_at is None:
row.activated_at = _now()
if derived == "checked" and row.checked_at is None:
row.checked_at = _now()
if derived == "rolled_back":
row.closed_at = _now()
self._audit(row, "MODEL_CHANGE_STATUS_REFRESHED", {"from": before, "to": derived})
self.session.commit()
self.session.refresh(row)
return self._response(row)
def reject(self, identifier: UUID, request: ModelChangeReject) -> ModelChangeResponse:
row = self._row(identifier, lock=True)
if row.version != request.expected_version:
raise self._error("MODEL_CHANGE_VERSION_CONFLICT", "Refresh the change before rejecting it")
if row.closed_at is not None:
return self._response(row)
if self._derived_status(row) in {"active", "checked"}:
raise self._error(
"MODEL_CHANGE_ALREADY_ACTIVE",
"An active change must be rolled back through the controlled lifecycle flow",
)
row.status = "rejected"
row.status_reason = request.reason
row.closed_at = _now()
row.updated_by = self.audit_context.actor_id
row.version += 1
self._audit(row, "MODEL_CHANGE_CASE_REJECTED", {"reason": request.reason})
self.session.commit()
self.session.refresh(row)
return self._response(row)
backend/src/modelforge_api/services/model_update_polling.py
+76
View File
@@ -0,0 +1,76 @@
"""Low-volume metadata checks for explicitly selected models; never model execution."""
import asyncio
import threading
import uuid
from datetime import UTC, datetime, timedelta
import structlog
from sqlalchemy import Engine, or_, select
from sqlalchemy.orm import Session
from modelforge_api.persistence.models import ModelUpdateWatch
from modelforge_api.services.acquisition import AcquisitionService
from modelforge_api.services.audit import AuditContext
from modelforge_api.services.model_updates import ModelUpdateService
from modelforge_api.services.registry import RegistryConflict
from modelforge_api.services.secret_slots import SecretAwareHuggingFaceProvider
from modelforge_api.settings import Settings
logger = structlog.get_logger()
class ModelUpdatePollingService:
"""One due watch per minute per API process, with database-fenced per-watch admission."""
def __init__(self, settings: Settings, engine: Engine) -> None:
self.settings = settings
self.engine = engine
self._stop = threading.Event()
def stop(self) -> None:
self._stop.set()
def service(self, session: Session) -> ModelUpdateService:
provider = SecretAwareHuggingFaceProvider(session, self.settings)
acquisition = AcquisitionService(
session, self.settings, provider,
credential_configured=provider.credential_configured,
)
return ModelUpdateService(session, acquisition, AuditContext(
"automation", "selected-model-update-monitor", str(uuid.uuid4()),
))
def run_once(self) -> None:
if self._stop.is_set() or not self.settings.model_update_monitor_enabled:
return
now = datetime.now(UTC)
cutoff = now - timedelta(seconds=self.settings.model_update_check_interval_seconds)
try:
with Session(self.engine) as session:
watch_id = session.scalar(select(ModelUpdateWatch.id).where(
ModelUpdateWatch.enabled.is_(True),
or_(ModelUpdateWatch.last_checked_at.is_(None),
ModelUpdateWatch.last_checked_at <= cutoff),
or_(ModelUpdateWatch.check_until.is_(None),
ModelUpdateWatch.check_until < now),
).order_by(ModelUpdateWatch.last_checked_at.asc().nulls_first(),
ModelUpdateWatch.id).limit(1))
if watch_id is None:
return
# The service rechecks due/enabled state in its atomic claim, so another
# API process completing this watch cannot cause an immediate second poll.
self.service(session).check(watch_id, if_checked_before=cutoff)
except RegistryConflict:
# A manual action or another process won admission. Its state remains authoritative.
return
except Exception as error: # noqa: BLE001 - keep the background loop alive; never log credentials
logger.error("model_update_monitor_check_failed", error_type=type(error).__name__)
async def run(self) -> None:
while not self._stop.is_set():
await asyncio.to_thread(self.run_once)
for _ in range(60):
if self._stop.is_set():
return
await asyncio.sleep(1)
backend/src/modelforge_api/services/model_updates.py
+195
View File
@@ -0,0 +1,195 @@
import uuid
from datetime import UTC, datetime, timedelta
from sqlalchemy import func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from modelforge_api.domain.model_updates import (
ModelUpdateNoticeResponse,
ModelWatchCreate,
ModelWatchResponse,
ModelWatchUpdate,
)
from modelforge_api.persistence.models import (
Model,
ModelRevision,
ModelUpdateNotice,
ModelUpdateWatch,
)
from modelforge_api.services.acquisition import AcquisitionService, ProviderBoundaryError
from modelforge_api.services.audit import AuditContext, AuditWriter
from modelforge_api.services.registry import RegistryConflict, RegistryNotFound
class ModelUpdateService:
def __init__(self, session: Session, acquisition: AcquisitionService, context: AuditContext) -> None:
self.session = session
self.acquisition = acquisition
self.audit = AuditWriter(session, context=context)
self.acquisition.audit = self.audit
def watches(self, limit: int = 50, offset: int = 0) -> list[ModelWatchResponse]:
return [ModelWatchResponse.model_validate(item) for item in self.session.scalars(
select(ModelUpdateWatch).order_by(ModelUpdateWatch.id).limit(limit).offset(offset)
)]
def create(self, request: ModelWatchCreate) -> ModelWatchResponse:
model = self.session.get(Model, request.model_id)
revision = self.session.get(ModelRevision, request.baseline_revision_id)
if model is None or revision is None:
raise RegistryNotFound("model or baseline revision not found")
if model.source_type != "huggingface" or revision.model_id != model.id:
raise RegistryConflict("watch requires a Hugging Face model and its own exact revision")
existing = self.session.scalar(select(ModelUpdateWatch).where(
ModelUpdateWatch.model_id == model.id,
ModelUpdateWatch.upstream_ref == request.upstream_ref,
))
if existing:
if existing.baseline_revision_id != revision.id:
raise RegistryConflict("watch baseline requires an explicit tracking update; review the existing watch")
return ModelWatchResponse.model_validate(existing)
item = ModelUpdateWatch(id=uuid.uuid4(), model_id=model.id,
baseline_revision_id=revision.id, baseline_commit_sha=revision.resolved_commit_sha,
upstream_ref=request.upstream_ref)
# The audit chain lock serializes admission, including the global watch limit.
self.audit.write("MODEL_WATCH_CREATED", "model_update_watch", str(item.id),
{"model_id": str(model.id), "baseline_revision_id": str(revision.id)})
if (self.session.scalar(select(func.count()).select_from(ModelUpdateWatch)) or 0) >= 100:
self.session.rollback()
raise RegistryConflict("maximum of 100 explicitly selected model watches reached")
self.session.add(item)
try:
self.session.commit()
except IntegrityError as exc:
self.session.rollback()
raise RegistryConflict("watch already exists; refresh the watch list") from exc
return ModelWatchResponse.model_validate(item)
def change(self, watch_id: uuid.UUID, request: ModelWatchUpdate) -> ModelWatchResponse:
item = self.session.scalar(select(ModelUpdateWatch).where(
ModelUpdateWatch.id == watch_id).with_for_update().execution_options(populate_existing=True))
if item is None:
raise RegistryNotFound("model watch not found")
if item.version != request.expected_version or item.baseline_revision_id != request.expected_baseline_revision_id:
raise RegistryConflict("model watch changed; refresh before changing tracking")
revision = self.session.get(ModelRevision, request.baseline_revision_id)
if revision is None:
raise RegistryNotFound("baseline revision not found")
if revision.model_id != item.model_id:
raise RegistryConflict("watch baseline must belong to the same model")
baseline_changed = revision.id != item.baseline_revision_id
until = item.check_until
if until is not None and until.tzinfo is None:
until = until.replace(tzinfo=UTC)
if baseline_changed and item.check_token is not None and until is not None and until > datetime.now(UTC):
raise RegistryConflict("wait for the active check or pause the watch before changing its baseline")
if not baseline_changed and item.enabled == request.enabled:
return ModelWatchResponse.model_validate(item)
previous = {"baseline_revision_id": str(item.baseline_revision_id),
"baseline_commit_sha": item.baseline_commit_sha, "enabled": item.enabled,
"version": item.version}
item.enabled = request.enabled
item.baseline_revision_id = revision.id
item.baseline_commit_sha = revision.resolved_commit_sha
item.version += 1
if not item.enabled or baseline_changed:
# Any result from the old check must fail its token fence, even after resume.
item.check_token = None
item.check_until = None
if baseline_changed:
item.last_checked_at = None
item.last_error_code = None
self.audit.write("MODEL_WATCH_TRACKING_CHANGED", "model_update_watch", str(item.id), {
"previous": previous, "baseline_revision_id": str(revision.id),
"baseline_commit_sha": item.baseline_commit_sha, "enabled": item.enabled,
"version": item.version, "reason": request.reason.strip(),
})
self.session.commit()
return ModelWatchResponse.model_validate(item)
def check(self, watch_id: uuid.UUID, *, if_checked_before: datetime | None = None) -> ModelWatchResponse:
item = self.session.get(ModelUpdateWatch, watch_id)
if item is None:
raise RegistryNotFound("model watch not found")
now = datetime.now(UTC)
token = uuid.uuid4()
claim = update(ModelUpdateWatch).where(
ModelUpdateWatch.id == watch_id,
ModelUpdateWatch.enabled.is_(True),
or_(ModelUpdateWatch.check_until.is_(None), ModelUpdateWatch.check_until < now),
)
if if_checked_before is not None:
claim = claim.where(or_(ModelUpdateWatch.last_checked_at.is_(None),
ModelUpdateWatch.last_checked_at <= if_checked_before))
claimed = self.session.execute(claim.values(check_until=now + timedelta(minutes=5), check_token=token)
.execution_options(synchronize_session="fetch").returning(ModelUpdateWatch.id)).first()
if claimed is None:
self.session.rollback()
raise RegistryConflict("model watch is paused, not due, or a check is already in progress")
self.session.commit()
self.session.refresh(item)
if not item.enabled or item.check_token != token:
raise RegistryConflict("model watch check superseded; refresh its status")
model_id, upstream_ref, baseline_sha = item.model_id, item.upstream_ref, item.baseline_commit_sha
error = None
candidate = None
try:
snapshot = self.acquisition.refresh_model(model_id, upstream_ref)
candidate = self.session.scalar(select(ModelRevision).where(
ModelRevision.model_id == model_id,
ModelRevision.resolved_commit_sha == snapshot.resolved_commit_sha,
))
if candidate is None:
raise RegistryConflict("upstream refresh did not resolve an exact revision")
except ProviderBoundaryError:
error = "upstream_unavailable"
except Exception:
self.session.rollback()
self.session.execute(update(ModelUpdateWatch).where(
ModelUpdateWatch.id == watch_id, ModelUpdateWatch.check_token == token,
).values(check_token=None, check_until=None, last_checked_at=datetime.now(UTC),
last_error_code="check_failed"))
self.audit.write("MODEL_WATCH_CHECK_FAILED", "model_update_watch", str(watch_id),
{"error_code": "check_failed"}, outcome="blocked")
self.session.commit()
raise
# An expired check cannot overwrite a subsequent check's state or create a stale notice.
finished = self.session.execute(update(ModelUpdateWatch).where(
ModelUpdateWatch.id == watch_id, ModelUpdateWatch.check_token == token,
).values(check_token=None, check_until=None, last_checked_at=datetime.now(UTC),
last_error_code=error).returning(ModelUpdateWatch.id)).first()
if finished is None:
self.session.rollback()
raise RegistryConflict("model watch check superseded; refresh its status")
if candidate and candidate.resolved_commit_sha != baseline_sha:
existing = self.session.scalar(select(ModelUpdateNotice).where(
ModelUpdateNotice.watch_id == watch_id,
ModelUpdateNotice.candidate_commit_sha == candidate.resolved_commit_sha,
))
if existing is None:
self.session.add(ModelUpdateNotice(watch_id=watch_id, model_id=model_id,
baseline_commit_sha=baseline_sha,
candidate_revision_id=candidate.id, candidate_commit_sha=candidate.resolved_commit_sha))
self.audit.write("MODEL_WATCH_CHECKED", "model_update_watch", str(watch_id),
{"error_code": error}, outcome="blocked" if error else "success")
self.session.commit()
self.session.refresh(item)
return ModelWatchResponse.model_validate(item)
def notices(self, limit: int = 50, offset: int = 0) -> list[ModelUpdateNoticeResponse]:
return [ModelUpdateNoticeResponse.model_validate(item) for item in self.session.scalars(
select(ModelUpdateNotice).order_by(ModelUpdateNotice.discovered_at.desc(), ModelUpdateNotice.id)
.limit(limit).offset(offset)
)]
def dismiss(self, notice_id: uuid.UUID) -> ModelUpdateNoticeResponse:
item = self.session.scalar(select(ModelUpdateNotice).where(
ModelUpdateNotice.id == notice_id).with_for_update())
if item is None:
raise RegistryNotFound("model update notice not found")
if item.status != "dismissed":
item.status = "dismissed"
self.audit.write("MODEL_UPDATE_DISMISSED", "model_update_notice", str(item.id), {})
self.session.commit()
return ModelUpdateNoticeResponse.model_validate(item)
backend/src/modelforge_api/services/node_agent.py
+36 -17
View File
@@ -361,6 +361,7 @@ class NodeAgentService:
raw: str,
*,
update_last_used: bool,
lock_node: bool = False,
) -> tuple[NodeCredential, ComputeNode]:
credential = self.repository.credential(credential_id)
if credential is None or credential.revoked_at is not None:
@@ -369,7 +370,11 @@ class NodeAgentService:
raise AgentAuthenticationError("invalid node credential")
if credential.scope != "node.publish":
raise AgentAuthorizationError("node credential scope is not authorized for publishing")
node = self.repository.node(credential.compute_node_id)
node = (
self.repository.node_for_update(credential.compute_node_id)
if lock_node
else self.repository.node(credential.compute_node_id)
)
if node is None:
raise AgentAuthenticationError("credential node no longer exists")
if node.decommissioned_at is not None:
@@ -396,8 +401,13 @@ class NodeAgentService:
)
def authenticate(self, authorization: str | None) -> tuple[NodeCredential, ComputeNode]:
identity, _evidence = self.authenticate_with_evidence(authorization)
return identity
credential_id, raw = self._parse_credential(authorization)
return self._resolve_authenticated_node(
credential_id,
raw,
update_last_used=True,
lock_node=True,
)
def reuse_authentication(
self,
@@ -415,6 +425,7 @@ class NodeAgentService:
credential_id,
raw,
update_last_used=False,
lock_node=True,
)
if node.id != evidence.node_id:
raise AgentAuthenticationError("credential node identity changed")
@@ -430,8 +441,23 @@ class NodeAgentService:
if node.decommissioned_at is not None:
raise AgentConflictError("decommissioned compute node cannot publish agent state")
def _locked_publishable_node(self, node: ComputeNode) -> ComputeNode:
"""Fence an authenticated agent write against concurrent lifecycle changes.
Agent routes also take this row lock while revalidating authentication. Keeping the same
node-first lock order here protects direct service callers and makes each state publisher
independently fail closed if its caller holds a stale ORM instance.
"""
current = self.repository.node_for_update(node.id)
if current is None:
raise AgentNotFoundError("compute node no longer exists")
self._assert_not_decommissioned(current)
if not current.enabled:
raise AgentAuthorizationError("compute node is disabled")
return current
def heartbeat(self, node: ComputeNode, request: HeartbeatRequest) -> ObservationAck:
self._assert_not_decommissioned(node)
node = self._locked_publishable_node(node)
self._assert_identity(node, request.identity_key)
if request.metadata.protocol_version != self.settings.agent_protocol_version:
raise AgentConflictError("agent protocol version is incompatible")
@@ -450,7 +476,7 @@ class NodeAgentService:
return ObservationAck(accepted=True, received_at=now)
def publish_inventory(self, node: ComputeNode, report: InventoryReport) -> ObservationAck:
self._assert_not_decommissioned(node)
node = self._locked_publishable_node(node)
self._assert_identity(node, report.identity_key)
if report.host.identity_key != node.key:
raise AgentAuthorizationError("host inventory identity does not match credential node")
@@ -517,7 +543,7 @@ class NodeAgentService:
return observation.value if observation.availability.value == "known" else None
def publish_telemetry(self, node: ComputeNode, report: TelemetryReport) -> ObservationAck:
self._assert_not_decommissioned(node)
node = self._locked_publishable_node(node)
self._assert_identity(node, report.identity_key)
if report.protocol_version != self.settings.agent_protocol_version:
raise AgentConflictError("agent protocol version is incompatible")
@@ -640,19 +666,12 @@ class NodeAgentService:
if node.decommissioned_at is not None:
raise AgentConflictError("decommissioned compute node cannot be updated")
values = update.model_dump(exclude_unset=True)
was_enabled = node.enabled
if "enabled" in values:
raise AgentConflictError(
"node enabled state requires an impact preview and typed confirmation"
)
for key, value in values.items():
setattr(node, key, value)
if "enabled" in values and values["enabled"] != was_enabled:
node.liveness_state = (
NodeLiveness.OFFLINE if values["enabled"] else NodeLiveness.DISABLED
)
self._audit(
"NODE_ENABLED" if values["enabled"] else "NODE_DISABLED",
"compute_node",
str(node.id),
{},
)
self.session.commit()
return node
backend/src/modelforge_api/services/node_decommission.py
+202 -2
View File
@@ -4,7 +4,7 @@ import hashlib
import json
import uuid
from datetime import UTC, datetime
from typing import Any
from typing import Any, Literal
from sqlalchemy import delete, func, select, text, update
from sqlalchemy.exc import IntegrityError
@@ -18,6 +18,13 @@ from modelforge_api.domain.node_decommission import (
NodeDecommissionPreview,
NodeDecommissionResult,
)
from modelforge_api.domain.node_lifecycle import (
NodeLifecycleBlocker,
NodeLifecycleChange,
NodeLifecyclePreview,
NodeLifecyclePreviewRequest,
NodeLifecycleResult,
)
from modelforge_api.persistence.models import (
Accelerator,
AcceleratorTelemetryLatest,
@@ -102,7 +109,7 @@ class NodeDecommissionService:
def _node(self, node_id: uuid.UUID, *, lock: bool = False) -> ComputeNode:
statement = select(ComputeNode).where(ComputeNode.id == node_id)
if lock:
statement = statement.with_for_update()
statement = statement.with_for_update().execution_options(populate_existing=True)
node = self.session.scalar(statement)
if node is None:
raise NodeDecommissionError(
@@ -740,6 +747,199 @@ class NodeDecommissionService:
dependent_records=dependent_records,
)
def _lifecycle_preview(
self, node: ComputeNode, request: NodeLifecyclePreviewRequest
) -> NodeLifecyclePreview:
dependency_snapshot = self._snapshot(node)
blockers: list[NodeLifecycleBlocker] = []
active_credential_ids = sorted(
str(value)
for value in self.session.scalars(
select(NodeCredential.id).where(
NodeCredential.compute_node_id == node.id,
NodeCredential.revoked_at.is_(None),
)
)
)
if node.decommissioned_at is not None:
blockers.append(
NodeLifecycleBlocker(
code="node_decommissioned",
message="decommissioned node identities are terminal and cannot be enabled or disabled",
record_type="compute_node",
count=1,
resource_ids=[str(node.id)],
)
)
elif node.enabled == request.target_enabled:
blockers.append(
NodeLifecycleBlocker(
code="already_in_requested_state",
message="node is already in the requested enabled state",
record_type="compute_node",
count=1,
resource_ids=[str(node.id)],
)
)
elif not request.target_enabled:
# A live heartbeat is expected before an administrator disables a node. Every other
# decommission dependency is conservatively treated as a blocker for temporary disable.
blockers.extend(
NodeLifecycleBlocker.model_validate(item.model_dump())
for item in dependency_snapshot.blockers
if item.code != "node_online"
)
elif node.observation_source == "remote_agent" and not active_credential_ids:
blockers.append(
NodeLifecycleBlocker(
code="active_node_credential_missing",
message="remote node has no active credential; rotate or re-enroll before enabling",
record_type="node_credential",
count=1,
resource_ids=[],
)
)
digest_payload = {
"base_dependency_digest": dependency_snapshot.dependency_digest,
"node_id": str(node.id),
"target_enabled": request.target_enabled,
"active_credential_ids": active_credential_ids,
"blockers": [item.model_dump(mode="json") for item in blockers],
}
dependency_digest = hashlib.sha256(
json.dumps(digest_payload, sort_keys=True, separators=(",", ":")).encode()
).hexdigest()
return NodeLifecyclePreview(
node_id=node.id,
persisted_identity=node.key,
hostname=node.hostname,
display_name=node.display_name or node.hostname,
current_enabled=node.enabled,
current_liveness=node.liveness_state,
target_enabled=request.target_enabled,
node_generation=node.generation,
dependency_digest=dependency_digest,
can_apply=not blockers,
blockers=blockers,
agent_evidence_after_enable=(
"awaiting_fresh_authenticated_heartbeat"
if request.target_enabled and node.observation_source == "remote_agent"
else "unchanged"
),
)
def preview_enabled_change(
self, node_id: uuid.UUID, request: NodeLifecyclePreviewRequest
) -> NodeLifecyclePreview:
return self._lifecycle_preview(self._node(node_id), request)
def apply_enabled_change(
self, node_id: uuid.UUID, request: NodeLifecycleChange
) -> NodeLifecycleResult:
node = self._node(node_id, lock=True)
if node.generation != request.expected_generation:
raise NodeDecommissionError(
409,
"node_generation_conflict",
"node generation changed after lifecycle preview",
{"expected": request.expected_generation, "actual": node.generation},
)
if request.confirmation not in {node.key, node.hostname, node.display_name}:
raise NodeDecommissionError(
409,
"node_lifecycle_confirmation_mismatch",
"typed confirmation must exactly match the persisted identity, hostname, or display name",
)
self._lock_dependency_tables()
preview = self._lifecycle_preview(
node, NodeLifecyclePreviewRequest(target_enabled=request.target_enabled)
)
if preview.dependency_digest != request.dependency_digest:
raise NodeDecommissionError(
409,
"node_lifecycle_preview_stale",
"node state or dependencies changed after impact review; generate a fresh preview",
{"preview": preview.model_dump(mode="json")},
)
if not preview.can_apply:
raise NodeDecommissionError(
409,
"node_lifecycle_blocked",
"node lifecycle transition is blocked until the listed dependencies are resolved",
{"blockers": [item.model_dump(mode="json") for item in preview.blockers]},
)
target_liveness: Literal["offline", "disabled"] = (
"offline" if request.target_enabled else "disabled"
)
values: dict[str, Any] = {
"enabled": request.target_enabled,
"generation": ComputeNode.generation + 1,
"liveness_state": target_liveness,
}
if request.target_enabled and node.observation_source == "remote_agent":
# Previous protocol/heartbeat facts must not make a re-enabled node look ready. A new
# authenticated, protocol-compatible heartbeat will publish current evidence again.
values.update(
agent_protocol_version=None,
agent_capabilities=[],
agent_version=None,
agent_started_at=None,
last_heartbeat_at=None,
last_connection_error="node enabled; awaiting fresh authenticated heartbeat",
)
claimed = rows_affected(
self.session.execute(
update(ComputeNode)
.where(
ComputeNode.id == node_id,
ComputeNode.generation == request.expected_generation,
ComputeNode.enabled == preview.current_enabled,
ComputeNode.decommissioned_at.is_(None),
)
.values(**values)
.execution_options(synchronize_session=False)
)
)
if not claimed:
self.session.rollback()
raise NodeDecommissionError(
409,
"node_generation_conflict",
"node changed while the lifecycle transition was being claimed",
{"expected": request.expected_generation},
)
AuditWriter(self.session, "operator", request.operator).write(
"NODE_ENABLED" if request.target_enabled else "NODE_DISABLED",
"compute_node",
str(node.id),
{
"operator": request.operator,
"reason": request.reason,
"previous_enabled": preview.current_enabled,
"enabled": request.target_enabled,
"generation": request.expected_generation + 1,
"dependency_digest": request.dependency_digest,
"liveness": target_liveness,
},
)
self.session.commit()
return NodeLifecycleResult(
node_id=node.id,
enabled=request.target_enabled,
generation=request.expected_generation + 1,
liveness=target_liveness,
agent_evidence_state=(
"awaiting_fresh_authenticated_heartbeat"
if request.target_enabled and node.observation_source == "remote_agent"
else "unchanged"
),
)
def preview(self, node_id: uuid.UUID) -> NodeDecommissionPreview:
return self._snapshot(self._node(node_id))
backend/src/modelforge_api/services/observability.py
+153 -20
View File
@@ -14,6 +14,8 @@ from sqlalchemy.orm import Session
from modelforge_api.db import rows_affected
from modelforge_api.domain.observability import (
ActiveSchedulerLeaseInventoryResponse,
ActiveSchedulerLeaseResponse,
AlertAction,
AlertResponse,
AlertRuleCreate,
@@ -74,6 +76,7 @@ from modelforge_api.persistence.models import (
StorageRoot,
StorageVolumeState,
)
from modelforge_api.services.alert_notifications import enqueue_alert_transition
from modelforge_api.services.audit import AuditWriter
from modelforge_api.settings import Settings, get_settings
@@ -1157,6 +1160,133 @@ class ObservabilityService:
)
]
def active_scheduler_leases(
self,
*,
limit: int = 100,
sort_by: Literal["expires_at", "created_at", "priority"] = "expires_at",
order: Literal["asc", "desc"] = "asc",
observed_at: datetime | None = None,
) -> ActiveSchedulerLeaseInventoryResponse:
"""Read current, unexpired scheduler leases without touching reconciliation state."""
now = observed_at or _now()
sort_column = {
"expires_at": ServingGpuLease.expires_at,
"created_at": ServingGpuLease.created_at,
"priority": ServingGpuLease.priority,
}[sort_by]
primary_order = sort_column.asc() if order == "asc" else sort_column.desc()
rows = self.session.execute(
select(
ServingGpuLease,
Accelerator.compute_node_id,
GatewayRequest.status,
GatewayRequest.capability_deployment_id,
GatewayRequest.compute_node_id,
ProjectBinding.project_id,
)
.outerjoin(Accelerator, Accelerator.id == ServingGpuLease.accelerator_id)
.outerjoin(GatewayRequest, GatewayRequest.request_id == ServingGpuLease.request_id)
.outerjoin(ServiceClient, ServiceClient.id == GatewayRequest.service_client_id)
.outerjoin(ProjectBinding, ProjectBinding.id == ServiceClient.project_binding_id)
.where(
ServingGpuLease.state.in_(("pending", "granted", "active", "releasing")),
ServingGpuLease.expires_at > now,
)
.order_by(primary_order, ServingGpuLease.id.asc())
.limit(limit + 1)
).all()
truncated = len(rows) > limit
selected = rows[:limit]
lease_states: dict[str, Literal["PENDING", "GRANTED", "ACTIVE", "RELEASING"]] = {
"pending": "PENDING",
"granted": "GRANTED",
"active": "ACTIVE",
"releasing": "RELEASING",
}
lease_types: dict[
str, Literal["RESIDENCY_RESERVATION", "REQUEST_EXECUTION", "LOAD_TRANSITION"]
] = {
"residency_reservation": "RESIDENCY_RESERVATION",
"request_execution": "REQUEST_EXECUTION",
"load_transition": "LOAD_TRANSITION",
}
known_priorities = {
"production",
"interactive",
"background",
"benchmark",
"lab",
"maintenance",
}
def linked_workload_status(
lease: ServingGpuLease,
request_status: str | None,
request_deployment_id: uuid.UUID | None,
request_node_id: uuid.UUID | None,
node_id: uuid.UUID | None,
) -> Literal["QUEUED", "RUNNING", "UNKNOWN"]:
if (
request_status is None
or request_deployment_id != lease.capability_deployment_id
or request_node_id is None
or request_node_id != node_id
or node_id is None
):
return "UNKNOWN"
status = request_status.casefold()
if status == "queued" and lease.state in {"pending", "granted"}:
return "QUEUED"
if status == "running" and lease.state == "active":
return "RUNNING"
return "UNKNOWN"
inventory = [
ActiveSchedulerLeaseResponse(
lease_id=lease.id,
project_id=project_id,
node_id=node_id,
accelerator_id=lease.accelerator_id,
deployment_id=lease.capability_deployment_id,
lease_state=lease_states[lease.state],
workload_status=linked_workload_status(
lease,
request_status,
request_deployment_id,
request_node_id,
node_id,
),
priority=(
lease.priority.upper()
if lease.priority in known_priorities
else "UNKNOWN"
),
lease_type=lease_types.get(lease.lease_type, "UNKNOWN"),
reserved_vram_bytes=lease.reserved_vram_bytes,
materialized_vram_bytes=lease.materialized_vram_bytes,
created_at=_aware(lease.created_at),
acquired_at=_aware(lease.acquired_at) if lease.acquired_at else None,
heartbeat_at=_aware(lease.heartbeat_at) if lease.heartbeat_at else None,
expires_at=_aware(lease.expires_at),
)
for (
lease,
node_id,
request_status,
request_deployment_id,
request_node_id,
project_id,
) in selected
]
return ActiveSchedulerLeaseInventoryResponse(
observed_at=now,
completeness="TRUNCATED" if truncated else "COMPLETE",
limit=limit,
returned_count=len(inventory),
leases=inventory,
)
def capacity_trend(self, node_id: uuid.UUID, hours: int = 24) -> TrendResponse:
start = _now() - timedelta(hours=hours)
rows = list(
@@ -1732,18 +1862,19 @@ class ObservabilityService:
)
self.session.add(alert)
self.session.flush()
self.session.add(
AlertHistoryEvent(
alert_id=alert.id,
from_state=None,
to_state=initial.value,
actor="observability-engine",
reason="Signal condition detected",
evidence=signal["details"],
occurred_at=now,
)
event = AlertHistoryEvent(
alert_id=alert.id,
from_state=None,
to_state=initial.value,
actor="observability-engine",
reason="Signal condition detected",
evidence=signal["details"],
occurred_at=now,
)
self.session.add(event)
self.session.flush()
if initial is AlertState.FIRING:
enqueue_alert_transition(self.session, self.settings, alert, event)
self._correlate_incident(alert, signal, now)
return
alert.last_seen_at = now
@@ -1817,17 +1948,19 @@ class ObservabilityService:
alert.resolved_at = now
rule = self.session.get(AlertRuleRevision, alert.rule_id)
alert.cooldown_until = now + timedelta(seconds=rule.cooldown_seconds if rule else 0)
self.session.add(
AlertHistoryEvent(
alert_id=alert.id,
from_state=previous,
to_state=state.value,
actor=actor,
reason=reason,
evidence=evidence,
occurred_at=now,
)
event = AlertHistoryEvent(
alert_id=alert.id,
from_state=previous,
to_state=state.value,
actor=actor,
reason=reason,
evidence=evidence,
occurred_at=now,
)
self.session.add(event)
if state in {AlertState.FIRING, AlertState.RESOLVED}:
self.session.flush()
enqueue_alert_transition(self.session, self.settings, alert, event)
def _suppressed_until(
self, rule: AlertRuleRevision, signal: dict[str, Any], now: datetime
backend/src/modelforge_api/services/observability_polling.py
+15 -5
View File
@@ -9,6 +9,7 @@ from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import Session
from modelforge_api.domain.observability import metrics
from modelforge_api.services.alert_notifications import dispatch_pending
from modelforge_api.services.observability import ObservabilityService
from modelforge_api.settings import Settings
@@ -30,11 +31,20 @@ class ObservabilityPollingService:
try:
with Session(self.engine) as session:
service = ObservabilityService(session, self.settings)
service.ensure_defaults()
service.collect_capacity()
service.evaluate_slos()
service.evaluate_alerts()
service.apply_retention()
evaluation_enabled = (
self.settings.observability_monitor_enabled
or self.settings.alert_notifications_enabled
)
if evaluation_enabled:
service.ensure_defaults()
if self.settings.observability_monitor_enabled:
service.collect_capacity()
service.evaluate_slos()
if evaluation_enabled:
service.evaluate_alerts()
if self.settings.observability_monitor_enabled:
service.apply_retention()
dispatch_pending(session, self.settings)
metrics.gauge("modelforge_observability_degraded", {}, 0)
except (SQLAlchemyError, OSError, RuntimeError, ValueError) as error:
metrics.gauge("modelforge_observability_degraded", {}, 1)
Loaded 100 of 434 files, more files were not shown because too many files have changed in this diff. Show more