ModelForge-Public/backend/src/modelforge_api/services/recovery.py

"""M15 backup, restore and disaster-recovery service.

The control plane owns its own recovery: classified assets, versioned policies, immutable
verified backup sets, guarded restore plans and a durable restore journal. Nothing here can
silently lose provenance, resurrect stale runtime truth, or bypass supply-chain verification.
"""

from __future__ import annotations

import hashlib
import json
import os
import re
import shutil
import time
import uuid
from collections.abc import Iterable, Iterator
from datetime import UTC, datetime, timedelta
from functools import lru_cache
from pathlib import Path
from typing import Any, cast

from sqlalchemy import create_engine, func, select
from sqlalchemy.orm import Session

from modelforge_api import __version__
from modelforge_api.domain.audit import (
    normalise_audit_timestamp,
)
from modelforge_api.domain.recovery import (
    POINT_IN_TIME_SUPPORT,
    RESTORE_ELIGIBLE_STATES,
    RESTORE_PHASE_ORDER,
    ArtifactRecoveryClass,
    ArtifactRecoveryCreate,
    ArtifactRecoveryResponse,
    ArtifactRecoveryState,
    BackupCapacityEstimate,
    BackupManifestEntryResponse,
    BackupMethod,
    BackupSetCreate,
    BackupSetResponse,
    BackupState,
    RecoveryAssetClass,
    RecoveryAssetResponse,
    RecoveryDashboard,
    RecoveryFailureCode,
    RecoveryPolicyCreate,
    RecoveryPolicyResponse,
    RecoveryReadiness,
    RecoveryReadinessEntry,
    RestoreAdvanceRequest,
    RestoreMode,
    RestoreOperationEventResponse,
    RestoreOperationResponse,
    RestorePlanCreate,
    RestorePlanResponse,
    RestorePlanState,
    RestoreState,
    SecretRecoveryClass,
    redact_database_url,
)
from modelforge_api.persistence.models import (
    ArtifactRecoveryOperation,
    ArtifactSet,
    ArtifactSetMember,
    AuditEvent,
    BackupManifestEntry,
    BackupSet,
    DerivedArtifactSource,
    Model,
    ModelArtifact,
    ModelRevision,
    RecoveryAssetRecord,
    RecoveryPolicyRevision,
    RestoreOperation,
    RestoreOperationEvent,
    RestorePlan,
    StorageRoot,
    UpstreamSnapshot,
)
from modelforge_api.services.audit import (
    AuditChainCheckpoint,
    AuditEventLike,
    AuditEventRecord,
    AuditWriter,
    audit_chain_violations,
)
from modelforge_api.services.recovery_crypto import (
    AesGcmBackupCipher,
    BackupCryptoError,
)
from modelforge_api.services.recovery_fingerprint import (
    CURRENT_TRUTH_TABLES,
    audit_fingerprint_differences_are_compatible,
    diff_fingerprints,
    fingerprint_compatibility,
    fingerprint_session,
    semantic_fingerprint,
)
from modelforge_api.services.recovery_paths import (
    RecoveryPathError,
    capacity_bytes,
    free_bytes,
    normalise_root,
    resolve_within,
    sha256_bytes,
    sha256_file,
)
from modelforge_api.services.recovery_postgres import (
    PostgresEngine,
    PostgresTarget,
    PostgresToolError,
    parse_target,
)
from modelforge_api.settings import Settings, get_settings

DATABASE_OBJECT = "database.dump"
ENCRYPTED_DATABASE_OBJECT = "database.dump.enc"
ARTIFACT_MANIFEST_OBJECT = "artifacts.json"
CONFIGURATION_MANIFEST_OBJECT = "configuration.json"
MANIFEST_OBJECT = "manifest.json"
MANIFEST_SCHEMA_VERSION = "m15.1"

AUDIT_RESOURCE = "recovery"

# Terminal stages of the M12 lifecycle and M13 cutover journals. A restored operation outside
# these stages was in flight at backup time and must be reconciled exactly once, never replayed.
LIFECYCLE_TERMINAL_STAGES = ("COMMITTED", "ROLLED_BACK", "FAILED")
CUTOVER_TERMINAL_STAGES = ("COMMITTED", "ROLLED_BACK", "FAILED")


def _quoted(values: tuple[str, ...]) -> str:
    return ",".join("'" + value.replace("'", "''") + "'" for value in values)


def _sql_literal(value: str) -> str:
    return "'" + value.replace("'", "''") + "'"


ACTIVE_BACKUP_STATES = (BackupState.CREATING.value, BackupState.VERIFYING.value)
ACTIVE_RESTORE_STATES = tuple(
    item.value
    for item in RestoreState
    if item
    not in {RestoreState.READY, RestoreState.FAILED, RestoreState.MANUAL_INTERVENTION_REQUIRED}
)


def _now() -> datetime:
    return datetime.now(UTC)


def _aware(value: datetime | None) -> datetime | None:
    if value is None:
        return None
    return value if value.tzinfo else value.replace(tzinfo=UTC)


def _psql_true(value: str) -> bool:
    """Accept PostgreSQL's native boolean text and explicit long-form test fixtures."""

    return value.strip().lower() in {"t", "true", "1"}


def _hash(payload: dict[str, Any]) -> str:
    return hashlib.sha256(
        json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode()
    ).hexdigest()


class RecoveryError(Exception):
    def __init__(
        self,
        status_code: int,
        code: str,
        message: str,
        details: dict[str, Any] | None = None,
    ) -> None:
        super().__init__(message)
        self.status_code = status_code
        self.code = code
        self.message = message
        self.details = details or {}


class RecoveryService:
    def __init__(
        self,
        session: Session,
        settings: Settings | None = None,
        actor_type: str = "operator",
        actor_id: str = "recovery",
    ) -> None:
        self.session = session
        self.settings = settings or get_settings()
        self.audit = AuditWriter(session, actor_type, actor_id)
        self.engine = PostgresEngine(
            pg_dump_path=self.settings.backup_pg_dump_path,
            pg_restore_path=self.settings.backup_pg_restore_path,
            psql_path=self.settings.backup_psql_path,
            timeout_seconds=self.settings.backup_command_timeout_seconds,
            base_environment=dict(os.environ),
        )

    # ------------------------------------------------------------------ policies

    def ensure_defaults(self) -> None:
        """Seed the versioned recovery policies and the classified asset inventory."""

        for request in _DEFAULT_POLICIES:
            existing = self.session.scalar(
                select(RecoveryPolicyRevision).where(
                    RecoveryPolicyRevision.key == request.key,
                    RecoveryPolicyRevision.active.is_(True),
                )
            )
            if existing is None:
                self._create_policy(request, audit=False)
        self.session.flush()
        policies = {
            item.key: item
            for item in self.session.scalars(
                select(RecoveryPolicyRevision).where(RecoveryPolicyRevision.active.is_(True))
            )
        }
        for spec in _DEFAULT_ASSETS:
            policy = policies.get(spec["policy_key"])
            if policy is None:
                continue
            record = self.session.scalar(
                select(RecoveryAssetRecord).where(RecoveryAssetRecord.key == spec["key"])
            )
            if record is None:
                record = RecoveryAssetRecord(key=str(spec["key"]))
                self.session.add(record)
            record.name = str(spec["name"])
            record.asset_class = policy.asset_class
            record.owner = str(spec["owner"])
            record.location = str(spec["location"])
            record.backup_method = policy.backup_method
            record.restore_method = str(spec["restore_method"])
            record.rebuild_method = spec.get("rebuild_method")
            record.rpo_seconds = policy.rpo_seconds
            record.readiness = str(spec["readiness"])
            record.dependencies = list(spec.get("dependencies", []))
            record.notes = str(spec.get("notes", ""))
            record.policy_revision_id = policy.id
        self.session.commit()

    def policies(self) -> list[RecoveryPolicyResponse]:
        return [
            self._policy_response(item)
            for item in self.session.scalars(
                select(RecoveryPolicyRevision).order_by(
                    RecoveryPolicyRevision.key, RecoveryPolicyRevision.revision
                )
            )
        ]

    def create_policy(self, request: RecoveryPolicyCreate) -> RecoveryPolicyResponse:
        revision = self._create_policy(request, audit=True)
        self.session.commit()
        return self._policy_response(revision)

    def _create_policy(
        self, request: RecoveryPolicyCreate, *, audit: bool
    ) -> RecoveryPolicyRevision:
        current = self.session.scalar(
            select(RecoveryPolicyRevision)
            .where(RecoveryPolicyRevision.key == request.key)
            .order_by(RecoveryPolicyRevision.revision.desc())
            .limit(1)
        )
        if current is not None:
            current.active = False
        payload = request.model_dump(mode="json", exclude={"created_by"})
        revision_number = (current.revision + 1) if current else 1
        record = RecoveryPolicyRevision(
            key=request.key,
            revision=revision_number,
            name=request.name,
            asset_class=request.asset_class.value,
            backup_method=request.backup_method.value,
            retention_days=request.retention_days,
            minimum_verified_backups=request.minimum_verified_backups,
            rpo_seconds=request.rpo_seconds,
            rto_target_seconds=request.rto_target_seconds,
            restore_verification=request.restore_verification,
            encryption_required=request.encryption_required,
            external_dependency=request.external_dependency,
            rehydration_allowed=request.rehydration_allowed,
            secret_class=request.secret_class.value if request.secret_class else None,
            rationale=request.rationale,
            active=True,
            fingerprint=_hash({**payload, "revision": revision_number}),
            created_by=request.created_by,
        )
        self.session.add(record)
        self.session.flush()
        if audit:
            self.audit.write(
                "RECOVERY_POLICY_CREATED",
                AUDIT_RESOURCE,
                str(record.id),
                {"key": record.key, "revision": record.revision, "asset_class": record.asset_class},
            )
        return record

    def assets(self) -> list[RecoveryAssetResponse]:
        rows = self.session.scalars(
            select(RecoveryAssetRecord).order_by(
                RecoveryAssetRecord.asset_class, RecoveryAssetRecord.key
            )
        )
        return [
            RecoveryAssetResponse(
                id=item.id,
                key=item.key,
                name=item.name,
                asset_class=RecoveryAssetClass(item.asset_class),
                owner=item.owner,
                location=item.location,
                backup_method=BackupMethod(item.backup_method),
                restore_method=item.restore_method,
                rebuild_method=item.rebuild_method,
                rpo_seconds=item.rpo_seconds,
                readiness=RecoveryReadiness(item.readiness),
                dependencies=list(item.dependencies or []),
                notes=item.notes,
                policy_key=item.policy_revision.key,
                updated_at=item.updated_at,
            )
            for item in rows
        ]

    # ------------------------------------------------------------------ backups

    def _database_policy(self) -> RecoveryPolicyRevision:
        policy = self.session.scalar(
            select(RecoveryPolicyRevision).where(
                RecoveryPolicyRevision.key == "control-plane.database",
                RecoveryPolicyRevision.active.is_(True),
            )
        )
        if policy is None:
            self.ensure_defaults()
            policy = self.session.scalar(
                select(RecoveryPolicyRevision).where(
                    RecoveryPolicyRevision.key == "control-plane.database",
                    RecoveryPolicyRevision.active.is_(True),
                )
            )
        if policy is None:  # pragma: no cover - defaults always seed this policy
            raise RecoveryError(500, "recovery_policy_missing", "no active database backup policy")
        return policy

    def _cipher(self, *, required: bool) -> AesGcmBackupCipher | None:
        secret = self.settings.backup_encryption_key
        if secret is None or not secret.get_secret_value().strip():
            if required:
                raise RecoveryError(
                    409,
                    RecoveryFailureCode.ENCRYPTION_KEY_UNAVAILABLE.value,
                    "the active recovery policy requires encryption but no backup key is configured",
                )
            return None
        return AesGcmBackupCipher(
            secret.get_secret_value(), self.settings.backup_encryption_key_id
        )

    def _backup_root(self) -> Path:
        try:
            return normalise_root(self.settings.backup_root)
        except (RecoveryPathError, OSError) as error:
            raise RecoveryError(
                500, RecoveryFailureCode.PATH_NOT_ALLOWED.value, str(error)
            ) from error

    def estimate_capacity(self) -> BackupCapacityEstimate:
        """Backups check headroom before writing; they never fill a disk to find out."""

        root = self._backup_root()
        database_bytes = self._estimate_database_bytes()
        rehydratable = int(
            self.session.scalar(select(func.coalesce(func.sum(ModelArtifact.size_bytes), 0))) or 0
        )
        available = free_bytes(root)
        capacity = capacity_bytes(root)
        required = int(database_bytes * self.settings.backup_capacity_headroom_ratio) + (
            self.settings.backup_minimum_free_bytes
        )
        sufficient = available is None or available >= required
        detail = (
            "capacity is unknown for this destination; the backup will still verify its payload"
            if available is None
            else (
                f"{available} bytes available for an estimated {required} byte requirement"
            )
        )
        return BackupCapacityEstimate(
            bytes_to_copy=database_bytes,
            bytes_manifest_only=rehydratable,
            estimated_protected_bytes=database_bytes,
            available_bytes=available,
            capacity_bytes=capacity,
            sufficient=sufficient,
            detail=detail,
        )

    def _estimate_database_bytes(self) -> int:
        try:
            value = self.session.scalar(
                select(func.pg_database_size(func.current_database()))
            )
            return int(value or 0)
        except Exception:  # noqa: BLE001 - non-PostgreSQL test databases have no such function
            self.session.rollback()
            return 0

    def _guard_no_concurrent_operation(self) -> None:
        active_backup = self.session.scalar(
            select(BackupSet).where(BackupSet.state.in_(ACTIVE_BACKUP_STATES)).limit(1)
        )
        if active_backup is not None:
            raise RecoveryError(
                409,
                RecoveryFailureCode.CONCURRENT_OPERATION.value,
                f"backup {active_backup.backup_id} is already {active_backup.state}",
            )
        active_restore = self.session.scalar(
            select(RestoreOperation)
            .where(RestoreOperation.state.in_(ACTIVE_RESTORE_STATES))
            .limit(1)
        )
        if active_restore is not None:
            raise RecoveryError(
                409,
                RecoveryFailureCode.CONCURRENT_OPERATION.value,
                f"restore operation {active_restore.id} is still {active_restore.state}",
            )

    def plan_backup(self, request: BackupSetCreate) -> BackupSet:
        existing = self.session.scalar(
            select(BackupSet).where(BackupSet.backup_id == request.backup_id)
        )
        if existing is not None:
            raise RecoveryError(
                409, "backup_already_exists", f"backup {request.backup_id} already exists"
            )
        self._guard_no_concurrent_operation()
        policy = self._database_policy()
        record = BackupSet(
            backup_id=request.backup_id,
            state=BackupState.PLANNED.value,
            policy_revision_id=policy.id,
            modelforge_version=__version__,
            modelforge_commit=_source_commit(),
            source_repository=_source_repository(),
            source_reference=_source_reference(),
            schema_revision=None,
            environment_fingerprint={},
            database_identity={},
            destination_root=str(self._backup_root()),
            included_asset_classes=[
                RecoveryAssetClass.AUTHORITATIVE.value,
                RecoveryAssetClass.REBUILDABLE.value,
            ],
            excluded_asset_classes=[
                RecoveryAssetClass.EPHEMERAL.value,
                RecoveryAssetClass.EXTERNAL.value,
                RecoveryAssetClass.SECRET.value,
            ],
            verification_details={},
            milestone=request.milestone,
            legal_hold=request.legal_hold,
            reason=request.reason,
            created_by=request.created_by,
        )
        self.session.add(record)
        self.session.flush()
        self.audit.write(
            "BACKUP_PLANNED",
            AUDIT_RESOURCE,
            str(record.id),
            {"backup_id": record.backup_id, "policy_revision": policy.revision},
        )
        self.session.commit()
        return record

    def create_backup(self, request: BackupSetCreate) -> BackupSetResponse:
        record = self.plan_backup(request)
        return self.execute_backup(record.id, include_artifact_manifest=request.include_artifact_manifest)

    def execute_backup(
        self, backup_set_id: uuid.UUID, *, include_artifact_manifest: bool = True
    ) -> BackupSetResponse:
        record = self._backup_row(backup_set_id)
        if record.state != BackupState.PLANNED.value:
            raise RecoveryError(
                409,
                "backup_not_planned",
                f"backup {record.backup_id} is {record.state} and cannot be created again",
            )
        self._guard_no_concurrent_operation()
        capacity = self.estimate_capacity()
        if not capacity.sufficient:
            self._fail_backup(
                record, RecoveryFailureCode.INSUFFICIENT_CAPACITY, capacity.detail
            )
            return self.backup(record.id)

        record.state = BackupState.CREATING.value
        record.started_at = _now()
        self.audit.write(
            "BACKUP_STARTED", AUDIT_RESOURCE, str(record.id), {"backup_id": record.backup_id}
        )
        self.session.commit()

        # Creating the destination directory is part of the backup, not a precondition of it: an
        # unwritable or vanished backup root must fail this backup closed with a typed code, not
        # escape as an unhandled error that leaves the set stuck in CREATING.
        directory: Path | None = None
        try:
            directory = self._backup_directory(record.backup_id)
            payload = self._write_backup_payload(
                record, directory, include_artifact_manifest=include_artifact_manifest
            )
        except (PostgresToolError, BackupCryptoError, RecoveryPathError) as error:
            if directory is not None:
                shutil.rmtree(directory, ignore_errors=True)
            self._fail_backup(record, RecoveryFailureCode(error.code), error.message)
            return self.backup(record.id)
        except OSError as error:
            if directory is not None:
                shutil.rmtree(directory, ignore_errors=True)
            self._fail_backup(
                record,
                RecoveryFailureCode.DESTINATION_UNAVAILABLE,
                f"the backup destination could not be written: {error}",
            )
            return self.backup(record.id)

        policy = record.policy_revision
        record.state = BackupState.CREATED.value
        record.completed_at = _now()
        record.expires_at = record.completed_at + timedelta(days=policy.retention_days)
        record.immutable_at = record.completed_at
        self.audit.write(
            "BACKUP_CREATED",
            AUDIT_RESOURCE,
            str(record.id),
            {
                "backup_id": record.backup_id,
                "payload_bytes": record.payload_bytes,
                "manifest_sha256": record.manifest_sha256,
                "encrypted": record.encrypted,
                "schema_revision": record.schema_revision,
                "duration_seconds": payload["duration_seconds"],
            },
        )
        self.session.commit()
        return self.backup(record.id)

    def _backup_directory(self, backup_id: str) -> Path:
        root = self._backup_root()
        directory = resolve_within(root, backup_id)
        directory.mkdir(parents=True, exist_ok=True)
        return directory

    def _write_backup_payload(
        self, record: BackupSet, directory: Path, *, include_artifact_manifest: bool
    ) -> dict[str, Any]:
        started = time.perf_counter()
        target = parse_target(self.settings.database_url)
        identity = self.engine.server_identity(target)
        alembic_revision = self.engine.alembic_revision(target)
        tool_version = self.engine.tool_version(self.settings.backup_pg_dump_path)

        # A fingerprint on each side of the dump bounds the concurrent-write window honestly.
        fingerprint_before = fingerprint_session(self.session)
        dump_path = directory / DATABASE_OBJECT
        dump_result = self.engine.dump(target, dump_path)
        fingerprint_after = fingerprint_session(self.session)

        plaintext_sha = sha256_file(dump_path)
        plaintext_bytes = dump_path.stat().st_size
        stored_object = DATABASE_OBJECT
        cipher = self._cipher(required=record.policy_revision.encryption_required)
        if cipher is not None:
            encrypted_path = directory / ENCRYPTED_DATABASE_OBJECT
            cipher.encrypt_file(dump_path, encrypted_path)
            dump_path.unlink(missing_ok=True)
            stored_object = ENCRYPTED_DATABASE_OBJECT
            record.encrypted = True
            record.encryption_algorithm = cipher.algorithm
            record.encryption_key_id = cipher.key_id
            stored_path = encrypted_path
        else:
            stored_path = dump_path

        entries: list[dict[str, Any]] = [
            {
                "logical_asset_type": "control_plane_database",
                "object_name": stored_object,
                "relative_path": f"{record.backup_id}/{stored_object}",
                "size_bytes": stored_path.stat().st_size,
                "sha256": sha256_file(stored_path),
                "source_generation": f"pg_dump:{alembic_revision or 'unknown'}",
                "schema_version": alembic_revision,
                "dependency_refs": {
                    "postgres_major": identity["major_version"],
                    "extensions": identity["extensions"],
                    "plaintext_sha256": plaintext_sha,
                    "plaintext_bytes": plaintext_bytes,
                },
            }
        ]

        if include_artifact_manifest:
            artifact_manifest = self._artifact_manifest()
            entries.append(
                self._write_json_object(
                    directory, record, ARTIFACT_MANIFEST_OBJECT, "artifact_inventory",
                    artifact_manifest,
                    {"rehydratable_bytes": artifact_manifest["totals"]["rehydratable_bytes"]},
                )
            )

        configuration = self._configuration_manifest()
        entries.append(
            self._write_json_object(
                directory, record, CONFIGURATION_MANIFEST_OBJECT, "configuration_reference",
                configuration,
                {"repository": configuration["repository"], "commit": configuration["commit"]},
            )
        )

        record.payload_relative_path = f"{record.backup_id}/{stored_object}"
        record.payload_sha256 = entries[0]["sha256"]
        record.payload_bytes = sum(int(item["size_bytes"]) for item in entries)
        record.schema_revision = alembic_revision
        record.database_identity = {
            "server_version": identity["server_version"],
            "server_version_num": identity["server_version_num"],
            "major_version": identity["major_version"],
            "database": identity["database"],
            "extensions": identity["extensions"],
            "backup_tool": tool_version,
            "backup_method": BackupMethod.POSTGRES_LOGICAL_CUSTOM.value,
            "connection": target.redacted,
        }
        record.environment_fingerprint = {
            "modelforge_version": __version__,
            "modelforge_commit": record.modelforge_commit,
            "source_reference": record.source_reference,
            "python": os.environ.get("PYTHON_VERSION", ""),
            "env": self.settings.env,
            "artifact_root": self.settings.artifact_root,
            "point_in_time_support": POINT_IN_TIME_SUPPORT,
        }

        manifest = {
            "manifest_schema_version": MANIFEST_SCHEMA_VERSION,
            "backup_id": record.backup_id,
            "created_at": _now().isoformat(),
            "modelforge_version": __version__,
            "modelforge_commit": record.modelforge_commit,
            "source_repository": record.source_repository,
            "source_reference": record.source_reference,
            "schema_revision": alembic_revision,
            "database_identity": record.database_identity,
            "environment_fingerprint": record.environment_fingerprint,
            "included_asset_classes": list(record.included_asset_classes),
            "excluded_asset_classes": list(record.excluded_asset_classes),
            "encryption": {
                "encrypted": record.encrypted,
                "algorithm": record.encryption_algorithm,
                "key_id": record.encryption_key_id,
            },
            "entries": entries,
        }
        manifest_bytes = json.dumps(manifest, sort_keys=True, indent=2).encode("utf-8")
        manifest_path = directory / MANIFEST_OBJECT
        manifest_path.write_bytes(manifest_bytes)
        record.manifest_relative_path = f"{record.backup_id}/{MANIFEST_OBJECT}"
        record.manifest_sha256 = sha256_bytes(manifest_bytes)

        for entry in entries:
            self.session.add(
                BackupManifestEntry(
                    backup_set_id=record.id,
                    logical_asset_type=entry["logical_asset_type"],
                    object_name=entry["object_name"],
                    relative_path=entry["relative_path"],
                    size_bytes=int(entry["size_bytes"]),
                    sha256=entry["sha256"],
                    source_generation=entry["source_generation"],
                    schema_version=entry["schema_version"],
                    dependency_refs=entry["dependency_refs"],
                )
            )

        duration = time.perf_counter() - started
        stable_groups = sorted(
            group
            for group in fingerprint_before["groups"]
            if fingerprint_before["groups"][group]["digest"]
            == fingerprint_after["groups"][group]["digest"]
        )
        record.verification_details = {
            "dump_duration_seconds": dump_result.duration_seconds,
            "backup_duration_seconds": duration,
            "source_fingerprint": fingerprint_after,
            "source_fingerprint_before_dump": {
                "digest": fingerprint_before["digest"],
                "groups": {
                    key: value["digest"] for key, value in fingerprint_before["groups"].items()
                },
            },
            "write_window_stable_groups": stable_groups,
            "write_window_active_groups": sorted(
                set(fingerprint_before["groups"]) - set(stable_groups)
            ),
        }
        return {"duration_seconds": duration}

    def _write_json_object(
        self,
        directory: Path,
        record: BackupSet,
        object_name: str,
        logical_type: str,
        payload: dict[str, Any],
        dependency_refs: dict[str, Any],
    ) -> dict[str, Any]:
        body = json.dumps(payload, sort_keys=True, indent=2).encode("utf-8")
        path = directory / object_name
        path.write_bytes(body)
        return {
            "logical_asset_type": logical_type,
            "object_name": object_name,
            "relative_path": f"{record.backup_id}/{object_name}",
            "size_bytes": len(body),
            "sha256": sha256_bytes(body),
            "source_generation": f"modelforge:{__version__}",
            "schema_version": MANIFEST_SCHEMA_VERSION,
            "dependency_refs": dependency_refs,
        }

    def _artifact_manifest(self) -> dict[str, Any]:
        """Large model weights are protected by exact identity, not by copying gigabytes."""

        sets: list[dict[str, Any]] = []
        rehydratable_bytes = 0
        non_rehydratable_bytes = 0
        for artifact_set in self.session.scalars(select(ArtifactSet)):
            revision = self.session.get(ModelRevision, artifact_set.revision_id)
            snapshot = self.session.get(UpstreamSnapshot, artifact_set.snapshot_id)
            members = self.session.scalars(
                select(ArtifactSetMember).where(
                    ArtifactSetMember.artifact_set_id == artifact_set.id
                )
            )
            files: list[dict[str, Any]] = []
            for member in members:
                artifact = self.session.get(ModelArtifact, member.artifact_id)
                if artifact is None:
                    continue
                files.append(
                    {
                        "artifact_id": str(artifact.id),
                        "filename": artifact.filename,
                        "sha256": artifact.sha256,
                        "size_bytes": artifact.size_bytes,
                        "artifact_type": artifact.artifact_type,
                        "serialization_format": artifact.serialization_format,
                        "security_status": artifact.security_status,
                        "license_status": artifact.license_status,
                    }
                )
            commit_sha = (revision.resolved_commit_sha if revision else None) or (
                snapshot.resolved_commit_sha if snapshot else None
            )
            repository = snapshot.repository_id if snapshot else None
            if repository is None and revision is not None:
                model = self.session.get(Model, revision.model_id)
                repository = model.upstream_source if model else None
            recovery_class = (
                ArtifactRecoveryClass.REHYDRATABLE
                if commit_sha and repository
                else ArtifactRecoveryClass.NON_REHYDRATABLE
            )
            total = sum(int(item["size_bytes"]) for item in files)
            if recovery_class is ArtifactRecoveryClass.REHYDRATABLE:
                rehydratable_bytes += total
            else:
                non_rehydratable_bytes += total
            sets.append(
                {
                    "artifact_set_id": str(artifact_set.id),
                    "revision_id": str(artifact_set.revision_id),
                    "recovery_class": recovery_class.value,
                    "upstream_repository": repository,
                    "upstream_commit_sha": commit_sha,
                    "file_count": len(files),
                    "total_bytes": total,
                    "files": sorted(files, key=lambda item: item["filename"]),
                }
            )
        return {
            "artifact_sets": sorted(sets, key=lambda item: item["artifact_set_id"]),
            "totals": {
                "artifact_set_count": len(sets),
                "rehydratable_bytes": rehydratable_bytes,
                "non_rehydratable_bytes": non_rehydratable_bytes,
            },
        }

    def _configuration_manifest(self) -> dict[str, Any]:
        config_root = Path(self.settings.config_root)
        files: list[dict[str, Any]] = []
        if config_root.is_dir():
            for path in sorted(config_root.rglob("*")):
                if path.is_file():
                    files.append(
                        {
                            "relative_path": path.relative_to(config_root).as_posix(),
                            "sha256": sha256_file(path),
                            "size_bytes": path.stat().st_size,
                            "classification": "SOURCE_CONTROLLED",
                        }
                    )
        return {
            "repository": _source_repository(),
            "commit": _source_commit(),
            "reference": _source_reference(),
            "config_root": str(config_root),
            "files": files,
            "host_local": [
                {"key": "MODELFORGE_BACKUP_ROOT", "classification": "HOST_LOCAL"},
                {"key": "MODELFORGE_ARTIFACT_ROOT", "classification": "HOST_LOCAL"},
                {"key": "MODELFORGE_QUARANTINE_ROOT", "classification": "HOST_LOCAL"},
            ],
            "secrets": [
                {"key": "MODELFORGE_OPERATOR_API_KEY", "recovery": "ROTATABLE_SECRET"},
                {"key": "MODELFORGE_BACKUP_ENCRYPTION_KEY", "recovery": "NON_EXPORTABLE_SECRET"},
                {"key": "MODELFORGE_HF_TOKEN", "recovery": "ROTATABLE_SECRET"},
            ],
        }

    def _fail_backup(
        self, record: BackupSet, code: RecoveryFailureCode, reason: str
    ) -> None:
        record.state = BackupState.FAILED.value
        record.failure_code = code.value
        record.failure_reason = reason[:2000]
        record.completed_at = _now()
        self.audit.write(
            "BACKUP_FAILED",
            AUDIT_RESOURCE,
            str(record.id),
            {"backup_id": record.backup_id, "failure_code": code.value},
            outcome="failure",
        )
        self.session.commit()

    def verify_backup(self, backup_set_id: uuid.UUID) -> BackupSetResponse:
        record = self._backup_row(backup_set_id)
        if record.state in {BackupState.PLANNED.value, BackupState.DELETED.value}:
            raise RecoveryError(
                409, "backup_not_verifiable", f"backup {record.backup_id} is {record.state}"
            )
        record.state = BackupState.VERIFYING.value
        self.audit.write(
            "BACKUP_VERIFICATION_STARTED",
            AUDIT_RESOURCE,
            str(record.id),
            {"backup_id": record.backup_id},
        )
        self.session.commit()

        outcome = self._verify_payload(record)
        details = dict(record.verification_details or {})
        details["verification"] = outcome
        record.verification_details = details
        if outcome["ok"]:
            record.state = BackupState.VERIFIED.value
            record.verified_at = _now()
            record.failure_code = None
            record.failure_reason = None
            self.audit.write(
                "BACKUP_VERIFIED",
                AUDIT_RESOURCE,
                str(record.id),
                {"backup_id": record.backup_id, "checks": outcome["checks"]},
            )
        else:
            record.state = BackupState.FAILED.value
            record.failure_code = outcome["failure_code"]
            record.failure_reason = outcome["failure_reason"]
            record.verified_at = None
            self.audit.write(
                "BACKUP_VERIFICATION_FAILED",
                AUDIT_RESOURCE,
                str(record.id),
                {"backup_id": record.backup_id, "failure_code": outcome["failure_code"]},
                outcome="failure",
            )
        self.session.commit()
        return self.backup(record.id)

    def _verify_payload(self, record: BackupSet) -> dict[str, Any]:
        checks: dict[str, str] = {}
        root = self._backup_root()
        try:
            manifest_path = resolve_within(
                root, record.manifest_relative_path or "", must_exist=True
            )
        except RecoveryPathError as error:
            return _verification_failure(
                RecoveryFailureCode.PAYLOAD_MISSING, f"manifest unavailable: {error.message}", checks
            )
        manifest_bytes = manifest_path.read_bytes()
        if sha256_bytes(manifest_bytes) != record.manifest_sha256:
            return _verification_failure(
                RecoveryFailureCode.MANIFEST_HASH_MISMATCH,
                "manifest hash does not match the recorded manifest identity",
                checks,
            )
        checks["manifest_hash"] = "MATCH"
        try:
            manifest = json.loads(manifest_bytes.decode("utf-8"))
        except (UnicodeDecodeError, json.JSONDecodeError) as error:
            return _verification_failure(
                RecoveryFailureCode.MANIFEST_INCOMPLETE, f"manifest is unreadable: {error}", checks
            )

        recorded = {
            item.object_name: item
            for item in self.session.scalars(
                select(BackupManifestEntry).where(BackupManifestEntry.backup_set_id == record.id)
            )
        }
        manifest_entries = {item["object_name"]: item for item in manifest.get("entries", [])}
        if set(recorded) != set(manifest_entries) or not manifest_entries:
            return _verification_failure(
                RecoveryFailureCode.MANIFEST_INCOMPLETE,
                "manifest objects do not match the journaled backup manifest entries",
                checks,
            )
        if not any(
            item["logical_asset_type"] == "control_plane_database"
            for item in manifest_entries.values()
        ):
            return _verification_failure(
                RecoveryFailureCode.MANIFEST_INCOMPLETE,
                "manifest has no control-plane database payload",
                checks,
            )
        checks["manifest_completeness"] = "COMPLETE"

        database_entry: dict[str, Any] | None = None
        for name, entry in sorted(manifest_entries.items()):
            journaled = recorded[name]
            if entry["sha256"] != journaled.sha256 or int(entry["size_bytes"]) != journaled.size_bytes:
                return _verification_failure(
                    RecoveryFailureCode.MANIFEST_HASH_MISMATCH,
                    f"manifest entry {name} disagrees with the journaled entry",
                    checks,
                )
            try:
                path = resolve_within(root, entry["relative_path"], must_exist=True)
            except RecoveryPathError as error:
                return _verification_failure(
                    RecoveryFailureCode(error.code), f"{name}: {error.message}", checks
                )
            if path.stat().st_size != int(entry["size_bytes"]):
                return _verification_failure(
                    RecoveryFailureCode.HASH_MISMATCH, f"{name} has an unexpected size", checks
                )
            if sha256_file(path) != entry["sha256"]:
                return _verification_failure(
                    RecoveryFailureCode.HASH_MISMATCH,
                    f"{name} payload hash does not match the manifest",
                    checks,
                )
            if entry["logical_asset_type"] == "control_plane_database":
                database_entry = {**entry, "path": path}
        checks["payload_hashes"] = "MATCH"

        if database_entry is None:  # pragma: no cover - guarded above
            return _verification_failure(
                RecoveryFailureCode.MANIFEST_INCOMPLETE, "no database payload", checks
            )
        try:
            readable = self._probe_database_payload(record, database_entry)
        except RecoveryError as error:
            return _verification_failure(
                RecoveryFailureCode(error.code)
                if error.code in RecoveryFailureCode.__members__.values()
                else RecoveryFailureCode.BACKUP_TOOL_FAILED,
                error.message,
                checks,
            )
        checks["archive_structure"] = f"READABLE:{readable}"
        return {
            "ok": True,
            "checks": checks,
            "failure_code": None,
            "failure_reason": None,
            "verified_at": _now().isoformat(),
        }

    def _probe_database_payload(self, record: BackupSet, entry: dict[str, Any]) -> int:
        path: Path = entry["path"]
        if not record.encrypted:
            return self.engine.list_dump_contents(path)
        cipher = self._cipher(required=True)
        if cipher is None:  # pragma: no cover - _cipher raises when required
            raise RecoveryError(
                409, RecoveryFailureCode.ENCRYPTION_KEY_UNAVAILABLE.value, "no key available"
            )
        staging = path.with_name(path.name + f".verify-{uuid.uuid4().hex[:8]}")
        try:
            cipher.decrypt_file(path, staging)
            expected = (entry.get("dependency_refs") or {}).get("plaintext_sha256")
            if expected and sha256_file(staging) != expected:
                raise RecoveryError(
                    409,
                    RecoveryFailureCode.HASH_MISMATCH.value,
                    "decrypted payload does not match the recorded plaintext hash",
                )
            return self.engine.list_dump_contents(staging)
        except BackupCryptoError as error:
            raise RecoveryError(409, error.code, error.message) from error
        except PostgresToolError as error:
            raise RecoveryError(409, error.code, error.message) from error
        finally:
            staging.unlink(missing_ok=True)

    def reconcile_interrupted_backups(self) -> int:
        """A process that died mid-backup must never leave a usable-looking backup behind."""

        interrupted = list(
            self.session.scalars(select(BackupSet).where(BackupSet.state.in_(ACTIVE_BACKUP_STATES)))
        )
        for record in interrupted:
            record.state = BackupState.FAILED.value
            record.failure_code = RecoveryFailureCode.MANIFEST_INCOMPLETE.value
            record.failure_reason = (
                "backup was interrupted before completion and is not restore eligible"
            )
            record.completed_at = _now()
            self.audit.write(
                "BACKUP_FAILED",
                AUDIT_RESOURCE,
                str(record.id),
                {"backup_id": record.backup_id, "reason": "interrupted"},
                outcome="failure",
            )
        if interrupted:
            self.session.commit()
        return len(interrupted)

    def apply_retention(self, observed_at: datetime | None = None) -> dict[str, int]:
        """Expire aged backups, but never drop the last verified recovery point."""

        now = observed_at or _now()
        expired = 0
        protected = 0
        verified = [
            item
            for item in self.session.scalars(
                select(BackupSet)
                .where(BackupSet.state == BackupState.VERIFIED.value)
                .order_by(BackupSet.verified_at.desc())
            )
        ]
        if self.session.scalar(
            select(RestoreOperation)
            .where(RestoreOperation.state.in_(ACTIVE_RESTORE_STATES))
            .limit(1)
        ):
            return {"expired": 0, "protected": len(verified), "skipped_active_restore": 1}
        policy = self._database_policy()
        keep = max(policy.minimum_verified_backups, 1)
        protected_ids = {item.id for item in verified[:keep]}
        for record in self.session.scalars(
            select(BackupSet).where(
                BackupSet.state.in_(
                    (BackupState.VERIFIED.value, BackupState.CREATED.value, BackupState.FAILED.value)
                )
            )
        ):
            expires_at = _aware(record.expires_at)
            if record.legal_hold or record.milestone:
                protected += 1
                continue
            if record.id in protected_ids:
                protected += 1
                continue
            if expires_at is None or expires_at > now:
                continue
            record.state = BackupState.EXPIRED.value
            expired += 1
            self.audit.write(
                "BACKUP_EXPIRED",
                AUDIT_RESOURCE,
                str(record.id),
                {"backup_id": record.backup_id, "expired_at": now.isoformat()},
            )
        self.session.commit()
        return {"expired": expired, "protected": protected, "skipped_active_restore": 0}

    def backups(self, limit: int = 100) -> list[BackupSetResponse]:
        rows = self.session.scalars(
            select(BackupSet).order_by(BackupSet.created_at.desc()).limit(limit)
        )
        return [self._backup_response(item) for item in rows]

    def backup(self, backup_set_id: uuid.UUID) -> BackupSetResponse:
        return self._backup_response(self._backup_row(backup_set_id))

    def _backup_row(self, backup_set_id: uuid.UUID) -> BackupSet:
        record = self.session.get(BackupSet, backup_set_id)
        if record is None:
            raise RecoveryError(404, "backup_not_found", "backup set does not exist")
        return record

    def _backup_response(self, record: BackupSet) -> BackupSetResponse:
        entries = self.session.scalars(
            select(BackupManifestEntry)
            .where(BackupManifestEntry.backup_set_id == record.id)
            .order_by(BackupManifestEntry.object_name)
        )
        return BackupSetResponse(
            id=record.id,
            backup_id=record.backup_id,
            state=BackupState(record.state),
            policy_key=record.policy_revision.key,
            policy_revision=record.policy_revision.revision,
            modelforge_version=record.modelforge_version,
            modelforge_commit=record.modelforge_commit,
            schema_revision=record.schema_revision,
            environment_fingerprint=dict(record.environment_fingerprint or {}),
            database_identity=dict(record.database_identity or {}),
            destination_root=record.destination_root,
            manifest_relative_path=record.manifest_relative_path,
            manifest_sha256=record.manifest_sha256,
            included_asset_classes=list(record.included_asset_classes or []),
            excluded_asset_classes=list(record.excluded_asset_classes or []),
            payload_bytes=record.payload_bytes,
            encrypted=record.encrypted,
            encryption_algorithm=record.encryption_algorithm,
            encryption_key_id=record.encryption_key_id,
            verification_details=_public_verification(record.verification_details),
            verified_at=record.verified_at,
            failure_code=record.failure_code,
            failure_reason=record.failure_reason,
            milestone=record.milestone,
            legal_hold=record.legal_hold,
            restore_eligible=BackupState(record.state) in RESTORE_ELIGIBLE_STATES,
            reason=record.reason,
            created_by=record.created_by,
            started_at=record.started_at,
            completed_at=record.completed_at,
            expires_at=record.expires_at,
            created_at=record.created_at,
            entries=[BackupManifestEntryResponse.model_validate(item) for item in entries],
        )

    def _policy_response(self, record: RecoveryPolicyRevision) -> RecoveryPolicyResponse:
        return RecoveryPolicyResponse(
            id=record.id,
            key=record.key,
            name=record.name,
            revision=record.revision,
            asset_class=RecoveryAssetClass(record.asset_class),
            backup_method=BackupMethod(record.backup_method),
            retention_days=record.retention_days,
            minimum_verified_backups=record.minimum_verified_backups,
            rpo_seconds=record.rpo_seconds,
            rto_target_seconds=record.rto_target_seconds,
            restore_verification=record.restore_verification,  # type: ignore[arg-type]
            encryption_required=record.encryption_required,
            external_dependency=record.external_dependency,
            rehydration_allowed=record.rehydration_allowed,
            secret_class=SecretRecoveryClass(record.secret_class) if record.secret_class else None,
            rationale=record.rationale,
            created_by=record.created_by,
            active=record.active,
            fingerprint=record.fingerprint,
            created_at=record.created_at,
        )

    # ------------------------------------------------------------------ restore

    def create_restore_plan(self, request: RestorePlanCreate) -> RestorePlanResponse:
        backup = self._backup_row(request.backup_set_id)
        if BackupState(backup.state) not in RESTORE_ELIGIBLE_STATES:
            raise RecoveryError(
                409,
                RecoveryFailureCode.BACKUP_NOT_RESTORE_ELIGIBLE.value,
                f"backup {backup.backup_id} is {backup.state} and is not restore eligible",
            )
        if request.target_environment == "PRODUCTION" and not (
            self.settings.restore_allow_production_target
        ):
            raise RecoveryError(
                409,
                RecoveryFailureCode.DESTINATION_NOT_ISOLATED.value,
                "production restore targets are disabled; use the operator runbook and CLI",
            )
        if _same_database(request.database_destination, self.settings.database_url):
            raise RecoveryError(
                409,
                RecoveryFailureCode.DESTINATION_NOT_ISOLATED.value,
                "a restore may not target the database this control plane is running on",
            )
        fingerprint = _hash(
            {
                "backup": str(backup.id),
                "mode": request.mode.value,
                "target": redact_database_url(request.database_destination),
                "label": request.target_label,
            }
        )
        existing = self.session.scalar(
            select(RestorePlan).where(RestorePlan.fingerprint == fingerprint)
        )
        if existing is not None:
            return self._plan_response(existing)
        record = RestorePlan(
            backup_set_id=backup.id,
            mode=request.mode.value,
            state=RestorePlanState.DRAFT.value,
            target_environment=request.target_environment,
            target_label=request.target_label,
            database_destination=request.database_destination,
            artifact_strategy=request.artifact_strategy,
            secret_strategy=request.secret_strategy,
            node_strategy=request.node_strategy,
            expected_modelforge_version=request.expected_modelforge_version,
            preflight={},
            validation_requirements=dict(request.validation_requirements),
            fingerprint=fingerprint,
            reason=request.reason,
            created_by=request.created_by,
        )
        self.session.add(record)
        self.session.flush()
        self.audit.write(
            "RESTORE_PLANNED",
            AUDIT_RESOURCE,
            str(record.id),
            {
                "backup_id": backup.backup_id,
                "mode": record.mode,
                "target": redact_database_url(record.database_destination),
            },
        )
        self.session.commit()
        return self._plan_response(record)

    def restore_plan(self, plan_id: uuid.UUID) -> RestorePlanResponse:
        return self._plan_response(self._plan_row(plan_id))

    def restore_plans(self, limit: int = 100) -> list[RestorePlanResponse]:
        return [
            self._plan_response(item)
            for item in self.session.scalars(
                select(RestorePlan).order_by(RestorePlan.created_at.desc()).limit(limit)
            )
        ]

    def _plan_row(self, plan_id: uuid.UUID) -> RestorePlan:
        record = self.session.get(RestorePlan, plan_id)
        if record is None:
            raise RecoveryError(404, "restore_plan_not_found", "restore plan does not exist")
        return record

    def _plan_response(self, record: RestorePlan) -> RestorePlanResponse:
        return RestorePlanResponse(
            id=record.id,
            backup_set_id=record.backup_set_id,
            backup_id=record.backup_set.backup_id,
            mode=RestoreMode(record.mode),
            state=RestorePlanState(record.state),
            target_environment=record.target_environment,
            target_label=record.target_label,
            database_destination_redacted=redact_database_url(record.database_destination),
            artifact_strategy=record.artifact_strategy,
            secret_strategy=record.secret_strategy,
            node_strategy=record.node_strategy,
            expected_modelforge_version=record.expected_modelforge_version,
            preflight=dict(record.preflight or {}),
            validation_requirements=dict(record.validation_requirements or {}),
            fingerprint=record.fingerprint,
            reason=record.reason,
            created_by=record.created_by,
            created_at=record.created_at,
        )

    def preflight(self, plan_id: uuid.UUID) -> RestorePlanResponse:
        plan = self._plan_row(plan_id)
        backup = plan.backup_set
        checks: list[dict[str, Any]] = []
        failures: list[str] = []

        def record_check(name: str, ok: bool, detail: str, code: str | None = None) -> None:
            checks.append({"check": name, "status": "PASS" if ok else "FAIL", "detail": detail})
            if not ok and code:
                failures.append(code)

        record_check(
            "backup_verified",
            BackupState(backup.state) in RESTORE_ELIGIBLE_STATES,
            f"backup state is {backup.state}",
            RecoveryFailureCode.BACKUP_NOT_RESTORE_ELIGIBLE.value,
        )
        verification: dict[str, Any] = (
            self._verify_payload(backup)
            if backup.manifest_sha256
            else {
                "ok": False,
                "failure_code": RecoveryFailureCode.MANIFEST_INCOMPLETE.value,
                "failure_reason": "backup has no manifest",
                "checks": {},
            }
        )
        record_check(
            "payload_hashes",
            bool(verification["ok"]),
            str(verification.get("failure_reason") or "all payload hashes match the manifest"),
            str(verification.get("failure_code") or RecoveryFailureCode.HASH_MISMATCH.value),
        )

        schema_ok, schema_detail = self._schema_compatibility(backup.schema_revision)
        record_check(
            "schema_compatibility",
            schema_ok,
            schema_detail,
            RecoveryFailureCode.SCHEMA_TOO_NEW.value,
        )

        key_ok = (not backup.encrypted) or self.settings.backup_encryption_key is not None
        record_check(
            "encryption_key",
            key_ok,
            "encryption key available" if key_ok else "no decryption key configured",
            RecoveryFailureCode.ENCRYPTION_KEY_UNAVAILABLE.value,
        )

        isolated = not _same_database(plan.database_destination, self.settings.database_url)
        record_check(
            "destination_isolation",
            isolated,
            redact_database_url(plan.database_destination),
            RecoveryFailureCode.DESTINATION_NOT_ISOLATED.value,
        )

        identity: dict[str, Any] = {}
        try:
            target = parse_target(plan.database_destination)
            reachable = True
            exists = self.engine.database_exists(target)
            empty = self.engine.table_is_empty(target) if exists else True
            identity = dict(self.engine.server_identity(target)) if exists else {}
        except PostgresToolError as error:
            reachable = False
            exists = False
            empty = False
            record_check("destination_reachable", False, error.message, error.code)
        if reachable:
            record_check(
                "destination_reachable", True, "destination PostgreSQL responded to a probe"
            )
            record_check(
                "destination_empty",
                empty or plan.mode != RestoreMode.VALIDATION.value,
                "destination public schema is empty" if empty else "destination already has tables",
                RecoveryFailureCode.DESTINATION_NOT_EMPTY.value,
            )
            source_major = int((backup.database_identity or {}).get("major_version") or 0)
            target_major = int(identity.get("major_version") or source_major)
            record_check(
                "postgres_compatibility",
                target_major >= source_major,
                f"source major {source_major} → destination major {target_major}",
                RecoveryFailureCode.POSTGRES_VERSION_INCOMPATIBLE.value,
            )

        root = self._backup_root()
        available = free_bytes(root)
        record_check(
            "disk_space",
            available is None or available >= backup.payload_bytes,
            f"{available} bytes available for a {backup.payload_bytes} byte payload",
            RecoveryFailureCode.INSUFFICIENT_CAPACITY.value,
        )

        commit = (backup.environment_fingerprint or {}).get("modelforge_commit")
        record_check(
            "source_control_revision",
            bool(commit),
            f"backup records ModelForge commit {commit}"
            if commit
            else "backup has no recorded source-control revision",
            RecoveryFailureCode.SOURCE_CONTROL_REVISION_UNAVAILABLE.value,
        )

        record_check(
            "external_dependencies",
            True,
            "ExampleRAG, ExampleVision and external GPU workloads are never modified by a restore",
        )

        passed = not failures
        plan.preflight = {
            "observed_at": _now().isoformat(),
            "status": "PASS" if passed else "FAIL",
            "checks": checks,
            "failure_codes": sorted(set(failures)),
        }
        plan.state = (
            RestorePlanState.PREFLIGHT_PASSED.value
            if passed
            else RestorePlanState.PREFLIGHT_FAILED.value
        )
        self.session.commit()
        return self._plan_response(plan)

    def _schema_compatibility(self, schema_revision: str | None) -> tuple[bool, str]:
        if not schema_revision:
            return False, "backup does not record an Alembic revision"
        known = _known_alembic_revisions(self.settings.alembic_directory)
        if not known:
            return False, "this ModelForge build ships no Alembic tree: SCHEMA_UNKNOWN"
        if schema_revision not in known:
            head_text = known[-1] if known else "unknown"
            return False, (
                f"backup schema {schema_revision} is unknown to this ModelForge build "
                f"(head {head_text}): SCHEMA_TOO_NEW"
            )
        head = known[-1]
        if schema_revision == head:
            return True, f"backup schema {schema_revision} matches the current head"
        return True, f"backup schema {schema_revision} will be migrated forward to {head}"

    def start_restore(
        self, plan_id: uuid.UUID, request: RestoreAdvanceRequest
    ) -> RestoreOperationResponse:
        plan = self._plan_row(plan_id)
        if plan.state != RestorePlanState.PREFLIGHT_PASSED.value:
            raise RecoveryError(
                409,
                "restore_preflight_required",
                f"restore plan is {plan.state}; a passing preflight is required",
            )
        existing = self.session.scalar(
            select(RestoreOperation)
            .where(RestoreOperation.plan_id == plan.id)
            .order_by(RestoreOperation.started_at.desc())
            .limit(1)
        )
        if existing is not None and RestoreState(existing.state) not in {
            RestoreState.FAILED,
            RestoreState.READY,
        }:
            # Idempotent retry: resume the journalled operation instead of creating a second one.
            return self._operation_response(existing)
        self._guard_no_concurrent_operation()
        attempt = (existing.attempt + 1) if existing else 1
        record = RestoreOperation(
            plan_id=plan.id,
            backup_set_id=plan.backup_set_id,
            state=RestoreState.PLANNED.value,
            attempt=attempt,
            idempotency_key=_hash({"plan": str(plan.id), "attempt": attempt}),
            preflight_result=dict(plan.preflight or {}),
            phase_durations={},
            source_fingerprint=(
                (plan.backup_set.verification_details or {}).get("source_fingerprint") or {}
            ),
            restored_fingerprint={},
            fingerprint_diff={},
            validation_result={},
        )
        self.session.add(record)
        self.session.flush()
        self._journal(
            record,
            None,
            RestoreState.PLANNED,
            "planned",
            request.actor,
            request.reason,
            {"attempt": attempt, "backup_id": plan.backup_set.backup_id},
        )
        self.audit.write(
            "RESTORE_STARTED",
            AUDIT_RESOURCE,
            str(record.id),
            {
                "backup_id": plan.backup_set.backup_id,
                "mode": plan.mode,
                "attempt": attempt,
                "target": redact_database_url(plan.database_destination),
            },
        )
        self.session.commit()
        return self._operation_response(record)

    def advance_restore(
        self, operation_id: uuid.UUID, request: RestoreAdvanceRequest
    ) -> RestoreOperationResponse:
        record = self._operation_row(operation_id)
        if RestoreState(record.state) in {RestoreState.READY, RestoreState.FAILED}:
            return self._operation_response(record)
        plan = record.plan
        durations: dict[str, Any] = dict(record.phase_durations or {})
        for phase in RESTORE_PHASE_ORDER:
            # READY is a fresh transition proof, never a cached inference from a journaled
            # VALIDATING duration (or even a stale READY duration left by malformed state).
            if phase is not RestoreState.READY and phase.value in durations:
                continue
            started = time.perf_counter()
            previous = RestoreState(record.state)
            record.state = phase.value
            self.session.flush()
            try:
                evidence = self._run_restore_phase(record, plan, phase)
            except (PostgresToolError, BackupCryptoError, RecoveryPathError) as error:
                self._fail_restore(record, previous, phase, error.code, error.message, request)
                return self._operation_response(record)
            except RecoveryError as error:
                self._fail_restore(
                    record, previous, phase, error.code, error.message, request
                )
                return self._operation_response(record)
            elapsed = time.perf_counter() - started
            durations[phase.value] = round(elapsed, 6)
            record.phase_durations = dict(durations)
            self._journal(
                record, previous, phase, phase.value, request.actor, request.reason, evidence
            )
            self.session.commit()
            if request.simulate_interruption_after is phase:
                # A deliberately interrupted rehearsal leaves a resumable journal, not a lie.
                self._journal(
                    record,
                    phase,
                    phase,
                    "interrupted",
                    request.actor,
                    "rehearsal interruption requested by the operator",
                    {"resumable": True},
                )
                self.session.commit()
                return self._operation_response(record)
            if request.stop_after is phase:
                return self._operation_response(record)
        record.rto_seconds = round(sum(float(value) for value in durations.values()), 6)
        record.ready_at = _now()
        self.audit.write(
            "RECOVERY_COMPLETED",
            AUDIT_RESOURCE,
            str(record.id),
            {
                "backup_id": plan.backup_set.backup_id,
                "rto_seconds": record.rto_seconds,
                "rpo_seconds": record.rpo_seconds,
            },
        )
        plan.state = RestorePlanState.CONSUMED.value
        self.session.commit()
        return self._operation_response(record)

    def _run_restore_phase(
        self, record: RestoreOperation, plan: RestorePlan, phase: RestoreState
    ) -> dict[str, Any]:
        if phase is RestoreState.PREFLIGHT:
            refreshed = self.preflight(plan.id)
            if refreshed.preflight.get("status") != "PASS":
                raise RecoveryError(
                    409,
                    RecoveryFailureCode.BACKUP_NOT_RESTORE_ELIGIBLE.value,
                    "restore preflight failed at execution time",
                    refreshed.preflight,
                )
            record.preflight_result = dict(refreshed.preflight)
            return {"checks": len(refreshed.preflight.get("checks", []))}
        if phase is RestoreState.RESTORING_DATABASE:
            return self._restore_database(record, plan)
        if phase is RestoreState.RESTORING_CONFIGURATION:
            return self._restore_configuration(record, plan)
        if phase is RestoreState.REHYDRATING_ARTIFACTS:
            return self._plan_artifact_recovery(record, plan)
        if phase is RestoreState.RECONCILING:
            return self._reconcile_restored(record, plan)
        if phase is RestoreState.VALIDATING:
            return self._validate_restored(record, plan)
        if phase is RestoreState.READY:
            checkpoint = self._verify_restored_audit_chain(
                parse_target(plan.database_destination)
            )
            return {
                "state": phase.value,
                "final_audit_verification": "PASS",
                "audit_event_count": checkpoint.event_count,
                "audit_last_sequence": checkpoint.last_sequence,
                "audit_last_event_hash": checkpoint.last_event_hash,
            }
        return {"state": phase.value}

    def _restore_database(self, record: RestoreOperation, plan: RestorePlan) -> dict[str, Any]:
        backup = plan.backup_set
        root = self._backup_root()
        entry = self.session.scalar(
            select(BackupManifestEntry).where(
                BackupManifestEntry.backup_set_id == backup.id,
                BackupManifestEntry.logical_asset_type == "control_plane_database",
            )
        )
        if entry is None:
            raise RecoveryError(
                409, RecoveryFailureCode.MANIFEST_INCOMPLETE.value, "no database payload recorded"
            )
        stored = resolve_within(root, entry.relative_path, must_exist=True)
        if sha256_file(stored) != entry.sha256:
            raise RecoveryError(
                409,
                RecoveryFailureCode.HASH_MISMATCH.value,
                "database payload hash changed since the backup was verified",
            )
        target = parse_target(plan.database_destination)
        if not self.engine.database_exists(target):
            self.engine.create_database(target)
        if not self.engine.table_is_empty(target):
            # Idempotent retry: a completed database phase is never replayed onto live data.
            raise RecoveryError(
                409,
                RecoveryFailureCode.DESTINATION_NOT_EMPTY.value,
                "restore destination already contains tables; drop or recreate it before retrying",
            )
        staging: Path | None = None
        try:
            if backup.encrypted:
                cipher = self._cipher(required=True)
                if cipher is None:  # pragma: no cover
                    raise RecoveryError(
                        409,
                        RecoveryFailureCode.ENCRYPTION_KEY_UNAVAILABLE.value,
                        "no decryption key",
                    )
                staging = stored.with_name(stored.name + f".restore-{uuid.uuid4().hex[:8]}")
                cipher.decrypt_file(stored, staging)
                expected = (entry.dependency_refs or {}).get("plaintext_sha256")
                if expected and sha256_file(staging) != expected:
                    raise RecoveryError(
                        409,
                        RecoveryFailureCode.HASH_MISMATCH.value,
                        "decrypted payload does not match the recorded plaintext hash",
                    )
                payload = staging
            else:
                payload = stored
            result = self.engine.restore(target, payload)
        finally:
            if staging is not None:
                staging.unlink(missing_ok=True)
        restored_revision = self.engine.alembic_revision(target)
        self.audit.write(
            "DATABASE_RESTORED",
            AUDIT_RESOURCE,
            str(record.id),
            {
                "backup_id": backup.backup_id,
                "target": redact_database_url(plan.database_destination),
                "schema_revision": restored_revision,
            },
        )
        return {
            "restore_seconds": result.duration_seconds,
            "schema_revision": restored_revision,
            "expected_schema_revision": backup.schema_revision,
            "database": target.redacted,
        }

    def _restore_configuration(
        self, record: RestoreOperation, plan: RestorePlan
    ) -> dict[str, Any]:
        backup = plan.backup_set
        root = self._backup_root()
        # Schema migration belongs to the restore phase, not to the optional presence of a
        # configuration manifest. Reconciliation relies on the current audit checkpoint schema.
        schema_target = self._migrate_destination(plan)
        entry = self.session.scalar(
            select(BackupManifestEntry).where(
                BackupManifestEntry.backup_set_id == backup.id,
                BackupManifestEntry.logical_asset_type == "configuration_reference",
            )
        )
        if entry is None:
            return {
                "configuration": "NOT_INCLUDED",
                "schema_after_migration": schema_target,
            }
        path = resolve_within(root, entry.relative_path, must_exist=True)
        payload = json.loads(path.read_text("utf-8"))
        current_root = Path(self.settings.config_root)
        matched = 0
        drifted: list[str] = []
        for item in payload.get("files", []):
            candidate = current_root / item["relative_path"]
            if candidate.is_file() and sha256_file(candidate) == item["sha256"]:
                matched += 1
            else:
                drifted.append(item["relative_path"])
        return {
            "repository": payload.get("repository"),
            "commit": payload.get("commit"),
            "reference": payload.get("reference"),
            "config_files_matched": matched,
            "config_files_drifted": drifted[:20],
            "secret_strategy": plan.secret_strategy,
            "schema_after_migration": schema_target,
        }

    def _migrate_destination(self, plan: RestorePlan) -> str | None:
        """Run only the forward migrations this build knows; never silently downgrade."""

        target = parse_target(plan.database_destination)
        current = self.engine.alembic_revision(target)
        known = _known_alembic_revisions(self.settings.alembic_directory)
        if current is None:
            return None
        if current not in known:
            raise RecoveryError(
                409,
                RecoveryFailureCode.SCHEMA_TOO_NEW.value,
                f"restored schema {current} is newer than this ModelForge build supports",
            )
        head = known[-1]
        if current == head:
            return current
        _run_alembic_upgrade(plan.database_destination, self.settings.alembic_directory)
        return self.engine.alembic_revision(target)

    def _plan_artifact_recovery(
        self, record: RestoreOperation, plan: RestorePlan
    ) -> dict[str, Any]:
        if plan.artifact_strategy == "NONE":
            return {"artifact_strategy": "NONE", "coverage": "NOT_EVALUATED"}
        root = self._backup_root()
        entry = self.session.scalar(
            select(BackupManifestEntry).where(
                BackupManifestEntry.backup_set_id == plan.backup_set_id,
                BackupManifestEntry.logical_asset_type == "artifact_inventory",
            )
        )
        if entry is None:
            return {"artifact_strategy": plan.artifact_strategy, "coverage": "NO_MANIFEST"}
        payload = json.loads(
            resolve_within(root, entry.relative_path, must_exist=True).read_text("utf-8")
        )
        rehydratable = [
            item
            for item in payload.get("artifact_sets", [])
            if item["recovery_class"] == ArtifactRecoveryClass.REHYDRATABLE.value
        ]
        non_rehydratable = [
            item
            for item in payload.get("artifact_sets", [])
            if item["recovery_class"] != ArtifactRecoveryClass.REHYDRATABLE.value
        ]
        self.audit.write(
            "ARTIFACT_REHYDRATION_STARTED",
            AUDIT_RESOURCE,
            str(record.id),
            {
                "artifact_sets": len(rehydratable),
                "strategy": plan.artifact_strategy,
            },
        )
        return {
            "artifact_strategy": plan.artifact_strategy,
            "rehydratable_sets": len(rehydratable),
            "non_rehydratable_sets": len(non_rehydratable),
            "rehydratable_bytes": payload.get("totals", {}).get("rehydratable_bytes", 0),
            "non_rehydratable_bytes": payload.get("totals", {}).get("non_rehydratable_bytes", 0),
            "coverage": "MANIFEST_RECORDED"
            if plan.artifact_strategy == "MANIFEST_ONLY"
            else "REHYDRATION_REQUIRED",
        }

    def _restored_audit_checkpoint(self, target: PostgresTarget) -> AuditChainCheckpoint:
        rows = self.engine.query_rows(
            target,
            "select singleton_id::text as singleton_id, event_count::text as event_count, "
            "last_sequence::text as last_sequence, coalesce(last_event_hash, '') as "
            "last_event_hash, (last_event_hash is null)::text as last_event_hash_is_null, "
            "hash_format, v2_start_sequence::text as v2_start_sequence, "
            "legacy_prefix_count::text as legacy_prefix_count, legacy_prefix_seal "
            "from audit_chain_heads where singleton_id = 1",
            max_rows=1,
        )
        if len(rows) != 1:
            raise RecoveryError(
                409,
                RecoveryFailureCode.AUDIT_CHAIN_CORRUPT.value,
                "restored audit checkpoint is missing or not a singleton",
            )
        row = rows[0]
        return AuditChainCheckpoint(
            singleton_id=int(row["singleton_id"]),
            event_count=int(row["event_count"]),
            last_sequence=int(row["last_sequence"]),
            last_event_hash=(
                None if _psql_true(row["last_event_hash_is_null"]) else row["last_event_hash"]
            ),
            hash_format=row["hash_format"],
            v2_start_sequence=int(row["v2_start_sequence"]),
            legacy_prefix_count=int(row["legacy_prefix_count"]),
            legacy_prefix_seal=row["legacy_prefix_seal"],
        )

    def _restored_audit_events(self, target: PostgresTarget) -> Iterator[AuditEventRecord]:
        after_cursor: tuple[int, str] | None = None
        page_size = 500
        while True:
            cursor_sql = ""
            if after_cursor is not None:
                after_sequence, after_id = after_cursor
                cursor_sql = (
                    "where (sequence > "
                    f"{after_sequence} or (sequence = {after_sequence} and "
                    f"id > {_sql_literal(after_id)}::uuid)) "
                )
            audit_page_sql = (
                "select id::text as id, sequence::text as sequence, "  # noqa: S608
                "to_char(occurred_at at time zone 'UTC', "
                "'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') as occurred_at, "
                "correlation_id, actor_type, actor_id, action, resource_type, "
                "coalesce(resource_id, '') as resource_id, "
                "(resource_id is null)::text as resource_id_is_null, outcome, "
                "details::text as details, coalesce(previous_event_hash, '') as "
                "previous_event_hash, (previous_event_hash is null)::text as "
                "previous_event_hash_is_null, event_hash, hash_format, "
                "coalesce(canonical_payload, '') as canonical_payload, "
                "(canonical_payload is null)::text as canonical_payload_is_null "
                "from audit_events "
                f"{cursor_sql}order by sequence, id limit {page_size}"
            )
            rows = self.engine.query_rows(
                target,
                audit_page_sql,
                max_rows=page_size,
            )
            if not rows:
                return
            for row in rows:
                details = json.loads(row["details"])
                if not isinstance(details, dict):
                    raise ValueError("restored audit details are not an object")
                sequence = int(row["sequence"])
                event_id = str(uuid.UUID(row["id"]))
                yield AuditEventRecord(
                    id=event_id,
                    sequence=sequence,
                    occurred_at=row["occurred_at"],
                    correlation_id=row["correlation_id"],
                    actor_type=row["actor_type"],
                    actor_id=row["actor_id"],
                    action=row["action"],
                    resource_type=row["resource_type"],
                    resource_id=(
                        None if _psql_true(row["resource_id_is_null"]) else row["resource_id"]
                    ),
                    outcome=row["outcome"],
                    details=details,
                    previous_event_hash=(
                        None
                        if _psql_true(row["previous_event_hash_is_null"])
                        else row["previous_event_hash"]
                    ),
                    event_hash=row["event_hash"],
                    hash_format=row["hash_format"],
                    canonical_payload=(
                        None
                        if _psql_true(row.get("canonical_payload_is_null", "t"))
                        else row.get("canonical_payload", "")
                    ),
                )
                after_cursor = (sequence, event_id)
            if len(rows) < page_size:
                return

    def _restored_audit_identity_counts(
        self, target: PostgresTarget
    ) -> tuple[int, int, int]:
        rows = self.engine.query_rows(
            target,
            "select count(*)::text as event_count, "
            "count(distinct id)::text as distinct_event_ids, "
            "count(distinct sequence)::text as distinct_sequences from audit_events",
            max_rows=1,
        )
        if len(rows) != 1:
            raise ValueError("restored audit identity counts are unavailable")
        row = rows[0]
        return (
            int(row["event_count"]),
            int(row["distinct_event_ids"]),
            int(row["distinct_sequences"]),
        )

    def _verify_restored_audit_chain(
        self, target: PostgresTarget
    ) -> AuditChainCheckpoint:
        try:
            checkpoint = self._restored_audit_checkpoint(target)
            event_count, distinct_event_ids, distinct_sequences = (
                self._restored_audit_identity_counts(target)
            )
            violations: list[str] = []
            if distinct_event_ids != event_count:
                violations.append("restored audit event UUIDs are not unique")
            if distinct_sequences != event_count:
                violations.append("restored audit event sequences are not unique")
            violations.extend(
                audit_chain_violations(
                    cast(
                        Iterable[AuditEventLike],
                        self._restored_audit_events(target),
                    ),
                    checkpoint,
                )
            )
        except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error:
            raise RecoveryError(
                409,
                RecoveryFailureCode.AUDIT_CHAIN_CORRUPT.value,
                f"restored audit chain cannot be parsed: {error}",
            ) from error
        if violations:
            raise RecoveryError(
                409,
                RecoveryFailureCode.AUDIT_CHAIN_CORRUPT.value,
                "restored audit chain or checkpoint failed strict verification",
                {"violations": violations[:25]},
            )
        return checkpoint

    def _append_restored_audit_marker(
        self,
        target: PostgresTarget,
        *,
        resource_id: str,
        expected_checkpoint: AuditChainCheckpoint,
    ) -> str:
        """Append a canonical marker to the isolated restore target without forking its chain.

        The target is reached through the recovery CLI adapter rather than this process' SQLAlchemy
        session. A transaction-scoped advisory lock and compare-and-set head predicate keep that
        exceptional path compatible with normal PostgreSQL audit writers.
        """

        resource_id = str(uuid.UUID(resource_id))
        correlation_id = str(uuid.uuid4())
        event_id = str(uuid.uuid4())
        occurred_at = datetime.now(UTC)
        sequence = expected_checkpoint.last_sequence + 1
        previous_sql = (
            "NULL"
            if expected_checkpoint.last_event_hash is None
            else _sql_literal(expected_checkpoint.last_event_hash)
        )
        timestamp_sql = _sql_literal(normalise_audit_timestamp(occurred_at))
        # The isolated restore uses the same database-owned operation as the live writer. The
        # function, not this process, constructs/hashes the canonical bytes and atomically advances
        # the locked checkpoint.
        insert_sql = (
            "select sequence::text || ':' || event_hash "  # noqa: S608
            "from modelforge_audit.append_event_v2("
            f"{_sql_literal(event_id)}::uuid, {timestamp_sql}::timestamptz, "
            f"{_sql_literal(correlation_id)}, 'control_plane', 'recovery', "
            "'RECOVERY_RECONCILIATION_COMPLETED', "
            f"{_sql_literal(AUDIT_RESOURCE)}, {_sql_literal(resource_id)}, 'success', "
            "'{}'::jsonb, "
            f"{expected_checkpoint.event_count}, {expected_checkpoint.last_sequence}, "
            f"{previous_sql}, {_sql_literal(expected_checkpoint.hash_format)}, "
            f"{expected_checkpoint.v2_start_sequence}, "
            f"{expected_checkpoint.legacy_prefix_count}, "
            f"{_sql_literal(expected_checkpoint.legacy_prefix_seal)})"
        )
        inserted = self.engine.scalar(target, insert_sql)
        inserted_sequence, separator, event_hash = inserted.partition(":")
        if separator != ":" or inserted_sequence != str(sequence) or len(event_hash) != 64:
            raise PostgresToolError(
                "BACKUP_TOOL_FAILED", "restored audit marker did not advance event and checkpoint"
            )
        return event_hash

    def _reconcile_restored(self, record: RestoreOperation, plan: RestorePlan) -> dict[str, Any]:
        """Clear stale current-truth rows exactly once; history is never rewritten."""

        target = parse_target(plan.database_destination)
        # Fail before the idempotency marker lookup and before any target mutation. A valid marker
        # on a corrupted chain is not authority to skip reconciliation.
        self._verify_restored_audit_chain(target)
        self.audit.write(
            "RECOVERY_RECONCILIATION_STARTED",
            AUDIT_RESOURCE,
            str(record.id),
            {"backup_id": plan.backup_set.backup_id},
        )
        # The marker must identify *this* restore operation. A backup legitimately carries the
        # reconciliation events of earlier restores, so matching on the action alone would make
        # every later restore believe it had already reconciled and silently keep stale leases.
        marker = self.engine.scalar(
            target,
            "select count(*) from audit_events where action = 'RECOVERY_RECONCILIATION_COMPLETED' "  # noqa: S608
            f"and resource_id = '{record.id}'",
        )
        if marker.strip() not in {"0", ""}:
            return {
                "reconciliation": "ALREADY_APPLIED",
                "applied_once": True,
                "restore_operation_id": str(record.id),
            }

        cleared: dict[str, int] = {}
        for table in sorted(CURRENT_TRUTH_TABLES):
            exists = self.engine.scalar(
                target,
                "select count(*) from information_schema.tables where table_schema='public' "  # noqa: S608
                f"and table_name='{table}'",
            )
            if exists.strip() in {"0", ""}:
                continue
            # The table name comes from the module-level CURRENT_TRUTH_TABLES allowlist.
            before = self.engine.scalar(target, f"select count(*) from {table}")  # noqa: S608
            self.engine.scalar(target, f"delete from {table}")  # noqa: S608
            cleared[table] = int(before or 0)

        # Lifecycle and migration operations use a `stage` vocabulary; anything that is not a
        # terminal stage was in flight when the backup was taken and needs reconciliation.
        pending_lifecycle = int(
            self.engine.scalar(
                target,
                "select count(*) from lifecycle_operations where stage not in "  # noqa: S608 - literal stage allowlist
                f"({_quoted(LIFECYCLE_TERMINAL_STAGES)})",
            )
            or 0
        )
        pending_migrations = int(
            self.engine.scalar(
                target,
                "select count(*) from migration_cutover_operations where stage not in "  # noqa: S608 - literal stage allowlist
                f"({_quoted(CUTOVER_TERMINAL_STAGES)})",
            )
            or 0
        )
        in_flight_lifecycle = [
            row
            for row in self.engine.scalar(
                target,
                "select coalesce(string_agg(id || ':' || stage, ','), '') from "  # noqa: S608 - literal stage allowlist
                "lifecycle_operations where stage not in "
                f"({_quoted(LIFECYCLE_TERMINAL_STAGES)})",
            ).split(",")
            if row
        ]
        in_flight_migrations = [
            row
            for row in self.engine.scalar(
                target,
                "select coalesce(string_agg(id || ':' || stage, ','), '') from "  # noqa: S608 - literal stage allowlist
                "migration_cutover_operations where stage not in "
                f"({_quoted(CUTOVER_TERMINAL_STAGES)})",
            ).split(",")
            if row
        ]
        firing_alerts = int(
            self.engine.scalar(
                target,
                "select count(*) from operational_alerts where state in ('PENDING','FIRING')",
            )
            or 0
        )
        # A restored alert is history, not current truth: it must be re-derived after startup.
        self.engine.scalar(
            target,
            "update operational_alerts set state='SUPPRESSED', "
            "suppressed_until = now() + interval '1 hour' where state in ('PENDING','FIRING')",
        )
        # Re-read immediately before the marker so concurrent target activity cannot make the
        # earlier proof stale. The marker CAS binds to this exact verified head.
        checkpoint = self._verify_restored_audit_chain(target)
        self._append_restored_audit_marker(
            target,
            resource_id=str(record.id),
            expected_checkpoint=checkpoint,
        )
        self.audit.write(
            "RECOVERY_RECONCILIATION_COMPLETED",
            AUDIT_RESOURCE,
            str(record.id),
            {
                "cleared_current_truth_rows": sum(cleared.values()),
                "pending_lifecycle_operations": pending_lifecycle,
                "pending_migration_cutovers": pending_migrations,
                "suppressed_alerts": firing_alerts,
            },
        )
        return {
            "reconciliation": "APPLIED",
            "applied_once": True,
            "cleared_current_truth": cleared,
            "pending_lifecycle_operations": pending_lifecycle,
            "pending_migration_cutovers": pending_migrations,
            "in_flight_lifecycle_operations": in_flight_lifecycle,
            "in_flight_migration_cutovers": in_flight_migrations,
            "restored_firing_alerts_suppressed": firing_alerts,
            "outcome": "RECOVERY_RECONCILIATION_REQUIRED"
            if pending_lifecycle or pending_migrations
            else "RECONCILED",
        }

    def _validate_restored(self, record: RestoreOperation, plan: RestorePlan) -> dict[str, Any]:
        target = parse_target(plan.database_destination)
        # This is an independent READY gate, not an inference from row-count fingerprints or the
        # fact that reconciliation happened earlier in the journal.
        audit_checkpoint = self._verify_restored_audit_chain(target)
        destination = create_engine(plan.database_destination, pool_pre_ping=True)
        try:
            with destination.connect() as connection:
                restored = semantic_fingerprint(connection)
        finally:
            destination.dispose()
        source = dict(record.source_fingerprint or {})
        diff: dict[str, Any] = (
            diff_fingerprints(source, restored) if source else {"identical": False}
        )
        fingerprint_contract = fingerprint_compatibility(
            source,
            restored,
            source_schema_revision=plan.backup_set.schema_revision,
        )
        record.restored_fingerprint = restored
        record.fingerprint_diff = diff

        constraint_count = int(
            self.engine.scalar(
                target,
                "select count(*) from information_schema.table_constraints "
                "where constraint_schema='public'",
            )
            or 0
        )
        sequence_count = int(
            self.engine.scalar(
                target, "select count(*) from information_schema.sequences where sequence_schema='public'"
            )
            or 0
        )
        index_count = int(
            self.engine.scalar(target, "select count(*) from pg_indexes where schemaname='public'")
            or 0
        )
        table_count = int(
            self.engine.scalar(
                target,
                "select count(*) from information_schema.tables where table_schema='public'",
            )
            or 0
        )
        audit_max = self.engine.scalar(target, "select coalesce(max(sequence),0) from audit_events")
        restored_latest_audit = self.engine.scalar(
            target,
            "select coalesce(to_char(max(occurred_at) at time zone 'UTC', "
            "'YYYY-MM-DD\"T\"HH24:MI:SS.USOF'), '') from audit_events "
            "where action <> 'RECOVERY_RECONCILIATION_COMPLETED'",
        )
        source_latest_audit = self.session.scalar(select(func.max(AuditEvent.occurred_at)))
        rpo_seconds: float | None = None
        restore_point = _aware(plan.backup_set.completed_at)
        if restored_latest_audit and source_latest_audit is not None:
            try:
                restored_moment = datetime.fromisoformat(restored_latest_audit)
            except ValueError:
                restored_moment = None
            restored_aware = _aware(restored_moment)
            source_aware = _aware(source_latest_audit)
            if restored_aware is not None and source_aware is not None:
                rpo_seconds = max(
                    0.0, source_aware.timestamp() - restored_aware.timestamp()
                )
        record.rpo_seconds = rpo_seconds

        requirements = dict(plan.validation_requirements or {})
        differences: list[dict[str, Any]] = list(diff.get("differences") or [])
        minimum_tables = int(requirements.get("minimum_tables", 100))
        stale_current_truth = {
            table: int(self.engine.scalar(target, f"select count(*) from {table}") or 0)  # noqa: S608
            for table in sorted(CURRENT_TRUTH_TABLES)
            if self.engine.scalar(
                target,
                "select count(*) from information_schema.tables where table_schema='public' "  # noqa: S608
                f"and table_name='{table}'",
            ).strip()
            not in {"0", ""}
        }
        gate = {
            "database_validated": table_count >= minimum_tables and constraint_count > 0,
            # A restore is not READY while a stale lease or NVML snapshot is still presented as
            # present-day truth; reconciliation must have actually run.
            "current_truth_reconciled": not any(stale_current_truth.values()),
            "schema_current": self.engine.alembic_revision(target)
            == _known_alembic_revisions(self.settings.alembic_directory)[-1],
            "audit_available": int(audit_max or 0) > 0,
            "audit_chain_intact": True,
            "fingerprint_compatible": fingerprint_contract.compatible,
            "no_unknown_corruption": audit_fingerprint_differences_are_compatible(
                source, restored, fingerprint_contract
            ),
        }
        result: dict[str, Any] = {
            "table_count": table_count,
            "constraint_count": constraint_count,
            "sequence_count": sequence_count,
            "index_count": index_count,
            "audit_max_sequence": int(audit_max or 0),
            "audit_checkpoint_event_count": audit_checkpoint.event_count,
            "audit_hash_format": audit_checkpoint.hash_format,
            "audit_v2_start_sequence": audit_checkpoint.v2_start_sequence,
            "restore_point": restore_point.isoformat() if restore_point else None,
            "observed_rpo_seconds": rpo_seconds,
            "fingerprint_identical": bool(diff.get("identical")),
            "fingerprint_compatibility_mode": fingerprint_contract.mode,
            "fingerprint_compatibility_reason": fingerprint_contract.reason,
            "differing_groups": diff.get("differing_groups", []),
            "difference_count": len(differences),
            "residual_current_truth_rows": stale_current_truth,
            "gate": gate,
        }
        record.validation_result = result
        if not all(gate.values()):
            raise RecoveryError(
                409,
                RecoveryFailureCode.RECOVERY_RECONCILIATION_REQUIRED.value,
                "restore validation gate did not pass",
                result,
            )
        self.audit.write(
            "RESTORE_VALIDATED",
            AUDIT_RESOURCE,
            str(record.id),
            {
                "backup_id": plan.backup_set.backup_id,
                "table_count": table_count,
                "fingerprint_identical": result["fingerprint_identical"],
            },
        )
        return result

    def _fail_restore(
        self,
        record: RestoreOperation,
        previous: RestoreState,
        phase: RestoreState,
        code: str,
        message: str,
        request: RestoreAdvanceRequest,
    ) -> None:
        terminal = (
            RestoreState.MANUAL_INTERVENTION_REQUIRED
            if code
            in {
                RecoveryFailureCode.AUDIT_CHAIN_CORRUPT.value,
                RecoveryFailureCode.RECOVERY_RECONCILIATION_REQUIRED.value,
                RecoveryFailureCode.DESTINATION_NOT_EMPTY.value,
            }
            else RestoreState.FAILED
        )
        record.state = terminal.value
        record.failure_code = code
        record.failure_reason = message[:2000]
        self._journal(
            record, previous, terminal, phase.value, request.actor, message, {"failure_code": code}
        )
        self.audit.write(
            "RESTORE_FAILED",
            AUDIT_RESOURCE,
            str(record.id),
            {"phase": phase.value, "failure_code": code},
            outcome="failure",
        )
        self.session.commit()

    def _journal(
        self,
        record: RestoreOperation,
        previous: RestoreState | None,
        to_state: RestoreState,
        phase: str,
        actor: str,
        reason: str,
        evidence: dict[str, Any],
    ) -> None:
        self.session.add(
            RestoreOperationEvent(
                restore_operation_id=record.id,
                from_state=previous.value if previous else None,
                to_state=to_state.value,
                phase=phase,
                actor=actor,
                reason=reason[:2000],
                evidence=evidence,
            )
        )
        self.session.flush()

    def restore_operations(self, limit: int = 100) -> list[RestoreOperationResponse]:
        return [
            self._operation_response(item)
            for item in self.session.scalars(
                select(RestoreOperation).order_by(RestoreOperation.started_at.desc()).limit(limit)
            )
        ]

    def restore_operation(self, operation_id: uuid.UUID) -> RestoreOperationResponse:
        return self._operation_response(self._operation_row(operation_id))

    def restore_events(self, operation_id: uuid.UUID) -> list[RestoreOperationEventResponse]:
        rows = self.session.scalars(
            select(RestoreOperationEvent)
            .where(RestoreOperationEvent.restore_operation_id == operation_id)
            .order_by(RestoreOperationEvent.occurred_at, RestoreOperationEvent.id)
        )
        return [RestoreOperationEventResponse.model_validate(item) for item in rows]

    def _operation_row(self, operation_id: uuid.UUID) -> RestoreOperation:
        record = self.session.get(RestoreOperation, operation_id)
        if record is None:
            raise RecoveryError(
                404, "restore_operation_not_found", "restore operation does not exist"
            )
        return record

    def _operation_response(self, record: RestoreOperation) -> RestoreOperationResponse:
        return RestoreOperationResponse(
            id=record.id,
            plan_id=record.plan_id,
            backup_set_id=record.backup_set_id,
            backup_id=record.backup_set.backup_id,
            state=RestoreState(record.state),
            mode=RestoreMode(record.plan.mode),
            attempt=record.attempt,
            idempotency_key=record.idempotency_key,
            preflight_result=dict(record.preflight_result or {}),
            phase_durations={
                key: float(value) for key, value in (record.phase_durations or {}).items()
            },
            source_fingerprint=_fingerprint_summary(record.source_fingerprint),
            restored_fingerprint=_fingerprint_summary(record.restored_fingerprint),
            fingerprint_diff=dict(record.fingerprint_diff or {}),
            validation_result=dict(record.validation_result or {}),
            rpo_seconds=record.rpo_seconds,
            rto_seconds=record.rto_seconds,
            failure_code=record.failure_code,
            failure_reason=record.failure_reason,
            started_at=record.started_at,
            updated_at=record.updated_at,
            ready_at=record.ready_at,
        )

    def reconcile_interrupted_restores(self) -> int:
        """A restore that died mid-phase is resumable, never silently 'done'."""

        pending = list(
            self.session.scalars(
                select(RestoreOperation).where(
                    RestoreOperation.state.in_(ACTIVE_RESTORE_STATES),
                    RestoreOperation.state != RestoreState.PLANNED.value,
                )
            )
        )
        for record in pending:
            previous = RestoreState(record.state)
            record.state = RestoreState.MANUAL_INTERVENTION_REQUIRED.value
            record.failure_code = RecoveryFailureCode.RECOVERY_RECONCILIATION_REQUIRED.value
            record.failure_reason = (
                f"restore was interrupted during {previous.value} and requires operator review"
            )
            self._journal(
                record,
                previous,
                RestoreState.MANUAL_INTERVENTION_REQUIRED,
                previous.value,
                "control-plane",
                "startup reconciliation found an interrupted restore",
                {"resumable_from": previous.value},
            )
        if pending:
            self.session.commit()
        return len(pending)

    # ------------------------------------------------------------------ artifacts

    def plan_artifact_recovery(
        self, request: ArtifactRecoveryCreate
    ) -> ArtifactRecoveryResponse:
        artifact_set = self.session.get(ArtifactSet, request.artifact_set_id)
        if artifact_set is None:
            raise RecoveryError(404, "artifact_set_not_found", "artifact set does not exist")
        storage_root = self.session.get(StorageRoot, request.target_storage_root_id)
        if storage_root is None:
            raise RecoveryError(404, "storage_root_not_found", "storage root does not exist")
        revision = self.session.get(ModelRevision, artifact_set.revision_id)
        if revision is None:  # pragma: no cover - FK guarantees this
            raise RecoveryError(404, "revision_not_found", "model revision does not exist")
        repository, commit_sha = _revision_identity(self.session, revision)
        members = list(
            self.session.scalars(
                select(ArtifactSetMember).where(
                    ArtifactSetMember.artifact_set_id == artifact_set.id
                )
            )
        )
        expected: list[dict[str, Any]] = []
        for member in members:
            artifact = self.session.get(ModelArtifact, member.artifact_id)
            if artifact is None:
                continue
            expected.append(
                {
                    "artifact_id": str(artifact.id),
                    "filename": artifact.filename,
                    "sha256": artifact.sha256,
                    "size_bytes": artifact.size_bytes,
                    "security_status": artifact.security_status,
                }
            )
        recovery_class = (
            ArtifactRecoveryClass.REHYDRATABLE
            if repository and commit_sha
            else ArtifactRecoveryClass.NON_REHYDRATABLE
        )
        record = ArtifactRecoveryOperation(
            restore_operation_id=request.restore_operation_id,
            artifact_set_id=artifact_set.id,
            model_revision_id=revision.id,
            recovery_class=recovery_class.value,
            state=ArtifactRecoveryState.PLANNED.value,
            upstream_repository=repository,
            upstream_commit_sha=commit_sha,
            target_storage_root_id=storage_root.id,
            expected_files=sorted(expected, key=lambda item: item["filename"]),
            verified_files=[],
            bytes_total=sum(int(item["size_bytes"]) for item in expected),
            lineage=_artifact_lineage(self.session, artifact_set, revision),
            reason=request.reason,
            created_by=request.created_by,
        )
        if recovery_class is ArtifactRecoveryClass.NON_REHYDRATABLE:
            record.state = ArtifactRecoveryState.BLOCKED.value
            record.failure_code = RecoveryFailureCode.ARTIFACT_NOT_REHYDRATABLE.value
            record.failure_reason = (
                "artifact set has no exact upstream revision; it must be protected by a payload "
                "backup rather than rehydration"
            )
        self.session.add(record)
        self.session.flush()
        self.audit.write(
            "ARTIFACT_REHYDRATION_STARTED",
            AUDIT_RESOURCE,
            str(record.id),
            {
                "artifact_set_id": str(artifact_set.id),
                "recovery_class": recovery_class.value,
                "commit_sha": commit_sha,
                "file_count": len(expected),
            },
        )
        self.session.commit()
        return self._artifact_response(record)

    def record_artifact_recovery(
        self,
        operation_id: uuid.UUID,
        *,
        state: ArtifactRecoveryState,
        verified_files: list[dict[str, Any]],
        bytes_recovered: int,
        duration_seconds: float | None,
        download_plan_id: uuid.UUID | None = None,
        artifact_job_id: uuid.UUID | None = None,
        failure_code: str | None = None,
        failure_reason: str | None = None,
    ) -> ArtifactRecoveryResponse:
        record = self.session.get(ArtifactRecoveryOperation, operation_id)
        if record is None:
            raise RecoveryError(
                404, "artifact_recovery_not_found", "artifact recovery operation does not exist"
            )
        expected = {item["filename"]: item for item in record.expected_files or []}
        mismatches = [
            item["filename"]
            for item in verified_files
            if item["filename"] in expected
            and item.get("sha256") != expected[item["filename"]]["sha256"]
        ]
        if mismatches and state is ArtifactRecoveryState.RECOVERED:
            state = ArtifactRecoveryState.FAILED
            failure_code = RecoveryFailureCode.ARTIFACT_HASH_MISMATCH.value
            failure_reason = f"rehydrated files did not match provenance: {', '.join(mismatches)}"
        record.state = state.value
        record.verified_files = verified_files
        record.bytes_recovered = bytes_recovered
        record.duration_seconds = duration_seconds
        record.download_plan_id = download_plan_id
        record.artifact_job_id = artifact_job_id
        record.failure_code = failure_code
        record.failure_reason = failure_reason
        record.completed_at = _now() if state in {
            ArtifactRecoveryState.RECOVERED,
            ArtifactRecoveryState.FAILED,
            ArtifactRecoveryState.BLOCKED,
        } else None
        self.audit.write(
            "ARTIFACT_REHYDRATED" if state is ArtifactRecoveryState.RECOVERED else "RESTORE_FAILED",
            AUDIT_RESOURCE,
            str(record.id),
            {
                "artifact_set_id": str(record.artifact_set_id),
                "state": state.value,
                "bytes_recovered": bytes_recovered,
                "failure_code": failure_code,
            },
            outcome="success" if state is ArtifactRecoveryState.RECOVERED else "failure",
        )
        self.session.commit()
        return self._artifact_response(record)

    def artifact_recoveries(self, limit: int = 100) -> list[ArtifactRecoveryResponse]:
        return [
            self._artifact_response(item)
            for item in self.session.scalars(
                select(ArtifactRecoveryOperation)
                .order_by(ArtifactRecoveryOperation.started_at.desc())
                .limit(limit)
            )
        ]

    def _artifact_response(self, record: ArtifactRecoveryOperation) -> ArtifactRecoveryResponse:
        return ArtifactRecoveryResponse(
            id=record.id,
            restore_operation_id=record.restore_operation_id,
            artifact_set_id=record.artifact_set_id,
            model_revision_id=record.model_revision_id,
            recovery_class=ArtifactRecoveryClass(record.recovery_class),
            state=ArtifactRecoveryState(record.state),
            upstream_repository=record.upstream_repository,
            upstream_commit_sha=record.upstream_commit_sha,
            target_storage_root_id=record.target_storage_root_id,
            expected_files=list(record.expected_files or []),
            verified_files=list(record.verified_files or []),
            bytes_total=record.bytes_total,
            bytes_recovered=record.bytes_recovered,
            download_plan_id=record.download_plan_id,
            artifact_job_id=record.artifact_job_id,
            lineage=dict(record.lineage or {}),
            duration_seconds=record.duration_seconds,
            failure_code=record.failure_code,
            failure_reason=record.failure_reason,
            started_at=record.started_at,
            completed_at=record.completed_at,
        )

    # ------------------------------------------------------------------ readiness

    def fingerprint(self) -> dict[str, Any]:
        return fingerprint_session(self.session)

    def readiness(self) -> list[RecoveryReadinessEntry]:
        latest = self._latest_verified_backup()
        entries: list[RecoveryReadinessEntry] = []
        for record in self.session.scalars(
            select(RecoveryAssetRecord).order_by(
                RecoveryAssetRecord.asset_class, RecoveryAssetRecord.key
            )
        ):
            declared = RecoveryReadiness(record.readiness)
            asset_class = RecoveryAssetClass(record.asset_class)
            readiness = declared
            detail = record.notes or record.restore_method
            if asset_class is RecoveryAssetClass.AUTHORITATIVE:
                if latest is None:
                    readiness = RecoveryReadiness.UNPROTECTED
                    detail = "no verified backup exists yet"
                else:
                    detail = f"protected by verified backup {latest.backup_id}"
            entries.append(
                RecoveryReadinessEntry(
                    asset_key=record.key,
                    asset_name=record.name,
                    asset_class=asset_class,
                    readiness=readiness,
                    policy_key=record.policy_revision.key,
                    rpo_seconds=record.rpo_seconds,
                    detail=detail,
                )
            )
        return entries

    def _latest_verified_backup(self) -> BackupSet | None:
        return self.session.scalar(
            select(BackupSet)
            .where(BackupSet.state == BackupState.VERIFIED.value)
            .order_by(BackupSet.verified_at.desc())
            .limit(1)
        )

    def dashboard(self) -> RecoveryDashboard:
        now = _now()
        latest = self._latest_verified_backup()
        verified_at = _aware(latest.verified_at) if latest else None
        age = (now - verified_at).total_seconds() if verified_at else None
        states: dict[str, int] = {}
        for state, count in self.session.execute(
            select(BackupSet.state, func.count()).group_by(BackupSet.state)
        ):
            states[str(state)] = int(count)
        readiness = self.readiness()
        unprotected = [
            item.asset_key for item in readiness if item.readiness is RecoveryReadiness.UNPROTECTED
        ]
        protected = sum(
            item.readiness
            in {
                RecoveryReadiness.PROTECTED,
                RecoveryReadiness.REHYDRATABLE,
                RecoveryReadiness.EXTERNAL_DEPENDENCY,
            }
            for item in readiness
        )
        last_rehearsal = self.session.scalar(
            select(RestoreOperation).order_by(RestoreOperation.started_at.desc()).limit(1)
        )
        capacity = self.estimate_capacity()
        artifact_bytes = int(
            self.session.scalar(select(func.coalesce(func.sum(ModelArtifact.size_bytes), 0))) or 0
        )
        return RecoveryDashboard(
            observed_at=now,
            point_in_time_support=POINT_IN_TIME_SUPPORT,
            latest_verified_backup_id=latest.backup_id if latest else None,
            latest_verified_backup_at=verified_at,
            latest_verified_backup_age_seconds=age,
            latest_verified_schema_revision=latest.schema_revision if latest else None,
            backup_states=states,
            verified_backup_count=states.get(BackupState.VERIFIED.value, 0),
            stale_backup=age is None or age > self.settings.backup_stale_after_seconds,
            backup_staleness_threshold_seconds=self.settings.backup_stale_after_seconds,
            last_restore_rehearsal_at=_aware(last_rehearsal.started_at) if last_rehearsal else None,
            last_restore_rehearsal_state=last_rehearsal.state if last_rehearsal else None,
            observed_restore_seconds=last_rehearsal.rto_seconds if last_rehearsal else None,
            observed_rpo_seconds=last_rehearsal.rpo_seconds if last_rehearsal else None,
            protected_asset_count=protected,
            unprotected_assets=unprotected,
            readiness=readiness,
            coverage_ratio=round(protected / len(readiness), 4) if readiness else 0.0,
            estimated_protected_bytes=capacity.estimated_protected_bytes,
            estimated_rehydratable_bytes=artifact_bytes,
            destination_capacity_bytes=capacity.capacity_bytes,
            destination_free_bytes=capacity.available_bytes,
        )


def _RECONCILE_MARKER_QUERY_MARKER() -> str:
    """The exactly-once reconciliation marker is scoped to one restore operation id."""

    return (
        "select count(*) from audit_events where action = 'RECOVERY_RECONCILIATION_COMPLETED' "
        "and resource_id = '<restore_operation_id>'"
    )


def _verification_failure(
    code: RecoveryFailureCode, reason: str, checks: dict[str, str]
) -> dict[str, Any]:
    return {
        "ok": False,
        "checks": checks,
        "failure_code": code.value,
        "failure_reason": reason,
        "verified_at": None,
    }


def _public_verification(details: dict[str, Any] | None) -> dict[str, Any]:
    """Fingerprints are large; the API exposes their digests rather than every table row hash."""

    payload = dict(details or {})
    source = payload.get("source_fingerprint")
    if isinstance(source, dict):
        payload["source_fingerprint"] = _fingerprint_summary(source)
    return payload


def _fingerprint_summary(fingerprint: dict[str, Any] | None) -> dict[str, Any]:
    payload = dict(fingerprint or {})
    if not payload:
        return {}
    groups = payload.get("groups", {})
    return {
        "version": payload.get("version"),
        "digest": payload.get("digest"),
        "row_total": payload.get("row_total"),
        "groups": {
            key: {"digest": value.get("digest"), "row_count": value.get("row_count")}
            for key, value in groups.items()
        },
    }


def _same_database(left: str, right: str) -> bool:
    try:
        first = parse_target(left)
        second = parse_target(right)
    except PostgresToolError:
        return left.strip() == right.strip()
    return (first.host, first.port, first.database) == (second.host, second.port, second.database)


def _revision_identity(session: Session, revision: ModelRevision) -> tuple[str | None, str | None]:
    """Rehydration needs the exact upstream repository and commit, never a floating ref."""

    commit_sha: str | None = revision.resolved_commit_sha or None
    snapshot = session.scalar(
        select(UpstreamSnapshot)
        .where(
            UpstreamSnapshot.model_id == revision.model_id,
            UpstreamSnapshot.resolved_commit_sha == revision.resolved_commit_sha,
        )
        .order_by(UpstreamSnapshot.observed_at.desc())
        .limit(1)
    )
    if snapshot is not None:
        return snapshot.repository_id, commit_sha or snapshot.resolved_commit_sha
    model = session.get(Model, revision.model_id)
    return (model.upstream_source if model else None), commit_sha


def _artifact_lineage(
    session: Session, artifact_set: ArtifactSet, revision: ModelRevision
) -> dict[str, Any]:
    derived = [
        {
            "derived_artifact_id": str(link.derived_artifact_id),
            "source_artifact_id": str(link.source_artifact_id),
            "source_sha256": link.source_sha256,
        }
        for link in session.scalars(
            select(DerivedArtifactSource).join(
                ModelArtifact, ModelArtifact.id == DerivedArtifactSource.source_artifact_id
            ).where(ModelArtifact.revision_id == revision.id)
        )
    ]
    return {
        "artifact_set_id": str(artifact_set.id),
        "artifact_set_variant": artifact_set.variant_key,
        "snapshot_id": str(artifact_set.snapshot_id),
        "model_revision_id": str(revision.id),
        "model_id": str(revision.model_id),
        "upstream_revision": revision.upstream_revision,
        "resolved_commit_sha": revision.resolved_commit_sha,
        "derived_lineage": derived,
        "lineage_preserved": True,
    }


_REVISION_PATTERN = re.compile(r"^revision(?::\s*str)?\s*=\s*[\"']([^\"']+)[\"']")
_DOWN_REVISION_PATTERN = re.compile(
    r"^down_revision(?::[^=]*)?\s*=\s*(?:[\"']([^\"']+)[\"']|None)"
)


def _alembic_root(configured: Path | None = None) -> Path | None:
    """Locate the shipped Alembic tree in both the source checkout and the installed image."""

    candidates: list[Path] = []
    if configured is not None:
        candidates.append(Path(configured))
    candidates.extend(
        [
            Path.cwd() / "alembic",
            Path(__file__).resolve().parents[3] / "alembic",
            Path(__file__).resolve().parents[4] / "backend" / "alembic",
            Path("/app/alembic"),
        ]
    )
    for candidate in candidates:
        if (candidate / "versions").is_dir():
            return candidate
    return None


@lru_cache(maxsize=8)
def _known_alembic_revisions(root: Path | None = None) -> tuple[str, ...]:
    """The Alembic chain this build ships, oldest first, walked by down_revision.

    Filename order is not migration order, and a backup taken from a newer build must be
    recognised as SCHEMA_TOO_NEW rather than silently accepted. An empty chain means the
    build cannot prove compatibility at all, which is itself a fail-closed condition.
    """

    resolved = _alembic_root(root)
    if resolved is None:
        return ()
    directory = resolved / "versions"
    if not directory.is_dir():
        return ()
    parents: dict[str, str | None] = {}
    for path in sorted(directory.glob("*.py")):
        revision: str | None = None
        down: str | None = None
        for line in path.read_text("utf-8").splitlines():
            match = _REVISION_PATTERN.match(line)
            if match and revision is None:
                revision = match.group(1)
                continue
            down_match = _DOWN_REVISION_PATTERN.match(line)
            if down_match and revision is not None:
                down = down_match.group(1)
                break
        if revision:
            parents[revision] = down
    children = {down: revision for revision, down in parents.items() if down is not None}
    ordered: list[str] = []
    current = next((revision for revision, down in parents.items() if down is None), None)
    while current is not None and current not in ordered:
        ordered.append(current)
        current = children.get(current)
    if len(ordered) != len(parents):  # pragma: no cover - a broken chain is a packaging error
        return tuple(sorted(parents))
    return tuple(ordered)


def _run_alembic_upgrade(database_url: str, root: Path | None = None) -> None:
    """Alembic is imported lazily so it stays out of the request hot path."""

    from alembic import command, config

    resolved = _alembic_root(root)
    if resolved is None:
        raise RecoveryError(
            500,
            RecoveryFailureCode.SCHEMA_UNKNOWN.value,
            "this ModelForge build ships no Alembic tree and cannot migrate a restored database",
        )
    alembic_config = config.Config(str(resolved.parent / "alembic.ini"))
    alembic_config.set_main_option("script_location", str(resolved))
    alembic_config.set_main_option("sqlalchemy.url", database_url)
    command.upgrade(alembic_config, "head")


def _git(*args: str) -> str | None:
    import subprocess

    root = Path(__file__).resolve().parents[4]
    try:
        completed = subprocess.run(  # noqa: S603 - fixed argv, no shell
            ["git", "-C", str(root), *args],  # noqa: S607 - git resolves from PATH
            capture_output=True,
            text=True,
            timeout=15,
            check=False,
        )
    except (FileNotFoundError, OSError, ValueError):
        return None
    value = (completed.stdout or "").strip()
    return value if completed.returncode == 0 and value else None


def _source_commit() -> str | None:
    return os.environ.get("MODELFORGE_SOURCE_COMMIT") or _git("rev-parse", "HEAD")


def _source_reference() -> str | None:
    return os.environ.get("MODELFORGE_SOURCE_REFERENCE") or _git(
        "rev-parse", "--abbrev-ref", "HEAD"
    )


def _source_repository() -> str | None:
    return os.environ.get("MODELFORGE_SOURCE_REPOSITORY") or _git(
        "config", "--get", "remote.origin.url"
    )


_DEFAULT_POLICIES: tuple[RecoveryPolicyCreate, ...] = (
    RecoveryPolicyCreate(
        key="control-plane.database",
        name="Control-plane PostgreSQL",
        asset_class=RecoveryAssetClass.AUTHORITATIVE,
        backup_method=BackupMethod.POSTGRES_LOGICAL_CUSTOM,
        retention_days=30,
        minimum_verified_backups=2,
        rpo_seconds=86_400,
        rto_target_seconds=3_600,
        restore_verification="FULL_RESTORE",
        encryption_required=True,
        rationale=(
            "Provenance, lifecycle, migration, approval and audit truth exist nowhere else. "
            "A consistent pg_dump snapshot bounds data loss to the backup interval and every "
            "backup must be restorable into a fresh database before it counts."
        ),
        created_by="modelforge",
    ),
    RecoveryPolicyCreate(
        key="artifacts.rehydratable",
        name="Rehydratable Hugging Face artifacts",
        asset_class=RecoveryAssetClass.REBUILDABLE,
        backup_method=BackupMethod.MANIFEST_ONLY,
        retention_days=30,
        minimum_verified_backups=2,
        rpo_seconds=None,
        rto_target_seconds=7_200,
        restore_verification="MANIFEST_ONLY",
        encryption_required=False,
        rehydration_allowed=True,
        rationale=(
            "Model weights are large and reproducible from an exact upstream commit plus the "
            "recorded per-file SHA-256. Recovery redownloads into quarantine and re-runs the "
            "supply-chain checks rather than copying hundreds of gigabytes on every backup."
        ),
        created_by="modelforge",
    ),
    RecoveryPolicyCreate(
        key="artifacts.non-rehydratable",
        name="Local and derived artifacts",
        asset_class=RecoveryAssetClass.AUTHORITATIVE,
        backup_method=BackupMethod.FILE_COPY,
        retention_days=30,
        minimum_verified_backups=1,
        rpo_seconds=604_800,
        rto_target_seconds=7_200,
        restore_verification="HASH_ONLY",
        encryption_required=True,
        rationale=(
            "Artifacts without an exact upstream identity, including derived and quantized "
            "outputs, cannot be redownloaded and therefore require a payload backup that "
            "preserves their lineage back to the exact source artifacts."
        ),
        created_by="modelforge",
    ),
    RecoveryPolicyCreate(
        key="runtime.ephemeral",
        name="Queues, leases and runtime residency",
        asset_class=RecoveryAssetClass.EPHEMERAL,
        backup_method=BackupMethod.NOT_BACKED_UP,
        retention_days=1,
        minimum_verified_backups=1,
        rpo_seconds=None,
        rto_target_seconds=300,
        restore_verification="NOT_APPLICABLE",
        encryption_required=False,
        rationale=(
            "Redis queues, GPU leases, residency and NVML telemetry describe the present, not "
            "history. After a restore they must be re-measured; restoring them would present a "
            "stale snapshot as current truth and could replay finished work."
        ),
        created_by="modelforge",
    ),
    RecoveryPolicyCreate(
        key="external.projects",
        name="External project and workload state",
        asset_class=RecoveryAssetClass.EXTERNAL,
        backup_method=BackupMethod.NOT_BACKED_UP,
        retention_days=1,
        minimum_verified_backups=1,
        rpo_seconds=None,
        rto_target_seconds=None,
        restore_verification="NOT_APPLICABLE",
        encryption_required=False,
        external_dependency=True,
        rationale=(
            "ExampleRAG Qdrant, ExampleVision, Ollama, Plex, Tdarr and Gitea are owned by their own "
            "projects. ModelForge records the dependency and never claims to back them up or "
            "modifies them during a recovery."
        ),
        created_by="modelforge",
    ),
    RecoveryPolicyCreate(
        key="secrets.credentials",
        name="Node, project and operator credentials",
        asset_class=RecoveryAssetClass.SECRET,
        backup_method=BackupMethod.MANIFEST_ONLY,
        retention_days=30,
        minimum_verified_backups=1,
        rpo_seconds=None,
        rto_target_seconds=1_800,
        restore_verification="NOT_APPLICABLE",
        encryption_required=True,
        secret_class=SecretRecoveryClass.ROTATABLE_SECRET,
        rationale=(
            "ModelForge stores only credential hashes. A restore reinstates the hashes as "
            "authoritative state, but plaintext cannot be recovered: consumers rotate. Backup "
            "encryption keys and operator break-glass secrets are never written into a backup."
        ),
        created_by="modelforge",
    ),
    RecoveryPolicyCreate(
        key="configuration.source-controlled",
        name="Source-controlled configuration and deployment manifests",
        asset_class=RecoveryAssetClass.REBUILDABLE,
        backup_method=BackupMethod.SOURCE_CONTROL_REFERENCE,
        retention_days=30,
        minimum_verified_backups=1,
        rpo_seconds=None,
        rto_target_seconds=900,
        restore_verification="HASH_ONLY",
        encryption_required=False,
        rehydration_allowed=True,
        rationale=(
            "Compose files, capability/project manifests and policy files live in Git. The "
            "backup records the canonical repository, commit and per-file hashes so a rebuild "
            "can prove it started from the same configuration."
        ),
        created_by="modelforge",
    ),
)


_DEFAULT_ASSETS: tuple[dict[str, Any], ...] = (
    {
        "key": "postgres.modelforge",
        "name": "ModelForge PostgreSQL database",
        "policy_key": "control-plane.database",
        "owner": "modelforge-control-plane",
        "location": "postgres:5432/modelforge",
        "restore_method": "pg_restore of a verified custom-format dump into a fresh database",
        "rebuild_method": None,
        "readiness": RecoveryReadiness.PROTECTED.value,
        "dependencies": ["backup.destination"],
        "notes": "Provenance, lifecycle, migration, approvals, audit and observability history.",
    },
    {
        "key": "artifacts.huggingface",
        "name": "Hugging Face model artifacts",
        "policy_key": "artifacts.rehydratable",
        "owner": "node-agent",
        "location": "GPU Node storage roots under the configured artifact root",
        "restore_method": "exact-revision redownload into quarantine, verify, then promote",
        "rebuild_method": "ModelRevision + commit SHA + expected file list + SHA-256",
        "readiness": RecoveryReadiness.REHYDRATABLE.value,
        "dependencies": ["postgres.modelforge", "external.huggingface"],
        "notes": "Never rehydrated from a floating revision; upstream outage reports blocked.",
    },
    {
        "key": "artifacts.derived",
        "name": "Derived and local-only artifacts",
        "policy_key": "artifacts.non-rehydratable",
        "owner": "node-agent",
        "location": "GPU Node storage roots",
        "restore_method": "payload restore preserving derived lineage to exact source artifacts",
        "rebuild_method": "re-derivation from the exact source artifacts and transformation identity",
        "readiness": RecoveryReadiness.PROTECTED.value,
        "dependencies": ["postgres.modelforge"],
        "notes": "No derived artifact may lose its lineage during a rebuild.",
    },
    {
        "key": "redis.queues",
        "name": "Redis queues and transient payloads",
        "policy_key": "runtime.ephemeral",
        "owner": "modelforge-control-plane",
        "location": "redis:6379/0",
        "restore_method": "not restored; queues start empty and leases are reconciled",
        "rebuild_method": "reconciliation from authoritative serving state",
        "readiness": RecoveryReadiness.PROTECTED.value,
        "dependencies": [],
        "notes": "Contains no authoritative state; a restore must not replay stale jobs.",
    },
    {
        "key": "runtime.residency",
        "name": "GPU leases, residency and NVML telemetry",
        "policy_key": "runtime.ephemeral",
        "owner": "scheduler",
        "location": "control-plane database current-truth tables",
        "restore_method": "cleared during reconciliation and re-measured from live GPU Node state",
        "rebuild_method": "Node Agent inventory and NVML polling",
        "readiness": RecoveryReadiness.PROTECTED.value,
        "dependencies": ["node.gpu_node"],
        "notes": "Restoring a stale NVML snapshot as current truth is explicitly forbidden.",
    },
    {
        "key": "node.gpu_node",
        "name": "GPU Node Node Agent identity and credential",
        "policy_key": "secrets.credentials",
        "owner": "node-agent",
        "location": "compute_nodes / node_credentials (hashes only)",
        "restore_method": "restored credential hash stays valid, or revoke and re-enroll",
        "rebuild_method": "operator-issued enrollment token and a new node credential",
        "readiness": RecoveryReadiness.ROTATION_REQUIRED.value,
        "dependencies": ["postgres.modelforge"],
        "notes": "Persisted node identity prevents a duplicate node claiming the same hardware.",
    },
    {
        "key": "credentials.project",
        "name": "Project gateway credentials",
        "policy_key": "secrets.credentials",
        "owner": "modelforge-control-plane",
        "location": "service_clients / service_credentials (hashes only)",
        "restore_method": "hashes restore as authoritative state; plaintext is unrecoverable",
        "rebuild_method": "issue a new scoped credential to the consuming project",
        "readiness": RecoveryReadiness.ROTATION_REQUIRED.value,
        "dependencies": ["postgres.modelforge"],
        "notes": "A consumer that lost its secret needs a new one; the hash alone cannot serve it.",
    },
    {
        "key": "credentials.operator",
        "name": "Operator API key and backup encryption key",
        "policy_key": "secrets.credentials",
        "owner": "operator",
        "location": "deployment secret mounts and environment",
        "restore_method": "never included in a backup; supplied by the operator during recovery",
        "rebuild_method": "break-glass bootstrap documented in the control-plane loss runbook",
        "readiness": RecoveryReadiness.ROTATION_REQUIRED.value,
        "dependencies": [],
        "notes": "There is no default administrator password anywhere in ModelForge.",
    },
    {
        "key": "configuration.repository",
        "name": "ModelForge source and deployment manifests",
        "policy_key": "configuration.source-controlled",
        "owner": "operator",
        "location": "canonical Gitea repository",
        "restore_method": "checkout of the exact commit recorded in the backup set",
        "rebuild_method": "git clone of the canonical repository at the recorded commit",
        "readiness": RecoveryReadiness.REHYDRATABLE.value,
        "dependencies": ["external.gitea"],
        "notes": "ModelForge does not claim to protect Gitea itself; that is platform operations.",
    },
    {
        "key": "external.examplerag",
        "name": "ExampleRAG Qdrant collections and aliases",
        "policy_key": "external.projects",
        "owner": "examplerag",
        "location": "ExampleRAG-owned Qdrant instance",
        "restore_method": "owned by ExampleRAG; ModelForge recovery never writes to it",
        "rebuild_method": "ExampleRAG reindex using a ModelForge capability",
        "readiness": RecoveryReadiness.EXTERNAL_DEPENDENCY.value,
        "dependencies": [],
        "notes": "A ModelForge restore must leave the production alias untouched.",
    },
    {
        "key": "external.examplevision",
        "name": "ExampleVision recognition state",
        "policy_key": "external.projects",
        "owner": "examplevision",
        "location": "ExampleVision-owned services",
        "restore_method": "owned by ExampleVision; ModelForge recovery never mutates it",
        "rebuild_method": "ExampleVision-owned procedures",
        "readiness": RecoveryReadiness.EXTERNAL_DEPENDENCY.value,
        "dependencies": [],
        "notes": "Production recognizer and VISION_AUTO_ACCEPT stay unchanged during recovery.",
    },
    {
        "key": "external.huggingface",
        "name": "Hugging Face upstream availability",
        "policy_key": "external.projects",
        "owner": "huggingface",
        "location": "https://huggingface.co",
        "restore_method": "not owned by ModelForge",
        "rebuild_method": None,
        "readiness": RecoveryReadiness.EXTERNAL_DEPENDENCY.value,
        "dependencies": [],
        "notes": "If upstream is unavailable, rehydration reports ARTIFACT_REHYDRATION_BLOCKED.",
    },
    {
        "key": "external.gitea",
        "name": "Canonical Gitea repository host",
        "policy_key": "external.projects",
        "owner": "platform-operations",
        "location": "ssh://git@192.0.2.10:222",
        "restore_method": "platform operations; a working clone plus remote suffices for DR drills",
        "rebuild_method": None,
        "readiness": RecoveryReadiness.EXTERNAL_DEPENDENCY.value,
        "dependencies": [],
        "notes": "ModelForge documents this dependency instead of pretending to protect Gitea.",
    },
    {
        "key": "backup.destination",
        "name": "Backup destination root",
        "policy_key": "configuration.source-controlled",
        "owner": "operator",
        "location": "configured MODELFORGE_BACKUP_ROOT",
        "restore_method": "operator-provided storage; paths are allowlisted and traversal-safe",
        "rebuild_method": "provision new storage and re-run a verified backup",
        "readiness": RecoveryReadiness.PROTECTED.value,
        "dependencies": [],
        "notes": "No developer path is hardcoded; the root is deployment configuration.",
    },
)