"""M15 backup, restore and disaster-recovery contracts. Recovery is a first-class control-plane concern: what ModelForge owns authoritatively, what it can rebuild from an exact upstream identity, what is deliberately discarded and what is never exportable are separate classifications with separate guarantees. """ from __future__ import annotations import re import uuid from datetime import datetime from enum import StrEnum from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator BACKUP_ID_PATTERN = r"^[a-z0-9][a-z0-9-]{6,62}$" SHA256_PATTERN = r"^[0-9a-f]{64}$" # ModelForge stores verified logical snapshots. WAL archiving and continuous point-in-time # recovery are deliberately out of scope for v1; the value is exported so operators and the # recovery dashboard never have to infer it. POINT_IN_TIME_SUPPORT: Literal["SUPPORTED", "NOT_SUPPORTED"] = "NOT_SUPPORTED" class RecoveryAssetClass(StrEnum): """Top-level classification that decides whether state must be copied at all.""" AUTHORITATIVE = "AUTHORITATIVE" REBUILDABLE = "REBUILDABLE" EPHEMERAL = "EPHEMERAL" EXTERNAL = "EXTERNAL" SECRET = "SECRET" # noqa: S105 - asset classification, not a credential class SecretRecoveryClass(StrEnum): RESTORABLE_SECRET = "RESTORABLE_SECRET" # noqa: S105 - asset classification, not a credential ROTATABLE_SECRET = "ROTATABLE_SECRET" # noqa: S105 - asset classification, not a credential NON_EXPORTABLE_SECRET = "NON_EXPORTABLE_SECRET" # noqa: S105 - asset classification, not a credential class ArtifactRecoveryClass(StrEnum): REHYDRATABLE = "REHYDRATABLE" NON_REHYDRATABLE = "NON_REHYDRATABLE" DERIVED = "DERIVED" LOCAL_ONLY = "LOCAL_ONLY" class ConfigurationClass(StrEnum): SOURCE_CONTROLLED = "SOURCE_CONTROLLED" SECRET = "SECRET" # noqa: S105 - asset classification, not a credential GENERATED = "GENERATED" HOST_LOCAL = "HOST_LOCAL" class BackupState(StrEnum): PLANNED = "PLANNED" CREATING = "CREATING" CREATED = "CREATED" VERIFYING = "VERIFYING" VERIFIED = "VERIFIED" FAILED = "FAILED" EXPIRED = "EXPIRED" DELETED = "DELETED" RESTORE_ELIGIBLE_STATES = frozenset({BackupState.VERIFIED}) BACKUP_TERMINAL_STATES = frozenset({BackupState.FAILED, BackupState.EXPIRED, BackupState.DELETED}) class BackupMethod(StrEnum): POSTGRES_LOGICAL_CUSTOM = "POSTGRES_LOGICAL_CUSTOM" MANIFEST_ONLY = "MANIFEST_ONLY" FILE_COPY = "FILE_COPY" SOURCE_CONTROL_REFERENCE = "SOURCE_CONTROL_REFERENCE" NOT_BACKED_UP = "NOT_BACKED_UP" class RestoreMode(StrEnum): """VALIDATION never touches an active target; the destructive modes are operator-gated.""" VALIDATION = "VALIDATION" REPLACEMENT = "REPLACEMENT" DISASTER_RECOVERY = "DISASTER_RECOVERY" class RestorePlanState(StrEnum): DRAFT = "DRAFT" PREFLIGHT_PASSED = "PREFLIGHT_PASSED" PREFLIGHT_FAILED = "PREFLIGHT_FAILED" CONSUMED = "CONSUMED" class RestoreState(StrEnum): PLANNED = "PLANNED" PREFLIGHT = "PREFLIGHT" RESTORING_DATABASE = "RESTORING_DATABASE" RESTORING_CONFIGURATION = "RESTORING_CONFIGURATION" REHYDRATING_ARTIFACTS = "REHYDRATING_ARTIFACTS" RECONCILING = "RECONCILING" VALIDATING = "VALIDATING" READY = "READY" FAILED = "FAILED" MANUAL_INTERVENTION_REQUIRED = "MANUAL_INTERVENTION_REQUIRED" RESTORE_TERMINAL_STATES = frozenset( {RestoreState.READY, RestoreState.FAILED, RestoreState.MANUAL_INTERVENTION_REQUIRED} ) RESTORE_PHASE_ORDER: tuple[RestoreState, ...] = ( RestoreState.PREFLIGHT, RestoreState.RESTORING_DATABASE, RestoreState.RESTORING_CONFIGURATION, RestoreState.REHYDRATING_ARTIFACTS, RestoreState.RECONCILING, RestoreState.VALIDATING, RestoreState.READY, ) class ArtifactRecoveryState(StrEnum): PLANNED = "PLANNED" REHYDRATING = "REHYDRATING" VERIFYING = "VERIFYING" RECOVERED = "RECOVERED" BLOCKED = "BLOCKED" FAILED = "FAILED" class RecoveryReadiness(StrEnum): PROTECTED = "PROTECTED" REHYDRATABLE = "REHYDRATABLE" ROTATION_REQUIRED = "ROTATION_REQUIRED" EXTERNAL_DEPENDENCY = "EXTERNAL_DEPENDENCY" UNPROTECTED = "UNPROTECTED" class RecoveryFailureCode(StrEnum): HASH_MISMATCH = "HASH_MISMATCH" MANIFEST_HASH_MISMATCH = "MANIFEST_HASH_MISMATCH" MANIFEST_INCOMPLETE = "MANIFEST_INCOMPLETE" PAYLOAD_MISSING = "PAYLOAD_MISSING" BACKUP_NOT_RESTORE_ELIGIBLE = "BACKUP_NOT_RESTORE_ELIGIBLE" ENCRYPTION_KEY_UNAVAILABLE = "ENCRYPTION_KEY_UNAVAILABLE" DECRYPTION_FAILED = "DECRYPTION_FAILED" SCHEMA_TOO_NEW = "SCHEMA_TOO_NEW" SCHEMA_UNKNOWN = "SCHEMA_UNKNOWN" POSTGRES_VERSION_INCOMPATIBLE = "POSTGRES_VERSION_INCOMPATIBLE" DESTINATION_NOT_ISOLATED = "DESTINATION_NOT_ISOLATED" DESTINATION_NOT_EMPTY = "DESTINATION_NOT_EMPTY" INSUFFICIENT_CAPACITY = "INSUFFICIENT_CAPACITY" DESTINATION_UNAVAILABLE = "DESTINATION_UNAVAILABLE" BACKUP_TOOL_UNAVAILABLE = "BACKUP_TOOL_UNAVAILABLE" BACKUP_TOOL_FAILED = "BACKUP_TOOL_FAILED" CONCURRENT_OPERATION = "CONCURRENT_OPERATION" PATH_NOT_ALLOWED = "PATH_NOT_ALLOWED" ARCHIVE_UNSAFE = "ARCHIVE_UNSAFE" ARTIFACT_REHYDRATION_BLOCKED = "ARTIFACT_REHYDRATION_BLOCKED" ARTIFACT_HASH_MISMATCH = "ARTIFACT_HASH_MISMATCH" ARTIFACT_NOT_REHYDRATABLE = "ARTIFACT_NOT_REHYDRATABLE" AUDIT_CHAIN_CORRUPT = "AUDIT_CHAIN_CORRUPT" RECOVERY_RECONCILIATION_REQUIRED = "RECOVERY_RECONCILIATION_REQUIRED" SOURCE_CONTROL_REVISION_UNAVAILABLE = "SOURCE_CONTROL_REVISION_UNAVAILABLE" class RecoveryModel(BaseModel): model_config = ConfigDict(extra="forbid", from_attributes=True) class RecoveryPolicyCreate(RecoveryModel): """A versioned recovery contract per asset class; never scattered inline rules.""" key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$") name: str = Field(min_length=3, max_length=255) asset_class: RecoveryAssetClass backup_method: BackupMethod retention_days: int = Field(ge=1, le=3650) minimum_verified_backups: int = Field(ge=1, le=100) rpo_seconds: int | None = Field(default=None, ge=0, le=2_592_000) rto_target_seconds: int | None = Field(default=None, ge=0, le=2_592_000) restore_verification: Literal["FULL_RESTORE", "HASH_ONLY", "MANIFEST_ONLY", "NOT_APPLICABLE"] encryption_required: bool external_dependency: bool = False rehydration_allowed: bool = False secret_class: SecretRecoveryClass | None = None rationale: str = Field(min_length=10, max_length=4000) created_by: str = Field(default="operator", min_length=1, max_length=255) @model_validator(mode="after") def coherent(self) -> RecoveryPolicyCreate: if self.asset_class is RecoveryAssetClass.AUTHORITATIVE: if self.backup_method in {BackupMethod.NOT_BACKED_UP, BackupMethod.MANIFEST_ONLY}: raise ValueError("authoritative state requires a payload-bearing backup method") if self.rpo_seconds is None: raise ValueError("authoritative state requires an explicit RPO target") if self.asset_class is RecoveryAssetClass.EPHEMERAL and self.backup_method not in { BackupMethod.NOT_BACKED_UP, BackupMethod.MANIFEST_ONLY, }: raise ValueError("ephemeral state must not claim a payload backup") if self.asset_class is RecoveryAssetClass.EXTERNAL and not self.external_dependency: raise ValueError("external state must be marked as an external dependency") if self.asset_class is RecoveryAssetClass.SECRET and self.secret_class is None: raise ValueError("secret state requires an explicit secret recovery class") if self.rehydration_allowed and self.asset_class is not RecoveryAssetClass.REBUILDABLE: raise ValueError("only rebuildable state may be rehydrated instead of copied") return self class RecoveryPolicyResponse(RecoveryPolicyCreate): id: uuid.UUID revision: int active: bool fingerprint: str created_at: datetime class RecoveryAssetResponse(RecoveryModel): id: uuid.UUID key: str name: str asset_class: RecoveryAssetClass owner: str location: str backup_method: BackupMethod restore_method: str rebuild_method: str | None rpo_seconds: int | None readiness: RecoveryReadiness dependencies: list[str] notes: str policy_key: str updated_at: datetime class BackupSetCreate(RecoveryModel): backup_id: str = Field(pattern=BACKUP_ID_PATTERN) reason: str = Field(min_length=5, max_length=2000) milestone: str | None = Field(default=None, max_length=64) legal_hold: bool = False include_artifact_manifest: bool = True created_by: str = Field(default="operator", min_length=1, max_length=255) class BackupManifestEntryResponse(RecoveryModel): id: uuid.UUID logical_asset_type: str object_name: str relative_path: str size_bytes: int sha256: str source_generation: str schema_version: str | None dependency_refs: dict[str, Any] class BackupSetResponse(RecoveryModel): id: uuid.UUID backup_id: str state: BackupState policy_key: str policy_revision: int modelforge_version: str modelforge_commit: str | None schema_revision: str | None environment_fingerprint: dict[str, Any] database_identity: dict[str, Any] destination_root: str manifest_relative_path: str | None manifest_sha256: str | None included_asset_classes: list[str] excluded_asset_classes: list[str] payload_bytes: int encrypted: bool encryption_algorithm: str | None encryption_key_id: str | None verification_details: dict[str, Any] verified_at: datetime | None failure_code: str | None failure_reason: str | None milestone: str | None legal_hold: bool restore_eligible: bool reason: str created_by: str started_at: datetime | None completed_at: datetime | None expires_at: datetime | None created_at: datetime entries: list[BackupManifestEntryResponse] = Field(default_factory=list) class RestorePlanCreate(RecoveryModel): backup_set_id: uuid.UUID mode: RestoreMode target_environment: Literal["ISOLATED", "STAGING", "PRODUCTION"] target_label: str = Field(min_length=3, max_length=128) database_destination: str = Field(min_length=8, max_length=1024) artifact_strategy: Literal["NONE", "MANIFEST_ONLY", "REHYDRATE_MISSING", "RESTORE_LOCAL"] secret_strategy: Literal["ROTATE", "RESTORE_HASHES", "MANUAL"] node_strategy: Literal["REUSE_CREDENTIAL", "RE_ENROLL", "NONE"] expected_modelforge_version: str | None = Field(default=None, max_length=64) validation_requirements: dict[str, Any] = Field(default_factory=dict) reason: str = Field(min_length=10, max_length=4000) created_by: str = Field(default="operator", min_length=1, max_length=255) @field_validator("database_destination") @classmethod def supported_destination(cls, value: str) -> str: if not value.startswith("postgresql+psycopg://"): raise ValueError("restore destinations must be PostgreSQL SQLAlchemy URLs") return value @model_validator(mode="after") def guarded(self) -> RestorePlanCreate: if self.mode is RestoreMode.VALIDATION and self.target_environment == "PRODUCTION": raise ValueError("validation restores must never target a production environment") if self.target_environment == "PRODUCTION" and self.mode is not RestoreMode.DISASTER_RECOVERY: raise ValueError("production targets require an explicit disaster-recovery restore") return self class RestorePlanResponse(RecoveryModel): id: uuid.UUID backup_set_id: uuid.UUID backup_id: str mode: RestoreMode state: RestorePlanState target_environment: str target_label: str database_destination_redacted: str artifact_strategy: str secret_strategy: str node_strategy: str expected_modelforge_version: str | None preflight: dict[str, Any] validation_requirements: dict[str, Any] fingerprint: str reason: str created_by: str created_at: datetime class RestoreOperationResponse(RecoveryModel): id: uuid.UUID plan_id: uuid.UUID backup_set_id: uuid.UUID backup_id: str state: RestoreState mode: RestoreMode attempt: int idempotency_key: str preflight_result: dict[str, Any] phase_durations: dict[str, float] source_fingerprint: dict[str, Any] restored_fingerprint: dict[str, Any] fingerprint_diff: dict[str, Any] validation_result: dict[str, Any] rpo_seconds: float | None rto_seconds: float | None failure_code: str | None failure_reason: str | None started_at: datetime updated_at: datetime ready_at: datetime | None class RestoreOperationEventResponse(RecoveryModel): id: uuid.UUID restore_operation_id: uuid.UUID from_state: str | None to_state: str phase: str actor: str reason: str evidence: dict[str, Any] occurred_at: datetime class RestoreAdvanceRequest(RecoveryModel): actor: str = Field(default="operator", min_length=1, max_length=255) reason: str = Field(min_length=5, max_length=2000) stop_after: RestoreState | None = None simulate_interruption_after: RestoreState | None = None class ArtifactRecoveryCreate(RecoveryModel): artifact_set_id: uuid.UUID target_storage_root_id: uuid.UUID reason: str = Field(min_length=10, max_length=2000) restore_operation_id: uuid.UUID | None = None created_by: str = Field(default="operator", min_length=1, max_length=255) class ArtifactRecoveryResponse(RecoveryModel): id: uuid.UUID restore_operation_id: uuid.UUID | None artifact_set_id: uuid.UUID model_revision_id: uuid.UUID recovery_class: ArtifactRecoveryClass state: ArtifactRecoveryState upstream_repository: str | None upstream_commit_sha: str | None target_storage_root_id: uuid.UUID expected_files: list[dict[str, Any]] verified_files: list[dict[str, Any]] bytes_total: int bytes_recovered: int download_plan_id: uuid.UUID | None artifact_job_id: uuid.UUID | None lineage: dict[str, Any] duration_seconds: float | None failure_code: str | None failure_reason: str | None started_at: datetime completed_at: datetime | None class RecoveryReadinessEntry(RecoveryModel): asset_key: str asset_name: str asset_class: RecoveryAssetClass readiness: RecoveryReadiness policy_key: str rpo_seconds: int | None detail: str class RecoveryDashboard(RecoveryModel): """Every field is measured; unknown values stay null instead of becoming a fiction.""" observed_at: datetime point_in_time_support: Literal["SUPPORTED", "NOT_SUPPORTED"] latest_verified_backup_id: str | None latest_verified_backup_at: datetime | None latest_verified_backup_age_seconds: float | None latest_verified_schema_revision: str | None backup_states: dict[str, int] verified_backup_count: int stale_backup: bool backup_staleness_threshold_seconds: int last_restore_rehearsal_at: datetime | None last_restore_rehearsal_state: str | None observed_restore_seconds: float | None observed_rpo_seconds: float | None protected_asset_count: int unprotected_assets: list[str] readiness: list[RecoveryReadinessEntry] coverage_ratio: float estimated_protected_bytes: int estimated_rehydratable_bytes: int destination_capacity_bytes: int | None destination_free_bytes: int | None class BackupCapacityEstimate(RecoveryModel): bytes_to_copy: int bytes_manifest_only: int estimated_protected_bytes: int available_bytes: int | None capacity_bytes: int | None sufficient: bool detail: str def redact_database_url(url: str) -> str: """Never journal or return a restore DSN that still carries its password.""" return re.sub(r"://([^:/@]+):[^@]*@", r"://\1:***@", url)