Initial public ModelForge release

backend/src/modelforge_api/domain/recovery.py

@@ -0,0 +1,471 @@
+"""M15 backup, restore and disaster-recovery contracts.
+
+Recovery is a first-class control-plane concern: what ModelForge owns authoritatively,
+what it can rebuild from an exact upstream identity, what is deliberately discarded and
+what is never exportable are separate classifications with separate guarantees.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from datetime import datetime
+from enum import StrEnum
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+BACKUP_ID_PATTERN = r"^[a-z0-9][a-z0-9-]{6,62}$"
+SHA256_PATTERN = r"^[0-9a-f]{64}$"
+
+# ModelForge stores verified logical snapshots. WAL archiving and continuous point-in-time
+# recovery are deliberately out of scope for v1; the value is exported so operators and the
+# recovery dashboard never have to infer it.
+POINT_IN_TIME_SUPPORT: Literal["SUPPORTED", "NOT_SUPPORTED"] = "NOT_SUPPORTED"
+
+
+class RecoveryAssetClass(StrEnum):
+    """Top-level classification that decides whether state must be copied at all."""
+
+    AUTHORITATIVE = "AUTHORITATIVE"
+    REBUILDABLE = "REBUILDABLE"
+    EPHEMERAL = "EPHEMERAL"
+    EXTERNAL = "EXTERNAL"
+    SECRET = "SECRET"  # noqa: S105 - asset classification, not a credential
+
+
+class SecretRecoveryClass(StrEnum):
+    RESTORABLE_SECRET = "RESTORABLE_SECRET"  # noqa: S105 - asset classification, not a credential
+    ROTATABLE_SECRET = "ROTATABLE_SECRET"  # noqa: S105 - asset classification, not a credential
+    NON_EXPORTABLE_SECRET = "NON_EXPORTABLE_SECRET"  # noqa: S105 - asset classification, not a credential
+
+
+class ArtifactRecoveryClass(StrEnum):
+    REHYDRATABLE = "REHYDRATABLE"
+    NON_REHYDRATABLE = "NON_REHYDRATABLE"
+    DERIVED = "DERIVED"
+    LOCAL_ONLY = "LOCAL_ONLY"
+
+
+class ConfigurationClass(StrEnum):
+    SOURCE_CONTROLLED = "SOURCE_CONTROLLED"
+    SECRET = "SECRET"  # noqa: S105 - asset classification, not a credential
+    GENERATED = "GENERATED"
+    HOST_LOCAL = "HOST_LOCAL"
+
+
+class BackupState(StrEnum):
+    PLANNED = "PLANNED"
+    CREATING = "CREATING"
+    CREATED = "CREATED"
+    VERIFYING = "VERIFYING"
+    VERIFIED = "VERIFIED"
+    FAILED = "FAILED"
+    EXPIRED = "EXPIRED"
+    DELETED = "DELETED"
+
+
+RESTORE_ELIGIBLE_STATES = frozenset({BackupState.VERIFIED})
+BACKUP_TERMINAL_STATES = frozenset({BackupState.FAILED, BackupState.EXPIRED, BackupState.DELETED})
+
+
+class BackupMethod(StrEnum):
+    POSTGRES_LOGICAL_CUSTOM = "POSTGRES_LOGICAL_CUSTOM"
+    MANIFEST_ONLY = "MANIFEST_ONLY"
+    FILE_COPY = "FILE_COPY"
+    SOURCE_CONTROL_REFERENCE = "SOURCE_CONTROL_REFERENCE"
+    NOT_BACKED_UP = "NOT_BACKED_UP"
+
+
+class RestoreMode(StrEnum):
+    """VALIDATION never touches an active target; the destructive modes are operator-gated."""
+
+    VALIDATION = "VALIDATION"
+    REPLACEMENT = "REPLACEMENT"
+    DISASTER_RECOVERY = "DISASTER_RECOVERY"
+
+
+class RestorePlanState(StrEnum):
+    DRAFT = "DRAFT"
+    PREFLIGHT_PASSED = "PREFLIGHT_PASSED"
+    PREFLIGHT_FAILED = "PREFLIGHT_FAILED"
+    CONSUMED = "CONSUMED"
+
+
+class RestoreState(StrEnum):
+    PLANNED = "PLANNED"
+    PREFLIGHT = "PREFLIGHT"
+    RESTORING_DATABASE = "RESTORING_DATABASE"
+    RESTORING_CONFIGURATION = "RESTORING_CONFIGURATION"
+    REHYDRATING_ARTIFACTS = "REHYDRATING_ARTIFACTS"
+    RECONCILING = "RECONCILING"
+    VALIDATING = "VALIDATING"
+    READY = "READY"
+    FAILED = "FAILED"
+    MANUAL_INTERVENTION_REQUIRED = "MANUAL_INTERVENTION_REQUIRED"
+
+
+RESTORE_TERMINAL_STATES = frozenset(
+    {RestoreState.READY, RestoreState.FAILED, RestoreState.MANUAL_INTERVENTION_REQUIRED}
+)
+
+RESTORE_PHASE_ORDER: tuple[RestoreState, ...] = (
+    RestoreState.PREFLIGHT,
+    RestoreState.RESTORING_DATABASE,
+    RestoreState.RESTORING_CONFIGURATION,
+    RestoreState.REHYDRATING_ARTIFACTS,
+    RestoreState.RECONCILING,
+    RestoreState.VALIDATING,
+    RestoreState.READY,
+)
+
+
+class ArtifactRecoveryState(StrEnum):
+    PLANNED = "PLANNED"
+    REHYDRATING = "REHYDRATING"
+    VERIFYING = "VERIFYING"
+    RECOVERED = "RECOVERED"
+    BLOCKED = "BLOCKED"
+    FAILED = "FAILED"
+
+
+class RecoveryReadiness(StrEnum):
+    PROTECTED = "PROTECTED"
+    REHYDRATABLE = "REHYDRATABLE"
+    ROTATION_REQUIRED = "ROTATION_REQUIRED"
+    EXTERNAL_DEPENDENCY = "EXTERNAL_DEPENDENCY"
+    UNPROTECTED = "UNPROTECTED"
+
+
+class RecoveryFailureCode(StrEnum):
+    HASH_MISMATCH = "HASH_MISMATCH"
+    MANIFEST_HASH_MISMATCH = "MANIFEST_HASH_MISMATCH"
+    MANIFEST_INCOMPLETE = "MANIFEST_INCOMPLETE"
+    PAYLOAD_MISSING = "PAYLOAD_MISSING"
+    BACKUP_NOT_RESTORE_ELIGIBLE = "BACKUP_NOT_RESTORE_ELIGIBLE"
+    ENCRYPTION_KEY_UNAVAILABLE = "ENCRYPTION_KEY_UNAVAILABLE"
+    DECRYPTION_FAILED = "DECRYPTION_FAILED"
+    SCHEMA_TOO_NEW = "SCHEMA_TOO_NEW"
+    SCHEMA_UNKNOWN = "SCHEMA_UNKNOWN"
+    POSTGRES_VERSION_INCOMPATIBLE = "POSTGRES_VERSION_INCOMPATIBLE"
+    DESTINATION_NOT_ISOLATED = "DESTINATION_NOT_ISOLATED"
+    DESTINATION_NOT_EMPTY = "DESTINATION_NOT_EMPTY"
+    INSUFFICIENT_CAPACITY = "INSUFFICIENT_CAPACITY"
+    DESTINATION_UNAVAILABLE = "DESTINATION_UNAVAILABLE"
+    BACKUP_TOOL_UNAVAILABLE = "BACKUP_TOOL_UNAVAILABLE"
+    BACKUP_TOOL_FAILED = "BACKUP_TOOL_FAILED"
+    CONCURRENT_OPERATION = "CONCURRENT_OPERATION"
+    PATH_NOT_ALLOWED = "PATH_NOT_ALLOWED"
+    ARCHIVE_UNSAFE = "ARCHIVE_UNSAFE"
+    ARTIFACT_REHYDRATION_BLOCKED = "ARTIFACT_REHYDRATION_BLOCKED"
+    ARTIFACT_HASH_MISMATCH = "ARTIFACT_HASH_MISMATCH"
+    ARTIFACT_NOT_REHYDRATABLE = "ARTIFACT_NOT_REHYDRATABLE"
+    AUDIT_CHAIN_CORRUPT = "AUDIT_CHAIN_CORRUPT"
+    RECOVERY_RECONCILIATION_REQUIRED = "RECOVERY_RECONCILIATION_REQUIRED"
+    SOURCE_CONTROL_REVISION_UNAVAILABLE = "SOURCE_CONTROL_REVISION_UNAVAILABLE"
+
+
+class RecoveryModel(BaseModel):
+    model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+
+class RecoveryPolicyCreate(RecoveryModel):
+    """A versioned recovery contract per asset class; never scattered inline rules."""
+
+    key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$")
+    name: str = Field(min_length=3, max_length=255)
+    asset_class: RecoveryAssetClass
+    backup_method: BackupMethod
+    retention_days: int = Field(ge=1, le=3650)
+    minimum_verified_backups: int = Field(ge=1, le=100)
+    rpo_seconds: int | None = Field(default=None, ge=0, le=2_592_000)
+    rto_target_seconds: int | None = Field(default=None, ge=0, le=2_592_000)
+    restore_verification: Literal["FULL_RESTORE", "HASH_ONLY", "MANIFEST_ONLY", "NOT_APPLICABLE"]
+    encryption_required: bool
+    external_dependency: bool = False
+    rehydration_allowed: bool = False
+    secret_class: SecretRecoveryClass | None = None
+    rationale: str = Field(min_length=10, max_length=4000)
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+    @model_validator(mode="after")
+    def coherent(self) -> RecoveryPolicyCreate:
+        if self.asset_class is RecoveryAssetClass.AUTHORITATIVE:
+            if self.backup_method in {BackupMethod.NOT_BACKED_UP, BackupMethod.MANIFEST_ONLY}:
+                raise ValueError("authoritative state requires a payload-bearing backup method")
+            if self.rpo_seconds is None:
+                raise ValueError("authoritative state requires an explicit RPO target")
+        if self.asset_class is RecoveryAssetClass.EPHEMERAL and self.backup_method not in {
+            BackupMethod.NOT_BACKED_UP,
+            BackupMethod.MANIFEST_ONLY,
+        }:
+            raise ValueError("ephemeral state must not claim a payload backup")
+        if self.asset_class is RecoveryAssetClass.EXTERNAL and not self.external_dependency:
+            raise ValueError("external state must be marked as an external dependency")
+        if self.asset_class is RecoveryAssetClass.SECRET and self.secret_class is None:
+            raise ValueError("secret state requires an explicit secret recovery class")
+        if self.rehydration_allowed and self.asset_class is not RecoveryAssetClass.REBUILDABLE:
+            raise ValueError("only rebuildable state may be rehydrated instead of copied")
+        return self
+
+
+class RecoveryPolicyResponse(RecoveryPolicyCreate):
+    id: uuid.UUID
+    revision: int
+    active: bool
+    fingerprint: str
+    created_at: datetime
+
+
+class RecoveryAssetResponse(RecoveryModel):
+    id: uuid.UUID
+    key: str
+    name: str
+    asset_class: RecoveryAssetClass
+    owner: str
+    location: str
+    backup_method: BackupMethod
+    restore_method: str
+    rebuild_method: str | None
+    rpo_seconds: int | None
+    readiness: RecoveryReadiness
+    dependencies: list[str]
+    notes: str
+    policy_key: str
+    updated_at: datetime
+
+
+class BackupSetCreate(RecoveryModel):
+    backup_id: str = Field(pattern=BACKUP_ID_PATTERN)
+    reason: str = Field(min_length=5, max_length=2000)
+    milestone: str | None = Field(default=None, max_length=64)
+    legal_hold: bool = False
+    include_artifact_manifest: bool = True
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+
+class BackupManifestEntryResponse(RecoveryModel):
+    id: uuid.UUID
+    logical_asset_type: str
+    object_name: str
+    relative_path: str
+    size_bytes: int
+    sha256: str
+    source_generation: str
+    schema_version: str | None
+    dependency_refs: dict[str, Any]
+
+
+class BackupSetResponse(RecoveryModel):
+    id: uuid.UUID
+    backup_id: str
+    state: BackupState
+    policy_key: str
+    policy_revision: int
+    modelforge_version: str
+    modelforge_commit: str | None
+    schema_revision: str | None
+    environment_fingerprint: dict[str, Any]
+    database_identity: dict[str, Any]
+    destination_root: str
+    manifest_relative_path: str | None
+    manifest_sha256: str | None
+    included_asset_classes: list[str]
+    excluded_asset_classes: list[str]
+    payload_bytes: int
+    encrypted: bool
+    encryption_algorithm: str | None
+    encryption_key_id: str | None
+    verification_details: dict[str, Any]
+    verified_at: datetime | None
+    failure_code: str | None
+    failure_reason: str | None
+    milestone: str | None
+    legal_hold: bool
+    restore_eligible: bool
+    reason: str
+    created_by: str
+    started_at: datetime | None
+    completed_at: datetime | None
+    expires_at: datetime | None
+    created_at: datetime
+    entries: list[BackupManifestEntryResponse] = Field(default_factory=list)
+
+
+class RestorePlanCreate(RecoveryModel):
+    backup_set_id: uuid.UUID
+    mode: RestoreMode
+    target_environment: Literal["ISOLATED", "STAGING", "PRODUCTION"]
+    target_label: str = Field(min_length=3, max_length=128)
+    database_destination: str = Field(min_length=8, max_length=1024)
+    artifact_strategy: Literal["NONE", "MANIFEST_ONLY", "REHYDRATE_MISSING", "RESTORE_LOCAL"]
+    secret_strategy: Literal["ROTATE", "RESTORE_HASHES", "MANUAL"]
+    node_strategy: Literal["REUSE_CREDENTIAL", "RE_ENROLL", "NONE"]
+    expected_modelforge_version: str | None = Field(default=None, max_length=64)
+    validation_requirements: dict[str, Any] = Field(default_factory=dict)
+    reason: str = Field(min_length=10, max_length=4000)
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+    @field_validator("database_destination")
+    @classmethod
+    def supported_destination(cls, value: str) -> str:
+        if not value.startswith("postgresql+psycopg://"):
+            raise ValueError("restore destinations must be PostgreSQL SQLAlchemy URLs")
+        return value
+
+    @model_validator(mode="after")
+    def guarded(self) -> RestorePlanCreate:
+        if self.mode is RestoreMode.VALIDATION and self.target_environment == "PRODUCTION":
+            raise ValueError("validation restores must never target a production environment")
+        if self.target_environment == "PRODUCTION" and self.mode is not RestoreMode.DISASTER_RECOVERY:
+            raise ValueError("production targets require an explicit disaster-recovery restore")
+        return self
+
+
+class RestorePlanResponse(RecoveryModel):
+    id: uuid.UUID
+    backup_set_id: uuid.UUID
+    backup_id: str
+    mode: RestoreMode
+    state: RestorePlanState
+    target_environment: str
+    target_label: str
+    database_destination_redacted: str
+    artifact_strategy: str
+    secret_strategy: str
+    node_strategy: str
+    expected_modelforge_version: str | None
+    preflight: dict[str, Any]
+    validation_requirements: dict[str, Any]
+    fingerprint: str
+    reason: str
+    created_by: str
+    created_at: datetime
+
+
+class RestoreOperationResponse(RecoveryModel):
+    id: uuid.UUID
+    plan_id: uuid.UUID
+    backup_set_id: uuid.UUID
+    backup_id: str
+    state: RestoreState
+    mode: RestoreMode
+    attempt: int
+    idempotency_key: str
+    preflight_result: dict[str, Any]
+    phase_durations: dict[str, float]
+    source_fingerprint: dict[str, Any]
+    restored_fingerprint: dict[str, Any]
+    fingerprint_diff: dict[str, Any]
+    validation_result: dict[str, Any]
+    rpo_seconds: float | None
+    rto_seconds: float | None
+    failure_code: str | None
+    failure_reason: str | None
+    started_at: datetime
+    updated_at: datetime
+    ready_at: datetime | None
+
+
+class RestoreOperationEventResponse(RecoveryModel):
+    id: uuid.UUID
+    restore_operation_id: uuid.UUID
+    from_state: str | None
+    to_state: str
+    phase: str
+    actor: str
+    reason: str
+    evidence: dict[str, Any]
+    occurred_at: datetime
+
+
+class RestoreAdvanceRequest(RecoveryModel):
+    actor: str = Field(default="operator", min_length=1, max_length=255)
+    reason: str = Field(min_length=5, max_length=2000)
+    stop_after: RestoreState | None = None
+    simulate_interruption_after: RestoreState | None = None
+
+
+class ArtifactRecoveryCreate(RecoveryModel):
+    artifact_set_id: uuid.UUID
+    target_storage_root_id: uuid.UUID
+    reason: str = Field(min_length=10, max_length=2000)
+    restore_operation_id: uuid.UUID | None = None
+    created_by: str = Field(default="operator", min_length=1, max_length=255)
+
+
+class ArtifactRecoveryResponse(RecoveryModel):
+    id: uuid.UUID
+    restore_operation_id: uuid.UUID | None
+    artifact_set_id: uuid.UUID
+    model_revision_id: uuid.UUID
+    recovery_class: ArtifactRecoveryClass
+    state: ArtifactRecoveryState
+    upstream_repository: str | None
+    upstream_commit_sha: str | None
+    target_storage_root_id: uuid.UUID
+    expected_files: list[dict[str, Any]]
+    verified_files: list[dict[str, Any]]
+    bytes_total: int
+    bytes_recovered: int
+    download_plan_id: uuid.UUID | None
+    artifact_job_id: uuid.UUID | None
+    lineage: dict[str, Any]
+    duration_seconds: float | None
+    failure_code: str | None
+    failure_reason: str | None
+    started_at: datetime
+    completed_at: datetime | None
+
+
+class RecoveryReadinessEntry(RecoveryModel):
+    asset_key: str
+    asset_name: str
+    asset_class: RecoveryAssetClass
+    readiness: RecoveryReadiness
+    policy_key: str
+    rpo_seconds: int | None
+    detail: str
+
+
+class RecoveryDashboard(RecoveryModel):
+    """Every field is measured; unknown values stay null instead of becoming a fiction."""
+
+    observed_at: datetime
+    point_in_time_support: Literal["SUPPORTED", "NOT_SUPPORTED"]
+    latest_verified_backup_id: str | None
+    latest_verified_backup_at: datetime | None
+    latest_verified_backup_age_seconds: float | None
+    latest_verified_schema_revision: str | None
+    backup_states: dict[str, int]
+    verified_backup_count: int
+    stale_backup: bool
+    backup_staleness_threshold_seconds: int
+    last_restore_rehearsal_at: datetime | None
+    last_restore_rehearsal_state: str | None
+    observed_restore_seconds: float | None
+    observed_rpo_seconds: float | None
+    protected_asset_count: int
+    unprotected_assets: list[str]
+    readiness: list[RecoveryReadinessEntry]
+    coverage_ratio: float
+    estimated_protected_bytes: int
+    estimated_rehydratable_bytes: int
+    destination_capacity_bytes: int | None
+    destination_free_bytes: int | None
+
+
+class BackupCapacityEstimate(RecoveryModel):
+    bytes_to_copy: int
+    bytes_manifest_only: int
+    estimated_protected_bytes: int
+    available_bytes: int | None
+    capacity_bytes: int | None
+    sufficient: bool
+    detail: str
+
+
+def redact_database_url(url: str) -> str:
+    """Never journal or return a restore DSN that still carries its password."""
+
+    return re.sub(r"://([^:/@]+):[^@]*@", r"://\1:***@", url)