472 lines
16 KiB
Python
472 lines
16 KiB
Python
"""M15 backup, restore and disaster-recovery contracts.
|
|
|
|
Recovery is a first-class control-plane concern: what ModelForge owns authoritatively,
|
|
what it can rebuild from an exact upstream identity, what is deliberately discarded and
|
|
what is never exportable are separate classifications with separate guarantees.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import uuid
|
|
from datetime import datetime
|
|
from enum import StrEnum
|
|
from typing import Any, Literal
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
|
|
BACKUP_ID_PATTERN = r"^[a-z0-9][a-z0-9-]{6,62}$"
|
|
SHA256_PATTERN = r"^[0-9a-f]{64}$"
|
|
|
|
# ModelForge stores verified logical snapshots. WAL archiving and continuous point-in-time
|
|
# recovery are deliberately out of scope for v1; the value is exported so operators and the
|
|
# recovery dashboard never have to infer it.
|
|
POINT_IN_TIME_SUPPORT: Literal["SUPPORTED", "NOT_SUPPORTED"] = "NOT_SUPPORTED"
|
|
|
|
|
|
class RecoveryAssetClass(StrEnum):
|
|
"""Top-level classification that decides whether state must be copied at all."""
|
|
|
|
AUTHORITATIVE = "AUTHORITATIVE"
|
|
REBUILDABLE = "REBUILDABLE"
|
|
EPHEMERAL = "EPHEMERAL"
|
|
EXTERNAL = "EXTERNAL"
|
|
SECRET = "SECRET" # noqa: S105 - asset classification, not a credential
|
|
|
|
|
|
class SecretRecoveryClass(StrEnum):
|
|
RESTORABLE_SECRET = "RESTORABLE_SECRET" # noqa: S105 - asset classification, not a credential
|
|
ROTATABLE_SECRET = "ROTATABLE_SECRET" # noqa: S105 - asset classification, not a credential
|
|
NON_EXPORTABLE_SECRET = "NON_EXPORTABLE_SECRET" # noqa: S105 - asset classification, not a credential
|
|
|
|
|
|
class ArtifactRecoveryClass(StrEnum):
|
|
REHYDRATABLE = "REHYDRATABLE"
|
|
NON_REHYDRATABLE = "NON_REHYDRATABLE"
|
|
DERIVED = "DERIVED"
|
|
LOCAL_ONLY = "LOCAL_ONLY"
|
|
|
|
|
|
class ConfigurationClass(StrEnum):
|
|
SOURCE_CONTROLLED = "SOURCE_CONTROLLED"
|
|
SECRET = "SECRET" # noqa: S105 - asset classification, not a credential
|
|
GENERATED = "GENERATED"
|
|
HOST_LOCAL = "HOST_LOCAL"
|
|
|
|
|
|
class BackupState(StrEnum):
|
|
PLANNED = "PLANNED"
|
|
CREATING = "CREATING"
|
|
CREATED = "CREATED"
|
|
VERIFYING = "VERIFYING"
|
|
VERIFIED = "VERIFIED"
|
|
FAILED = "FAILED"
|
|
EXPIRED = "EXPIRED"
|
|
DELETED = "DELETED"
|
|
|
|
|
|
RESTORE_ELIGIBLE_STATES = frozenset({BackupState.VERIFIED})
|
|
BACKUP_TERMINAL_STATES = frozenset({BackupState.FAILED, BackupState.EXPIRED, BackupState.DELETED})
|
|
|
|
|
|
class BackupMethod(StrEnum):
|
|
POSTGRES_LOGICAL_CUSTOM = "POSTGRES_LOGICAL_CUSTOM"
|
|
MANIFEST_ONLY = "MANIFEST_ONLY"
|
|
FILE_COPY = "FILE_COPY"
|
|
SOURCE_CONTROL_REFERENCE = "SOURCE_CONTROL_REFERENCE"
|
|
NOT_BACKED_UP = "NOT_BACKED_UP"
|
|
|
|
|
|
class RestoreMode(StrEnum):
|
|
"""VALIDATION never touches an active target; the destructive modes are operator-gated."""
|
|
|
|
VALIDATION = "VALIDATION"
|
|
REPLACEMENT = "REPLACEMENT"
|
|
DISASTER_RECOVERY = "DISASTER_RECOVERY"
|
|
|
|
|
|
class RestorePlanState(StrEnum):
|
|
DRAFT = "DRAFT"
|
|
PREFLIGHT_PASSED = "PREFLIGHT_PASSED"
|
|
PREFLIGHT_FAILED = "PREFLIGHT_FAILED"
|
|
CONSUMED = "CONSUMED"
|
|
|
|
|
|
class RestoreState(StrEnum):
|
|
PLANNED = "PLANNED"
|
|
PREFLIGHT = "PREFLIGHT"
|
|
RESTORING_DATABASE = "RESTORING_DATABASE"
|
|
RESTORING_CONFIGURATION = "RESTORING_CONFIGURATION"
|
|
REHYDRATING_ARTIFACTS = "REHYDRATING_ARTIFACTS"
|
|
RECONCILING = "RECONCILING"
|
|
VALIDATING = "VALIDATING"
|
|
READY = "READY"
|
|
FAILED = "FAILED"
|
|
MANUAL_INTERVENTION_REQUIRED = "MANUAL_INTERVENTION_REQUIRED"
|
|
|
|
|
|
RESTORE_TERMINAL_STATES = frozenset(
|
|
{RestoreState.READY, RestoreState.FAILED, RestoreState.MANUAL_INTERVENTION_REQUIRED}
|
|
)
|
|
|
|
RESTORE_PHASE_ORDER: tuple[RestoreState, ...] = (
|
|
RestoreState.PREFLIGHT,
|
|
RestoreState.RESTORING_DATABASE,
|
|
RestoreState.RESTORING_CONFIGURATION,
|
|
RestoreState.REHYDRATING_ARTIFACTS,
|
|
RestoreState.RECONCILING,
|
|
RestoreState.VALIDATING,
|
|
RestoreState.READY,
|
|
)
|
|
|
|
|
|
class ArtifactRecoveryState(StrEnum):
|
|
PLANNED = "PLANNED"
|
|
REHYDRATING = "REHYDRATING"
|
|
VERIFYING = "VERIFYING"
|
|
RECOVERED = "RECOVERED"
|
|
BLOCKED = "BLOCKED"
|
|
FAILED = "FAILED"
|
|
|
|
|
|
class RecoveryReadiness(StrEnum):
|
|
PROTECTED = "PROTECTED"
|
|
REHYDRATABLE = "REHYDRATABLE"
|
|
ROTATION_REQUIRED = "ROTATION_REQUIRED"
|
|
EXTERNAL_DEPENDENCY = "EXTERNAL_DEPENDENCY"
|
|
UNPROTECTED = "UNPROTECTED"
|
|
|
|
|
|
class RecoveryFailureCode(StrEnum):
|
|
HASH_MISMATCH = "HASH_MISMATCH"
|
|
MANIFEST_HASH_MISMATCH = "MANIFEST_HASH_MISMATCH"
|
|
MANIFEST_INCOMPLETE = "MANIFEST_INCOMPLETE"
|
|
PAYLOAD_MISSING = "PAYLOAD_MISSING"
|
|
BACKUP_NOT_RESTORE_ELIGIBLE = "BACKUP_NOT_RESTORE_ELIGIBLE"
|
|
ENCRYPTION_KEY_UNAVAILABLE = "ENCRYPTION_KEY_UNAVAILABLE"
|
|
DECRYPTION_FAILED = "DECRYPTION_FAILED"
|
|
SCHEMA_TOO_NEW = "SCHEMA_TOO_NEW"
|
|
SCHEMA_UNKNOWN = "SCHEMA_UNKNOWN"
|
|
POSTGRES_VERSION_INCOMPATIBLE = "POSTGRES_VERSION_INCOMPATIBLE"
|
|
DESTINATION_NOT_ISOLATED = "DESTINATION_NOT_ISOLATED"
|
|
DESTINATION_NOT_EMPTY = "DESTINATION_NOT_EMPTY"
|
|
INSUFFICIENT_CAPACITY = "INSUFFICIENT_CAPACITY"
|
|
DESTINATION_UNAVAILABLE = "DESTINATION_UNAVAILABLE"
|
|
BACKUP_TOOL_UNAVAILABLE = "BACKUP_TOOL_UNAVAILABLE"
|
|
BACKUP_TOOL_FAILED = "BACKUP_TOOL_FAILED"
|
|
CONCURRENT_OPERATION = "CONCURRENT_OPERATION"
|
|
PATH_NOT_ALLOWED = "PATH_NOT_ALLOWED"
|
|
ARCHIVE_UNSAFE = "ARCHIVE_UNSAFE"
|
|
ARTIFACT_REHYDRATION_BLOCKED = "ARTIFACT_REHYDRATION_BLOCKED"
|
|
ARTIFACT_HASH_MISMATCH = "ARTIFACT_HASH_MISMATCH"
|
|
ARTIFACT_NOT_REHYDRATABLE = "ARTIFACT_NOT_REHYDRATABLE"
|
|
AUDIT_CHAIN_CORRUPT = "AUDIT_CHAIN_CORRUPT"
|
|
RECOVERY_RECONCILIATION_REQUIRED = "RECOVERY_RECONCILIATION_REQUIRED"
|
|
SOURCE_CONTROL_REVISION_UNAVAILABLE = "SOURCE_CONTROL_REVISION_UNAVAILABLE"
|
|
|
|
|
|
class RecoveryModel(BaseModel):
|
|
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
|
|
|
|
|
class RecoveryPolicyCreate(RecoveryModel):
|
|
"""A versioned recovery contract per asset class; never scattered inline rules."""
|
|
|
|
key: str = Field(pattern=r"^[a-z][a-z0-9_.-]{2,127}$")
|
|
name: str = Field(min_length=3, max_length=255)
|
|
asset_class: RecoveryAssetClass
|
|
backup_method: BackupMethod
|
|
retention_days: int = Field(ge=1, le=3650)
|
|
minimum_verified_backups: int = Field(ge=1, le=100)
|
|
rpo_seconds: int | None = Field(default=None, ge=0, le=2_592_000)
|
|
rto_target_seconds: int | None = Field(default=None, ge=0, le=2_592_000)
|
|
restore_verification: Literal["FULL_RESTORE", "HASH_ONLY", "MANIFEST_ONLY", "NOT_APPLICABLE"]
|
|
encryption_required: bool
|
|
external_dependency: bool = False
|
|
rehydration_allowed: bool = False
|
|
secret_class: SecretRecoveryClass | None = None
|
|
rationale: str = Field(min_length=10, max_length=4000)
|
|
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
|
|
|
@model_validator(mode="after")
|
|
def coherent(self) -> RecoveryPolicyCreate:
|
|
if self.asset_class is RecoveryAssetClass.AUTHORITATIVE:
|
|
if self.backup_method in {BackupMethod.NOT_BACKED_UP, BackupMethod.MANIFEST_ONLY}:
|
|
raise ValueError("authoritative state requires a payload-bearing backup method")
|
|
if self.rpo_seconds is None:
|
|
raise ValueError("authoritative state requires an explicit RPO target")
|
|
if self.asset_class is RecoveryAssetClass.EPHEMERAL and self.backup_method not in {
|
|
BackupMethod.NOT_BACKED_UP,
|
|
BackupMethod.MANIFEST_ONLY,
|
|
}:
|
|
raise ValueError("ephemeral state must not claim a payload backup")
|
|
if self.asset_class is RecoveryAssetClass.EXTERNAL and not self.external_dependency:
|
|
raise ValueError("external state must be marked as an external dependency")
|
|
if self.asset_class is RecoveryAssetClass.SECRET and self.secret_class is None:
|
|
raise ValueError("secret state requires an explicit secret recovery class")
|
|
if self.rehydration_allowed and self.asset_class is not RecoveryAssetClass.REBUILDABLE:
|
|
raise ValueError("only rebuildable state may be rehydrated instead of copied")
|
|
return self
|
|
|
|
|
|
class RecoveryPolicyResponse(RecoveryPolicyCreate):
|
|
id: uuid.UUID
|
|
revision: int
|
|
active: bool
|
|
fingerprint: str
|
|
created_at: datetime
|
|
|
|
|
|
class RecoveryAssetResponse(RecoveryModel):
|
|
id: uuid.UUID
|
|
key: str
|
|
name: str
|
|
asset_class: RecoveryAssetClass
|
|
owner: str
|
|
location: str
|
|
backup_method: BackupMethod
|
|
restore_method: str
|
|
rebuild_method: str | None
|
|
rpo_seconds: int | None
|
|
readiness: RecoveryReadiness
|
|
dependencies: list[str]
|
|
notes: str
|
|
policy_key: str
|
|
updated_at: datetime
|
|
|
|
|
|
class BackupSetCreate(RecoveryModel):
|
|
backup_id: str = Field(pattern=BACKUP_ID_PATTERN)
|
|
reason: str = Field(min_length=5, max_length=2000)
|
|
milestone: str | None = Field(default=None, max_length=64)
|
|
legal_hold: bool = False
|
|
include_artifact_manifest: bool = True
|
|
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
|
|
|
|
|
class BackupManifestEntryResponse(RecoveryModel):
|
|
id: uuid.UUID
|
|
logical_asset_type: str
|
|
object_name: str
|
|
relative_path: str
|
|
size_bytes: int
|
|
sha256: str
|
|
source_generation: str
|
|
schema_version: str | None
|
|
dependency_refs: dict[str, Any]
|
|
|
|
|
|
class BackupSetResponse(RecoveryModel):
|
|
id: uuid.UUID
|
|
backup_id: str
|
|
state: BackupState
|
|
policy_key: str
|
|
policy_revision: int
|
|
modelforge_version: str
|
|
modelforge_commit: str | None
|
|
schema_revision: str | None
|
|
environment_fingerprint: dict[str, Any]
|
|
database_identity: dict[str, Any]
|
|
destination_root: str
|
|
manifest_relative_path: str | None
|
|
manifest_sha256: str | None
|
|
included_asset_classes: list[str]
|
|
excluded_asset_classes: list[str]
|
|
payload_bytes: int
|
|
encrypted: bool
|
|
encryption_algorithm: str | None
|
|
encryption_key_id: str | None
|
|
verification_details: dict[str, Any]
|
|
verified_at: datetime | None
|
|
failure_code: str | None
|
|
failure_reason: str | None
|
|
milestone: str | None
|
|
legal_hold: bool
|
|
restore_eligible: bool
|
|
reason: str
|
|
created_by: str
|
|
started_at: datetime | None
|
|
completed_at: datetime | None
|
|
expires_at: datetime | None
|
|
created_at: datetime
|
|
entries: list[BackupManifestEntryResponse] = Field(default_factory=list)
|
|
|
|
|
|
class RestorePlanCreate(RecoveryModel):
|
|
backup_set_id: uuid.UUID
|
|
mode: RestoreMode
|
|
target_environment: Literal["ISOLATED", "STAGING", "PRODUCTION"]
|
|
target_label: str = Field(min_length=3, max_length=128)
|
|
database_destination: str = Field(min_length=8, max_length=1024)
|
|
artifact_strategy: Literal["NONE", "MANIFEST_ONLY", "REHYDRATE_MISSING", "RESTORE_LOCAL"]
|
|
secret_strategy: Literal["ROTATE", "RESTORE_HASHES", "MANUAL"]
|
|
node_strategy: Literal["REUSE_CREDENTIAL", "RE_ENROLL", "NONE"]
|
|
expected_modelforge_version: str | None = Field(default=None, max_length=64)
|
|
validation_requirements: dict[str, Any] = Field(default_factory=dict)
|
|
reason: str = Field(min_length=10, max_length=4000)
|
|
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
|
|
|
@field_validator("database_destination")
|
|
@classmethod
|
|
def supported_destination(cls, value: str) -> str:
|
|
if not value.startswith("postgresql+psycopg://"):
|
|
raise ValueError("restore destinations must be PostgreSQL SQLAlchemy URLs")
|
|
return value
|
|
|
|
@model_validator(mode="after")
|
|
def guarded(self) -> RestorePlanCreate:
|
|
if self.mode is RestoreMode.VALIDATION and self.target_environment == "PRODUCTION":
|
|
raise ValueError("validation restores must never target a production environment")
|
|
if self.target_environment == "PRODUCTION" and self.mode is not RestoreMode.DISASTER_RECOVERY:
|
|
raise ValueError("production targets require an explicit disaster-recovery restore")
|
|
return self
|
|
|
|
|
|
class RestorePlanResponse(RecoveryModel):
|
|
id: uuid.UUID
|
|
backup_set_id: uuid.UUID
|
|
backup_id: str
|
|
mode: RestoreMode
|
|
state: RestorePlanState
|
|
target_environment: str
|
|
target_label: str
|
|
database_destination_redacted: str
|
|
artifact_strategy: str
|
|
secret_strategy: str
|
|
node_strategy: str
|
|
expected_modelforge_version: str | None
|
|
preflight: dict[str, Any]
|
|
validation_requirements: dict[str, Any]
|
|
fingerprint: str
|
|
reason: str
|
|
created_by: str
|
|
created_at: datetime
|
|
|
|
|
|
class RestoreOperationResponse(RecoveryModel):
|
|
id: uuid.UUID
|
|
plan_id: uuid.UUID
|
|
backup_set_id: uuid.UUID
|
|
backup_id: str
|
|
state: RestoreState
|
|
mode: RestoreMode
|
|
attempt: int
|
|
idempotency_key: str
|
|
preflight_result: dict[str, Any]
|
|
phase_durations: dict[str, float]
|
|
source_fingerprint: dict[str, Any]
|
|
restored_fingerprint: dict[str, Any]
|
|
fingerprint_diff: dict[str, Any]
|
|
validation_result: dict[str, Any]
|
|
rpo_seconds: float | None
|
|
rto_seconds: float | None
|
|
failure_code: str | None
|
|
failure_reason: str | None
|
|
started_at: datetime
|
|
updated_at: datetime
|
|
ready_at: datetime | None
|
|
|
|
|
|
class RestoreOperationEventResponse(RecoveryModel):
|
|
id: uuid.UUID
|
|
restore_operation_id: uuid.UUID
|
|
from_state: str | None
|
|
to_state: str
|
|
phase: str
|
|
actor: str
|
|
reason: str
|
|
evidence: dict[str, Any]
|
|
occurred_at: datetime
|
|
|
|
|
|
class RestoreAdvanceRequest(RecoveryModel):
|
|
actor: str = Field(default="operator", min_length=1, max_length=255)
|
|
reason: str = Field(min_length=5, max_length=2000)
|
|
stop_after: RestoreState | None = None
|
|
simulate_interruption_after: RestoreState | None = None
|
|
|
|
|
|
class ArtifactRecoveryCreate(RecoveryModel):
|
|
artifact_set_id: uuid.UUID
|
|
target_storage_root_id: uuid.UUID
|
|
reason: str = Field(min_length=10, max_length=2000)
|
|
restore_operation_id: uuid.UUID | None = None
|
|
created_by: str = Field(default="operator", min_length=1, max_length=255)
|
|
|
|
|
|
class ArtifactRecoveryResponse(RecoveryModel):
|
|
id: uuid.UUID
|
|
restore_operation_id: uuid.UUID | None
|
|
artifact_set_id: uuid.UUID
|
|
model_revision_id: uuid.UUID
|
|
recovery_class: ArtifactRecoveryClass
|
|
state: ArtifactRecoveryState
|
|
upstream_repository: str | None
|
|
upstream_commit_sha: str | None
|
|
target_storage_root_id: uuid.UUID
|
|
expected_files: list[dict[str, Any]]
|
|
verified_files: list[dict[str, Any]]
|
|
bytes_total: int
|
|
bytes_recovered: int
|
|
download_plan_id: uuid.UUID | None
|
|
artifact_job_id: uuid.UUID | None
|
|
lineage: dict[str, Any]
|
|
duration_seconds: float | None
|
|
failure_code: str | None
|
|
failure_reason: str | None
|
|
started_at: datetime
|
|
completed_at: datetime | None
|
|
|
|
|
|
class RecoveryReadinessEntry(RecoveryModel):
|
|
asset_key: str
|
|
asset_name: str
|
|
asset_class: RecoveryAssetClass
|
|
readiness: RecoveryReadiness
|
|
policy_key: str
|
|
rpo_seconds: int | None
|
|
detail: str
|
|
|
|
|
|
class RecoveryDashboard(RecoveryModel):
|
|
"""Every field is measured; unknown values stay null instead of becoming a fiction."""
|
|
|
|
observed_at: datetime
|
|
point_in_time_support: Literal["SUPPORTED", "NOT_SUPPORTED"]
|
|
latest_verified_backup_id: str | None
|
|
latest_verified_backup_at: datetime | None
|
|
latest_verified_backup_age_seconds: float | None
|
|
latest_verified_schema_revision: str | None
|
|
backup_states: dict[str, int]
|
|
verified_backup_count: int
|
|
stale_backup: bool
|
|
backup_staleness_threshold_seconds: int
|
|
last_restore_rehearsal_at: datetime | None
|
|
last_restore_rehearsal_state: str | None
|
|
observed_restore_seconds: float | None
|
|
observed_rpo_seconds: float | None
|
|
protected_asset_count: int
|
|
unprotected_assets: list[str]
|
|
readiness: list[RecoveryReadinessEntry]
|
|
coverage_ratio: float
|
|
estimated_protected_bytes: int
|
|
estimated_rehydratable_bytes: int
|
|
destination_capacity_bytes: int | None
|
|
destination_free_bytes: int | None
|
|
|
|
|
|
class BackupCapacityEstimate(RecoveryModel):
|
|
bytes_to_copy: int
|
|
bytes_manifest_only: int
|
|
estimated_protected_bytes: int
|
|
available_bytes: int | None
|
|
capacity_bytes: int | None
|
|
sufficient: bool
|
|
detail: str
|
|
|
|
|
|
def redact_database_url(url: str) -> str:
|
|
"""Never journal or return a restore DSN that still carries its password."""
|
|
|
|
return re.sub(r"://([^:/@]+):[^@]*@", r"://\1:***@", url)
|