Initial curated ITWorx_MCP_Hub source from 368fba691ea8d23403b4d56b53b5ec670d7c5020
This commit is contained in:
Public Source Publisher committed
commit
cb5be898ea
663 files changed
+82674
No files matched your search
@@ -0,0 +1,243 @@
|
||||
"""Hash-verified, clean-room backup and restore primitives.
|
||||
|
||||
Archives contain only an explicit allowlist of non-secret inputs. Database
|
||||
dump commands use mounted service configuration and never accept a DSN on the
|
||||
command line, keeping credentials out of process listings and evidence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from packages.hub_observability import sanitize_fields
|
||||
|
||||
|
||||
class BackupError(ValueError):
|
||||
"""Raised for unsafe, corrupt, or incomplete backup operations."""
|
||||
|
||||
|
||||
_SECRET_NAME = {".env", ".env.example", "secrets", "secret", "password", "credentials"}
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _safe_relative(root: Path, candidate: Path) -> Path:
|
||||
try:
|
||||
return candidate.resolve().relative_to(root.resolve())
|
||||
except ValueError as exc:
|
||||
raise BackupError("backup input must remain below its registered root") from exc
|
||||
|
||||
|
||||
def _is_secret_path(relative: Path) -> bool:
|
||||
return any(
|
||||
any(
|
||||
term in part.lower()
|
||||
for term in ("secret", "password", "credential", "token", "private_key")
|
||||
)
|
||||
or part.lower() in _SECRET_NAME
|
||||
for part in relative.parts
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackupPolicy:
|
||||
retention_days: int = 30
|
||||
max_artifacts: int = 14
|
||||
encryption_required_before_export: bool = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not 1 <= self.retention_days <= 3650:
|
||||
raise ValueError("retention_days must be between 1 and 3650")
|
||||
if not 1 <= self.max_artifacts <= 1000:
|
||||
raise ValueError("max_artifacts must be between 1 and 1000")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackupArtifact:
|
||||
archive: Path
|
||||
manifest: Path
|
||||
sha256: str
|
||||
bytes: int
|
||||
entries: tuple[str, ...]
|
||||
created_at: str
|
||||
|
||||
|
||||
class BackupManager:
|
||||
def __init__(self, backup_root: Path, *, policy: BackupPolicy | None = None) -> None:
|
||||
self.backup_root = backup_root.resolve()
|
||||
self.policy = policy or BackupPolicy()
|
||||
|
||||
def create_archive(
|
||||
self,
|
||||
source_root: Path,
|
||||
*,
|
||||
name: str,
|
||||
include: Iterable[Path],
|
||||
metadata: Mapping[str, Any] | None = None,
|
||||
) -> BackupArtifact:
|
||||
if not name or Path(name).name != name or name in {".", ".."}:
|
||||
raise BackupError("backup name must be a single safe path component")
|
||||
root = source_root.resolve()
|
||||
self.backup_root.mkdir(parents=True, exist_ok=True)
|
||||
archive = self.backup_root / f"{name}.tar.gz"
|
||||
manifest_path = self.backup_root / f"{name}.manifest.json"
|
||||
entries: list[str] = []
|
||||
with tarfile.open(archive, "w:gz") as bundle:
|
||||
for raw in include:
|
||||
candidate = (root / raw).resolve() if not raw.is_absolute() else raw.resolve()
|
||||
relative = _safe_relative(root, candidate)
|
||||
if _is_secret_path(relative):
|
||||
raise BackupError(
|
||||
f"secret-bearing path is not eligible for ordinary backup: {relative}"
|
||||
)
|
||||
if not candidate.exists():
|
||||
raise BackupError(f"backup input does not exist: {relative}")
|
||||
if candidate.is_symlink() or (
|
||||
candidate.is_dir() and any(item.is_symlink() for item in candidate.rglob("*"))
|
||||
):
|
||||
raise BackupError("symlink backup inputs are not allowed")
|
||||
bundle.add(candidate, arcname=relative.as_posix(), recursive=True)
|
||||
entries.append(relative.as_posix())
|
||||
entries = sorted(set(entries))
|
||||
created_at = datetime.now(UTC).isoformat()
|
||||
digest = _sha256(archive)
|
||||
manifest = {
|
||||
"schemaVersion": "1.0",
|
||||
"artifact": archive.name,
|
||||
"sha256": digest,
|
||||
"bytes": archive.stat().st_size,
|
||||
"createdAt": created_at,
|
||||
"entries": entries,
|
||||
"metadata": sanitize_fields(dict(metadata or {})),
|
||||
"encryption": "required-before-export"
|
||||
if self.policy.encryption_required_before_export
|
||||
else "not-required",
|
||||
"retention": {
|
||||
"days": self.policy.retention_days,
|
||||
"maxArtifacts": self.policy.max_artifacts,
|
||||
},
|
||||
}
|
||||
manifest_path.write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||
)
|
||||
archive.with_suffix(archive.suffix + ".sha256").write_text(
|
||||
f"{digest} {archive.name}\n", encoding="utf-8"
|
||||
)
|
||||
return BackupArtifact(
|
||||
archive, manifest_path, digest, archive.stat().st_size, tuple(entries), created_at
|
||||
)
|
||||
|
||||
def verify(self, artifact: BackupArtifact | Path) -> BackupArtifact:
|
||||
archive = artifact.archive if isinstance(artifact, BackupArtifact) else artifact
|
||||
manifest_path = archive.with_name(archive.name.removesuffix(".tar.gz") + ".manifest.json")
|
||||
if not archive.is_file() or not manifest_path.is_file():
|
||||
raise BackupError("backup archive and manifest are both required")
|
||||
raw = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
digest = _sha256(archive)
|
||||
if raw.get("sha256") != digest or raw.get("bytes") != archive.stat().st_size:
|
||||
raise BackupError("backup integrity metadata does not match archive")
|
||||
entries = tuple(str(item) for item in raw.get("entries", []))
|
||||
if any(Path(item).is_absolute() or ".." in Path(item).parts for item in entries):
|
||||
raise BackupError("backup manifest contains an unsafe entry")
|
||||
return BackupArtifact(
|
||||
archive, manifest_path, digest, archive.stat().st_size, entries, str(raw["createdAt"])
|
||||
)
|
||||
|
||||
def restore_clean(self, artifact: BackupArtifact | Path, target_root: Path) -> BackupArtifact:
|
||||
verified = self.verify(artifact)
|
||||
target = target_root.resolve()
|
||||
if target.exists() and any(target.iterdir()):
|
||||
raise BackupError("clean restore target must not contain existing files")
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(verified.archive, "r:gz") as bundle:
|
||||
members = bundle.getmembers()
|
||||
for member in members:
|
||||
member_path = Path(member.name)
|
||||
if (
|
||||
member_path.is_absolute()
|
||||
or ".." in member_path.parts
|
||||
or member.issym()
|
||||
or member.islnk()
|
||||
):
|
||||
raise BackupError("archive contains an unsafe member")
|
||||
if member.isdir():
|
||||
continue
|
||||
destination = (target / member_path).resolve()
|
||||
try:
|
||||
destination.relative_to(target)
|
||||
except ValueError as exc:
|
||||
raise BackupError("archive member escapes restore target") from exc
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
source = bundle.extractfile(member)
|
||||
if source is None:
|
||||
raise BackupError("archive member could not be read")
|
||||
with destination.open("wb") as output:
|
||||
shutil.copyfileobj(source, output)
|
||||
return verified
|
||||
|
||||
def prune(self, *, now: datetime | None = None) -> list[Path]:
|
||||
current = now or datetime.now(UTC)
|
||||
manifests = sorted(
|
||||
self.backup_root.glob("*.manifest.json"),
|
||||
key=lambda item: item.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
removed: list[Path] = []
|
||||
for index, manifest in enumerate(manifests):
|
||||
too_old = datetime.fromtimestamp(manifest.stat().st_mtime, UTC) < current - timedelta(
|
||||
days=self.policy.retention_days
|
||||
)
|
||||
if index >= self.policy.max_artifacts or too_old:
|
||||
archive = manifest.with_name(
|
||||
manifest.name.removesuffix(".manifest.json") + ".tar.gz"
|
||||
)
|
||||
for path in (manifest, archive, archive.with_suffix(archive.suffix + ".sha256")):
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
removed.append(path)
|
||||
return removed
|
||||
|
||||
|
||||
def build_pg_dump_command(
|
||||
database_name: str, output_path: Path, *, executable: str = "pg_dump"
|
||||
) -> list[str]:
|
||||
"""Build a password-free command; libpq reads credentials from its mounted service file."""
|
||||
|
||||
if not database_name or database_name.startswith("-") or "/" in database_name:
|
||||
raise BackupError("database name is not a safe identifier")
|
||||
if output_path.name != output_path.name.replace(" ", "") or output_path.suffix != ".dump":
|
||||
raise BackupError("database dump output must be a simple .dump filename")
|
||||
return [
|
||||
executable,
|
||||
"--format=custom",
|
||||
"--no-owner",
|
||||
"--no-privileges",
|
||||
"--dbname",
|
||||
database_name,
|
||||
"--file",
|
||||
str(output_path),
|
||||
]
|
||||
|
||||
|
||||
def run_pg_dump(
|
||||
database_name: str, output_path: Path, *, environment: Mapping[str, str] | None = None
|
||||
) -> None:
|
||||
command = build_pg_dump_command(database_name, output_path)
|
||||
safe_environment = dict(environment or os.environ)
|
||||
subprocess.run(command, check=True, env=safe_environment, stdout=subprocess.DEVNULL) # noqa: S603
|
||||
Reference in new issue
Block a user