"""Deterministic secret exclusion and safe text normalization for untrusted content.""" from __future__ import annotations import re import unicodedata _SECRET_PATTERNS = ( re.compile(r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----"), re.compile( r"(?i)\b(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[:=]\s*[^\s,;]{8,}" ), re.compile(r"(?i)\b(?:password|passwd|secret)\s*[:=]\s*['\"]?[^\s'\",;]{8,}"), re.compile(r"\bAKIA[0-9A-Z]{16}\b"), re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"), ) _CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") _INLINE_SECRET_PATTERNS = ( re.compile( r"(?i)(\b(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token|" r"password|passwd|secret|credential|private[_-]?key)\s*[:=]\s*['\"]?)" r"[^\s'\",;]{4,}" ), re.compile( r"(?i)(\b[A-Z][A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASSWD|API_KEY|CREDENTIAL)" r"\s*=\s*['\"]?)[^\s'\",;]{4,}" ), re.compile( r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|gitea_[A-Za-z0-9_-]{20,})\b" ), re.compile(r"(?i)(https?://[^\s/:@]+:)[^\s/@]+@"), ) def normalize_text(value: str) -> str: """Normalize Unicode and controls without interpreting content.""" normalized = unicodedata.normalize("NFC", value).replace("\r\n", "\n").replace("\r", "\n") return _CONTROL_RE.sub("", normalized) def is_secret_like(value: str) -> bool: return any(pattern.search(value) is not None for pattern in _SECRET_PATTERNS) def safe_content(value: str) -> str | None: """Return normalized content or None when the whole document must be excluded.""" normalized = normalize_text(value) return None if is_secret_like(normalized) else normalized def safe_snippet(value: str, *, max_bytes: int = 2000) -> str: normalized = normalize_text(value) encoded = normalized.encode("utf-8") if len(encoded) <= max_bytes: return normalized return encoded[: max_bytes - 3].decode("utf-8", errors="ignore") + "..." def redact_sensitive_text(value: str, *, max_bytes: int = 2000) -> str: """Normalize, truncate and redact credential-shaped inline values.""" result = safe_snippet(value, max_bytes=max_bytes) for pattern in _INLINE_SECRET_PATTERNS: result = pattern.sub( lambda match: f"{match.group(1)}" if match.lastindex else "", result, ) return result