71 lines
2.5 KiB
Python
71 lines
2.5 KiB
Python
"""Deterministic secret exclusion and safe text normalization for untrusted content."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import unicodedata
|
|
|
|
_SECRET_PATTERNS = (
|
|
re.compile(r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----"),
|
|
re.compile(
|
|
r"(?i)\b(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[:=]\s*[^\s,;]{8,}"
|
|
),
|
|
re.compile(r"(?i)\b(?:password|passwd|secret)\s*[:=]\s*['\"]?[^\s'\",;]{8,}"),
|
|
re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
|
|
re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
|
|
)
|
|
_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
|
_INLINE_SECRET_PATTERNS = (
|
|
re.compile(
|
|
r"(?i)(\b(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token|"
|
|
r"password|passwd|secret|credential|private[_-]?key)\s*[:=]\s*['\"]?)"
|
|
r"[^\s'\",;]{4,}"
|
|
),
|
|
re.compile(
|
|
r"(?i)(\b[A-Z][A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASSWD|API_KEY|CREDENTIAL)"
|
|
r"\s*=\s*['\"]?)[^\s'\",;]{4,}"
|
|
),
|
|
re.compile(
|
|
r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|gitea_[A-Za-z0-9_-]{20,})\b"
|
|
),
|
|
re.compile(r"(?i)(https?://[^\s/:@]+:)[^\s/@]+@"),
|
|
)
|
|
|
|
|
|
def normalize_text(value: str) -> str:
|
|
"""Normalize Unicode and controls without interpreting content."""
|
|
|
|
normalized = unicodedata.normalize("NFC", value).replace("\r\n", "\n").replace("\r", "\n")
|
|
return _CONTROL_RE.sub("", normalized)
|
|
|
|
|
|
def is_secret_like(value: str) -> bool:
|
|
return any(pattern.search(value) is not None for pattern in _SECRET_PATTERNS)
|
|
|
|
|
|
def safe_content(value: str) -> str | None:
|
|
"""Return normalized content or None when the whole document must be excluded."""
|
|
|
|
normalized = normalize_text(value)
|
|
return None if is_secret_like(normalized) else normalized
|
|
|
|
|
|
def safe_snippet(value: str, *, max_bytes: int = 2000) -> str:
|
|
normalized = normalize_text(value)
|
|
encoded = normalized.encode("utf-8")
|
|
if len(encoded) <= max_bytes:
|
|
return normalized
|
|
return encoded[: max_bytes - 3].decode("utf-8", errors="ignore") + "..."
|
|
|
|
|
|
def redact_sensitive_text(value: str, *, max_bytes: int = 2000) -> str:
|
|
"""Normalize, truncate and redact credential-shaped inline values."""
|
|
|
|
result = safe_snippet(value, max_bytes=max_bytes)
|
|
for pattern in _INLINE_SECRET_PATTERNS:
|
|
result = pattern.sub(
|
|
lambda match: f"{match.group(1)}<redacted>" if match.lastindex else "<redacted>",
|
|
result,
|
|
)
|
|
return result
|