Initial curated ITWorx_MCP_Hub source from 368fba691ea8d23403b4d56b53b5ec670d7c5020

packages/connector_kit/redaction.py

@@ -0,0 +1,70 @@
+"""Deterministic secret exclusion and safe text normalization for untrusted content."""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+
+_SECRET_PATTERNS = (
+    re.compile(r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----"),
+    re.compile(
+        r"(?i)\b(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[:=]\s*[^\s,;]{8,}"
+    ),
+    re.compile(r"(?i)\b(?:password|passwd|secret)\s*[:=]\s*['\"]?[^\s'\",;]{8,}"),
+    re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
+    re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
+)
+_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
+_INLINE_SECRET_PATTERNS = (
+    re.compile(
+        r"(?i)(\b(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token|"
+        r"password|passwd|secret|credential|private[_-]?key)\s*[:=]\s*['\"]?)"
+        r"[^\s'\",;]{4,}"
+    ),
+    re.compile(
+        r"(?i)(\b[A-Z][A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASSWD|API_KEY|CREDENTIAL)"
+        r"\s*=\s*['\"]?)[^\s'\",;]{4,}"
+    ),
+    re.compile(
+        r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|gitea_[A-Za-z0-9_-]{20,})\b"
+    ),
+    re.compile(r"(?i)(https?://[^\s/:@]+:)[^\s/@]+@"),
+)
+
+
+def normalize_text(value: str) -> str:
+    """Normalize Unicode and controls without interpreting content."""
+
+    normalized = unicodedata.normalize("NFC", value).replace("\r\n", "\n").replace("\r", "\n")
+    return _CONTROL_RE.sub("", normalized)
+
+
+def is_secret_like(value: str) -> bool:
+    return any(pattern.search(value) is not None for pattern in _SECRET_PATTERNS)
+
+
+def safe_content(value: str) -> str | None:
+    """Return normalized content or None when the whole document must be excluded."""
+
+    normalized = normalize_text(value)
+    return None if is_secret_like(normalized) else normalized
+
+
+def safe_snippet(value: str, *, max_bytes: int = 2000) -> str:
+    normalized = normalize_text(value)
+    encoded = normalized.encode("utf-8")
+    if len(encoded) <= max_bytes:
+        return normalized
+    return encoded[: max_bytes - 3].decode("utf-8", errors="ignore") + "..."
+
+
+def redact_sensitive_text(value: str, *, max_bytes: int = 2000) -> str:
+    """Normalize, truncate and redact credential-shaped inline values."""
+
+    result = safe_snippet(value, max_bytes=max_bytes)
+    for pattern in _INLINE_SECRET_PATTERNS:
+        result = pattern.sub(
+            lambda match: f"{match.group(1)}<redacted>" if match.lastindex else "<redacted>",
+            result,
+        )
+    return result