Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/security/content/test_content_security.py

@@ -0,0 +1,119 @@
+"""Adversarial tests for the pre-parser content security gate."""
+
+from __future__ import annotations
+
+import io
+import zipfile
+from uuid import uuid4
+
+import pytest
+
+from ragcore.application.security import (
+    ContentSecurityRequest,
+    QuarantinedContentError,
+    QuarantineReason,
+    SecurityDisposition,
+)
+from ragcore.domain.audit import AuditEventType
+from ragcore.infrastructure.security import ContentSecurityGateService, ContentSecurityPolicy
+
+
+class RecordingAudit:
+    def __init__(self) -> None:
+        self.events: list[dict[str, object]] = []
+
+    async def record(self, **event: object) -> None:
+        self.events.append(event)
+
+
+def request(
+    content: bytes, *, media_type: str = "text/plain", name: str = "note.txt"
+) -> ContentSecurityRequest:
+    return ContentSecurityRequest(
+        workspace_id=uuid4(),
+        source_id=uuid4(),
+        content=content,
+        declared_media_type=media_type,
+        original_name=name,
+        trace_id="trace-content-security",
+    )
+
+
+@pytest.mark.asyncio
+async def test_mime_mismatch_and_disallowed_type_are_quarantined() -> None:
+    audit = RecordingAudit()
+    gate = ContentSecurityGateService(audit)
+
+    mismatch = await gate.inspect(request(b"%PDF-1.7", media_type="text/plain", name="note.txt"))
+    disallowed = await gate.inspect(
+        request(b"#!/bin/sh\necho unsafe", media_type="text/x-shellscript", name="run.sh")
+    )
+
+    assert mismatch.disposition is SecurityDisposition.QUARANTINED
+    assert mismatch.reason is QuarantineReason.MIME_MISMATCH
+    assert disallowed.reason is QuarantineReason.DISALLOWED_MEDIA_TYPE
+    with pytest.raises(QuarantinedContentError):
+        mismatch.release_for_parser()
+    assert len(audit.events) == 2
+    assert all(event["event_type"] is AuditEventType.SECURITY_FAILURE for event in audit.events)
+
+
+def _zip_with_entry(name: str, content: bytes, *, compression: int = zipfile.ZIP_DEFLATED) -> bytes:
+    output = io.BytesIO()
+    with zipfile.ZipFile(output, "w", compression=compression) as archive:
+        archive.writestr(name, content)
+    return output.getvalue()
+
+
+@pytest.mark.asyncio
+async def test_archive_ratio_and_size_limits_quarantine_before_release() -> None:
+    audit = RecordingAudit()
+    ratio_gate = ContentSecurityGateService(
+        audit,
+        policy=ContentSecurityPolicy(max_archive_ratio=2.0, max_archive_uncompressed_bytes=256),
+    )
+    size_gate = ContentSecurityGateService(
+        audit,
+        policy=ContentSecurityPolicy(max_archive_ratio=2.0, max_archive_uncompressed_bytes=32),
+    )
+
+    ratio = await ratio_gate.inspect(
+        request(
+            _zip_with_entry("data.txt", b"a" * 128), media_type="application/zip", name="data.zip"
+        )
+    )
+    size = await size_gate.inspect(
+        request(
+            _zip_with_entry("data.txt", b"b" * 40, compression=zipfile.ZIP_STORED),
+            media_type="application/zip",
+            name="data.zip",
+        )
+    )
+
+    assert ratio.reason is QuarantineReason.ARCHIVE_RATIO_LIMIT
+    assert size.reason is QuarantineReason.ARCHIVE_SIZE_LIMIT
+    for decision in (ratio, size):
+        with pytest.raises(QuarantinedContentError):
+            decision.release_for_parser()
+
+
+@pytest.mark.asyncio
+async def test_archive_depth_limit_and_quarantine_cannot_reach_parser_or_index() -> None:
+    audit = RecordingAudit()
+    policy = ContentSecurityPolicy(max_archive_depth=1, max_archive_ratio=10_000.0)
+    gate = ContentSecurityGateService(audit, policy=policy)
+    nested = _zip_with_entry("inner.zip", _zip_with_entry("note.txt", b"safe"))
+
+    decision = await gate.inspect(request(nested, media_type="application/zip", name="outer.zip"))
+
+    assert decision.reason is QuarantineReason.ARCHIVE_DEPTH_LIMIT
+    parser_calls = 0
+
+    def parser_or_index(content: bytes) -> None:
+        nonlocal parser_calls
+        parser_calls += 1
+        assert content
+
+    with pytest.raises(QuarantinedContentError):
+        parser_or_index(decision.release_for_parser())
+    assert parser_calls == 0

tests/security/content/test_production_policy.py

@@ -0,0 +1,167 @@
+"""The production content-security policy must accept exactly what the upload
+API accepts and quarantine what only full-content inspection can catch.
+
+Regression guard for the wiring of ``ContentSecurityGateService`` into normal
+ingestion: the gate must never silently re-block a rich document type the
+product ships (PDF/DOCX/PPTX/XLSX/HTML), code/YAML/TOML/log text, or
+Markdown -- and it must still catch a PDF masquerading as text, a renamed
+plain ZIP posing as an Office document, and NUL-laden "text".
+"""
+
+from __future__ import annotations
+
+import io
+import zipfile
+from uuid import uuid4
+
+import pytest
+
+from ragcore.application.security import (
+    ContentSecurityRequest,
+    QuarantineReason,
+    SecurityDisposition,
+)
+from ragcore.domain.audit import AuditEventType
+from ragcore.infrastructure.connectors.upload.policy import allowed_upload_types
+from ragcore.infrastructure.parsers.docling_runtime import docling_available
+from ragcore.infrastructure.security import (
+    ContentSecurityGateService,
+    production_content_security_policy,
+)
+
+
+class RecordingAudit:
+    def __init__(self) -> None:
+        self.events: list[dict[str, object]] = []
+
+    async def record(self, **event: object) -> None:
+        self.events.append(event)
+
+
+def _request(content: bytes, *, media_type: str, name: str) -> ContentSecurityRequest:
+    return ContentSecurityRequest(
+        workspace_id=uuid4(),
+        source_id=uuid4(),
+        content=content,
+        declared_media_type=media_type,
+        original_name=name,
+        trace_id="trace-production-policy",
+    )
+
+
+def _office_container(member_prefix: str) -> bytes:
+    output = io.BytesIO()
+    with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
+        archive.writestr("[Content_Types].xml", "<Types/>")
+        archive.writestr(f"{member_prefix}document.xml", "<w:document/>")
+    return output.getvalue()
+
+
+def _plain_zip() -> bytes:
+    output = io.BytesIO()
+    with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
+        archive.writestr("readme.txt", "not an office document")
+    return output.getvalue()
+
+
+def test_production_policy_mirrors_the_upload_policy_exactly() -> None:
+    policy = production_content_security_policy()
+    upload_types = allowed_upload_types()
+    assert dict(policy.extension_media_types) == upload_types
+    assert policy.allowed_media_types == frozenset(
+        mime for mimes in upload_types.values() for mime in mimes
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("content", "media_type", "name"),
+    [
+        (b"# Title\n\nBody text.\n", "text/markdown", "notes.md"),
+        (b"plain notes\n", "text/plain", "notes.txt"),
+        (b"def main() -> None:\n    return None\n", "text/plain", "module.py"),
+        (b"key: value\n", "text/yaml", "config.yaml"),
+        (b"[tool]\nname = 'x'\n", "application/toml", "config.toml"),
+        (b"2026-01-01 INFO started\n", "text/x-log", "service.log"),
+        (b"a,b\n1,2\n", "text/csv", "table.csv"),
+        (b'{"ok": true}', "application/json", "data.json"),
+    ],
+)
+async def test_every_text_type_the_product_accepts_passes_the_gate(
+    content: bytes, media_type: str, name: str
+) -> None:
+    audit = RecordingAudit()
+    gate = ContentSecurityGateService(audit, policy=production_content_security_policy())
+    decision = await gate.inspect(_request(content, media_type=media_type, name=name))
+    assert decision.disposition is SecurityDisposition.ACCEPTED, (name, decision.reason)
+    assert decision.release_for_parser() == content
+    # Accepted content is not a security *failure*; only quarantines are audited.
+    assert audit.events == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not docling_available(), reason="rich document types need Docling")
+@pytest.mark.parametrize(
+    ("content", "media_type", "name"),
+    [
+        (b"%PDF-1.7\n1 0 obj<</Type/Catalog>>endobj\n", "application/pdf", "report.pdf"),
+        (
+            _office_container("word/"),
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "design.docx",
+        ),
+        (
+            _office_container("ppt/"),
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+            "deck.pptx",
+        ),
+        (
+            _office_container("xl/"),
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+            "sheet.xlsx",
+        ),
+        (b"<html><body><p>hi</p></body></html>", "text/html", "page.html"),
+    ],
+)
+async def test_every_rich_type_the_product_accepts_passes_the_gate(
+    content: bytes, media_type: str, name: str
+) -> None:
+    gate = ContentSecurityGateService(RecordingAudit(), policy=production_content_security_policy())
+    decision = await gate.inspect(_request(content, media_type=media_type, name=name))
+    assert decision.disposition is SecurityDisposition.ACCEPTED, (name, decision.reason)
+
+
+@pytest.mark.asyncio
+async def test_content_that_only_full_inspection_can_catch_is_quarantined_and_audited() -> None:
+    audit = RecordingAudit()
+    gate = ContentSecurityGateService(audit, policy=production_content_security_policy())
+
+    pdf_as_text = await gate.inspect(
+        _request(b"%PDF-1.7 binary", media_type="text/plain", name="innocent.txt")
+    )
+    nul_text = await gate.inspect(
+        _request(b"looks\x00like\x00text", media_type="text/plain", name="weird.txt")
+    )
+    disallowed = await gate.inspect(
+        _request(b"MZ\x90\x00", media_type="application/octet-stream", name="tool.exe")
+    )
+
+    assert pdf_as_text.reason is QuarantineReason.MIME_MISMATCH
+    assert nul_text.reason is QuarantineReason.MIME_MISMATCH
+    assert disallowed.reason is QuarantineReason.DISALLOWED_MEDIA_TYPE
+    assert len(audit.events) == 3
+    assert all(event["event_type"] is AuditEventType.SECURITY_FAILURE for event in audit.events)
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not docling_available(), reason="rich document types need Docling")
+async def test_a_plain_zip_renamed_to_an_office_extension_is_quarantined() -> None:
+    gate = ContentSecurityGateService(RecordingAudit(), policy=production_content_security_policy())
+    decision = await gate.inspect(
+        _request(
+            _plain_zip(),
+            media_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            name="fake.docx",
+        )
+    )
+    assert decision.reason is QuarantineReason.MIME_MISMATCH