Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/security/connectors/folder/test_folder_connector.py

@@ -0,0 +1,209 @@
+"""Adversarial tests for the read-only folder connector."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+
+from ragcore.domain.sources import FolderSourceConfig, SourceCursor, SourceValidationError
+from ragcore.infrastructure.connectors.folder import (
+    FolderConnector,
+    FolderTraversalError,
+)
+
+
+def connector(
+    root: Path, *, max_items: int = 256, allowed_roots: tuple[Path, ...] | None = None
+) -> FolderConnector:
+    """Traversal tests configure the root under test as its own allowed root:
+    they are about escaping *this* root, and the allowlist is exercised on its
+    own below."""
+
+    return FolderConnector(
+        uuid4(),
+        FolderSourceConfig(root),
+        allowed_roots=(root,) if allowed_roots is None else allowed_roots,
+        max_items=max_items,
+    )
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_scan_is_deterministic_cursorable_and_partial_scans_cannot_delete() -> None:
+    root = (Path(".") / "folder-test-root").resolve()
+    root.mkdir()
+    try:
+        (root / "a.txt").write_text("alpha", encoding="utf-8")
+        (root / "nested").mkdir()
+        (root / "nested" / "b.md").write_text("bravo", encoding="utf-8")
+        current = connector(root, max_items=1)
+
+        first = await current.scan()
+        assert first.complete is False
+        assert first.next_cursor is not None
+        assert first.items[0].external_id == "a.txt"
+        assert first.deleted_item_ids({"a.txt", "nested/b.md", "deleted.txt"}) == frozenset()
+
+        second = await current.scan(first.next_cursor)
+        assert second.complete is False
+        assert second.next_cursor is None
+        assert [item.external_id for item in second.items] == ["nested/b.md"]
+        assert second.deleted_item_ids({"a.txt", "nested/b.md", "deleted.txt"}) == frozenset()
+    finally:
+        (root / "nested" / "b.md").unlink(missing_ok=True)
+        (root / "nested").rmdir()
+        (root / "a.txt").unlink(missing_ok=True)
+        root.rmdir()
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_symlink_and_traversal_are_denied() -> None:
+    root = (Path(".") / "folder-symlink-root").resolve()
+    outside = (Path(".") / "folder-symlink-outside.txt").resolve()
+    root.mkdir()
+    outside.write_text("outside", encoding="utf-8")
+    link = root / "escape.txt"
+    try:
+        link.symlink_to(outside)
+        current = connector(root)
+        with pytest.raises(FolderTraversalError):
+            await current.scan()
+        with pytest.raises(FolderTraversalError):
+            await current.scan(SourceCursor("../escape.txt"))
+    finally:
+        link.unlink(missing_ok=True)
+        outside.unlink(missing_ok=True)
+        root.rmdir()
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_acquisition_is_read_only_and_rejects_cross_root_item() -> None:
+    root = (Path(".") / "folder-readonly-root").resolve()
+    root.mkdir()
+    original = b"read only source"
+    source = root / "source.txt"
+    source.write_bytes(original)
+    try:
+        current = connector(root)
+        result = await current.scan()
+        acquisition = await current.acquire(result.items[0])
+        assert b"".join([chunk async for chunk in acquisition.stream]) == original
+        assert source.read_bytes() == original
+        assert list(root.iterdir()) == [source]
+
+        with pytest.raises(FolderTraversalError):
+            await current.acquire(
+                result.items[0].__class__(
+                    source_id=result.items[0].source_id,
+                    external_id="../escape.txt",
+                    locator="../escape.txt",
+                    name="escape.txt",
+                )
+            )
+    finally:
+        source.unlink(missing_ok=True)
+        root.rmdir()
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_unchanged_items_skip_acquisition_by_fingerprint() -> None:
+    root = (Path(".") / "folder-fingerprint-root").resolve()
+    root.mkdir()
+    source = root / "source.txt"
+    source.write_text("first", encoding="utf-8")
+    try:
+        current = connector(root)
+        first = (await current.scan()).items[0]
+        fingerprint = current.fingerprint(first)
+        assert current.should_acquire(first, fingerprint) is False
+
+        source.write_text("second", encoding="utf-8")
+        changed = (await current.scan()).items[0]
+        assert current.should_acquire(changed, fingerprint) is True
+    finally:
+        source.unlink(missing_ok=True)
+        root.rmdir()
+
+
+@pytest.mark.security
+def test_folder_config_rejects_symlink_root() -> None:
+    root = (Path(".") / "folder-config-root").resolve()
+    target = (Path(".") / "folder-config-target").resolve()
+    target.mkdir()
+    try:
+        root.symlink_to(target, target_is_directory=True)
+        with pytest.raises(SourceValidationError):
+            FolderSourceConfig(root)
+    finally:
+        root.unlink(missing_ok=True)
+        target.rmdir()
+
+
+# -- the allowlist, enforced by the connector itself -------------------------
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_a_root_outside_every_allowed_root_cannot_be_scanned(tmp_path: Path) -> None:
+    """A perfectly well-formed root -- absolute, no traversal, no symlink, a
+    real directory -- is still refused when no operator ever allowed it."""
+
+    allowed = tmp_path / "corpora"
+    allowed.mkdir()
+    elsewhere = tmp_path / "secrets"
+    elsewhere.mkdir()
+    (elsewhere / "keys.txt").write_text("shhh", encoding="utf-8")
+
+    current = connector(elsewhere, allowed_roots=(allowed,))
+
+    with pytest.raises(FolderTraversalError, match="not an allowed source root"):
+        await current.scan()
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_no_configured_roots_means_no_folder_can_be_scanned(tmp_path: Path) -> None:
+    root = tmp_path / "corpora"
+    root.mkdir()
+    (root / "a.txt").write_text("alpha", encoding="utf-8")
+
+    current = connector(root, allowed_roots=())
+
+    with pytest.raises(FolderTraversalError, match="not an allowed source root"):
+        await current.scan()
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_a_directory_below_an_allowed_root_is_readable(tmp_path: Path) -> None:
+    allowed = tmp_path / "corpora"
+    root = allowed / "runbooks"
+    root.mkdir(parents=True)
+    (root / "a.txt").write_text("alpha", encoding="utf-8")
+
+    result = await connector(root, allowed_roots=(allowed,)).scan()
+
+    assert [item.external_id for item in result.items] == ["a.txt"]
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_the_allowlist_is_re_checked_on_acquire_not_only_on_scan(tmp_path: Path) -> None:
+    """Every read revalidates. A connector that passed the check once must not
+    keep reading if the answer would be different now."""
+
+    root = tmp_path / "corpora"
+    root.mkdir()
+    (root / "a.txt").write_text("alpha", encoding="utf-8")
+    scanned = await connector(root, allowed_roots=(root,)).scan()
+    [item] = scanned.items
+
+    disallowed = connector(root, allowed_roots=(tmp_path / "somewhere-else",))
+
+    with pytest.raises(FolderTraversalError, match="not an allowed source root"):
+        await disallowed.acquire(item)

tests/security/connectors/git/test_git_security.py

@@ -0,0 +1,88 @@
+"""Adversarial tests for Git connector content and execution boundaries."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+
+from ragcore.domain.sources import GitSourceConfig, SourceItem
+from ragcore.infrastructure.connectors.git import GitConnector, GitPathError
+
+
+def git(root: Path, *arguments: str) -> str:
+    git_executable = shutil.which("git")
+    if git_executable is None:
+        raise RuntimeError("git is required for this integration test")
+    result = subprocess.run(  # noqa: S603 - test arguments are passed without a shell
+        [git_executable, *arguments],
+        cwd=root,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return result.stdout.strip()
+
+
+def repository_with_untrusted_content(tmp_path: Path) -> tuple[Path, str, Path]:
+    root = tmp_path / "repository"
+    root.mkdir()
+    git(root, "init", "--quiet")
+    git(root, "config", "user.email", "tests@example.invalid")
+    git(root, "config", "user.name", "RAGCore Tests")
+    (root / ".env").write_text("TOKEN=not-for-ingestion\n", encoding="utf-8")
+    (root / "secret.txt").write_text("password=not-for-ingestion\n", encoding="utf-8")
+    (root / "binary.bin").write_bytes(b"safe prefix\x00binary")
+    executable = root / "run.sh"
+    executable.write_text("#!/bin/sh\necho should-not-run\n", encoding="utf-8")
+    executable.chmod(0o755)
+    git(root, "add", ".")
+    git(root, "commit", "--quiet", "-m", "untrusted")
+    marker = root / "hook-ran.txt"
+    hook = root / ".git" / "hooks" / "pre-commit"
+    hook.write_text(f"#!/bin/sh\ntouch {marker}\n", encoding="utf-8")
+    hook.chmod(0o755)
+    return root, git(root, "rev-parse", "HEAD"), marker
+
+
+def current(root: Path, commit: str) -> GitConnector:
+    # These tests are about untrusted *repository content*; the checkout is
+    # configured as its own allowed root so the allowlist (covered separately
+    # in tests/security/sources) never masks what they are checking.
+    return GitConnector(
+        uuid4(),
+        GitSourceConfig("local-checkout", commit, root.resolve()),
+        allowed_roots=(root.resolve(),),
+    )
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_binary_secret_and_executable_patterns_are_excluded_without_running_hooks(
+    tmp_path: Path,
+) -> None:
+    root, commit, marker = repository_with_untrusted_content(tmp_path)
+    result = await current(root, commit).scan()
+
+    assert result.items == ()
+    assert not marker.exists()
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_forged_path_or_commit_provenance_is_denied(tmp_path: Path) -> None:
+    root, commit, _ = repository_with_untrusted_content(tmp_path)
+    connector = current(root, commit)
+    item = SourceItem(
+        source_id=connector.source_id,
+        external_id=f"{commit}:../outside.txt",
+        locator="../outside.txt",
+        name="outside.txt",
+        metadata={"commit": commit, "path": "../outside.txt", "object_id": "0" * 40},
+    )
+
+    with pytest.raises(GitPathError):
+        await connector.acquire(item)

tests/security/connectors/git/test_git_ssrf.py

@@ -0,0 +1,156 @@
+"""SSRF/unsafe-source-URL coverage for the Git connector (M6-07).
+
+GitConnector never performs its own network fetch -- `repository` is
+provenance metadata the operator supplies (the actual `git clone`/`fetch`
+happens outside RAGCore, before a pinned local checkout is handed to this
+connector), and every `_run_git` call only ever runs local, read-only git
+plumbing against `config.checkout_root`. This test proves that directly:
+even an attacker-shaped `repository` value (a cloud metadata endpoint, a
+`file://` URL, a loopback address) is never passed to `subprocess.run`, is
+never used as a network target, and has no effect on connector behavior.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+
+from ragcore.domain.sources import GitSourceConfig, SourceValidationError, WebsiteSourceConfig
+from ragcore.infrastructure.connectors.git import GitConnector
+
+
+def _git(root: Path, *arguments: str) -> str:
+    git_executable = shutil.which("git")
+    if git_executable is None:
+        raise RuntimeError("git is required for this integration test")
+    result = subprocess.run(  # noqa: S603 - test arguments are passed without a shell
+        [git_executable, *arguments],
+        cwd=root,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return result.stdout.strip()
+
+
+def repository_with_untrusted_content(tmp_path: Path) -> tuple[Path, str, Path]:
+    root = tmp_path / "repository"
+    root.mkdir()
+    _git(root, "init", "--quiet")
+    _git(root, "config", "user.email", "tests@example.invalid")
+    _git(root, "config", "user.name", "RAGCore Tests")
+    (root / "readme.md").write_text("# hello\n", encoding="utf-8")
+    _git(root, "add", ".")
+    _git(root, "commit", "--quiet", "-m", "initial")
+    return root, _git(root, "rev-parse", "HEAD"), root / "unused-marker.txt"
+
+
+SSRF_SHAPED_REPOSITORY_VALUES = (
+    "http://169.254.169.254/latest/meta-data/",
+    "file:///etc/passwd",
+    "http://localhost:6333/collections",
+    "http://[::1]:22/",
+    "gopher://internal-service:1234/_evil",
+)
+
+
+@pytest.mark.parametrize("repository", SSRF_SHAPED_REPOSITORY_VALUES)
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_ssrf_shaped_repository_value_is_never_dereferenced(
+    tmp_path: Path, repository: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    root, commit, _marker = repository_with_untrusted_content(tmp_path)
+
+    try:
+        config = GitSourceConfig(repository, commit, root.resolve())
+    except SourceValidationError:
+        # Rejected upfront by the unrelated secret-material check -- also a
+        # safe outcome (never even reaches a connector), nothing further to
+        # prove for this particular value.
+        return
+    connector = GitConnector(uuid4(), config, allowed_roots=(root.resolve(),))
+
+    calls: list[tuple[list[str], Path | str | None]] = []
+    real_run = subprocess.run
+
+    def recording_run(args, **kwargs):  # type: ignore[no-untyped-def]
+        calls.append((list(args), kwargs.get("cwd")))
+        return real_run(args, **kwargs)
+
+    monkeypatch.setattr(subprocess, "run", recording_run)
+
+    result = await connector.scan()
+
+    assert calls, "expected at least one git subprocess call"
+    for args, cwd in calls:
+        assert repository not in args, "the repository value must never be a subprocess argument"
+        assert cwd == root.resolve(), "every git call must run against the local checkout only"
+        assert args[0].endswith("git") or "git" in args[0].lower()
+    # Behavior (one scanned readme.md) is identical regardless of the
+    # attacker-shaped repository value -- it never influences what gets read.
+    assert [item.name for item in result.items] == ["readme.md"]
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_git_connector_never_calls_clone_fetch_or_pull(tmp_path: Path) -> None:
+    root, commit, _marker = repository_with_untrusted_content(tmp_path)
+    connector = GitConnector(
+        uuid4(),
+        GitSourceConfig("https://example.invalid/repo.git", commit, root.resolve()),
+        allowed_roots=(root.resolve(),),
+    )
+
+    network_subcommands = {"clone", "fetch", "pull", "remote", "push"}
+    original_run = subprocess.run
+
+    def guarding_run(args, **kwargs):  # type: ignore[no-untyped-def]
+        subcommand = args[1] if len(args) > 1 else None
+        assert subcommand not in network_subcommands, f"unexpected network git subcommand: {args}"
+        return original_run(args, **kwargs)
+
+    import ragcore.infrastructure.connectors.git.connector as connector_module
+
+    connector_module.subprocess.run = guarding_run  # type: ignore[attr-defined]
+    try:
+        await connector.scan()
+    finally:
+        connector_module.subprocess.run = original_run  # type: ignore[attr-defined]
+
+
+@pytest.mark.security
+@pytest.mark.parametrize(
+    "sitemap_url",
+    [
+        "file:///etc/passwd",
+        "javascript:alert(1)",
+        "ftp://example.com/sitemap.xml",
+        "http://169.254.169.254/sitemap.xml",
+        "gopher://internal/",
+    ],
+)
+def test_website_source_config_rejects_unsafe_schemes_and_unlisted_hosts(sitemap_url: str) -> None:
+    with pytest.raises(SourceValidationError):
+        WebsiteSourceConfig(allowed_domains=("example.com",), sitemap_url=sitemap_url)
+
+
+@pytest.mark.security
+def test_website_source_config_requires_sitemap_host_in_allowlist() -> None:
+    with pytest.raises(SourceValidationError):
+        WebsiteSourceConfig(
+            allowed_domains=("example.com",),
+            sitemap_url="https://attacker.example.net/sitemap.xml",
+        )
+
+
+@pytest.mark.security
+def test_website_source_config_accepts_a_correctly_scoped_https_sitemap() -> None:
+    config = WebsiteSourceConfig(
+        allowed_domains=("example.com",), sitemap_url="https://example.com/sitemap.xml"
+    )
+    assert config.sitemap_url == "https://example.com/sitemap.xml"