Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/security/connectors/git/test_git_security.py

@@ -0,0 +1,88 @@
+"""Adversarial tests for Git connector content and execution boundaries."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+
+from ragcore.domain.sources import GitSourceConfig, SourceItem
+from ragcore.infrastructure.connectors.git import GitConnector, GitPathError
+
+
+def git(root: Path, *arguments: str) -> str:
+    git_executable = shutil.which("git")
+    if git_executable is None:
+        raise RuntimeError("git is required for this integration test")
+    result = subprocess.run(  # noqa: S603 - test arguments are passed without a shell
+        [git_executable, *arguments],
+        cwd=root,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return result.stdout.strip()
+
+
+def repository_with_untrusted_content(tmp_path: Path) -> tuple[Path, str, Path]:
+    root = tmp_path / "repository"
+    root.mkdir()
+    git(root, "init", "--quiet")
+    git(root, "config", "user.email", "tests@example.invalid")
+    git(root, "config", "user.name", "RAGCore Tests")
+    (root / ".env").write_text("TOKEN=not-for-ingestion\n", encoding="utf-8")
+    (root / "secret.txt").write_text("password=not-for-ingestion\n", encoding="utf-8")
+    (root / "binary.bin").write_bytes(b"safe prefix\x00binary")
+    executable = root / "run.sh"
+    executable.write_text("#!/bin/sh\necho should-not-run\n", encoding="utf-8")
+    executable.chmod(0o755)
+    git(root, "add", ".")
+    git(root, "commit", "--quiet", "-m", "untrusted")
+    marker = root / "hook-ran.txt"
+    hook = root / ".git" / "hooks" / "pre-commit"
+    hook.write_text(f"#!/bin/sh\ntouch {marker}\n", encoding="utf-8")
+    hook.chmod(0o755)
+    return root, git(root, "rev-parse", "HEAD"), marker
+
+
+def current(root: Path, commit: str) -> GitConnector:
+    # These tests are about untrusted *repository content*; the checkout is
+    # configured as its own allowed root so the allowlist (covered separately
+    # in tests/security/sources) never masks what they are checking.
+    return GitConnector(
+        uuid4(),
+        GitSourceConfig("local-checkout", commit, root.resolve()),
+        allowed_roots=(root.resolve(),),
+    )
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_binary_secret_and_executable_patterns_are_excluded_without_running_hooks(
+    tmp_path: Path,
+) -> None:
+    root, commit, marker = repository_with_untrusted_content(tmp_path)
+    result = await current(root, commit).scan()
+
+    assert result.items == ()
+    assert not marker.exists()
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_forged_path_or_commit_provenance_is_denied(tmp_path: Path) -> None:
+    root, commit, _ = repository_with_untrusted_content(tmp_path)
+    connector = current(root, commit)
+    item = SourceItem(
+        source_id=connector.source_id,
+        external_id=f"{commit}:../outside.txt",
+        locator="../outside.txt",
+        name="outside.txt",
+        metadata={"commit": commit, "path": "../outside.txt", "object_id": "0" * 40},
+    )
+
+    with pytest.raises(GitPathError):
+        await connector.acquire(item)

tests/security/connectors/git/test_git_ssrf.py

@@ -0,0 +1,156 @@
+"""SSRF/unsafe-source-URL coverage for the Git connector (M6-07).
+
+GitConnector never performs its own network fetch -- `repository` is
+provenance metadata the operator supplies (the actual `git clone`/`fetch`
+happens outside RAGCore, before a pinned local checkout is handed to this
+connector), and every `_run_git` call only ever runs local, read-only git
+plumbing against `config.checkout_root`. This test proves that directly:
+even an attacker-shaped `repository` value (a cloud metadata endpoint, a
+`file://` URL, a loopback address) is never passed to `subprocess.run`, is
+never used as a network target, and has no effect on connector behavior.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+
+from ragcore.domain.sources import GitSourceConfig, SourceValidationError, WebsiteSourceConfig
+from ragcore.infrastructure.connectors.git import GitConnector
+
+
+def _git(root: Path, *arguments: str) -> str:
+    git_executable = shutil.which("git")
+    if git_executable is None:
+        raise RuntimeError("git is required for this integration test")
+    result = subprocess.run(  # noqa: S603 - test arguments are passed without a shell
+        [git_executable, *arguments],
+        cwd=root,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return result.stdout.strip()
+
+
+def repository_with_untrusted_content(tmp_path: Path) -> tuple[Path, str, Path]:
+    root = tmp_path / "repository"
+    root.mkdir()
+    _git(root, "init", "--quiet")
+    _git(root, "config", "user.email", "tests@example.invalid")
+    _git(root, "config", "user.name", "RAGCore Tests")
+    (root / "readme.md").write_text("# hello\n", encoding="utf-8")
+    _git(root, "add", ".")
+    _git(root, "commit", "--quiet", "-m", "initial")
+    return root, _git(root, "rev-parse", "HEAD"), root / "unused-marker.txt"
+
+
+SSRF_SHAPED_REPOSITORY_VALUES = (
+    "http://169.254.169.254/latest/meta-data/",
+    "file:///etc/passwd",
+    "http://localhost:6333/collections",
+    "http://[::1]:22/",
+    "gopher://internal-service:1234/_evil",
+)
+
+
+@pytest.mark.parametrize("repository", SSRF_SHAPED_REPOSITORY_VALUES)
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_ssrf_shaped_repository_value_is_never_dereferenced(
+    tmp_path: Path, repository: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    root, commit, _marker = repository_with_untrusted_content(tmp_path)
+
+    try:
+        config = GitSourceConfig(repository, commit, root.resolve())
+    except SourceValidationError:
+        # Rejected upfront by the unrelated secret-material check -- also a
+        # safe outcome (never even reaches a connector), nothing further to
+        # prove for this particular value.
+        return
+    connector = GitConnector(uuid4(), config, allowed_roots=(root.resolve(),))
+
+    calls: list[tuple[list[str], Path | str | None]] = []
+    real_run = subprocess.run
+
+    def recording_run(args, **kwargs):  # type: ignore[no-untyped-def]
+        calls.append((list(args), kwargs.get("cwd")))
+        return real_run(args, **kwargs)
+
+    monkeypatch.setattr(subprocess, "run", recording_run)
+
+    result = await connector.scan()
+
+    assert calls, "expected at least one git subprocess call"
+    for args, cwd in calls:
+        assert repository not in args, "the repository value must never be a subprocess argument"
+        assert cwd == root.resolve(), "every git call must run against the local checkout only"
+        assert args[0].endswith("git") or "git" in args[0].lower()
+    # Behavior (one scanned readme.md) is identical regardless of the
+    # attacker-shaped repository value -- it never influences what gets read.
+    assert [item.name for item in result.items] == ["readme.md"]
+
+
+@pytest.mark.security
+@pytest.mark.anyio
+async def test_git_connector_never_calls_clone_fetch_or_pull(tmp_path: Path) -> None:
+    root, commit, _marker = repository_with_untrusted_content(tmp_path)
+    connector = GitConnector(
+        uuid4(),
+        GitSourceConfig("https://example.invalid/repo.git", commit, root.resolve()),
+        allowed_roots=(root.resolve(),),
+    )
+
+    network_subcommands = {"clone", "fetch", "pull", "remote", "push"}
+    original_run = subprocess.run
+
+    def guarding_run(args, **kwargs):  # type: ignore[no-untyped-def]
+        subcommand = args[1] if len(args) > 1 else None
+        assert subcommand not in network_subcommands, f"unexpected network git subcommand: {args}"
+        return original_run(args, **kwargs)
+
+    import ragcore.infrastructure.connectors.git.connector as connector_module
+
+    connector_module.subprocess.run = guarding_run  # type: ignore[attr-defined]
+    try:
+        await connector.scan()
+    finally:
+        connector_module.subprocess.run = original_run  # type: ignore[attr-defined]
+
+
+@pytest.mark.security
+@pytest.mark.parametrize(
+    "sitemap_url",
+    [
+        "file:///etc/passwd",
+        "javascript:alert(1)",
+        "ftp://example.com/sitemap.xml",
+        "http://169.254.169.254/sitemap.xml",
+        "gopher://internal/",
+    ],
+)
+def test_website_source_config_rejects_unsafe_schemes_and_unlisted_hosts(sitemap_url: str) -> None:
+    with pytest.raises(SourceValidationError):
+        WebsiteSourceConfig(allowed_domains=("example.com",), sitemap_url=sitemap_url)
+
+
+@pytest.mark.security
+def test_website_source_config_requires_sitemap_host_in_allowlist() -> None:
+    with pytest.raises(SourceValidationError):
+        WebsiteSourceConfig(
+            allowed_domains=("example.com",),
+            sitemap_url="https://attacker.example.net/sitemap.xml",
+        )
+
+
+@pytest.mark.security
+def test_website_source_config_accepts_a_correctly_scoped_https_sitemap() -> None:
+    config = WebsiteSourceConfig(
+        allowed_domains=("example.com",), sitemap_url="https://example.com/sitemap.xml"
+    )
+    assert config.sitemap_url == "https://example.com/sitemap.xml"