Initial curated RepoPulse source from 2524ccc3c1c27739996dccb2b7992989f88cd26d

2026-10-01 01:34:52 +02:00
commit 0adf8ef5d9
@@ -0,0 +1,232 @@
+from __future__ import annotations
+
+import subprocess
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import pytest
+
+from app.acquisition import AcquisitionError, GitAcquirer
+from app.gitea import GiteaClient
+from app.jobs import ScanJobRunner
+from app.models import Repository, ScanJobStatus, ScanSource, ScanTrigger, SourceOrigin
+from app.scanner import scan_repository
+from app.store import ScanStore
+
+
+def _git_repository(root: Path) -> tuple[Path, str]:
+    root.mkdir()
+    (root / "README.md").write_text("# Demo\n", encoding="utf-8")
+    (root / "app.py").write_text("print('hello')\n", encoding="utf-8")
+    subprocess.run(["git", "init", "--quiet", str(root)], check=True)
+    subprocess.run(["git", "-C", str(root), "config", "user.email", "test@example.com"], check=True)
+    subprocess.run(["git", "-C", str(root), "config", "user.name", "RepoPulse Test"], check=True)
+    subprocess.run(["git", "-C", str(root), "add", "."], check=True)
+    subprocess.run(["git", "-C", str(root), "commit", "--quiet", "-m", "initial"], check=True)
+    revision = subprocess.run(
+        ["git", "-C", str(root), "rev-parse", "HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    return root, revision
+
+
+def _catalog_repository(source: Path, revision: str) -> Repository:
+    return Repository(
+        id="gitea:64",
+        provider="gitea",
+        provider_repository_id="64",
+        owner="Jens",
+        name="Demo",
+        full_name="Jens/Demo",
+        clone_url=source.as_uri(),
+        default_branch="master",
+        latest_revision=revision,
+    )
+
+
+def test_acquirer_scans_detached_exact_revision_and_cleans_up(tmp_path: Path) -> None:
+    source, revision = _git_repository(tmp_path / "source")
+    repository = _catalog_repository(source, revision)
+    acquirer = GitAcquirer(tmp_path / "cache", timeout=30)
+
+    acquired = acquirer.acquire(repository, "job-1", revision)
+
+    checked_out = subprocess.run(
+        ["git", "-C", str(acquired.path), "rev-parse", "HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    assert checked_out == revision
+    assert (acquired.path / "app.py").read_text(encoding="utf-8") == "print('hello')\n"
+    acquirer.cleanup(acquired, repository)
+    assert not acquired.path.exists()
+
+
+def test_acquirer_recovers_a_corrupt_mirror_and_rejects_missing_sha(tmp_path: Path) -> None:
+    source, revision = _git_repository(tmp_path / "source")
+    repository = _catalog_repository(source, revision)
+    acquirer = GitAcquirer(tmp_path / "cache", timeout=30)
+    mirror = acquirer._mirror_path(repository)
+    mirror.parent.mkdir(parents=True, exist_ok=True)
+    mirror.write_text("corrupt mirror", encoding="utf-8")
+
+    with pytest.raises(AcquisitionError, match="Requested commit was not found"):
+        acquirer.acquire(repository, "job-missing", "abcdef0123456789abcdef0123456789abcdef01")
+
+    acquired = acquirer.acquire(repository, "job-recovered", revision)
+    assert acquired.actual_revision == revision
+    acquirer.cleanup(acquired, repository)
+
+
+def test_durable_runner_completes_and_deduplicates_revision(tmp_path: Path) -> None:
+    source, revision = _git_repository(tmp_path / "source")
+    store = ScanStore(str(tmp_path / "history.db"))
+    repository = _catalog_repository(source, revision)
+    store.upsert_repository(repository)
+    runner = ScanJobRunner(
+        store,
+        gitea=GiteaClient("http://gitea.local", "test-token", transport=lambda *_: (500, b"")),
+        acquirer=GitAcquirer(tmp_path / "cache", timeout=30),
+    )
+
+    first = runner.enqueue(repository)
+    completed = runner.process_one()
+    reused = runner.enqueue(repository)
+
+    assert completed is not None
+    assert completed.status == ScanJobStatus.COMPLETED
+    assert completed.scan_id is not None
+    assert reused.id == first.id
+    assert reused.deduplicated is True
+    report = store.get(completed.scan_id)
+    assert report is not None
+    assert report.source.actual_revision == revision
+    assert report.source.trigger == ScanTrigger.MANUAL
+    assert completed.duration_ms is not None
+
+
+def test_expired_job_lease_is_recovered_for_another_claim(tmp_path: Path) -> None:
+    store = ScanStore(str(tmp_path / "history.db"))
+    repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567")
+    store.upsert_repository(repository)
+    queued = store.enqueue_job(repository.id, repository.latest_revision, ScanTrigger.SCHEDULED)
+    store.update_job(
+        queued.id,
+        status=ScanJobStatus.SCANNING,
+        phase="scanning",
+        lease_until=datetime.now(UTC) - timedelta(seconds=5),
+    )
+
+    claimed = store.claim_next_job()
+
+    assert claimed is not None
+    assert claimed.id == queued.id
+    assert claimed.status == ScanJobStatus.ACQUIRING_SOURCE
+    assert claimed.attempt == 1
+    assert claimed.message == "Acquiring exact repository revision"
+
+
+def test_active_job_lease_renews_but_terminal_jobs_are_not_resurrected(tmp_path: Path) -> None:
+    store = ScanStore(str(tmp_path / "history.db"))
+    repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567")
+    store.upsert_repository(repository)
+    queued = store.enqueue_job(repository.id, repository.latest_revision, ScanTrigger.SCHEDULED)
+    claimed = store.claim_next_job()
+
+    assert claimed is not None
+    with store._connection() as db:
+        initial_lease = db.execute("SELECT lease_until FROM scan_jobs WHERE id = ?", (queued.id,)).fetchone()[0]
+    assert initial_lease is not None
+    renewed = store.renew_job_lease(queued.id)
+    assert renewed is not None
+    with store._connection() as db:
+        renewed_lease = db.execute("SELECT lease_until FROM scan_jobs WHERE id = ?", (queued.id,)).fetchone()[0]
+    assert renewed_lease is not None and renewed_lease > initial_lease
+
+    completed = store.update_job(queued.id, status=ScanJobStatus.COMPLETED, phase="completed", progress=100)
+    assert completed is not None
+    after_completion = store.renew_job_lease(queued.id)
+    assert after_completion is not None
+    with store._connection() as db:
+        terminal_lease = db.execute("SELECT lease_until FROM scan_jobs WHERE id = ?", (queued.id,)).fetchone()[0]
+    assert terminal_lease is None
+
+
+def test_atomic_claim_allows_only_one_worker_to_own_a_queued_job(tmp_path: Path) -> None:
+    store = ScanStore(str(tmp_path / "history.db"))
+    repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567")
+    store.upsert_repository(repository)
+    store.enqueue_job(repository.id, repository.latest_revision, ScanTrigger.SCHEDULED)
+
+    with ThreadPoolExecutor(max_workers=2) as workers:
+        claims = list(workers.map(lambda _unused: store.claim_next_job(), range(2)))
+
+    successful = [claim for claim in claims if claim is not None]
+    assert len(successful) == 1
+    assert successful[0].status == ScanJobStatus.ACQUIRING_SOURCE
+
+
+def test_scheduler_queues_unscanned_and_changed_repositories_only(tmp_path: Path) -> None:
+    source, revision = _git_repository(tmp_path / "source")
+    store = ScanStore(str(tmp_path / "history.db"))
+    repository = _catalog_repository(source, revision)
+    store.upsert_repository(repository)
+    runner = ScanJobRunner(store, gitea=GiteaClient())
+
+    runner._schedule_changed_repositories()
+    first = store.claim_next_job()
+    assert first is not None
+    assert first.requested_revision == revision
+    store.update_job(first.id, status=ScanJobStatus.SUPERSEDED)
+
+    report = scan_repository(
+        str(source),
+        source=ScanSource(
+            origin=SourceOrigin.GITEA,
+            provider="gitea",
+            repository_id=repository.id,
+            repository_full_name=repository.full_name,
+            branch=repository.default_branch,
+            requested_revision=revision,
+            actual_revision=revision,
+            trigger=ScanTrigger.MANUAL,
+        ),
+    )
+    store.save(report)
+    runner._schedule_changed_repositories()
+    assert store.queue_stats()["queued"] == 0
+
+    changed = "abcdef0123456789abcdef0123456789abcdef01"
+    store.upsert_repository(repository.model_copy(update={"latest_revision": changed}))
+    runner._schedule_changed_repositories()
+    assert store.queue_stats()["queued"] == 1
+
+    disabled = repository.model_copy(
+        update={
+            "id": "gitea:65",
+            "provider_repository_id": "65",
+            "latest_revision": "1234567890abcdef1234567890abcdef12345678",
+            "policy": repository.policy.model_copy(update={"auto_scan": False}),
+        }
+    )
+    store.upsert_repository(disabled)
+    runner._schedule_changed_repositories()
+    assert store.queue_stats()["queued"] == 1
+
+
+def test_acquisition_rejects_token_bearing_clone_outside_gitea_origin(tmp_path: Path) -> None:
+    repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567").model_copy(
+        update={"clone_url": "https://evil.example/Jens/Demo.git"}
+    )
+    acquirer = GitAcquirer(
+        tmp_path / "cache",
+        token="test-token",
+        allowed_clone_origin="https://gitea.local",
+    )
+
+    with pytest.raises(AcquisitionError, match="outside the configured Gitea origin"):
+        acquirer.acquire(repository, "job-1", repository.latest_revision)