233 lines
9.2 KiB
Python
233 lines
9.2 KiB
Python
from __future__ import annotations
|
|
|
|
import subprocess
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.acquisition import AcquisitionError, GitAcquirer
|
|
from app.gitea import GiteaClient
|
|
from app.jobs import ScanJobRunner
|
|
from app.models import Repository, ScanJobStatus, ScanSource, ScanTrigger, SourceOrigin
|
|
from app.scanner import scan_repository
|
|
from app.store import ScanStore
|
|
|
|
|
|
def _git_repository(root: Path) -> tuple[Path, str]:
|
|
root.mkdir()
|
|
(root / "README.md").write_text("# Demo\n", encoding="utf-8")
|
|
(root / "app.py").write_text("print('hello')\n", encoding="utf-8")
|
|
subprocess.run(["git", "init", "--quiet", str(root)], check=True)
|
|
subprocess.run(["git", "-C", str(root), "config", "user.email", "test@example.com"], check=True)
|
|
subprocess.run(["git", "-C", str(root), "config", "user.name", "RepoPulse Test"], check=True)
|
|
subprocess.run(["git", "-C", str(root), "add", "."], check=True)
|
|
subprocess.run(["git", "-C", str(root), "commit", "--quiet", "-m", "initial"], check=True)
|
|
revision = subprocess.run(
|
|
["git", "-C", str(root), "rev-parse", "HEAD"],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
return root, revision
|
|
|
|
|
|
def _catalog_repository(source: Path, revision: str) -> Repository:
|
|
return Repository(
|
|
id="gitea:64",
|
|
provider="gitea",
|
|
provider_repository_id="64",
|
|
owner="Jens",
|
|
name="Demo",
|
|
full_name="Jens/Demo",
|
|
clone_url=source.as_uri(),
|
|
default_branch="master",
|
|
latest_revision=revision,
|
|
)
|
|
|
|
|
|
def test_acquirer_scans_detached_exact_revision_and_cleans_up(tmp_path: Path) -> None:
|
|
source, revision = _git_repository(tmp_path / "source")
|
|
repository = _catalog_repository(source, revision)
|
|
acquirer = GitAcquirer(tmp_path / "cache", timeout=30)
|
|
|
|
acquired = acquirer.acquire(repository, "job-1", revision)
|
|
|
|
checked_out = subprocess.run(
|
|
["git", "-C", str(acquired.path), "rev-parse", "HEAD"],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
assert checked_out == revision
|
|
assert (acquired.path / "app.py").read_text(encoding="utf-8") == "print('hello')\n"
|
|
acquirer.cleanup(acquired, repository)
|
|
assert not acquired.path.exists()
|
|
|
|
|
|
def test_acquirer_recovers_a_corrupt_mirror_and_rejects_missing_sha(tmp_path: Path) -> None:
|
|
source, revision = _git_repository(tmp_path / "source")
|
|
repository = _catalog_repository(source, revision)
|
|
acquirer = GitAcquirer(tmp_path / "cache", timeout=30)
|
|
mirror = acquirer._mirror_path(repository)
|
|
mirror.parent.mkdir(parents=True, exist_ok=True)
|
|
mirror.write_text("corrupt mirror", encoding="utf-8")
|
|
|
|
with pytest.raises(AcquisitionError, match="Requested commit was not found"):
|
|
acquirer.acquire(repository, "job-missing", "abcdef0123456789abcdef0123456789abcdef01")
|
|
|
|
acquired = acquirer.acquire(repository, "job-recovered", revision)
|
|
assert acquired.actual_revision == revision
|
|
acquirer.cleanup(acquired, repository)
|
|
|
|
|
|
def test_durable_runner_completes_and_deduplicates_revision(tmp_path: Path) -> None:
|
|
source, revision = _git_repository(tmp_path / "source")
|
|
store = ScanStore(str(tmp_path / "history.db"))
|
|
repository = _catalog_repository(source, revision)
|
|
store.upsert_repository(repository)
|
|
runner = ScanJobRunner(
|
|
store,
|
|
gitea=GiteaClient("http://gitea.local", "test-token", transport=lambda *_: (500, b"")),
|
|
acquirer=GitAcquirer(tmp_path / "cache", timeout=30),
|
|
)
|
|
|
|
first = runner.enqueue(repository)
|
|
completed = runner.process_one()
|
|
reused = runner.enqueue(repository)
|
|
|
|
assert completed is not None
|
|
assert completed.status == ScanJobStatus.COMPLETED
|
|
assert completed.scan_id is not None
|
|
assert reused.id == first.id
|
|
assert reused.deduplicated is True
|
|
report = store.get(completed.scan_id)
|
|
assert report is not None
|
|
assert report.source.actual_revision == revision
|
|
assert report.source.trigger == ScanTrigger.MANUAL
|
|
assert completed.duration_ms is not None
|
|
|
|
|
|
def test_expired_job_lease_is_recovered_for_another_claim(tmp_path: Path) -> None:
|
|
store = ScanStore(str(tmp_path / "history.db"))
|
|
repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567")
|
|
store.upsert_repository(repository)
|
|
queued = store.enqueue_job(repository.id, repository.latest_revision, ScanTrigger.SCHEDULED)
|
|
store.update_job(
|
|
queued.id,
|
|
status=ScanJobStatus.SCANNING,
|
|
phase="scanning",
|
|
lease_until=datetime.now(UTC) - timedelta(seconds=5),
|
|
)
|
|
|
|
claimed = store.claim_next_job()
|
|
|
|
assert claimed is not None
|
|
assert claimed.id == queued.id
|
|
assert claimed.status == ScanJobStatus.ACQUIRING_SOURCE
|
|
assert claimed.attempt == 1
|
|
assert claimed.message == "Acquiring exact repository revision"
|
|
|
|
|
|
def test_active_job_lease_renews_but_terminal_jobs_are_not_resurrected(tmp_path: Path) -> None:
|
|
store = ScanStore(str(tmp_path / "history.db"))
|
|
repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567")
|
|
store.upsert_repository(repository)
|
|
queued = store.enqueue_job(repository.id, repository.latest_revision, ScanTrigger.SCHEDULED)
|
|
claimed = store.claim_next_job()
|
|
|
|
assert claimed is not None
|
|
with store._connection() as db:
|
|
initial_lease = db.execute("SELECT lease_until FROM scan_jobs WHERE id = ?", (queued.id,)).fetchone()[0]
|
|
assert initial_lease is not None
|
|
renewed = store.renew_job_lease(queued.id)
|
|
assert renewed is not None
|
|
with store._connection() as db:
|
|
renewed_lease = db.execute("SELECT lease_until FROM scan_jobs WHERE id = ?", (queued.id,)).fetchone()[0]
|
|
assert renewed_lease is not None and renewed_lease > initial_lease
|
|
|
|
completed = store.update_job(queued.id, status=ScanJobStatus.COMPLETED, phase="completed", progress=100)
|
|
assert completed is not None
|
|
after_completion = store.renew_job_lease(queued.id)
|
|
assert after_completion is not None
|
|
with store._connection() as db:
|
|
terminal_lease = db.execute("SELECT lease_until FROM scan_jobs WHERE id = ?", (queued.id,)).fetchone()[0]
|
|
assert terminal_lease is None
|
|
|
|
|
|
def test_atomic_claim_allows_only_one_worker_to_own_a_queued_job(tmp_path: Path) -> None:
|
|
store = ScanStore(str(tmp_path / "history.db"))
|
|
repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567")
|
|
store.upsert_repository(repository)
|
|
store.enqueue_job(repository.id, repository.latest_revision, ScanTrigger.SCHEDULED)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as workers:
|
|
claims = list(workers.map(lambda _unused: store.claim_next_job(), range(2)))
|
|
|
|
successful = [claim for claim in claims if claim is not None]
|
|
assert len(successful) == 1
|
|
assert successful[0].status == ScanJobStatus.ACQUIRING_SOURCE
|
|
|
|
|
|
def test_scheduler_queues_unscanned_and_changed_repositories_only(tmp_path: Path) -> None:
|
|
source, revision = _git_repository(tmp_path / "source")
|
|
store = ScanStore(str(tmp_path / "history.db"))
|
|
repository = _catalog_repository(source, revision)
|
|
store.upsert_repository(repository)
|
|
runner = ScanJobRunner(store, gitea=GiteaClient())
|
|
|
|
runner._schedule_changed_repositories()
|
|
first = store.claim_next_job()
|
|
assert first is not None
|
|
assert first.requested_revision == revision
|
|
store.update_job(first.id, status=ScanJobStatus.SUPERSEDED)
|
|
|
|
report = scan_repository(
|
|
str(source),
|
|
source=ScanSource(
|
|
origin=SourceOrigin.GITEA,
|
|
provider="gitea",
|
|
repository_id=repository.id,
|
|
repository_full_name=repository.full_name,
|
|
branch=repository.default_branch,
|
|
requested_revision=revision,
|
|
actual_revision=revision,
|
|
trigger=ScanTrigger.MANUAL,
|
|
),
|
|
)
|
|
store.save(report)
|
|
runner._schedule_changed_repositories()
|
|
assert store.queue_stats()["queued"] == 0
|
|
|
|
changed = "abcdef0123456789abcdef0123456789abcdef01"
|
|
store.upsert_repository(repository.model_copy(update={"latest_revision": changed}))
|
|
runner._schedule_changed_repositories()
|
|
assert store.queue_stats()["queued"] == 1
|
|
|
|
disabled = repository.model_copy(
|
|
update={
|
|
"id": "gitea:65",
|
|
"provider_repository_id": "65",
|
|
"latest_revision": "1234567890abcdef1234567890abcdef12345678",
|
|
"policy": repository.policy.model_copy(update={"auto_scan": False}),
|
|
}
|
|
)
|
|
store.upsert_repository(disabled)
|
|
runner._schedule_changed_repositories()
|
|
assert store.queue_stats()["queued"] == 1
|
|
|
|
|
|
def test_acquisition_rejects_token_bearing_clone_outside_gitea_origin(tmp_path: Path) -> None:
|
|
repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567").model_copy(
|
|
update={"clone_url": "https://evil.example/Jens/Demo.git"}
|
|
)
|
|
acquirer = GitAcquirer(
|
|
tmp_path / "cache",
|
|
token="test-token",
|
|
allowed_clone_origin="https://gitea.local",
|
|
)
|
|
|
|
with pytest.raises(AcquisitionError, match="outside the configured Gitea origin"):
|
|
acquirer.acquire(repository, "job-1", repository.latest_revision)
|