Initial curated RepoPulse source from 2524ccc3c1c27739996dccb2b7992989f88cd26d
This commit is contained in:
Public Source Publisher committed
commit
0adf8ef5d9
90 files changed
+14315
No files matched your search
@@ -0,0 +1,264 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Lock, RLock
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .models import Repository
|
||||
|
||||
SAFE_REVISION = re.compile(r"^[0-9a-fA-F]{7,64}$")
|
||||
SAFE_BRANCH = re.compile(r"^[A-Za-z0-9._/-]+$")
|
||||
SAFE_JOB_ID = re.compile(r"^[A-Za-z0-9-]+$")
|
||||
|
||||
|
||||
class AcquisitionError(RuntimeError):
|
||||
def __init__(self, code: str, detail: str) -> None:
|
||||
super().__init__(detail)
|
||||
self.code = code
|
||||
self.detail = detail
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AcquiredSource:
|
||||
path: Path
|
||||
actual_revision: str
|
||||
branch: str
|
||||
|
||||
|
||||
class GitAcquirer:
|
||||
"""Maintain bounded bare mirrors and scan detached exact-revision worktrees."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cache_root: str | Path | None = None,
|
||||
token: str | None = None,
|
||||
*,
|
||||
timeout: int = 180,
|
||||
allowed_clone_origin: str | None = None,
|
||||
) -> None:
|
||||
raw_root = cache_root or os.getenv("REPOPULSE_REPOSITORY_CACHE", "data/repos")
|
||||
self.cache_root = Path(raw_root).expanduser().resolve()
|
||||
self.token = token if token is not None else os.getenv("REPOPULSE_GITEA_TOKEN", "")
|
||||
self.timeout = max(10, timeout)
|
||||
configured_origin = allowed_clone_origin or os.getenv("REPOPULSE_GITEA_URL", "")
|
||||
self.allowed_clone_origin = self._origin(configured_origin)
|
||||
self.mirrors_root = self.cache_root / "mirrors"
|
||||
self.worktrees_root = self.cache_root / "worktrees"
|
||||
self.mirrors_root.mkdir(parents=True, exist_ok=True)
|
||||
self.worktrees_root.mkdir(parents=True, exist_ok=True)
|
||||
self._lock_guard = Lock()
|
||||
self._repository_locks: dict[str, RLock] = {}
|
||||
|
||||
@staticmethod
|
||||
def _origin(value: str) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
return None
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
except ValueError:
|
||||
return None
|
||||
return f"{parsed.scheme.casefold()}://{parsed.hostname.casefold()}:{port}"
|
||||
|
||||
def _repository_lock(self, repository_id: str) -> RLock:
|
||||
with self._lock_guard:
|
||||
return self._repository_locks.setdefault(repository_id, RLock())
|
||||
|
||||
def _git_env(self) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
if self.token:
|
||||
env.update(
|
||||
{
|
||||
"GIT_CONFIG_COUNT": "1",
|
||||
"GIT_CONFIG_KEY_0": "http.extraHeader",
|
||||
"GIT_CONFIG_VALUE_0": f"Authorization: token {self.token}",
|
||||
}
|
||||
)
|
||||
return env
|
||||
|
||||
def _run(self, args: list[str]) -> str:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
args,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
errors="replace",
|
||||
timeout=self.timeout,
|
||||
env=self._git_env(),
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise AcquisitionError("timeout", "Git source acquisition timed out") from exc
|
||||
except (OSError, subprocess.CalledProcessError) as exc:
|
||||
# Git stderr may contain remote URLs or server-specific details; do
|
||||
# not persist or expose it because the job API is user-visible.
|
||||
raise AcquisitionError("git_failed", "Git source acquisition failed") from exc
|
||||
return result.stdout.strip()
|
||||
|
||||
def _mirror_path(self, repository: Repository) -> Path:
|
||||
identity = f"{repository.provider}-{repository.provider_repository_id}"
|
||||
safe = re.sub(r"[^A-Za-z0-9._-]", "_", identity)
|
||||
return self.mirrors_root / f"{safe}.git"
|
||||
|
||||
def _worktree_path(self, job_id: str) -> Path:
|
||||
if not SAFE_JOB_ID.fullmatch(job_id):
|
||||
raise AcquisitionError("invalid_job", "Scan job identity is invalid")
|
||||
return self.worktrees_root / job_id
|
||||
|
||||
@staticmethod
|
||||
def _validate_branch(branch: str) -> None:
|
||||
if (
|
||||
not SAFE_BRANCH.fullmatch(branch)
|
||||
or branch.startswith("-")
|
||||
or ".." in branch
|
||||
or branch.endswith(("/", "."))
|
||||
):
|
||||
raise AcquisitionError("invalid_branch", "Repository default branch is invalid")
|
||||
|
||||
def _validate_clone_url(self, clone_url: str | None) -> str:
|
||||
if not clone_url or "\x00" in clone_url:
|
||||
raise AcquisitionError("missing_clone_url", "Repository clone URL is unavailable")
|
||||
try:
|
||||
parsed = urlsplit(clone_url)
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise AcquisitionError("unsafe_clone_url", "Repository clone URL contains unsafe credentials or metadata")
|
||||
if parsed.scheme in {"http", "https"}:
|
||||
origin = self._origin(clone_url)
|
||||
if self.token and (self.allowed_clone_origin is None or origin != self.allowed_clone_origin):
|
||||
raise AcquisitionError("unsafe_clone_url", "Repository clone URL is outside the configured Gitea origin")
|
||||
elif parsed.scheme != "file":
|
||||
raise AcquisitionError("unsafe_clone_url", "Repository clone URL uses an unsupported scheme")
|
||||
except ValueError as exc:
|
||||
raise AcquisitionError("unsafe_clone_url", "Repository clone URL is invalid") from exc
|
||||
return clone_url
|
||||
|
||||
def acquire(
|
||||
self,
|
||||
repository: Repository,
|
||||
job_id: str,
|
||||
requested_revision: str | None = None,
|
||||
) -> AcquiredSource:
|
||||
with self._repository_lock(repository.id):
|
||||
return self._acquire(repository, job_id, requested_revision)
|
||||
|
||||
def _acquire(
|
||||
self,
|
||||
repository: Repository,
|
||||
job_id: str,
|
||||
requested_revision: str | None = None,
|
||||
) -> AcquiredSource:
|
||||
branch = repository.default_branch or "main"
|
||||
self._validate_branch(branch)
|
||||
if requested_revision is not None and not SAFE_REVISION.fullmatch(requested_revision):
|
||||
raise AcquisitionError("invalid_revision", "Requested revision is not a valid commit identifier")
|
||||
clone_url = self._validate_clone_url(repository.clone_url)
|
||||
mirror = self._mirror_path(repository)
|
||||
if mirror.exists() and not self._mirror_is_usable(mirror):
|
||||
self._remove_mirror_if_owned(mirror)
|
||||
if not mirror.exists():
|
||||
self._run(["git", "clone", "--mirror", clone_url, str(mirror)])
|
||||
else:
|
||||
self._run(["git", "--git-dir", str(mirror), "remote", "set-url", "origin", clone_url])
|
||||
self._run(
|
||||
[
|
||||
"git",
|
||||
"--git-dir",
|
||||
str(mirror),
|
||||
"fetch",
|
||||
"--prune",
|
||||
"--no-tags",
|
||||
"origin",
|
||||
f"+refs/heads/{branch}:refs/remotes/origin/{branch}",
|
||||
]
|
||||
)
|
||||
if requested_revision:
|
||||
try:
|
||||
actual_revision = self._run(
|
||||
["git", "--git-dir", str(mirror), "rev-parse", f"{requested_revision}^{{commit}}"]
|
||||
)
|
||||
except AcquisitionError as exc:
|
||||
raise AcquisitionError("revision_unavailable", "Requested commit was not found in the mirror") from exc
|
||||
if actual_revision.casefold() != requested_revision.casefold() and len(requested_revision) == 40:
|
||||
raise AcquisitionError("revision_unavailable", "Requested commit was not found in the mirror")
|
||||
else:
|
||||
actual_revision = self._run(
|
||||
[
|
||||
"git",
|
||||
"--git-dir",
|
||||
str(mirror),
|
||||
"rev-parse",
|
||||
f"refs/remotes/origin/{branch}^{{commit}}",
|
||||
]
|
||||
)
|
||||
if not SAFE_REVISION.fullmatch(actual_revision):
|
||||
raise AcquisitionError("invalid_revision", "Git did not return a usable commit identifier")
|
||||
|
||||
worktree = self._worktree_path(job_id)
|
||||
self._remove_worktree_if_owned(mirror, worktree)
|
||||
try:
|
||||
self._run(
|
||||
[
|
||||
"git",
|
||||
"--git-dir",
|
||||
str(mirror),
|
||||
"worktree",
|
||||
"add",
|
||||
"--detach",
|
||||
str(worktree),
|
||||
actual_revision,
|
||||
]
|
||||
)
|
||||
except AcquisitionError:
|
||||
self._remove_worktree_if_owned(mirror, worktree)
|
||||
raise
|
||||
return AcquiredSource(path=worktree, actual_revision=actual_revision, branch=branch)
|
||||
|
||||
def cleanup(self, source: AcquiredSource, repository: Repository) -> None:
|
||||
with self._repository_lock(repository.id):
|
||||
mirror = self._mirror_path(repository)
|
||||
self._remove_worktree_if_owned(mirror, source.path)
|
||||
|
||||
def _mirror_is_usable(self, mirror: Path) -> bool:
|
||||
try:
|
||||
return self._run(["git", "--git-dir", str(mirror), "rev-parse", "--is-bare-repository"]) == "true"
|
||||
except AcquisitionError:
|
||||
return False
|
||||
|
||||
def _remove_mirror_if_owned(self, mirror: Path) -> None:
|
||||
try:
|
||||
relative = mirror.resolve().relative_to(self.mirrors_root.resolve())
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
raise AcquisitionError("unsafe_mirror", "Repository mirror path escaped the repository cache")
|
||||
if not relative.parts:
|
||||
raise AcquisitionError("unsafe_mirror", "Repository mirror path is invalid")
|
||||
if mirror.is_dir() and not mirror.is_symlink():
|
||||
shutil.rmtree(mirror)
|
||||
elif mirror.exists() or mirror.is_symlink():
|
||||
mirror.unlink()
|
||||
|
||||
def _remove_worktree_if_owned(self, mirror: Path, worktree: Path) -> None:
|
||||
try:
|
||||
relative = worktree.resolve().relative_to(self.worktrees_root.resolve())
|
||||
except (OSError, RuntimeError, ValueError):
|
||||
raise AcquisitionError("unsafe_worktree", "Worktree path escaped the repository cache")
|
||||
if not relative.parts:
|
||||
raise AcquisitionError("unsafe_worktree", "Worktree path is invalid")
|
||||
if worktree.exists():
|
||||
try:
|
||||
self._run(["git", "--git-dir", str(mirror), "worktree", "remove", "--force", str(worktree)])
|
||||
except AcquisitionError:
|
||||
# A stale registration must not prevent a safe bounded cleanup.
|
||||
pass
|
||||
if worktree.exists():
|
||||
shutil.rmtree(worktree)
|
||||
try:
|
||||
self._run(["git", "--git-dir", str(mirror), "worktree", "prune"])
|
||||
except AcquisitionError:
|
||||
pass
|
||||
Reference in new issue
Block a user