265 lines
11 KiB
Python
265 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from threading import Lock, RLock
|
|
from urllib.parse import urlsplit
|
|
|
|
from .models import Repository
|
|
|
|
SAFE_REVISION = re.compile(r"^[0-9a-fA-F]{7,64}$")
|
|
SAFE_BRANCH = re.compile(r"^[A-Za-z0-9._/-]+$")
|
|
SAFE_JOB_ID = re.compile(r"^[A-Za-z0-9-]+$")
|
|
|
|
|
|
class AcquisitionError(RuntimeError):
|
|
def __init__(self, code: str, detail: str) -> None:
|
|
super().__init__(detail)
|
|
self.code = code
|
|
self.detail = detail
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AcquiredSource:
|
|
path: Path
|
|
actual_revision: str
|
|
branch: str
|
|
|
|
|
|
class GitAcquirer:
|
|
"""Maintain bounded bare mirrors and scan detached exact-revision worktrees."""
|
|
|
|
def __init__(
|
|
self,
|
|
cache_root: str | Path | None = None,
|
|
token: str | None = None,
|
|
*,
|
|
timeout: int = 180,
|
|
allowed_clone_origin: str | None = None,
|
|
) -> None:
|
|
raw_root = cache_root or os.getenv("REPOPULSE_REPOSITORY_CACHE", "data/repos")
|
|
self.cache_root = Path(raw_root).expanduser().resolve()
|
|
self.token = token if token is not None else os.getenv("REPOPULSE_GITEA_TOKEN", "")
|
|
self.timeout = max(10, timeout)
|
|
configured_origin = allowed_clone_origin or os.getenv("REPOPULSE_GITEA_URL", "")
|
|
self.allowed_clone_origin = self._origin(configured_origin)
|
|
self.mirrors_root = self.cache_root / "mirrors"
|
|
self.worktrees_root = self.cache_root / "worktrees"
|
|
self.mirrors_root.mkdir(parents=True, exist_ok=True)
|
|
self.worktrees_root.mkdir(parents=True, exist_ok=True)
|
|
self._lock_guard = Lock()
|
|
self._repository_locks: dict[str, RLock] = {}
|
|
|
|
@staticmethod
|
|
def _origin(value: str) -> str | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
parsed = urlsplit(value)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
return None
|
|
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
|
except ValueError:
|
|
return None
|
|
return f"{parsed.scheme.casefold()}://{parsed.hostname.casefold()}:{port}"
|
|
|
|
def _repository_lock(self, repository_id: str) -> RLock:
|
|
with self._lock_guard:
|
|
return self._repository_locks.setdefault(repository_id, RLock())
|
|
|
|
def _git_env(self) -> dict[str, str]:
|
|
env = os.environ.copy()
|
|
if self.token:
|
|
env.update(
|
|
{
|
|
"GIT_CONFIG_COUNT": "1",
|
|
"GIT_CONFIG_KEY_0": "http.extraHeader",
|
|
"GIT_CONFIG_VALUE_0": f"Authorization: token {self.token}",
|
|
}
|
|
)
|
|
return env
|
|
|
|
def _run(self, args: list[str]) -> str:
|
|
try:
|
|
result = subprocess.run(
|
|
args,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
errors="replace",
|
|
timeout=self.timeout,
|
|
env=self._git_env(),
|
|
)
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise AcquisitionError("timeout", "Git source acquisition timed out") from exc
|
|
except (OSError, subprocess.CalledProcessError) as exc:
|
|
# Git stderr may contain remote URLs or server-specific details; do
|
|
# not persist or expose it because the job API is user-visible.
|
|
raise AcquisitionError("git_failed", "Git source acquisition failed") from exc
|
|
return result.stdout.strip()
|
|
|
|
def _mirror_path(self, repository: Repository) -> Path:
|
|
identity = f"{repository.provider}-{repository.provider_repository_id}"
|
|
safe = re.sub(r"[^A-Za-z0-9._-]", "_", identity)
|
|
return self.mirrors_root / f"{safe}.git"
|
|
|
|
def _worktree_path(self, job_id: str) -> Path:
|
|
if not SAFE_JOB_ID.fullmatch(job_id):
|
|
raise AcquisitionError("invalid_job", "Scan job identity is invalid")
|
|
return self.worktrees_root / job_id
|
|
|
|
@staticmethod
|
|
def _validate_branch(branch: str) -> None:
|
|
if (
|
|
not SAFE_BRANCH.fullmatch(branch)
|
|
or branch.startswith("-")
|
|
or ".." in branch
|
|
or branch.endswith(("/", "."))
|
|
):
|
|
raise AcquisitionError("invalid_branch", "Repository default branch is invalid")
|
|
|
|
def _validate_clone_url(self, clone_url: str | None) -> str:
|
|
if not clone_url or "\x00" in clone_url:
|
|
raise AcquisitionError("missing_clone_url", "Repository clone URL is unavailable")
|
|
try:
|
|
parsed = urlsplit(clone_url)
|
|
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
|
raise AcquisitionError("unsafe_clone_url", "Repository clone URL contains unsafe credentials or metadata")
|
|
if parsed.scheme in {"http", "https"}:
|
|
origin = self._origin(clone_url)
|
|
if self.token and (self.allowed_clone_origin is None or origin != self.allowed_clone_origin):
|
|
raise AcquisitionError("unsafe_clone_url", "Repository clone URL is outside the configured Gitea origin")
|
|
elif parsed.scheme != "file":
|
|
raise AcquisitionError("unsafe_clone_url", "Repository clone URL uses an unsupported scheme")
|
|
except ValueError as exc:
|
|
raise AcquisitionError("unsafe_clone_url", "Repository clone URL is invalid") from exc
|
|
return clone_url
|
|
|
|
def acquire(
|
|
self,
|
|
repository: Repository,
|
|
job_id: str,
|
|
requested_revision: str | None = None,
|
|
) -> AcquiredSource:
|
|
with self._repository_lock(repository.id):
|
|
return self._acquire(repository, job_id, requested_revision)
|
|
|
|
def _acquire(
|
|
self,
|
|
repository: Repository,
|
|
job_id: str,
|
|
requested_revision: str | None = None,
|
|
) -> AcquiredSource:
|
|
branch = repository.default_branch or "main"
|
|
self._validate_branch(branch)
|
|
if requested_revision is not None and not SAFE_REVISION.fullmatch(requested_revision):
|
|
raise AcquisitionError("invalid_revision", "Requested revision is not a valid commit identifier")
|
|
clone_url = self._validate_clone_url(repository.clone_url)
|
|
mirror = self._mirror_path(repository)
|
|
if mirror.exists() and not self._mirror_is_usable(mirror):
|
|
self._remove_mirror_if_owned(mirror)
|
|
if not mirror.exists():
|
|
self._run(["git", "clone", "--mirror", clone_url, str(mirror)])
|
|
else:
|
|
self._run(["git", "--git-dir", str(mirror), "remote", "set-url", "origin", clone_url])
|
|
self._run(
|
|
[
|
|
"git",
|
|
"--git-dir",
|
|
str(mirror),
|
|
"fetch",
|
|
"--prune",
|
|
"--no-tags",
|
|
"origin",
|
|
f"+refs/heads/{branch}:refs/remotes/origin/{branch}",
|
|
]
|
|
)
|
|
if requested_revision:
|
|
try:
|
|
actual_revision = self._run(
|
|
["git", "--git-dir", str(mirror), "rev-parse", f"{requested_revision}^{{commit}}"]
|
|
)
|
|
except AcquisitionError as exc:
|
|
raise AcquisitionError("revision_unavailable", "Requested commit was not found in the mirror") from exc
|
|
if actual_revision.casefold() != requested_revision.casefold() and len(requested_revision) == 40:
|
|
raise AcquisitionError("revision_unavailable", "Requested commit was not found in the mirror")
|
|
else:
|
|
actual_revision = self._run(
|
|
[
|
|
"git",
|
|
"--git-dir",
|
|
str(mirror),
|
|
"rev-parse",
|
|
f"refs/remotes/origin/{branch}^{{commit}}",
|
|
]
|
|
)
|
|
if not SAFE_REVISION.fullmatch(actual_revision):
|
|
raise AcquisitionError("invalid_revision", "Git did not return a usable commit identifier")
|
|
|
|
worktree = self._worktree_path(job_id)
|
|
self._remove_worktree_if_owned(mirror, worktree)
|
|
try:
|
|
self._run(
|
|
[
|
|
"git",
|
|
"--git-dir",
|
|
str(mirror),
|
|
"worktree",
|
|
"add",
|
|
"--detach",
|
|
str(worktree),
|
|
actual_revision,
|
|
]
|
|
)
|
|
except AcquisitionError:
|
|
self._remove_worktree_if_owned(mirror, worktree)
|
|
raise
|
|
return AcquiredSource(path=worktree, actual_revision=actual_revision, branch=branch)
|
|
|
|
def cleanup(self, source: AcquiredSource, repository: Repository) -> None:
|
|
with self._repository_lock(repository.id):
|
|
mirror = self._mirror_path(repository)
|
|
self._remove_worktree_if_owned(mirror, source.path)
|
|
|
|
def _mirror_is_usable(self, mirror: Path) -> bool:
|
|
try:
|
|
return self._run(["git", "--git-dir", str(mirror), "rev-parse", "--is-bare-repository"]) == "true"
|
|
except AcquisitionError:
|
|
return False
|
|
|
|
def _remove_mirror_if_owned(self, mirror: Path) -> None:
|
|
try:
|
|
relative = mirror.resolve().relative_to(self.mirrors_root.resolve())
|
|
except (OSError, RuntimeError, ValueError):
|
|
raise AcquisitionError("unsafe_mirror", "Repository mirror path escaped the repository cache")
|
|
if not relative.parts:
|
|
raise AcquisitionError("unsafe_mirror", "Repository mirror path is invalid")
|
|
if mirror.is_dir() and not mirror.is_symlink():
|
|
shutil.rmtree(mirror)
|
|
elif mirror.exists() or mirror.is_symlink():
|
|
mirror.unlink()
|
|
|
|
def _remove_worktree_if_owned(self, mirror: Path, worktree: Path) -> None:
|
|
try:
|
|
relative = worktree.resolve().relative_to(self.worktrees_root.resolve())
|
|
except (OSError, RuntimeError, ValueError):
|
|
raise AcquisitionError("unsafe_worktree", "Worktree path escaped the repository cache")
|
|
if not relative.parts:
|
|
raise AcquisitionError("unsafe_worktree", "Worktree path is invalid")
|
|
if worktree.exists():
|
|
try:
|
|
self._run(["git", "--git-dir", str(mirror), "worktree", "remove", "--force", str(worktree)])
|
|
except AcquisitionError:
|
|
# A stale registration must not prevent a safe bounded cleanup.
|
|
pass
|
|
if worktree.exists():
|
|
shutil.rmtree(worktree)
|
|
try:
|
|
self._run(["git", "--git-dir", str(mirror), "worktree", "prune"])
|
|
except AcquisitionError:
|
|
pass
|