Initial curated RepoPulse source from 2524ccc3c1c27739996dccb2b7992989f88cd26d
This commit is contained in:
Public Source Publisher committed
commit
0adf8ef5d9
90 files changed
+14315
No files matched your search
@@ -0,0 +1,314 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from .models import Repository, RepositoryPolicy, RepositorySyncResult
|
||||
|
||||
JsonTransport = Callable[[str, dict[str, str]], tuple[int, bytes]]
|
||||
|
||||
|
||||
class GiteaError(RuntimeError):
|
||||
def __init__(self, code: str, detail: str, *, status_code: int | None = None) -> None:
|
||||
super().__init__(detail)
|
||||
self.code = code
|
||||
self.detail = detail
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
def _parse_time(value: object) -> datetime | None:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed.replace(tzinfo=UTC) if parsed.tzinfo is None else parsed
|
||||
|
||||
|
||||
class GiteaClient:
|
||||
"""Small, read-only Gitea API client with bounded pagination and safe errors."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str | None = None,
|
||||
token: str | None = None,
|
||||
*,
|
||||
timeout: float = 10.0,
|
||||
page_size: int = 50,
|
||||
max_pages: int = 20,
|
||||
max_repositories: int = 500,
|
||||
max_branch_workers: int | None = None,
|
||||
transport: JsonTransport | None = None,
|
||||
) -> None:
|
||||
self.base_url = (base_url or os.getenv("REPOPULSE_GITEA_URL", "")).rstrip("/")
|
||||
self.token = token if token is not None else os.getenv("REPOPULSE_GITEA_TOKEN", "")
|
||||
self.timeout = max(1.0, timeout)
|
||||
self.page_size = max(1, min(page_size, 100))
|
||||
self.max_pages = max(1, max_pages)
|
||||
self.max_repositories = max(1, max_repositories)
|
||||
if max_branch_workers is None:
|
||||
try:
|
||||
configured_workers = int(os.getenv("REPOPULSE_GITEA_BRANCH_WORKERS", "4"))
|
||||
except ValueError:
|
||||
configured_workers = 4
|
||||
else:
|
||||
configured_workers = max_branch_workers
|
||||
self.max_branch_workers = min(8, max(1, configured_workers))
|
||||
self.transport = transport
|
||||
self._listing_complete = True
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self.base_url and self.token)
|
||||
|
||||
def _request_json(self, path: str, params: dict[str, str | int] | None = None) -> Any:
|
||||
if not self.base_url:
|
||||
raise GiteaError("not_configured", "Gitea URL is not configured")
|
||||
query = urllib.parse.urlencode(params or {})
|
||||
url = f"{self.base_url}/api/v1/{path.lstrip('/')}"
|
||||
if query:
|
||||
url = f"{url}?{query}"
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "RepoPulse/1.5",
|
||||
}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"token {self.token}"
|
||||
try:
|
||||
if self.transport:
|
||||
status_code, raw = self.transport(url, headers)
|
||||
if status_code >= 400:
|
||||
self._raise_for_status(status_code)
|
||||
else:
|
||||
request = urllib.request.Request(url, headers=headers, method="GET")
|
||||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||||
status_code = int(response.status)
|
||||
raw = response.read(2_000_001)
|
||||
if status_code >= 400:
|
||||
self._raise_for_status(status_code)
|
||||
except GiteaError:
|
||||
raise
|
||||
except urllib.error.HTTPError as exc:
|
||||
self._raise_for_status(exc.code)
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise GiteaError("unreachable", "Gitea could not be reached") from exc
|
||||
if not isinstance(raw, bytes):
|
||||
raise GiteaError("invalid_response", "Gitea returned an invalid response body")
|
||||
if len(raw) > 2_000_000:
|
||||
raise GiteaError("response_too_large", "Gitea response exceeded the configured bound")
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise GiteaError("invalid_response", "Gitea returned invalid JSON") from exc
|
||||
|
||||
@staticmethod
|
||||
def _raise_for_status(status_code: int) -> None:
|
||||
if status_code in {401, 403}:
|
||||
raise GiteaError("authentication", "Gitea authentication was rejected", status_code=status_code)
|
||||
if status_code == 404:
|
||||
raise GiteaError("not_found", "The requested Gitea resource was not found", status_code=status_code)
|
||||
if status_code == 429:
|
||||
raise GiteaError("rate_limited", "Gitea rate-limited the request", status_code=status_code)
|
||||
if status_code >= 500:
|
||||
raise GiteaError("upstream", "Gitea returned a server error", status_code=status_code)
|
||||
raise GiteaError("request_failed", "Gitea rejected the request", status_code=status_code)
|
||||
|
||||
def _repository_from_payload(self, payload: object) -> Repository:
|
||||
if not isinstance(payload, dict):
|
||||
raise GiteaError("invalid_repository", "Gitea returned an invalid repository record")
|
||||
raw_id = payload.get("id")
|
||||
if isinstance(raw_id, bool) or not isinstance(raw_id, (int, str)) or not str(raw_id).isdigit():
|
||||
raise GiteaError("invalid_repository", "Gitea repository identity is invalid")
|
||||
owner_payload = payload.get("owner")
|
||||
owner = owner_payload.get("login") if isinstance(owner_payload, dict) else None
|
||||
owner = owner or payload.get("owner_name")
|
||||
name = payload.get("name")
|
||||
full_name = payload.get("full_name")
|
||||
if not isinstance(owner, str) or not isinstance(name, str):
|
||||
if isinstance(full_name, str) and "/" in full_name:
|
||||
owner, name = full_name.split("/", 1)
|
||||
else:
|
||||
raise GiteaError("invalid_repository", "Gitea repository name is invalid")
|
||||
full_name = full_name if isinstance(full_name, str) else f"{owner}/{name}"
|
||||
repository_id = str(raw_id)
|
||||
# Do not trust a provider-supplied clone host when constructing the
|
||||
# credential-bearing acquisition URL. The configured Gitea origin is
|
||||
# the only network destination that may receive the server token.
|
||||
clone_url = (
|
||||
f"{self.base_url}/{urllib.parse.quote(owner, safe='')}/"
|
||||
f"{urllib.parse.quote(name, safe='')}.git"
|
||||
)
|
||||
html_url = payload.get("html_url")
|
||||
default_branch = payload.get("default_branch") or "main"
|
||||
return Repository(
|
||||
id=f"gitea:{repository_id}",
|
||||
provider="gitea",
|
||||
provider_repository_id=repository_id,
|
||||
owner=owner,
|
||||
name=name,
|
||||
full_name=full_name,
|
||||
description=payload.get("description") if isinstance(payload.get("description"), str) else None,
|
||||
html_url=html_url if isinstance(html_url, str) else None,
|
||||
clone_url=clone_url if isinstance(clone_url, str) else None,
|
||||
default_branch=str(default_branch),
|
||||
private=bool(payload.get("private", False)),
|
||||
archived=bool(payload.get("archived", False)),
|
||||
latest_revision_at=_parse_time(payload.get("updated_at")),
|
||||
policy=RepositoryPolicy(),
|
||||
)
|
||||
|
||||
def list_repositories(self) -> list[Repository]:
|
||||
if not self.configured:
|
||||
raise GiteaError("not_configured", "Gitea connector is not configured")
|
||||
repositories: list[Repository] = []
|
||||
self._listing_complete = False
|
||||
for page in range(1, self.max_pages + 1):
|
||||
payload = self._request_json(
|
||||
"user/repos",
|
||||
{"limit": self.page_size, "page": page, "sort": "updated", "direction": "desc"},
|
||||
)
|
||||
if not isinstance(payload, list):
|
||||
raise GiteaError("invalid_response", "Gitea repository listing was not a list")
|
||||
for item in payload:
|
||||
if len(repositories) >= self.max_repositories:
|
||||
return repositories
|
||||
repositories.append(self._repository_from_payload(item))
|
||||
if len(payload) < self.page_size:
|
||||
self._listing_complete = True
|
||||
break
|
||||
return repositories
|
||||
|
||||
def branch_revision(self, repository: Repository) -> str:
|
||||
owner = urllib.parse.quote(repository.owner, safe="")
|
||||
name = urllib.parse.quote(repository.name, safe="")
|
||||
branch = urllib.parse.quote(repository.default_branch, safe="")
|
||||
payload = self._request_json(f"repos/{owner}/{name}/branches/{branch}")
|
||||
if not isinstance(payload, dict):
|
||||
raise GiteaError("invalid_response", "Gitea branch response was invalid")
|
||||
commit = payload.get("commit")
|
||||
revision = None
|
||||
if isinstance(commit, dict):
|
||||
revision = commit.get("id") or commit.get("sha")
|
||||
if (
|
||||
not isinstance(revision, str)
|
||||
or not 7 <= len(revision) <= 64
|
||||
or not all(char in "0123456789abcdefABCDEF" for char in revision)
|
||||
):
|
||||
raise GiteaError("invalid_revision", "Gitea did not return a usable branch revision")
|
||||
return revision
|
||||
|
||||
def discover_with_errors(self) -> tuple[list[Repository], list[tuple[Repository, str]], list[str]]:
|
||||
"""Resolve branch tips independently so one repository cannot abort a catalog sync."""
|
||||
|
||||
repositories = self.list_repositories()
|
||||
|
||||
def resolve(repository: Repository) -> tuple[Repository, str | None, str | None]:
|
||||
try:
|
||||
return repository, self.branch_revision(repository), None
|
||||
except GiteaError as exc:
|
||||
return repository, None, f"{repository.full_name}: {exc.code}: {exc.detail}"
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(self.max_branch_workers, len(repositories) or 1)) as executor:
|
||||
outcomes = list(executor.map(resolve, repositories))
|
||||
discovered = [
|
||||
(repository.model_copy(update={"latest_revision": revision}), revision)
|
||||
for repository, revision, error in outcomes
|
||||
if revision is not None and error is None
|
||||
]
|
||||
errors = [error for _, _, error in outcomes if error]
|
||||
return repositories, discovered, errors
|
||||
|
||||
def discover(self) -> list[tuple[Repository, str]]:
|
||||
"""Compatibility helper for callers needing only successfully resolved repositories."""
|
||||
|
||||
_, discovered, _ = self.discover_with_errors()
|
||||
return discovered
|
||||
|
||||
def sync(self, store) -> RepositorySyncResult:
|
||||
synced_at = datetime.now(UTC)
|
||||
try:
|
||||
listed, discovered, branch_errors = self.discover_with_errors()
|
||||
except GiteaError as exc:
|
||||
error = f"{exc.code}: {exc.detail}"
|
||||
for existing in store.list_repositories(limit=1000):
|
||||
if existing.provider == "gitea" and existing.available:
|
||||
store.update_repository_sync(existing, synced_at=synced_at, error=error)
|
||||
result = RepositorySyncResult(
|
||||
provider="gitea",
|
||||
errors=[error],
|
||||
synced_at=synced_at,
|
||||
)
|
||||
store.record_sync_result(result)
|
||||
return result
|
||||
created = 0
|
||||
updated = 0
|
||||
listed_ids = {repository.id for repository in listed}
|
||||
seen_ids: set[str] = set()
|
||||
for repository, revision in discovered:
|
||||
seen_ids.add(repository.id)
|
||||
before = store.get_repository(repository.id)
|
||||
merged = repository.model_copy(
|
||||
update={
|
||||
"available": True,
|
||||
"last_synced_at": synced_at,
|
||||
"sync_error": None,
|
||||
"latest_revision": revision,
|
||||
"policy": before.policy if before else RepositoryPolicy(),
|
||||
}
|
||||
)
|
||||
store.upsert_repository(merged)
|
||||
if before is None:
|
||||
created += 1
|
||||
else:
|
||||
updated += 1
|
||||
for repository in listed:
|
||||
if repository.id in seen_ids:
|
||||
continue
|
||||
before = store.get_repository(repository.id)
|
||||
error = next((item for item in branch_errors if item.startswith(f"{repository.full_name}:")), None)
|
||||
merged = repository.model_copy(
|
||||
update={
|
||||
"available": True,
|
||||
"last_synced_at": synced_at,
|
||||
"sync_error": error or "branch: revision could not be resolved",
|
||||
"latest_revision": before.latest_revision if before else None,
|
||||
"policy": before.policy if before else RepositoryPolicy(),
|
||||
}
|
||||
)
|
||||
store.upsert_repository(merged)
|
||||
if before is None:
|
||||
created += 1
|
||||
else:
|
||||
updated += 1
|
||||
unavailable = 0
|
||||
if self._listing_complete:
|
||||
for existing in store.list_repositories(limit=1000):
|
||||
if existing.provider == "gitea" and existing.id not in listed_ids and existing.available:
|
||||
store.update_repository_sync(
|
||||
existing.model_copy(update={"available": False}),
|
||||
synced_at=synced_at,
|
||||
error="not_discovered",
|
||||
)
|
||||
unavailable += 1
|
||||
result = RepositorySyncResult(
|
||||
provider="gitea",
|
||||
discovered=len(discovered),
|
||||
created=created,
|
||||
updated=updated,
|
||||
unavailable=unavailable,
|
||||
errors=branch_errors,
|
||||
partial=bool(branch_errors) or not self._listing_complete,
|
||||
branch_failures=len(branch_errors),
|
||||
listing_complete=self._listing_complete,
|
||||
synced_at=synced_at,
|
||||
)
|
||||
store.record_sync_result(result)
|
||||
return result
|
||||
Reference in new issue
Block a user