315 lines
14 KiB
Python
315 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from collections.abc import Callable
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from .models import Repository, RepositoryPolicy, RepositorySyncResult
|
|
|
|
JsonTransport = Callable[[str, dict[str, str]], tuple[int, bytes]]
|
|
|
|
|
|
class GiteaError(RuntimeError):
|
|
def __init__(self, code: str, detail: str, *, status_code: int | None = None) -> None:
|
|
super().__init__(detail)
|
|
self.code = code
|
|
self.detail = detail
|
|
self.status_code = status_code
|
|
|
|
|
|
def _parse_time(value: object) -> datetime | None:
|
|
if not isinstance(value, str) or not value:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(value)
|
|
except ValueError:
|
|
return None
|
|
return parsed.replace(tzinfo=UTC) if parsed.tzinfo is None else parsed
|
|
|
|
|
|
class GiteaClient:
|
|
"""Small, read-only Gitea API client with bounded pagination and safe errors."""
|
|
|
|
def __init__(
|
|
self,
|
|
base_url: str | None = None,
|
|
token: str | None = None,
|
|
*,
|
|
timeout: float = 10.0,
|
|
page_size: int = 50,
|
|
max_pages: int = 20,
|
|
max_repositories: int = 500,
|
|
max_branch_workers: int | None = None,
|
|
transport: JsonTransport | None = None,
|
|
) -> None:
|
|
self.base_url = (base_url or os.getenv("REPOPULSE_GITEA_URL", "")).rstrip("/")
|
|
self.token = token if token is not None else os.getenv("REPOPULSE_GITEA_TOKEN", "")
|
|
self.timeout = max(1.0, timeout)
|
|
self.page_size = max(1, min(page_size, 100))
|
|
self.max_pages = max(1, max_pages)
|
|
self.max_repositories = max(1, max_repositories)
|
|
if max_branch_workers is None:
|
|
try:
|
|
configured_workers = int(os.getenv("REPOPULSE_GITEA_BRANCH_WORKERS", "4"))
|
|
except ValueError:
|
|
configured_workers = 4
|
|
else:
|
|
configured_workers = max_branch_workers
|
|
self.max_branch_workers = min(8, max(1, configured_workers))
|
|
self.transport = transport
|
|
self._listing_complete = True
|
|
|
|
@property
|
|
def configured(self) -> bool:
|
|
return bool(self.base_url and self.token)
|
|
|
|
def _request_json(self, path: str, params: dict[str, str | int] | None = None) -> Any:
|
|
if not self.base_url:
|
|
raise GiteaError("not_configured", "Gitea URL is not configured")
|
|
query = urllib.parse.urlencode(params or {})
|
|
url = f"{self.base_url}/api/v1/{path.lstrip('/')}"
|
|
if query:
|
|
url = f"{url}?{query}"
|
|
headers = {
|
|
"Accept": "application/json",
|
|
"User-Agent": "RepoPulse/1.5",
|
|
}
|
|
if self.token:
|
|
headers["Authorization"] = f"token {self.token}"
|
|
try:
|
|
if self.transport:
|
|
status_code, raw = self.transport(url, headers)
|
|
if status_code >= 400:
|
|
self._raise_for_status(status_code)
|
|
else:
|
|
request = urllib.request.Request(url, headers=headers, method="GET")
|
|
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
|
status_code = int(response.status)
|
|
raw = response.read(2_000_001)
|
|
if status_code >= 400:
|
|
self._raise_for_status(status_code)
|
|
except GiteaError:
|
|
raise
|
|
except urllib.error.HTTPError as exc:
|
|
self._raise_for_status(exc.code)
|
|
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
raise GiteaError("unreachable", "Gitea could not be reached") from exc
|
|
if not isinstance(raw, bytes):
|
|
raise GiteaError("invalid_response", "Gitea returned an invalid response body")
|
|
if len(raw) > 2_000_000:
|
|
raise GiteaError("response_too_large", "Gitea response exceeded the configured bound")
|
|
try:
|
|
return json.loads(raw.decode("utf-8"))
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise GiteaError("invalid_response", "Gitea returned invalid JSON") from exc
|
|
|
|
@staticmethod
|
|
def _raise_for_status(status_code: int) -> None:
|
|
if status_code in {401, 403}:
|
|
raise GiteaError("authentication", "Gitea authentication was rejected", status_code=status_code)
|
|
if status_code == 404:
|
|
raise GiteaError("not_found", "The requested Gitea resource was not found", status_code=status_code)
|
|
if status_code == 429:
|
|
raise GiteaError("rate_limited", "Gitea rate-limited the request", status_code=status_code)
|
|
if status_code >= 500:
|
|
raise GiteaError("upstream", "Gitea returned a server error", status_code=status_code)
|
|
raise GiteaError("request_failed", "Gitea rejected the request", status_code=status_code)
|
|
|
|
def _repository_from_payload(self, payload: object) -> Repository:
|
|
if not isinstance(payload, dict):
|
|
raise GiteaError("invalid_repository", "Gitea returned an invalid repository record")
|
|
raw_id = payload.get("id")
|
|
if isinstance(raw_id, bool) or not isinstance(raw_id, (int, str)) or not str(raw_id).isdigit():
|
|
raise GiteaError("invalid_repository", "Gitea repository identity is invalid")
|
|
owner_payload = payload.get("owner")
|
|
owner = owner_payload.get("login") if isinstance(owner_payload, dict) else None
|
|
owner = owner or payload.get("owner_name")
|
|
name = payload.get("name")
|
|
full_name = payload.get("full_name")
|
|
if not isinstance(owner, str) or not isinstance(name, str):
|
|
if isinstance(full_name, str) and "/" in full_name:
|
|
owner, name = full_name.split("/", 1)
|
|
else:
|
|
raise GiteaError("invalid_repository", "Gitea repository name is invalid")
|
|
full_name = full_name if isinstance(full_name, str) else f"{owner}/{name}"
|
|
repository_id = str(raw_id)
|
|
# Do not trust a provider-supplied clone host when constructing the
|
|
# credential-bearing acquisition URL. The configured Gitea origin is
|
|
# the only network destination that may receive the server token.
|
|
clone_url = (
|
|
f"{self.base_url}/{urllib.parse.quote(owner, safe='')}/"
|
|
f"{urllib.parse.quote(name, safe='')}.git"
|
|
)
|
|
html_url = payload.get("html_url")
|
|
default_branch = payload.get("default_branch") or "main"
|
|
return Repository(
|
|
id=f"gitea:{repository_id}",
|
|
provider="gitea",
|
|
provider_repository_id=repository_id,
|
|
owner=owner,
|
|
name=name,
|
|
full_name=full_name,
|
|
description=payload.get("description") if isinstance(payload.get("description"), str) else None,
|
|
html_url=html_url if isinstance(html_url, str) else None,
|
|
clone_url=clone_url if isinstance(clone_url, str) else None,
|
|
default_branch=str(default_branch),
|
|
private=bool(payload.get("private", False)),
|
|
archived=bool(payload.get("archived", False)),
|
|
latest_revision_at=_parse_time(payload.get("updated_at")),
|
|
policy=RepositoryPolicy(),
|
|
)
|
|
|
|
def list_repositories(self) -> list[Repository]:
|
|
if not self.configured:
|
|
raise GiteaError("not_configured", "Gitea connector is not configured")
|
|
repositories: list[Repository] = []
|
|
self._listing_complete = False
|
|
for page in range(1, self.max_pages + 1):
|
|
payload = self._request_json(
|
|
"user/repos",
|
|
{"limit": self.page_size, "page": page, "sort": "updated", "direction": "desc"},
|
|
)
|
|
if not isinstance(payload, list):
|
|
raise GiteaError("invalid_response", "Gitea repository listing was not a list")
|
|
for item in payload:
|
|
if len(repositories) >= self.max_repositories:
|
|
return repositories
|
|
repositories.append(self._repository_from_payload(item))
|
|
if len(payload) < self.page_size:
|
|
self._listing_complete = True
|
|
break
|
|
return repositories
|
|
|
|
def branch_revision(self, repository: Repository) -> str:
|
|
owner = urllib.parse.quote(repository.owner, safe="")
|
|
name = urllib.parse.quote(repository.name, safe="")
|
|
branch = urllib.parse.quote(repository.default_branch, safe="")
|
|
payload = self._request_json(f"repos/{owner}/{name}/branches/{branch}")
|
|
if not isinstance(payload, dict):
|
|
raise GiteaError("invalid_response", "Gitea branch response was invalid")
|
|
commit = payload.get("commit")
|
|
revision = None
|
|
if isinstance(commit, dict):
|
|
revision = commit.get("id") or commit.get("sha")
|
|
if (
|
|
not isinstance(revision, str)
|
|
or not 7 <= len(revision) <= 64
|
|
or not all(char in "0123456789abcdefABCDEF" for char in revision)
|
|
):
|
|
raise GiteaError("invalid_revision", "Gitea did not return a usable branch revision")
|
|
return revision
|
|
|
|
def discover_with_errors(self) -> tuple[list[Repository], list[tuple[Repository, str]], list[str]]:
|
|
"""Resolve branch tips independently so one repository cannot abort a catalog sync."""
|
|
|
|
repositories = self.list_repositories()
|
|
|
|
def resolve(repository: Repository) -> tuple[Repository, str | None, str | None]:
|
|
try:
|
|
return repository, self.branch_revision(repository), None
|
|
except GiteaError as exc:
|
|
return repository, None, f"{repository.full_name}: {exc.code}: {exc.detail}"
|
|
|
|
with ThreadPoolExecutor(max_workers=min(self.max_branch_workers, len(repositories) or 1)) as executor:
|
|
outcomes = list(executor.map(resolve, repositories))
|
|
discovered = [
|
|
(repository.model_copy(update={"latest_revision": revision}), revision)
|
|
for repository, revision, error in outcomes
|
|
if revision is not None and error is None
|
|
]
|
|
errors = [error for _, _, error in outcomes if error]
|
|
return repositories, discovered, errors
|
|
|
|
def discover(self) -> list[tuple[Repository, str]]:
|
|
"""Compatibility helper for callers needing only successfully resolved repositories."""
|
|
|
|
_, discovered, _ = self.discover_with_errors()
|
|
return discovered
|
|
|
|
def sync(self, store) -> RepositorySyncResult:
|
|
synced_at = datetime.now(UTC)
|
|
try:
|
|
listed, discovered, branch_errors = self.discover_with_errors()
|
|
except GiteaError as exc:
|
|
error = f"{exc.code}: {exc.detail}"
|
|
for existing in store.list_repositories(limit=1000):
|
|
if existing.provider == "gitea" and existing.available:
|
|
store.update_repository_sync(existing, synced_at=synced_at, error=error)
|
|
result = RepositorySyncResult(
|
|
provider="gitea",
|
|
errors=[error],
|
|
synced_at=synced_at,
|
|
)
|
|
store.record_sync_result(result)
|
|
return result
|
|
created = 0
|
|
updated = 0
|
|
listed_ids = {repository.id for repository in listed}
|
|
seen_ids: set[str] = set()
|
|
for repository, revision in discovered:
|
|
seen_ids.add(repository.id)
|
|
before = store.get_repository(repository.id)
|
|
merged = repository.model_copy(
|
|
update={
|
|
"available": True,
|
|
"last_synced_at": synced_at,
|
|
"sync_error": None,
|
|
"latest_revision": revision,
|
|
"policy": before.policy if before else RepositoryPolicy(),
|
|
}
|
|
)
|
|
store.upsert_repository(merged)
|
|
if before is None:
|
|
created += 1
|
|
else:
|
|
updated += 1
|
|
for repository in listed:
|
|
if repository.id in seen_ids:
|
|
continue
|
|
before = store.get_repository(repository.id)
|
|
error = next((item for item in branch_errors if item.startswith(f"{repository.full_name}:")), None)
|
|
merged = repository.model_copy(
|
|
update={
|
|
"available": True,
|
|
"last_synced_at": synced_at,
|
|
"sync_error": error or "branch: revision could not be resolved",
|
|
"latest_revision": before.latest_revision if before else None,
|
|
"policy": before.policy if before else RepositoryPolicy(),
|
|
}
|
|
)
|
|
store.upsert_repository(merged)
|
|
if before is None:
|
|
created += 1
|
|
else:
|
|
updated += 1
|
|
unavailable = 0
|
|
if self._listing_complete:
|
|
for existing in store.list_repositories(limit=1000):
|
|
if existing.provider == "gitea" and existing.id not in listed_ids and existing.available:
|
|
store.update_repository_sync(
|
|
existing.model_copy(update={"available": False}),
|
|
synced_at=synced_at,
|
|
error="not_discovered",
|
|
)
|
|
unavailable += 1
|
|
result = RepositorySyncResult(
|
|
provider="gitea",
|
|
discovered=len(discovered),
|
|
created=created,
|
|
updated=updated,
|
|
unavailable=unavailable,
|
|
errors=branch_errors,
|
|
partial=bool(branch_errors) or not self._listing_complete,
|
|
branch_failures=len(branch_errors),
|
|
listing_complete=self._listing_complete,
|
|
synced_at=synced_at,
|
|
)
|
|
store.record_sync_result(result)
|
|
return result
|