from __future__ import annotations import json import os import urllib.error import urllib.parse import urllib.request from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime from typing import Any from .models import Repository, RepositoryPolicy, RepositorySyncResult JsonTransport = Callable[[str, dict[str, str]], tuple[int, bytes]] class GiteaError(RuntimeError): def __init__(self, code: str, detail: str, *, status_code: int | None = None) -> None: super().__init__(detail) self.code = code self.detail = detail self.status_code = status_code def _parse_time(value: object) -> datetime | None: if not isinstance(value, str) or not value: return None try: parsed = datetime.fromisoformat(value) except ValueError: return None return parsed.replace(tzinfo=UTC) if parsed.tzinfo is None else parsed class GiteaClient: """Small, read-only Gitea API client with bounded pagination and safe errors.""" def __init__( self, base_url: str | None = None, token: str | None = None, *, timeout: float = 10.0, page_size: int = 50, max_pages: int = 20, max_repositories: int = 500, max_branch_workers: int | None = None, transport: JsonTransport | None = None, ) -> None: self.base_url = (base_url or os.getenv("REPOPULSE_GITEA_URL", "")).rstrip("/") self.token = token if token is not None else os.getenv("REPOPULSE_GITEA_TOKEN", "") self.timeout = max(1.0, timeout) self.page_size = max(1, min(page_size, 100)) self.max_pages = max(1, max_pages) self.max_repositories = max(1, max_repositories) if max_branch_workers is None: try: configured_workers = int(os.getenv("REPOPULSE_GITEA_BRANCH_WORKERS", "4")) except ValueError: configured_workers = 4 else: configured_workers = max_branch_workers self.max_branch_workers = min(8, max(1, configured_workers)) self.transport = transport self._listing_complete = True @property def configured(self) -> bool: return bool(self.base_url and self.token) def _request_json(self, path: str, params: dict[str, str | int] | None = None) -> Any: if not self.base_url: raise GiteaError("not_configured", "Gitea URL is not configured") query = urllib.parse.urlencode(params or {}) url = f"{self.base_url}/api/v1/{path.lstrip('/')}" if query: url = f"{url}?{query}" headers = { "Accept": "application/json", "User-Agent": "RepoPulse/1.5", } if self.token: headers["Authorization"] = f"token {self.token}" try: if self.transport: status_code, raw = self.transport(url, headers) if status_code >= 400: self._raise_for_status(status_code) else: request = urllib.request.Request(url, headers=headers, method="GET") with urllib.request.urlopen(request, timeout=self.timeout) as response: status_code = int(response.status) raw = response.read(2_000_001) if status_code >= 400: self._raise_for_status(status_code) except GiteaError: raise except urllib.error.HTTPError as exc: self._raise_for_status(exc.code) except (urllib.error.URLError, TimeoutError, OSError) as exc: raise GiteaError("unreachable", "Gitea could not be reached") from exc if not isinstance(raw, bytes): raise GiteaError("invalid_response", "Gitea returned an invalid response body") if len(raw) > 2_000_000: raise GiteaError("response_too_large", "Gitea response exceeded the configured bound") try: return json.loads(raw.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise GiteaError("invalid_response", "Gitea returned invalid JSON") from exc @staticmethod def _raise_for_status(status_code: int) -> None: if status_code in {401, 403}: raise GiteaError("authentication", "Gitea authentication was rejected", status_code=status_code) if status_code == 404: raise GiteaError("not_found", "The requested Gitea resource was not found", status_code=status_code) if status_code == 429: raise GiteaError("rate_limited", "Gitea rate-limited the request", status_code=status_code) if status_code >= 500: raise GiteaError("upstream", "Gitea returned a server error", status_code=status_code) raise GiteaError("request_failed", "Gitea rejected the request", status_code=status_code) def _repository_from_payload(self, payload: object) -> Repository: if not isinstance(payload, dict): raise GiteaError("invalid_repository", "Gitea returned an invalid repository record") raw_id = payload.get("id") if isinstance(raw_id, bool) or not isinstance(raw_id, (int, str)) or not str(raw_id).isdigit(): raise GiteaError("invalid_repository", "Gitea repository identity is invalid") owner_payload = payload.get("owner") owner = owner_payload.get("login") if isinstance(owner_payload, dict) else None owner = owner or payload.get("owner_name") name = payload.get("name") full_name = payload.get("full_name") if not isinstance(owner, str) or not isinstance(name, str): if isinstance(full_name, str) and "/" in full_name: owner, name = full_name.split("/", 1) else: raise GiteaError("invalid_repository", "Gitea repository name is invalid") full_name = full_name if isinstance(full_name, str) else f"{owner}/{name}" repository_id = str(raw_id) # Do not trust a provider-supplied clone host when constructing the # credential-bearing acquisition URL. The configured Gitea origin is # the only network destination that may receive the server token. clone_url = ( f"{self.base_url}/{urllib.parse.quote(owner, safe='')}/" f"{urllib.parse.quote(name, safe='')}.git" ) html_url = payload.get("html_url") default_branch = payload.get("default_branch") or "main" return Repository( id=f"gitea:{repository_id}", provider="gitea", provider_repository_id=repository_id, owner=owner, name=name, full_name=full_name, description=payload.get("description") if isinstance(payload.get("description"), str) else None, html_url=html_url if isinstance(html_url, str) else None, clone_url=clone_url if isinstance(clone_url, str) else None, default_branch=str(default_branch), private=bool(payload.get("private", False)), archived=bool(payload.get("archived", False)), latest_revision_at=_parse_time(payload.get("updated_at")), policy=RepositoryPolicy(), ) def list_repositories(self) -> list[Repository]: if not self.configured: raise GiteaError("not_configured", "Gitea connector is not configured") repositories: list[Repository] = [] self._listing_complete = False for page in range(1, self.max_pages + 1): payload = self._request_json( "user/repos", {"limit": self.page_size, "page": page, "sort": "updated", "direction": "desc"}, ) if not isinstance(payload, list): raise GiteaError("invalid_response", "Gitea repository listing was not a list") for item in payload: if len(repositories) >= self.max_repositories: return repositories repositories.append(self._repository_from_payload(item)) if len(payload) < self.page_size: self._listing_complete = True break return repositories def branch_revision(self, repository: Repository) -> str: owner = urllib.parse.quote(repository.owner, safe="") name = urllib.parse.quote(repository.name, safe="") branch = urllib.parse.quote(repository.default_branch, safe="") payload = self._request_json(f"repos/{owner}/{name}/branches/{branch}") if not isinstance(payload, dict): raise GiteaError("invalid_response", "Gitea branch response was invalid") commit = payload.get("commit") revision = None if isinstance(commit, dict): revision = commit.get("id") or commit.get("sha") if ( not isinstance(revision, str) or not 7 <= len(revision) <= 64 or not all(char in "0123456789abcdefABCDEF" for char in revision) ): raise GiteaError("invalid_revision", "Gitea did not return a usable branch revision") return revision def discover_with_errors(self) -> tuple[list[Repository], list[tuple[Repository, str]], list[str]]: """Resolve branch tips independently so one repository cannot abort a catalog sync.""" repositories = self.list_repositories() def resolve(repository: Repository) -> tuple[Repository, str | None, str | None]: try: return repository, self.branch_revision(repository), None except GiteaError as exc: return repository, None, f"{repository.full_name}: {exc.code}: {exc.detail}" with ThreadPoolExecutor(max_workers=min(self.max_branch_workers, len(repositories) or 1)) as executor: outcomes = list(executor.map(resolve, repositories)) discovered = [ (repository.model_copy(update={"latest_revision": revision}), revision) for repository, revision, error in outcomes if revision is not None and error is None ] errors = [error for _, _, error in outcomes if error] return repositories, discovered, errors def discover(self) -> list[tuple[Repository, str]]: """Compatibility helper for callers needing only successfully resolved repositories.""" _, discovered, _ = self.discover_with_errors() return discovered def sync(self, store) -> RepositorySyncResult: synced_at = datetime.now(UTC) try: listed, discovered, branch_errors = self.discover_with_errors() except GiteaError as exc: error = f"{exc.code}: {exc.detail}" for existing in store.list_repositories(limit=1000): if existing.provider == "gitea" and existing.available: store.update_repository_sync(existing, synced_at=synced_at, error=error) result = RepositorySyncResult( provider="gitea", errors=[error], synced_at=synced_at, ) store.record_sync_result(result) return result created = 0 updated = 0 listed_ids = {repository.id for repository in listed} seen_ids: set[str] = set() for repository, revision in discovered: seen_ids.add(repository.id) before = store.get_repository(repository.id) merged = repository.model_copy( update={ "available": True, "last_synced_at": synced_at, "sync_error": None, "latest_revision": revision, "policy": before.policy if before else RepositoryPolicy(), } ) store.upsert_repository(merged) if before is None: created += 1 else: updated += 1 for repository in listed: if repository.id in seen_ids: continue before = store.get_repository(repository.id) error = next((item for item in branch_errors if item.startswith(f"{repository.full_name}:")), None) merged = repository.model_copy( update={ "available": True, "last_synced_at": synced_at, "sync_error": error or "branch: revision could not be resolved", "latest_revision": before.latest_revision if before else None, "policy": before.policy if before else RepositoryPolicy(), } ) store.upsert_repository(merged) if before is None: created += 1 else: updated += 1 unavailable = 0 if self._listing_complete: for existing in store.list_repositories(limit=1000): if existing.provider == "gitea" and existing.id not in listed_ids and existing.available: store.update_repository_sync( existing.model_copy(update={"available": False}), synced_at=synced_at, error="not_discovered", ) unavailable += 1 result = RepositorySyncResult( provider="gitea", discovered=len(discovered), created=created, updated=updated, unavailable=unavailable, errors=branch_errors, partial=bool(branch_errors) or not self._listing_complete, branch_failures=len(branch_errors), listing_complete=self._listing_complete, synced_at=synced_at, ) store.record_sync_result(result) return result