"""Typed, fixed-route client for the separate Knowledge Hub API.""" from __future__ import annotations import ipaddress import json import os import re from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path from typing import Any, Literal from urllib.parse import parse_qsl, urlsplit from uuid import UUID import httpx from packages.hub_runtime import SecretFileError, read_secret_file from services.runtime.ingestion_auth import ( ActorContext, authorize_actor, authorize_workspace_actor, ) PROPOSALS_PATH = "/api/v1/ingestion-proposals" _GUID = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") _ETAG = re.compile(r'^"[\x21\x23-\x7e]{1,254}"$') _IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") _UNSAFE_HOSTS = frozenset( { "localhost", "metadata", "metadata.google.internal", "instance-data.ec2.internal", } ) _CREATE_FIELDS = frozenset({"manifest", "correlationId", "publication"}) _PUBLICATION_FIELDS = frozenset( { "title", "markdown", "documentType", "documentStatus", "mutationKind", "parentPageId", "targetPageId", "expectedTargetETag", } ) _SOURCE_MIME_TYPES = frozenset( { "text/plain", "application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", } ) class HubClientError(RuntimeError): """Redacted Hub error containing only stable classification metadata.""" def __init__( self, code: str, correlation_id: str, *, status_code: int | None = None, retry_after_seconds: int | None = None, ) -> None: self.code = code self.status_code = status_code self.correlation_id = correlation_id self.retry_after_seconds = retry_after_seconds super().__init__("Knowledge Hub request failed.") class HubConflictError(HubClientError): def __init__(self, correlation_id: str) -> None: super().__init__("hub_conflict", correlation_id, status_code=409) class HubPreconditionFailed(HubClientError): def __init__(self, correlation_id: str) -> None: super().__init__("hub_precondition_failed", correlation_id, status_code=412) class HubRateLimitedError(HubClientError): def __init__(self, correlation_id: str, retry_after_seconds: int | None) -> None: super().__init__( "hub_rate_limited", correlation_id, status_code=429, retry_after_seconds=retry_after_seconds, ) class HubResponseTooLarge(HubClientError): def __init__(self, correlation_id: str, *, status_code: int | None = None) -> None: super().__init__("hub_response_too_large", correlation_id, status_code=status_code) class HubUnavailable(HubClientError): def __init__(self, correlation_id: str) -> None: super().__init__("hub_unavailable", correlation_id) @dataclass(frozen=True) class HubResponse: data: dict[str, Any] status_code: int etag: str | None correlation_id: str @property def id(self) -> str | None: value = self.data.get("id") return value if isinstance(value, str) else None @dataclass(frozen=True) class SourceFile: file_name: str content_type: str content: bytes def _validate_endpoint(endpoint: str) -> str: if not isinstance(endpoint, str) or len(endpoint) > 2048: raise ValueError("Knowledge Hub endpoint is invalid") try: parsed = urlsplit(endpoint.strip()) port = parsed.port except ValueError as exc: raise ValueError("Knowledge Hub endpoint is invalid") from exc hostname = parsed.hostname if not ( parsed.scheme.lower() == "https" and hostname and not parsed.username and not parsed.password and parsed.path in {"", "/"} and not parsed.query and not parsed.fragment and port in {None, 443} and not any(ord(character) < 0x21 for character in endpoint) ): raise ValueError("Knowledge Hub endpoint must be a safe HTTPS origin") host = hostname.rstrip(".").lower() if host in _UNSAFE_HOSTS or host.endswith((".localhost", ".local")): raise ValueError("Knowledge Hub endpoint must not target a local host") try: address = ipaddress.ip_address(host) except ValueError: address = None if address is not None and (not address.is_global or address.is_multicast): raise ValueError("Knowledge Hub endpoint must not target a private or reserved address") return f"https://{parsed.netloc.rstrip('/')}" def _canonical_guid(value: str) -> str: if not isinstance(value, str) or not _GUID.fullmatch(value): raise ValueError("Knowledge Hub route is not allow-listed") return str(UUID(value)) def _validate_etag(etag: str | None, *, required: bool) -> str | None: if etag is None: if required: raise ValueError("current Hub ETag is required") return None if not isinstance(etag, str) or not _ETAG.fullmatch(etag): raise ValueError("current Hub ETag is invalid") return etag def _validate_idempotency_key(key: str | None, *, required: bool) -> str | None: if key is None: if required: raise ValueError("Hub idempotency key is required") return None if not isinstance(key, str) or not _IDEMPOTENCY_KEY.fullmatch(key): raise ValueError("Hub idempotency key is invalid") return key def _parse_route(path: str) -> tuple[str, list[tuple[str, str]]]: if not isinstance(path, str) or not 1 <= len(path) <= 4096: raise ValueError("Knowledge Hub route is not allow-listed") try: parsed = urlsplit(path) query = parse_qsl(parsed.query, keep_blank_values=True, strict_parsing=True) except ValueError as exc: raise ValueError("Knowledge Hub route is not allow-listed") from exc if ( parsed.scheme or parsed.netloc or parsed.fragment or not parsed.path.startswith("/api/v1/") or parsed.path.startswith("//") or "\\" in parsed.path or "%" in parsed.path or "//" in parsed.path or any(part in {".", ".."} for part in parsed.path.split("/")) or any(ord(character) < 0x20 for character in path) or len(query) > 8 or len(parsed.query) > 2048 ): raise ValueError("Knowledge Hub route is not allow-listed") if len({key for key, _ in query}) != len(query): raise ValueError("Knowledge Hub route is not allow-listed") return parsed.path, query def _validate_query( query: list[tuple[str, str]], *, allowed: frozenset[str], max_values: Mapping[str, int], integer_values: frozenset[str], ) -> str: values: list[tuple[str, str]] = [] for key, value in query: if key not in allowed or len(value) > max_values.get(key, 0): raise ValueError("Knowledge Hub query is not allow-listed") if any(ord(character) < 0x20 or ord(character) == 0x7F for character in value): raise ValueError("Knowledge Hub query is not allow-listed") if key == "cursor" and not value: raise ValueError("Knowledge Hub query is not allow-listed") if key in integer_values: try: number = int(value) except ValueError as exc: raise ValueError("Knowledge Hub query is not allow-listed") from exc if str(number) != value or not 0 <= number <= max_values[key]: raise ValueError("Knowledge Hub query is not allow-listed") elif key in {"limit", "updates", "decisions", "documents"}: try: number = int(value) except ValueError as exc: raise ValueError("Knowledge Hub query is not allow-listed") from exc upper = max_values[key] lower = 1 if key == "limit" else 0 if str(number) != value or not lower <= number <= upper: raise ValueError("Knowledge Hub query is not allow-listed") values.append((key, value)) return str(httpx.QueryParams(dict(values))) class HubClient: """Only the typed search/context/proposal routes cross this API boundary.""" def __init__( self, endpoint: str, service_token: str, *, transport: httpx.AsyncBaseTransport | None = None, timeout_seconds: float = 20.0, max_response_bytes: int = 512 * 1024, max_upload_bytes: int = 16 * 1024 * 1024, workspace_enabled: bool = False, ) -> None: self._base_url = _validate_endpoint(endpoint) if ( not isinstance(service_token, str) or not service_token or len(service_token) > 8192 or any(ord(character) < 0x21 or ord(character) > 0x7E for character in service_token) ): raise ValueError("Knowledge Hub service credential is invalid") if not 0 < timeout_seconds <= 120: raise ValueError("Knowledge Hub timeout is invalid") if not 1 <= max_response_bytes <= 2 * 1024 * 1024: raise ValueError("Knowledge Hub response limit is invalid") if not 1 <= max_upload_bytes <= 64 * 1024 * 1024: raise ValueError("Knowledge Hub upload limit is invalid") if not isinstance(workspace_enabled, bool): raise ValueError("Knowledge Hub workspace policy is invalid") self._service_token = service_token self._workspace_enabled = workspace_enabled self._max_response_bytes = max_response_bytes self._max_upload_bytes = max_upload_bytes timeout = httpx.Timeout(timeout_seconds, connect=min(5.0, timeout_seconds)) self._http = httpx.AsyncClient( base_url=self._base_url, timeout=timeout, follow_redirects=False, trust_env=False, transport=transport, headers={"Accept": "application/json"}, ) def __repr__(self) -> str: return f"HubClient(endpoint={self._base_url!r})" @classmethod def from_environment( cls, *, environ: Mapping[str, str] | None = None, secret_root: Path = Path("/run/secrets"), transport: httpx.AsyncBaseTransport | None = None, ) -> HubClient: values = environ if environ is not None else os.environ endpoint = values.get("KNOWLEDGE_HUB_ENDPOINT", "").strip() token_path = Path( values.get( "KNOWLEDGE_HUB_INGESTION_TOKEN_FILE", "/run/secrets/knowledge_hub_ingestion_token", ) ) try: token = read_secret_file(token_path, root=secret_root) except SecretFileError as exc: raise ValueError("Knowledge Hub service credential is unavailable") from exc workspace_setting = ( values.get("KNOWLEDGE_HUB_WORKSPACE_READ_ENABLED", "false").strip().lower() ) if workspace_setting not in {"true", "false"}: raise ValueError("Knowledge Hub workspace policy is invalid") return cls( endpoint, token, transport=transport, workspace_enabled=workspace_setting == "true" ) async def aclose(self) -> None: await self._http.aclose() async def request_json( self, method: Literal["GET", "POST", "PATCH"], path: str, *, actor: ActorContext, json_body: Mapping[str, object] | None, etag: str | None, idempotency_key: str | None, project_id: str | None = None, ) -> HubResponse: normalized_method = method.upper() if isinstance(method, str) else "" if normalized_method not in {"GET", "POST", "PATCH"}: raise ValueError("Knowledge Hub method is not allow-listed") route, query = _parse_route(path) query_string = "" selected_project: str | None = None mutating = normalized_method in {"POST", "PATCH"} required_etag = False required_idempotency = False require_response_etag = mutating if normalized_method == "GET" and route == "/api/v1/search": query_string = _validate_query( query, allowed=frozenset({"q", "tag", "type", "cursor", "limit"}), max_values={"q": 1000, "tag": 128, "type": 64, "cursor": 500, "limit": 100}, integer_values=frozenset(), ) if not any(key == "q" and value.strip() for key, value in query): raise ValueError("Knowledge Hub search query is invalid") selected_project = _optional_project(project_id) self._authorize_target(actor, "knowledge:read", selected_project) if json_body is not None or etag is not None or idempotency_key is not None: raise ValueError("Knowledge Hub read request is invalid") elif normalized_method == "GET" and ( match := re.fullmatch(r"/api/v1/projects/([^/]+)/context", route) ): selected_project = _canonical_guid(match.group(1)) if _optional_project(project_id) != selected_project: raise ValueError("Knowledge Hub route is not allow-listed") query_string = _validate_query( query, allowed=frozenset({"updates", "decisions", "documents"}), max_values={"updates": 25, "decisions": 25, "documents": 25}, integer_values=frozenset(), ) authorize_actor(actor, capability="knowledge:read", project=selected_project) if json_body is not None or etag is not None or idempotency_key is not None: raise ValueError("Knowledge Hub read request is invalid") elif normalized_method == "GET" and route == "/api/v1/pages/roots": if project_id is not None: raise ValueError("Knowledge Hub workspace page request is invalid") query_string = _validate_query( query, allowed=frozenset({"cursor", "limit"}), max_values={"cursor": 500, "limit": 100}, integer_values=frozenset(), ) self._authorize_target(actor, "knowledge:read", None) if json_body is not None or etag is not None or idempotency_key is not None: raise ValueError("Knowledge Hub workspace page request is invalid") elif normalized_method == "GET" and ( match := re.fullmatch(r"/api/v1/pages/([^/]+)/children", route) ): page_id = _canonical_guid(match.group(1)) if project_id is not None: raise ValueError("Knowledge Hub workspace page request is invalid") query_string = _validate_query( query, allowed=frozenset({"cursor", "limit"}), max_values={"cursor": 500, "limit": 100}, integer_values=frozenset(), ) self._authorize_target(actor, "knowledge:read", None) if json_body is not None or etag is not None or idempotency_key is not None: raise ValueError("Knowledge Hub workspace page request is invalid") route = f"/api/v1/pages/{page_id}/children" elif normalized_method == "GET" and ( match := re.fullmatch(r"/api/v1/pages/([^/]+)", route) ): page_id = _canonical_guid(match.group(1)) if query or project_id is not None: raise ValueError("Knowledge Hub workspace page request is invalid") self._authorize_target(actor, "knowledge:read", None) if json_body is not None or etag is not None or idempotency_key is not None: raise ValueError("Knowledge Hub workspace page request is invalid") route = f"/api/v1/pages/{page_id}" require_response_etag = True elif normalized_method == "POST" and route == PROPOSALS_PATH: if query: raise ValueError("Knowledge Hub route is not allow-listed") selected_project = _optional_project(project_id) self._authorize_target(actor, "ingestion:write", selected_project) _validate_create_body(json_body, actor=actor, project_id=selected_project) if etag is not None: raise ValueError("Knowledge Hub create request is invalid") required_idempotency = True elif normalized_method == "GET" and ( match := re.fullmatch(r"/api/v1/ingestion-proposals/([^/]+)", route) ): proposal_id = _canonical_guid(match.group(1)) if query: raise ValueError("Knowledge Hub route is not allow-listed") selected_project = _optional_project(project_id) self._authorize_target(actor, "ingestion:write", selected_project) if json_body is not None or etag is not None or idempotency_key is not None: raise ValueError("Knowledge Hub status request is invalid") route = f"{PROPOSALS_PATH}/{proposal_id}" elif normalized_method == "PATCH" and ( match := re.fullmatch(r"/api/v1/ingestion-proposals/([^/]+)", route) ): proposal_id = _canonical_guid(match.group(1)) if query: raise ValueError("Knowledge Hub route is not allow-listed") selected_project = _optional_project(project_id) self._authorize_target(actor, "ingestion:write", selected_project) _validate_analysis_body(json_body) required_etag = True if idempotency_key is not None: raise ValueError("Knowledge Hub analysis request is invalid") route = f"{PROPOSALS_PATH}/{proposal_id}" elif normalized_method == "POST" and ( match := re.fullmatch(r"/api/v1/ingestion-proposals/([^/]+)/retry", route) ): proposal_id = _canonical_guid(match.group(1)) if query or json_body is not None: raise ValueError("Knowledge Hub retry request is invalid") selected_project = _required_project(project_id) authorize_actor(actor, capability="ingestion:write", project=selected_project) required_etag = True required_idempotency = True route = f"{PROPOSALS_PATH}/{proposal_id}/retry" else: raise ValueError("Knowledge Hub route is not allow-listed") validated_etag = _validate_etag(etag, required=required_etag) validated_idempotency = _validate_idempotency_key( idempotency_key, required=required_idempotency ) if normalized_method == "GET" and route == "/api/v1/search": pass if json_body is not None: try: encoded_body = json.dumps(json_body, ensure_ascii=False, separators=(",", ":")) except (TypeError, ValueError) as exc: raise ValueError("Knowledge Hub request body is invalid") from exc if len(encoded_body.encode("utf-8")) > 256 * 1024: raise ValueError("Knowledge Hub request body is too large") request_path = route + (f"?{query_string}" if query_string else "") headers = self._headers(actor) if validated_etag is not None: headers["If-Match"] = validated_etag if validated_idempotency is not None: headers["Idempotency-Key"] = validated_idempotency response = await self._send_json( normalized_method, request_path, headers=headers, json_body=json_body, correlation_id=actor.correlation_id, require_response_etag=require_response_etag, ) if normalized_method == "GET" and route.startswith(PROPOSALS_PATH + "/"): _require_response_project(response.data, selected_project) return response async def upload_source( self, proposal_id: str, source: SourceFile, *, actor: ActorContext, etag: str | None, idempotency_key: str, project_id: str | None = None, ) -> HubResponse: normalized_id = _canonical_guid(proposal_id) selected_project = _optional_project(project_id) self._authorize_target(actor, "ingestion:write", selected_project) current_etag = _validate_etag(etag, required=True) key = _validate_idempotency_key(idempotency_key, required=True) filename = _validate_source(source, self._max_upload_bytes) route = f"{PROPOSALS_PATH}/{normalized_id}/sources" headers = self._headers(actor) if current_etag is not None: headers["If-Match"] = current_etag if key is not None: headers["Idempotency-Key"] = key response = await self._send_multipart( route, headers=headers, source=source, filename=filename, correlation_id=actor.correlation_id, ) _require_response_project(response.data, selected_project) return response def _authorize_target( self, actor: ActorContext, capability: Literal["knowledge:read", "ingestion:write"], project_id: str | None, ) -> None: if project_id is None: if not self._workspace_enabled: raise PermissionError("Knowledge Hub workspace access is disabled") authorize_workspace_actor(actor) authorize_actor(actor, capability=capability, project=project_id) def _headers(self, actor: ActorContext) -> dict[str, str]: return { "Authorization": f"Bearer {self._service_token}", "X-Correlation-ID": actor.correlation_id, } async def _send_json( self, method: str, path: str, *, headers: dict[str, str], json_body: Mapping[str, object] | None, correlation_id: str, require_response_etag: bool, ) -> HubResponse: try: async with self._http.stream( method, path, headers=headers, json=json_body, ) as response: self._raise_status(response.status_code, response.headers, correlation_id) raw = await self._read_response(response, correlation_id) return self._decode_response( response, raw, correlation_id, require_etag=require_response_etag, ) except HubClientError: raise except httpx.HTTPError: raise HubUnavailable(correlation_id) from None async def _send_multipart( self, path: str, *, headers: dict[str, str], source: SourceFile, filename: str, correlation_id: str, ) -> HubResponse: try: async with self._http.stream( "POST", path, headers=headers, files={"file": (filename, source.content, source.content_type)}, ) as response: self._raise_status(response.status_code, response.headers, correlation_id) raw = await self._read_response(response, correlation_id) result = self._decode_response(response, raw, correlation_id, require_etag=True) return result except HubClientError: raise except httpx.HTTPError: raise HubUnavailable(correlation_id) from None async def _read_response(self, response: httpx.Response, correlation_id: str) -> bytes: length = response.headers.get("content-length") if length is not None: try: if int(length) > self._max_response_bytes: raise HubResponseTooLarge(correlation_id, status_code=response.status_code) except ValueError as exc: raise HubClientError( "hub_invalid_response", correlation_id, status_code=response.status_code ) from exc content = bytearray() async for chunk in response.aiter_bytes(): if len(content) + len(chunk) > self._max_response_bytes: raise HubResponseTooLarge(correlation_id, status_code=response.status_code) content.extend(chunk) return bytes(content) def _decode_response( self, response: httpx.Response, raw: bytes, correlation_id: str, *, require_etag: bool, ) -> HubResponse: try: data = json.loads(raw) except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: raise HubClientError( "hub_invalid_response", correlation_id, status_code=response.status_code ) from exc if not isinstance(data, dict): raise HubClientError( "hub_invalid_response", correlation_id, status_code=response.status_code ) raw_etag = response.headers.get("etag") if raw_etag is not None and not _ETAG.fullmatch(raw_etag): raise HubClientError( "hub_invalid_response", correlation_id, status_code=response.status_code ) if require_etag and raw_etag is None: raise HubClientError( "hub_missing_etag", correlation_id, status_code=response.status_code ) return HubResponse( data=data, status_code=response.status_code, etag=raw_etag, correlation_id=correlation_id, ) @staticmethod def _raise_status( status_code: int, headers: httpx.Headers, correlation_id: str, ) -> None: if 200 <= status_code < 300: return if status_code == 409: raise HubConflictError(correlation_id) if status_code == 412: raise HubPreconditionFailed(correlation_id) if status_code == 429: raw_retry = headers.get("retry-after", "") try: retry_after = max(1, min(60, int(raw_retry))) if raw_retry else None except ValueError: retry_after = None raise HubRateLimitedError(correlation_id, retry_after) code = "hub_unavailable" if status_code >= 500 else "hub_request_rejected" raise HubClientError(code, correlation_id, status_code=status_code) def _optional_project(project_id: str | None) -> str | None: if project_id is None: return None try: return str(UUID(project_id)) except (TypeError, ValueError, AttributeError) as exc: raise ValueError("Knowledge Hub project is invalid") from exc def _required_project(project_id: str | None) -> str: normalized = _optional_project(project_id) if normalized is None: raise ValueError("Knowledge Hub project is required") return normalized def _validate_create_body( body: Mapping[str, object] | None, *, actor: ActorContext, project_id: str | None, ) -> None: if not isinstance(body, Mapping) or set(body) - _CREATE_FIELDS: raise ValueError("Knowledge Hub proposal request is invalid") manifest = body.get("manifest") if not isinstance(manifest, str) or not manifest.strip() or len(manifest) > 200_000: raise ValueError("Knowledge Hub proposal request is invalid") if body.get("correlationId") not in {None, actor.correlation_id}: raise ValueError("Knowledge Hub proposal request is invalid") publication = body.get("publication") if publication is not None: if not isinstance(publication, Mapping) or set(publication) - _PUBLICATION_FIELDS: raise ValueError("Knowledge Hub proposal request is invalid") for key, value in publication.items(): string_fields = { "title", "markdown", "documentType", "documentStatus", "mutationKind", "expectedTargetETag", } if key in string_fields and ( not isinstance(value, str) or len(value) > (32_000 if key == "markdown" else 100_000) ): raise ValueError("Knowledge Hub proposal request is invalid") try: decoded = json.loads(manifest) except json.JSONDecodeError: decoded = None manifest_project = _extract_project_id(decoded) if ( not isinstance(decoded, Mapping) or ( project_id is None and (decoded.get("workspace") != "knowledge-hub" or "projectId" in decoded) ) or (project_id is not None and manifest_project != project_id) ): raise PermissionError("ingestion caller is not authorized") def _validate_analysis_body(body: Mapping[str, object] | None) -> None: analysis = body.get("analysis") if isinstance(body, Mapping) else None if ( not isinstance(body, Mapping) or set(body) != {"analysis"} or not isinstance(analysis, str) or not analysis.strip() or len(analysis) > 32_000 or "\x00" in analysis ): raise ValueError("Knowledge Hub analysis request is invalid") def _validate_source(source: SourceFile, max_upload_bytes: int) -> str: if not isinstance(source, SourceFile): raise ValueError("Knowledge Hub source is invalid") filename = source.file_name if ( not isinstance(filename, str) or not 1 <= len(filename) <= 255 or filename in {".", ".."} or "/" in filename or "\\" in filename or '"' in filename or any(ord(character) < 0x20 for character in filename) or not isinstance(source.content_type, str) or source.content_type.lower() not in _SOURCE_MIME_TYPES or not isinstance(source.content, bytes) or not 1 <= len(source.content) <= max_upload_bytes ): raise ValueError("Knowledge Hub source is invalid") return filename def _extract_project_id(value: object) -> str | None: if not isinstance(value, Mapping): return None project = value.get("projectId") if isinstance(project, str): try: return str(UUID(project)) except ValueError: return None for key in ("destination", "metadata", "project"): nested = value.get(key) if isinstance(nested, Mapping): found = _extract_project_id(nested) if found is not None: return found return None def _require_response_project(data: Mapping[str, Any], expected_project: str | None) -> None: manifest = data.get("manifest") if isinstance(manifest, str): try: manifest = json.loads(manifest) except json.JSONDecodeError: return if expected_project is None: if not isinstance(manifest, Mapping) or ( manifest.get("workspace") != "knowledge-hub" or "projectId" in manifest ): raise PermissionError("ingestion caller is not authorized") return actual = _extract_project_id(manifest) if actual is None: metadata = data.get("metadata") actual = _extract_project_id(metadata) if actual is not None and actual != expected_project: raise PermissionError("ingestion caller is not authorized")