809 lines
31 KiB
Python
809 lines
31 KiB
Python
"""Typed, fixed-route client for the separate Knowledge Hub API."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
import re
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Literal
|
|
from urllib.parse import parse_qsl, urlsplit
|
|
from uuid import UUID
|
|
|
|
import httpx
|
|
|
|
from packages.hub_runtime import SecretFileError, read_secret_file
|
|
from services.runtime.ingestion_auth import (
|
|
ActorContext,
|
|
authorize_actor,
|
|
authorize_workspace_actor,
|
|
)
|
|
|
|
PROPOSALS_PATH = "/api/v1/ingestion-proposals"
|
|
_GUID = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$")
|
|
_ETAG = re.compile(r'^"[\x21\x23-\x7e]{1,254}"$')
|
|
_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
|
_UNSAFE_HOSTS = frozenset(
|
|
{
|
|
"localhost",
|
|
"metadata",
|
|
"metadata.google.internal",
|
|
"instance-data.ec2.internal",
|
|
}
|
|
)
|
|
_CREATE_FIELDS = frozenset({"manifest", "correlationId", "publication"})
|
|
_PUBLICATION_FIELDS = frozenset(
|
|
{
|
|
"title",
|
|
"markdown",
|
|
"documentType",
|
|
"documentStatus",
|
|
"mutationKind",
|
|
"parentPageId",
|
|
"targetPageId",
|
|
"expectedTargetETag",
|
|
}
|
|
)
|
|
_SOURCE_MIME_TYPES = frozenset(
|
|
{
|
|
"text/plain",
|
|
"application/pdf",
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
}
|
|
)
|
|
|
|
|
|
class HubClientError(RuntimeError):
|
|
"""Redacted Hub error containing only stable classification metadata."""
|
|
|
|
def __init__(
|
|
self,
|
|
code: str,
|
|
correlation_id: str,
|
|
*,
|
|
status_code: int | None = None,
|
|
retry_after_seconds: int | None = None,
|
|
) -> None:
|
|
self.code = code
|
|
self.status_code = status_code
|
|
self.correlation_id = correlation_id
|
|
self.retry_after_seconds = retry_after_seconds
|
|
super().__init__("Knowledge Hub request failed.")
|
|
|
|
|
|
class HubConflictError(HubClientError):
|
|
def __init__(self, correlation_id: str) -> None:
|
|
super().__init__("hub_conflict", correlation_id, status_code=409)
|
|
|
|
|
|
class HubPreconditionFailed(HubClientError):
|
|
def __init__(self, correlation_id: str) -> None:
|
|
super().__init__("hub_precondition_failed", correlation_id, status_code=412)
|
|
|
|
|
|
class HubRateLimitedError(HubClientError):
|
|
def __init__(self, correlation_id: str, retry_after_seconds: int | None) -> None:
|
|
super().__init__(
|
|
"hub_rate_limited",
|
|
correlation_id,
|
|
status_code=429,
|
|
retry_after_seconds=retry_after_seconds,
|
|
)
|
|
|
|
|
|
class HubResponseTooLarge(HubClientError):
|
|
def __init__(self, correlation_id: str, *, status_code: int | None = None) -> None:
|
|
super().__init__("hub_response_too_large", correlation_id, status_code=status_code)
|
|
|
|
|
|
class HubUnavailable(HubClientError):
|
|
def __init__(self, correlation_id: str) -> None:
|
|
super().__init__("hub_unavailable", correlation_id)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class HubResponse:
|
|
data: dict[str, Any]
|
|
status_code: int
|
|
etag: str | None
|
|
correlation_id: str
|
|
|
|
@property
|
|
def id(self) -> str | None:
|
|
value = self.data.get("id")
|
|
return value if isinstance(value, str) else None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SourceFile:
|
|
file_name: str
|
|
content_type: str
|
|
content: bytes
|
|
|
|
|
|
def _validate_endpoint(endpoint: str) -> str:
|
|
if not isinstance(endpoint, str) or len(endpoint) > 2048:
|
|
raise ValueError("Knowledge Hub endpoint is invalid")
|
|
try:
|
|
parsed = urlsplit(endpoint.strip())
|
|
port = parsed.port
|
|
except ValueError as exc:
|
|
raise ValueError("Knowledge Hub endpoint is invalid") from exc
|
|
hostname = parsed.hostname
|
|
if not (
|
|
parsed.scheme.lower() == "https"
|
|
and hostname
|
|
and not parsed.username
|
|
and not parsed.password
|
|
and parsed.path in {"", "/"}
|
|
and not parsed.query
|
|
and not parsed.fragment
|
|
and port in {None, 443}
|
|
and not any(ord(character) < 0x21 for character in endpoint)
|
|
):
|
|
raise ValueError("Knowledge Hub endpoint must be a safe HTTPS origin")
|
|
host = hostname.rstrip(".").lower()
|
|
if host in _UNSAFE_HOSTS or host.endswith((".localhost", ".local")):
|
|
raise ValueError("Knowledge Hub endpoint must not target a local host")
|
|
try:
|
|
address = ipaddress.ip_address(host)
|
|
except ValueError:
|
|
address = None
|
|
if address is not None and (not address.is_global or address.is_multicast):
|
|
raise ValueError("Knowledge Hub endpoint must not target a private or reserved address")
|
|
return f"https://{parsed.netloc.rstrip('/')}"
|
|
|
|
|
|
def _canonical_guid(value: str) -> str:
|
|
if not isinstance(value, str) or not _GUID.fullmatch(value):
|
|
raise ValueError("Knowledge Hub route is not allow-listed")
|
|
return str(UUID(value))
|
|
|
|
|
|
def _validate_etag(etag: str | None, *, required: bool) -> str | None:
|
|
if etag is None:
|
|
if required:
|
|
raise ValueError("current Hub ETag is required")
|
|
return None
|
|
if not isinstance(etag, str) or not _ETAG.fullmatch(etag):
|
|
raise ValueError("current Hub ETag is invalid")
|
|
return etag
|
|
|
|
|
|
def _validate_idempotency_key(key: str | None, *, required: bool) -> str | None:
|
|
if key is None:
|
|
if required:
|
|
raise ValueError("Hub idempotency key is required")
|
|
return None
|
|
if not isinstance(key, str) or not _IDEMPOTENCY_KEY.fullmatch(key):
|
|
raise ValueError("Hub idempotency key is invalid")
|
|
return key
|
|
|
|
|
|
def _parse_route(path: str) -> tuple[str, list[tuple[str, str]]]:
|
|
if not isinstance(path, str) or not 1 <= len(path) <= 4096:
|
|
raise ValueError("Knowledge Hub route is not allow-listed")
|
|
try:
|
|
parsed = urlsplit(path)
|
|
query = parse_qsl(parsed.query, keep_blank_values=True, strict_parsing=True)
|
|
except ValueError as exc:
|
|
raise ValueError("Knowledge Hub route is not allow-listed") from exc
|
|
if (
|
|
parsed.scheme
|
|
or parsed.netloc
|
|
or parsed.fragment
|
|
or not parsed.path.startswith("/api/v1/")
|
|
or parsed.path.startswith("//")
|
|
or "\\" in parsed.path
|
|
or "%" in parsed.path
|
|
or "//" in parsed.path
|
|
or any(part in {".", ".."} for part in parsed.path.split("/"))
|
|
or any(ord(character) < 0x20 for character in path)
|
|
or len(query) > 8
|
|
or len(parsed.query) > 2048
|
|
):
|
|
raise ValueError("Knowledge Hub route is not allow-listed")
|
|
if len({key for key, _ in query}) != len(query):
|
|
raise ValueError("Knowledge Hub route is not allow-listed")
|
|
return parsed.path, query
|
|
|
|
|
|
def _validate_query(
|
|
query: list[tuple[str, str]],
|
|
*,
|
|
allowed: frozenset[str],
|
|
max_values: Mapping[str, int],
|
|
integer_values: frozenset[str],
|
|
) -> str:
|
|
values: list[tuple[str, str]] = []
|
|
for key, value in query:
|
|
if key not in allowed or len(value) > max_values.get(key, 0):
|
|
raise ValueError("Knowledge Hub query is not allow-listed")
|
|
if any(ord(character) < 0x20 or ord(character) == 0x7F for character in value):
|
|
raise ValueError("Knowledge Hub query is not allow-listed")
|
|
if key == "cursor" and not value:
|
|
raise ValueError("Knowledge Hub query is not allow-listed")
|
|
if key in integer_values:
|
|
try:
|
|
number = int(value)
|
|
except ValueError as exc:
|
|
raise ValueError("Knowledge Hub query is not allow-listed") from exc
|
|
if str(number) != value or not 0 <= number <= max_values[key]:
|
|
raise ValueError("Knowledge Hub query is not allow-listed")
|
|
elif key in {"limit", "updates", "decisions", "documents"}:
|
|
try:
|
|
number = int(value)
|
|
except ValueError as exc:
|
|
raise ValueError("Knowledge Hub query is not allow-listed") from exc
|
|
upper = max_values[key]
|
|
lower = 1 if key == "limit" else 0
|
|
if str(number) != value or not lower <= number <= upper:
|
|
raise ValueError("Knowledge Hub query is not allow-listed")
|
|
values.append((key, value))
|
|
return str(httpx.QueryParams(dict(values)))
|
|
|
|
|
|
class HubClient:
|
|
"""Only the typed search/context/proposal routes cross this API boundary."""
|
|
|
|
def __init__(
|
|
self,
|
|
endpoint: str,
|
|
service_token: str,
|
|
*,
|
|
transport: httpx.AsyncBaseTransport | None = None,
|
|
timeout_seconds: float = 20.0,
|
|
max_response_bytes: int = 512 * 1024,
|
|
max_upload_bytes: int = 16 * 1024 * 1024,
|
|
workspace_enabled: bool = False,
|
|
) -> None:
|
|
self._base_url = _validate_endpoint(endpoint)
|
|
if (
|
|
not isinstance(service_token, str)
|
|
or not service_token
|
|
or len(service_token) > 8192
|
|
or any(ord(character) < 0x21 or ord(character) > 0x7E for character in service_token)
|
|
):
|
|
raise ValueError("Knowledge Hub service credential is invalid")
|
|
if not 0 < timeout_seconds <= 120:
|
|
raise ValueError("Knowledge Hub timeout is invalid")
|
|
if not 1 <= max_response_bytes <= 2 * 1024 * 1024:
|
|
raise ValueError("Knowledge Hub response limit is invalid")
|
|
if not 1 <= max_upload_bytes <= 64 * 1024 * 1024:
|
|
raise ValueError("Knowledge Hub upload limit is invalid")
|
|
if not isinstance(workspace_enabled, bool):
|
|
raise ValueError("Knowledge Hub workspace policy is invalid")
|
|
self._service_token = service_token
|
|
self._workspace_enabled = workspace_enabled
|
|
self._max_response_bytes = max_response_bytes
|
|
self._max_upload_bytes = max_upload_bytes
|
|
timeout = httpx.Timeout(timeout_seconds, connect=min(5.0, timeout_seconds))
|
|
self._http = httpx.AsyncClient(
|
|
base_url=self._base_url,
|
|
timeout=timeout,
|
|
follow_redirects=False,
|
|
trust_env=False,
|
|
transport=transport,
|
|
headers={"Accept": "application/json"},
|
|
)
|
|
|
|
def __repr__(self) -> str:
|
|
return f"HubClient(endpoint={self._base_url!r})"
|
|
|
|
@classmethod
|
|
def from_environment(
|
|
cls,
|
|
*,
|
|
environ: Mapping[str, str] | None = None,
|
|
secret_root: Path = Path("/run/secrets"),
|
|
transport: httpx.AsyncBaseTransport | None = None,
|
|
) -> HubClient:
|
|
values = environ if environ is not None else os.environ
|
|
endpoint = values.get("KNOWLEDGE_HUB_ENDPOINT", "").strip()
|
|
token_path = Path(
|
|
values.get(
|
|
"KNOWLEDGE_HUB_INGESTION_TOKEN_FILE",
|
|
"/run/secrets/knowledge_hub_ingestion_token",
|
|
)
|
|
)
|
|
try:
|
|
token = read_secret_file(token_path, root=secret_root)
|
|
except SecretFileError as exc:
|
|
raise ValueError("Knowledge Hub service credential is unavailable") from exc
|
|
workspace_setting = (
|
|
values.get("KNOWLEDGE_HUB_WORKSPACE_READ_ENABLED", "false").strip().lower()
|
|
)
|
|
if workspace_setting not in {"true", "false"}:
|
|
raise ValueError("Knowledge Hub workspace policy is invalid")
|
|
return cls(
|
|
endpoint, token, transport=transport, workspace_enabled=workspace_setting == "true"
|
|
)
|
|
|
|
async def aclose(self) -> None:
|
|
await self._http.aclose()
|
|
|
|
async def request_json(
|
|
self,
|
|
method: Literal["GET", "POST", "PATCH"],
|
|
path: str,
|
|
*,
|
|
actor: ActorContext,
|
|
json_body: Mapping[str, object] | None,
|
|
etag: str | None,
|
|
idempotency_key: str | None,
|
|
project_id: str | None = None,
|
|
) -> HubResponse:
|
|
normalized_method = method.upper() if isinstance(method, str) else ""
|
|
if normalized_method not in {"GET", "POST", "PATCH"}:
|
|
raise ValueError("Knowledge Hub method is not allow-listed")
|
|
route, query = _parse_route(path)
|
|
query_string = ""
|
|
selected_project: str | None = None
|
|
mutating = normalized_method in {"POST", "PATCH"}
|
|
required_etag = False
|
|
required_idempotency = False
|
|
require_response_etag = mutating
|
|
|
|
if normalized_method == "GET" and route == "/api/v1/search":
|
|
query_string = _validate_query(
|
|
query,
|
|
allowed=frozenset({"q", "tag", "type", "cursor", "limit"}),
|
|
max_values={"q": 1000, "tag": 128, "type": 64, "cursor": 500, "limit": 100},
|
|
integer_values=frozenset(),
|
|
)
|
|
if not any(key == "q" and value.strip() for key, value in query):
|
|
raise ValueError("Knowledge Hub search query is invalid")
|
|
selected_project = _optional_project(project_id)
|
|
self._authorize_target(actor, "knowledge:read", selected_project)
|
|
if json_body is not None or etag is not None or idempotency_key is not None:
|
|
raise ValueError("Knowledge Hub read request is invalid")
|
|
elif normalized_method == "GET" and (
|
|
match := re.fullmatch(r"/api/v1/projects/([^/]+)/context", route)
|
|
):
|
|
selected_project = _canonical_guid(match.group(1))
|
|
if _optional_project(project_id) != selected_project:
|
|
raise ValueError("Knowledge Hub route is not allow-listed")
|
|
query_string = _validate_query(
|
|
query,
|
|
allowed=frozenset({"updates", "decisions", "documents"}),
|
|
max_values={"updates": 25, "decisions": 25, "documents": 25},
|
|
integer_values=frozenset(),
|
|
)
|
|
authorize_actor(actor, capability="knowledge:read", project=selected_project)
|
|
if json_body is not None or etag is not None or idempotency_key is not None:
|
|
raise ValueError("Knowledge Hub read request is invalid")
|
|
elif normalized_method == "GET" and route == "/api/v1/pages/roots":
|
|
if project_id is not None:
|
|
raise ValueError("Knowledge Hub workspace page request is invalid")
|
|
query_string = _validate_query(
|
|
query,
|
|
allowed=frozenset({"cursor", "limit"}),
|
|
max_values={"cursor": 500, "limit": 100},
|
|
integer_values=frozenset(),
|
|
)
|
|
self._authorize_target(actor, "knowledge:read", None)
|
|
if json_body is not None or etag is not None or idempotency_key is not None:
|
|
raise ValueError("Knowledge Hub workspace page request is invalid")
|
|
elif normalized_method == "GET" and (
|
|
match := re.fullmatch(r"/api/v1/pages/([^/]+)/children", route)
|
|
):
|
|
page_id = _canonical_guid(match.group(1))
|
|
if project_id is not None:
|
|
raise ValueError("Knowledge Hub workspace page request is invalid")
|
|
query_string = _validate_query(
|
|
query,
|
|
allowed=frozenset({"cursor", "limit"}),
|
|
max_values={"cursor": 500, "limit": 100},
|
|
integer_values=frozenset(),
|
|
)
|
|
self._authorize_target(actor, "knowledge:read", None)
|
|
if json_body is not None or etag is not None or idempotency_key is not None:
|
|
raise ValueError("Knowledge Hub workspace page request is invalid")
|
|
route = f"/api/v1/pages/{page_id}/children"
|
|
elif normalized_method == "GET" and (
|
|
match := re.fullmatch(r"/api/v1/pages/([^/]+)", route)
|
|
):
|
|
page_id = _canonical_guid(match.group(1))
|
|
if query or project_id is not None:
|
|
raise ValueError("Knowledge Hub workspace page request is invalid")
|
|
self._authorize_target(actor, "knowledge:read", None)
|
|
if json_body is not None or etag is not None or idempotency_key is not None:
|
|
raise ValueError("Knowledge Hub workspace page request is invalid")
|
|
route = f"/api/v1/pages/{page_id}"
|
|
require_response_etag = True
|
|
elif normalized_method == "POST" and route == PROPOSALS_PATH:
|
|
if query:
|
|
raise ValueError("Knowledge Hub route is not allow-listed")
|
|
selected_project = _optional_project(project_id)
|
|
self._authorize_target(actor, "ingestion:write", selected_project)
|
|
_validate_create_body(json_body, actor=actor, project_id=selected_project)
|
|
if etag is not None:
|
|
raise ValueError("Knowledge Hub create request is invalid")
|
|
required_idempotency = True
|
|
elif normalized_method == "GET" and (
|
|
match := re.fullmatch(r"/api/v1/ingestion-proposals/([^/]+)", route)
|
|
):
|
|
proposal_id = _canonical_guid(match.group(1))
|
|
if query:
|
|
raise ValueError("Knowledge Hub route is not allow-listed")
|
|
selected_project = _optional_project(project_id)
|
|
self._authorize_target(actor, "ingestion:write", selected_project)
|
|
if json_body is not None or etag is not None or idempotency_key is not None:
|
|
raise ValueError("Knowledge Hub status request is invalid")
|
|
route = f"{PROPOSALS_PATH}/{proposal_id}"
|
|
elif normalized_method == "PATCH" and (
|
|
match := re.fullmatch(r"/api/v1/ingestion-proposals/([^/]+)", route)
|
|
):
|
|
proposal_id = _canonical_guid(match.group(1))
|
|
if query:
|
|
raise ValueError("Knowledge Hub route is not allow-listed")
|
|
selected_project = _optional_project(project_id)
|
|
self._authorize_target(actor, "ingestion:write", selected_project)
|
|
_validate_analysis_body(json_body)
|
|
required_etag = True
|
|
if idempotency_key is not None:
|
|
raise ValueError("Knowledge Hub analysis request is invalid")
|
|
route = f"{PROPOSALS_PATH}/{proposal_id}"
|
|
elif normalized_method == "POST" and (
|
|
match := re.fullmatch(r"/api/v1/ingestion-proposals/([^/]+)/retry", route)
|
|
):
|
|
proposal_id = _canonical_guid(match.group(1))
|
|
if query or json_body is not None:
|
|
raise ValueError("Knowledge Hub retry request is invalid")
|
|
selected_project = _required_project(project_id)
|
|
authorize_actor(actor, capability="ingestion:write", project=selected_project)
|
|
required_etag = True
|
|
required_idempotency = True
|
|
route = f"{PROPOSALS_PATH}/{proposal_id}/retry"
|
|
else:
|
|
raise ValueError("Knowledge Hub route is not allow-listed")
|
|
|
|
validated_etag = _validate_etag(etag, required=required_etag)
|
|
validated_idempotency = _validate_idempotency_key(
|
|
idempotency_key, required=required_idempotency
|
|
)
|
|
if normalized_method == "GET" and route == "/api/v1/search":
|
|
pass
|
|
if json_body is not None:
|
|
try:
|
|
encoded_body = json.dumps(json_body, ensure_ascii=False, separators=(",", ":"))
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("Knowledge Hub request body is invalid") from exc
|
|
if len(encoded_body.encode("utf-8")) > 256 * 1024:
|
|
raise ValueError("Knowledge Hub request body is too large")
|
|
|
|
request_path = route + (f"?{query_string}" if query_string else "")
|
|
headers = self._headers(actor)
|
|
if validated_etag is not None:
|
|
headers["If-Match"] = validated_etag
|
|
if validated_idempotency is not None:
|
|
headers["Idempotency-Key"] = validated_idempotency
|
|
response = await self._send_json(
|
|
normalized_method,
|
|
request_path,
|
|
headers=headers,
|
|
json_body=json_body,
|
|
correlation_id=actor.correlation_id,
|
|
require_response_etag=require_response_etag,
|
|
)
|
|
if normalized_method == "GET" and route.startswith(PROPOSALS_PATH + "/"):
|
|
_require_response_project(response.data, selected_project)
|
|
return response
|
|
|
|
async def upload_source(
|
|
self,
|
|
proposal_id: str,
|
|
source: SourceFile,
|
|
*,
|
|
actor: ActorContext,
|
|
etag: str | None,
|
|
idempotency_key: str,
|
|
project_id: str | None = None,
|
|
) -> HubResponse:
|
|
normalized_id = _canonical_guid(proposal_id)
|
|
selected_project = _optional_project(project_id)
|
|
self._authorize_target(actor, "ingestion:write", selected_project)
|
|
current_etag = _validate_etag(etag, required=True)
|
|
key = _validate_idempotency_key(idempotency_key, required=True)
|
|
filename = _validate_source(source, self._max_upload_bytes)
|
|
route = f"{PROPOSALS_PATH}/{normalized_id}/sources"
|
|
headers = self._headers(actor)
|
|
if current_etag is not None:
|
|
headers["If-Match"] = current_etag
|
|
if key is not None:
|
|
headers["Idempotency-Key"] = key
|
|
response = await self._send_multipart(
|
|
route,
|
|
headers=headers,
|
|
source=source,
|
|
filename=filename,
|
|
correlation_id=actor.correlation_id,
|
|
)
|
|
_require_response_project(response.data, selected_project)
|
|
return response
|
|
|
|
def _authorize_target(
|
|
self,
|
|
actor: ActorContext,
|
|
capability: Literal["knowledge:read", "ingestion:write"],
|
|
project_id: str | None,
|
|
) -> None:
|
|
if project_id is None:
|
|
if not self._workspace_enabled:
|
|
raise PermissionError("Knowledge Hub workspace access is disabled")
|
|
authorize_workspace_actor(actor)
|
|
authorize_actor(actor, capability=capability, project=project_id)
|
|
|
|
def _headers(self, actor: ActorContext) -> dict[str, str]:
|
|
return {
|
|
"Authorization": f"Bearer {self._service_token}",
|
|
"X-Correlation-ID": actor.correlation_id,
|
|
}
|
|
|
|
async def _send_json(
|
|
self,
|
|
method: str,
|
|
path: str,
|
|
*,
|
|
headers: dict[str, str],
|
|
json_body: Mapping[str, object] | None,
|
|
correlation_id: str,
|
|
require_response_etag: bool,
|
|
) -> HubResponse:
|
|
try:
|
|
async with self._http.stream(
|
|
method,
|
|
path,
|
|
headers=headers,
|
|
json=json_body,
|
|
) as response:
|
|
self._raise_status(response.status_code, response.headers, correlation_id)
|
|
raw = await self._read_response(response, correlation_id)
|
|
return self._decode_response(
|
|
response,
|
|
raw,
|
|
correlation_id,
|
|
require_etag=require_response_etag,
|
|
)
|
|
except HubClientError:
|
|
raise
|
|
except httpx.HTTPError:
|
|
raise HubUnavailable(correlation_id) from None
|
|
|
|
async def _send_multipart(
|
|
self,
|
|
path: str,
|
|
*,
|
|
headers: dict[str, str],
|
|
source: SourceFile,
|
|
filename: str,
|
|
correlation_id: str,
|
|
) -> HubResponse:
|
|
try:
|
|
async with self._http.stream(
|
|
"POST",
|
|
path,
|
|
headers=headers,
|
|
files={"file": (filename, source.content, source.content_type)},
|
|
) as response:
|
|
self._raise_status(response.status_code, response.headers, correlation_id)
|
|
raw = await self._read_response(response, correlation_id)
|
|
result = self._decode_response(response, raw, correlation_id, require_etag=True)
|
|
return result
|
|
except HubClientError:
|
|
raise
|
|
except httpx.HTTPError:
|
|
raise HubUnavailable(correlation_id) from None
|
|
|
|
async def _read_response(self, response: httpx.Response, correlation_id: str) -> bytes:
|
|
length = response.headers.get("content-length")
|
|
if length is not None:
|
|
try:
|
|
if int(length) > self._max_response_bytes:
|
|
raise HubResponseTooLarge(correlation_id, status_code=response.status_code)
|
|
except ValueError as exc:
|
|
raise HubClientError(
|
|
"hub_invalid_response", correlation_id, status_code=response.status_code
|
|
) from exc
|
|
content = bytearray()
|
|
async for chunk in response.aiter_bytes():
|
|
if len(content) + len(chunk) > self._max_response_bytes:
|
|
raise HubResponseTooLarge(correlation_id, status_code=response.status_code)
|
|
content.extend(chunk)
|
|
return bytes(content)
|
|
|
|
def _decode_response(
|
|
self,
|
|
response: httpx.Response,
|
|
raw: bytes,
|
|
correlation_id: str,
|
|
*,
|
|
require_etag: bool,
|
|
) -> HubResponse:
|
|
try:
|
|
data = json.loads(raw)
|
|
except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
|
|
raise HubClientError(
|
|
"hub_invalid_response", correlation_id, status_code=response.status_code
|
|
) from exc
|
|
if not isinstance(data, dict):
|
|
raise HubClientError(
|
|
"hub_invalid_response", correlation_id, status_code=response.status_code
|
|
)
|
|
raw_etag = response.headers.get("etag")
|
|
if raw_etag is not None and not _ETAG.fullmatch(raw_etag):
|
|
raise HubClientError(
|
|
"hub_invalid_response", correlation_id, status_code=response.status_code
|
|
)
|
|
if require_etag and raw_etag is None:
|
|
raise HubClientError(
|
|
"hub_missing_etag", correlation_id, status_code=response.status_code
|
|
)
|
|
return HubResponse(
|
|
data=data,
|
|
status_code=response.status_code,
|
|
etag=raw_etag,
|
|
correlation_id=correlation_id,
|
|
)
|
|
|
|
@staticmethod
|
|
def _raise_status(
|
|
status_code: int,
|
|
headers: httpx.Headers,
|
|
correlation_id: str,
|
|
) -> None:
|
|
if 200 <= status_code < 300:
|
|
return
|
|
if status_code == 409:
|
|
raise HubConflictError(correlation_id)
|
|
if status_code == 412:
|
|
raise HubPreconditionFailed(correlation_id)
|
|
if status_code == 429:
|
|
raw_retry = headers.get("retry-after", "")
|
|
try:
|
|
retry_after = max(1, min(60, int(raw_retry))) if raw_retry else None
|
|
except ValueError:
|
|
retry_after = None
|
|
raise HubRateLimitedError(correlation_id, retry_after)
|
|
code = "hub_unavailable" if status_code >= 500 else "hub_request_rejected"
|
|
raise HubClientError(code, correlation_id, status_code=status_code)
|
|
|
|
|
|
def _optional_project(project_id: str | None) -> str | None:
|
|
if project_id is None:
|
|
return None
|
|
try:
|
|
return str(UUID(project_id))
|
|
except (TypeError, ValueError, AttributeError) as exc:
|
|
raise ValueError("Knowledge Hub project is invalid") from exc
|
|
|
|
|
|
def _required_project(project_id: str | None) -> str:
|
|
normalized = _optional_project(project_id)
|
|
if normalized is None:
|
|
raise ValueError("Knowledge Hub project is required")
|
|
return normalized
|
|
|
|
|
|
def _validate_create_body(
|
|
body: Mapping[str, object] | None,
|
|
*,
|
|
actor: ActorContext,
|
|
project_id: str | None,
|
|
) -> None:
|
|
if not isinstance(body, Mapping) or set(body) - _CREATE_FIELDS:
|
|
raise ValueError("Knowledge Hub proposal request is invalid")
|
|
manifest = body.get("manifest")
|
|
if not isinstance(manifest, str) or not manifest.strip() or len(manifest) > 200_000:
|
|
raise ValueError("Knowledge Hub proposal request is invalid")
|
|
if body.get("correlationId") not in {None, actor.correlation_id}:
|
|
raise ValueError("Knowledge Hub proposal request is invalid")
|
|
publication = body.get("publication")
|
|
if publication is not None:
|
|
if not isinstance(publication, Mapping) or set(publication) - _PUBLICATION_FIELDS:
|
|
raise ValueError("Knowledge Hub proposal request is invalid")
|
|
for key, value in publication.items():
|
|
string_fields = {
|
|
"title",
|
|
"markdown",
|
|
"documentType",
|
|
"documentStatus",
|
|
"mutationKind",
|
|
"expectedTargetETag",
|
|
}
|
|
if key in string_fields and (
|
|
not isinstance(value, str)
|
|
or len(value) > (32_000 if key == "markdown" else 100_000)
|
|
):
|
|
raise ValueError("Knowledge Hub proposal request is invalid")
|
|
try:
|
|
decoded = json.loads(manifest)
|
|
except json.JSONDecodeError:
|
|
decoded = None
|
|
manifest_project = _extract_project_id(decoded)
|
|
if (
|
|
not isinstance(decoded, Mapping)
|
|
or (
|
|
project_id is None
|
|
and (decoded.get("workspace") != "knowledge-hub" or "projectId" in decoded)
|
|
)
|
|
or (project_id is not None and manifest_project != project_id)
|
|
):
|
|
raise PermissionError("ingestion caller is not authorized")
|
|
|
|
|
|
def _validate_analysis_body(body: Mapping[str, object] | None) -> None:
|
|
analysis = body.get("analysis") if isinstance(body, Mapping) else None
|
|
if (
|
|
not isinstance(body, Mapping)
|
|
or set(body) != {"analysis"}
|
|
or not isinstance(analysis, str)
|
|
or not analysis.strip()
|
|
or len(analysis) > 32_000
|
|
or "\x00" in analysis
|
|
):
|
|
raise ValueError("Knowledge Hub analysis request is invalid")
|
|
|
|
|
|
def _validate_source(source: SourceFile, max_upload_bytes: int) -> str:
|
|
if not isinstance(source, SourceFile):
|
|
raise ValueError("Knowledge Hub source is invalid")
|
|
filename = source.file_name
|
|
if (
|
|
not isinstance(filename, str)
|
|
or not 1 <= len(filename) <= 255
|
|
or filename in {".", ".."}
|
|
or "/" in filename
|
|
or "\\" in filename
|
|
or '"' in filename
|
|
or any(ord(character) < 0x20 for character in filename)
|
|
or not isinstance(source.content_type, str)
|
|
or source.content_type.lower() not in _SOURCE_MIME_TYPES
|
|
or not isinstance(source.content, bytes)
|
|
or not 1 <= len(source.content) <= max_upload_bytes
|
|
):
|
|
raise ValueError("Knowledge Hub source is invalid")
|
|
return filename
|
|
|
|
|
|
def _extract_project_id(value: object) -> str | None:
|
|
if not isinstance(value, Mapping):
|
|
return None
|
|
project = value.get("projectId")
|
|
if isinstance(project, str):
|
|
try:
|
|
return str(UUID(project))
|
|
except ValueError:
|
|
return None
|
|
for key in ("destination", "metadata", "project"):
|
|
nested = value.get(key)
|
|
if isinstance(nested, Mapping):
|
|
found = _extract_project_id(nested)
|
|
if found is not None:
|
|
return found
|
|
return None
|
|
|
|
|
|
def _require_response_project(data: Mapping[str, Any], expected_project: str | None) -> None:
|
|
manifest = data.get("manifest")
|
|
if isinstance(manifest, str):
|
|
try:
|
|
manifest = json.loads(manifest)
|
|
except json.JSONDecodeError:
|
|
return
|
|
if expected_project is None:
|
|
if not isinstance(manifest, Mapping) or (
|
|
manifest.get("workspace") != "knowledge-hub" or "projectId" in manifest
|
|
):
|
|
raise PermissionError("ingestion caller is not authorized")
|
|
return
|
|
actual = _extract_project_id(manifest)
|
|
if actual is None:
|
|
metadata = data.get("metadata")
|
|
actual = _extract_project_id(metadata)
|
|
if actual is not None and actual != expected_project:
|
|
raise PermissionError("ingestion caller is not authorized")
|