"""Scoped Knowledge Hub lookup and review-only ingestion workflows.""" from __future__ import annotations import hashlib import hmac import json import re from base64 import b64decode, urlsafe_b64encode from binascii import Error as Base64Error from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import Any, Literal from urllib.parse import unquote, urlencode, urlsplit from .ingestion_auth import ActorContext, authorize_actor, authorize_workspace_actor from .ingestion_files import ( FileIngestionError, FilePolicy, ProcessedSource, acquire_scan_extract, scan_text_source, ) from .ingestion_hub_client import HubClient, SourceFile KNOWN_STATUSES = frozenset( { "Received", "Analyzing", "ReadyForReview", "ChangesRequested", "Approved", "Publishing", "Published", "Rejected", "Failed", } ) _TERMINAL_STATUSES = frozenset({"Published", "Rejected"}) _MUTABLE_ANALYSIS_STATUSES = frozenset({"Received", "ChangesRequested"}) _UUID = re.compile( r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$", re.I ) _SAFE_CORRELATION = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") _SAFE_CODE = re.compile(r"^[a-z][a-z0-9_]{0,47}$") _MAX_ANALYSIS_CHARS = 32_000 _MAX_MANIFEST_CHARS = 10_000 _MAX_MARKDOWN_CHARS = 32_000 _MAX_SOURCE_TEXT_CHARS = 100_000 _MAX_SEARCH_RESULTS = 20 _MAX_CONTEXT_TEXT = 4_000 _MAX_REVIEWER_FEEDBACK = 4_000 _MAX_EXTRACT_PAGE_CHARS = 12_000 _MAX_PAGE_READ_CHARS = 20_000 _MAX_KNOWLEDGE_PAGE_CHARS = 1_000_000 _MAX_TREE_PAGE_SIZE = 100 _WORKSPACE_ETAG = re.compile(r'^"[\x21\x23-\x7e]{1,254}"$') _GITEA_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") _GIT_REF = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,299}$") _GIT_SHA = re.compile(r"^[0-9a-fA-F]{7,64}$") _GIT_BLOB_SHA = re.compile(r"^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$") _SOURCE_FILE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._ -]{0,127}$") ActorAuthorizer = Callable[..., None] FileAcquirer = Callable[..., Awaitable[tuple[ProcessedSource, ...]]] TextSourceScanner = Callable[..., Awaitable[ProcessedSource]] class IngestionWorkflowError(Exception): """A stable workflow failure with no raw upstream or source data.""" def __init__( self, code: str, message: str, *, action: str, proposal_id: str | None = None, retry_after_seconds: int | None = None, ) -> None: super().__init__(message) self.code = code self.safe_message = message self.action = action self.proposal_id = proposal_id self.retry_after_seconds = retry_after_seconds class IngestionWorkflow: """Orchestrate bounded MCP operations over the public Hub API only.""" def __init__( self, client: HubClient, *, file_policy: FilePolicy | None = None, file_acquirer: FileAcquirer = acquire_scan_extract, text_source_scanner: TextSourceScanner = scan_text_source, workspace_read_enabled: bool = False, cursor_signing_key: bytes | None = None, actor_authorizer: ActorAuthorizer = authorize_actor, ) -> None: self.client = client self.file_policy = file_policy if file_policy is not None else FilePolicy.from_environment() self.file_acquirer = file_acquirer self.text_source_scanner = text_source_scanner if not isinstance(workspace_read_enabled, bool): raise ValueError("workspace read configuration is invalid") self.workspace_read_enabled = workspace_read_enabled if cursor_signing_key is not None and ( not isinstance(cursor_signing_key, bytes) or len(cursor_signing_key) < 32 ): raise ValueError("attachment cursor signing key is invalid") self.cursor_signing_key = cursor_signing_key self.actor_authorizer = actor_authorizer async def search_knowledge( self, *, project_id: str | None, query: str, actor: ActorContext, limit: int = 10, cursor: str | None = None, ) -> dict[str, Any]: project_id = self._resolve_project(actor, "knowledge:read", project_id) normalized_query = query.strip() if isinstance(query, str) else "" if not 1 <= len(normalized_query) <= 500: raise ValueError("search query is invalid") if ( isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= _MAX_SEARCH_RESULTS ): raise ValueError("search limit is invalid") if cursor is not None and (not isinstance(cursor, str) or len(cursor) > 500): raise ValueError("search cursor is invalid") params: dict[str, str] = {"q": normalized_query, "limit": str(limit)} if cursor: params["cursor"] = cursor query_path = f"/api/v1/search?{urlencode(params)}" search_response = await self.client.request_json( "GET", query_path, actor=actor, json_body=None, etag=None, idempotency_key=None, project_id=project_id, ) allowed_citations: dict[str, dict[str, Any]] | None = None if project_id is not None: context_response = await self.client.request_json( "GET", self._project_context_path(project_id), actor=actor, json_body=None, etag=None, idempotency_key=None, project_id=project_id, ) allowed_citations = self._project_citations(context_response.data) raw_items = search_response.data.get("items") items = raw_items if isinstance(raw_items, list) else [] results: list[dict[str, Any]] = [] for item in items: if not isinstance(item, Mapping): continue page_id = self._string(item.get("id"), max_length=64) if not page_id: continue citation = ( allowed_citations.get(page_id) if allowed_citations is not None else {"pageId": page_id} ) if citation is None: continue results.append( { "id": page_id, "title": self._string(item.get("title"), max_length=300) or "Untitled page", "slug": self._string(item.get("slug"), max_length=200), "documentType": self._string(item.get("documentType"), max_length=64), "documentStatus": self._string(item.get("documentStatus"), max_length=64), "snippet": self._string(item.get("snippet"), max_length=1_000), "updatedAt": self._string(item.get("updatedAt"), max_length=64), "citation": citation, } ) if len(results) >= limit: break next_cursor = self._string(search_response.data.get("nextCursor"), max_length=500) return { "results": results, "nextCursor": next_cursor, "projectId": project_id, "correlationId": self._correlation(search_response.correlation_id, actor), } async def get_knowledge_context( self, *, project_id: str, actor: ActorContext, ) -> dict[str, Any]: project_id = self._validate_uuid(project_id, "project identifier") self._authorize(actor, "knowledge:read", project_id) response = await self.client.request_json( "GET", self._project_context_path(project_id), actor=actor, json_body=None, etag=None, idempotency_key=None, project_id=project_id, ) return self._project_context_projection(response.data, response.correlation_id, actor) async def extract_knowledge_files( self, *, project_id: str | None, files: Sequence[Mapping[str, Any]], actor: ActorContext, page_cursor: str | None = None, ) -> dict[str, Any]: """Return one scanned text page from the attachments without creating a proposal.""" project_id = self._resolve_project(actor, "ingestion:write", project_id) if self.cursor_signing_key is None: raise ValueError("attachment cursor signing key is unavailable") self._validate_file_refs(files) if page_cursor is not None and (not isinstance(page_cursor, str) or len(page_cursor) > 256): raise ValueError("attachment page cursor is invalid") processed = await self.file_acquirer(files, actor=actor, policy=self.file_policy) if not processed or len(processed) != len(files): raise FileIngestionError("source_limit") if any(not isinstance(source.text, str) or not source.text.strip() for source in processed): raise FileIngestionError("extraction_failed") requested_ids = [item.get("file_id") for item in files] if any( source.file_id != expected for source, expected in zip(processed, requested_ids, strict=True) ): raise FileIngestionError("invalid_input", input_reason="acquired_file_id_mismatch") file_set_digest = self._extraction_file_set_digest(processed) file_index, start_offset = self._decode_extraction_cursor( page_cursor, file_set_digest, actor=actor, project_id=project_id ) if not 0 <= file_index < len(processed): raise ValueError("attachment page cursor is invalid") selected = processed[file_index] if not isinstance(selected.text, str) or not 0 <= start_offset < max(len(selected.text), 1): raise ValueError("attachment page cursor is invalid") end_offset = min(start_offset + _MAX_EXTRACT_PAGE_CHARS, len(selected.text)) next_index, next_offset = file_index, end_offset if end_offset >= len(selected.text): next_index, next_offset = file_index + 1, 0 next_cursor = ( self._encode_extraction_cursor( file_set_digest, next_index, next_offset, actor=actor, project_id=project_id ) if next_index < len(processed) else None ) coverage: list[dict[str, Any]] = [] for index, source in enumerate(processed): total_chars = len(source.text) covered_chars = ( total_chars if index < file_index else min(end_offset, total_chars) if index == file_index else 0 ) coverage.append( { "fileId": source.file_id, "fileName": source.file_name, "sha256": source.sha256, "totalChars": total_chars, "coveredChars": covered_chars, "complete": covered_chars == total_chars, } ) page = { "fileId": selected.file_id, "fileName": selected.file_name, "mimeType": selected.mime_type, "byteSize": selected.byte_size, "sha256": selected.sha256, "startOffset": start_offset, "endOffset": end_offset, "totalChars": len(selected.text), "text": selected.text[start_offset:end_offset], "complete": end_offset >= len(selected.text), } return { "page": page, "fileCoverage": coverage, "nextCursor": next_cursor, "allFilesComplete": next_cursor is None, "proposalCreated": False, "correlationId": actor.correlation_id, } async def get_workspace_roots( self, *, actor: ActorContext, limit: int = 50, cursor: str | None = None, ) -> dict[str, Any]: """Read one bounded page of roots from the token-scoped Knowledge Hub workspace.""" self._authorize_workspace_read(actor) self._validate_tree_page(limit, cursor) params: dict[str, str] = {"limit": str(limit)} if cursor: params["cursor"] = cursor response = await self.client.request_json( "GET", f"/api/v1/pages/roots?{urlencode(params)}", actor=actor, json_body=None, etag=None, idempotency_key=None, ) return self._workspace_tree_projection(response.data, response.correlation_id, actor) async def get_workspace_children( self, *, parent_page_id: str, actor: ActorContext, limit: int = 50, cursor: str | None = None, ) -> dict[str, Any]: """Read one bounded child-page slice beneath an explicitly selected page.""" self._authorize_workspace_read(actor) parent_page_id = self._validate_uuid(parent_page_id, "parent page identifier") self._validate_tree_page(limit, cursor) params: dict[str, str] = {"limit": str(limit)} if cursor: params["cursor"] = cursor response = await self.client.request_json( "GET", f"/api/v1/pages/{parent_page_id}/children?{urlencode(params)}", actor=actor, json_body=None, etag=None, idempotency_key=None, ) result = self._workspace_tree_projection(response.data, response.correlation_id, actor) result["parentPageId"] = parent_page_id return result async def get_workspace_page( self, *, page_id: str, actor: ActorContext, offset_chars: int = 0, max_chars: int = _MAX_PAGE_READ_CHARS, expected_etag: str | None = None, ) -> dict[str, Any]: """Read a version-checked Markdown slice and its current destination ETag.""" self._authorize_workspace_read(actor) page_id = self._validate_uuid(page_id, "page identifier") if ( isinstance(offset_chars, bool) or not isinstance(offset_chars, int) or not 0 <= offset_chars <= _MAX_KNOWLEDGE_PAGE_CHARS or isinstance(max_chars, bool) or not isinstance(max_chars, int) or not 1 <= max_chars <= _MAX_PAGE_READ_CHARS ): raise ValueError("page content range is invalid") if offset_chars > 0 and expected_etag is None: raise ValueError("page pagination requires the current ETag") if expected_etag is not None and ( not isinstance(expected_etag, str) or not _WORKSPACE_ETAG.fullmatch(expected_etag) ): raise ValueError("page ETag is invalid") response = await self.client.request_json( "GET", f"/api/v1/pages/{page_id}", actor=actor, json_body=None, etag=None, idempotency_key=None, ) if expected_etag is not None and response.etag != expected_etag: from .ingestion_hub_client import HubPreconditionFailed raise HubPreconditionFailed(response.correlation_id) if response.data.get("id") != page_id: raise IngestionWorkflowError( "page_unavailable", "The requested Knowledge Hub page could not be confirmed.", action="refresh", ) markdown = response.data.get("markdown") if not isinstance(markdown, str) or len(markdown) > _MAX_KNOWLEDGE_PAGE_CHARS: raise IngestionWorkflowError( "page_too_large", "This page exceeds the comparison limit and cannot be used for an update proposal.", action="return", ) if offset_chars > len(markdown): raise ValueError("page content offset is outside the page") end_offset = min(offset_chars + max_chars, len(markdown)) breadcrumbs = response.data.get("breadcrumb", response.data.get("breadcrumbs")) if not isinstance(breadcrumbs, list) or len(breadcrumbs) > 50: raise IngestionWorkflowError( "page_unavailable", "The requested page breadcrumbs could not be confirmed.", action="refresh", ) page_breadcrumbs = [ { "id": self._validate_uuid(item.get("id"), "breadcrumb page identifier"), "title": self._string(item.get("title"), max_length=300) or "Untitled page", } for item in breadcrumbs if isinstance(item, Mapping) ] if len(page_breadcrumbs) != len(breadcrumbs): raise IngestionWorkflowError( "page_unavailable", "The requested page breadcrumbs could not be confirmed.", action="refresh", ) return { "pageId": page_id, "title": self._string(response.data.get("title"), max_length=300) or "Untitled page", "slug": self._string(response.data.get("slug"), max_length=200), "documentType": self._string(response.data.get("documentType"), max_length=64), "documentStatus": self._string(response.data.get("documentStatus"), max_length=64), "parentPageId": self._string(response.data.get("parentPageId"), max_length=64), "currentRevisionNumber": response.data.get("currentRevisionNumber"), "breadcrumbs": page_breadcrumbs, "etag": response.etag, "offsetChars": offset_chars, "nextOffset": end_offset if end_offset < len(markdown) else None, "totalChars": len(markdown), "markdown": markdown[offset_chars:end_offset], "complete": end_offset >= len(markdown), "truncated": end_offset < len(markdown), "correlationId": self._correlation(response.correlation_id, actor), } async def analyze_knowledge_files( self, *, project_id: str | None, files: Sequence[Mapping[str, Any]], manifest: str, publication: Mapping[str, Any], actor: ActorContext, ) -> dict[str, Any]: project_id = self._resolve_project(actor, "ingestion:write", project_id) normalized_manifest = self._validate_manifest(manifest) normalized_publication = self._validate_publication(publication) self._validate_file_refs(files) # Reject any file that cannot be fully validated, scanned and extracted # before a durable proposal shell exists. The file service returns only # bounded bytes after its temporary copies have been securely removed. processed = await self.file_acquirer(files, actor=actor, policy=self.file_policy) if not processed: raise FileIngestionError("source_limit") if any(not isinstance(source.text, str) or not source.text.strip() for source in processed): raise FileIngestionError("extraction_failed") idempotency_key = self._proposal_idempotency_key( actor=actor, project_id=project_id, files=files, manifest=normalized_manifest, publication=normalized_publication, ) # The public API's manifest is the only persisted field available for a # project id on a machine-created shell. Keep it structured and exclude # ChatGPT's short-lived URLs and file bodies entirely. manifest_envelope = json.dumps( self._proposal_manifest(project_id, normalized_manifest), ensure_ascii=False, separators=(",", ":"), ) shell = await self.client.request_json( "POST", "/api/v1/ingestion-proposals", actor=actor, json_body={ "manifest": manifest_envelope, "correlationId": actor.correlation_id, "publication": normalized_publication, }, etag=None, idempotency_key=idempotency_key, project_id=project_id, ) proposal_id = self._response_id(shell.data) current = await self.client.request_json( "GET", f"/api/v1/ingestion-proposals/{proposal_id}", actor=actor, json_body=None, etag=None, idempotency_key=None, project_id=project_id, ) self._require_proposal_project(current.data, project_id) current_status = self._status(current.data.get("status")) if current_status == "ReadyForReview" or current_status in { "Approved", "Publishing", "Published", "Rejected", }: return self._status_projection(current.data, current.correlation_id, actor) if current_status not in {"Received", "ChangesRequested", "Analyzing"}: raise IngestionWorkflowError( "proposal_unavailable", "This proposal cannot accept additional sources in its current state.", action="refresh", proposal_id=proposal_id, ) current_etag = current.etag or shell.etag if not current_etag: raise IngestionWorkflowError( "version_unavailable", "The proposal version could not be confirmed. Refresh the proposal and retry.", action="refresh", proposal_id=proposal_id, ) existing_digests = self._existing_source_digests(current.data) uploaded: list[dict[str, Any]] = [] for source in processed: if source.sha256 in existing_digests: continue upload_response = await self.client.upload_source( proposal_id, SourceFile(source.file_name, source.mime_type, source.content), actor=actor, etag=current_etag, idempotency_key=self._source_idempotency_key(idempotency_key, source.sha256), project_id=project_id, ) current_etag = upload_response.etag or "" if not current_etag: raise IngestionWorkflowError( "version_unavailable", ( "A source was accepted but its current proposal version " "could not be confirmed." ), action="refresh", proposal_id=proposal_id, ) existing_digests.add(source.sha256) uploaded.append(self._source_projection(source)) return { "proposalId": proposal_id, "status": current_status if current_status in KNOWN_STATUSES else "Unknown", "sourceCount": len(existing_digests), "uploadedSources": uploaded, "nextAction": "submit_ingestion_proposal", "correlationId": self._correlation(current.correlation_id, actor), } async def create_text_source_proposal( self, *, project_id: str | None, source_file_name: str, source_text: str, source_complete: bool, source_uri: str, source_ref: str, source_commit: str, source_sha: str, manifest: str, publication: Mapping[str, Any], actor: ActorContext, ) -> dict[str, Any]: """Create a complete review proposal from text already read through Gitea MCP.""" project_id = self._resolve_project(actor, "ingestion:write", project_id) if source_complete is not True: raise ValueError("text source must be complete before proposal creation") provenance = self._validate_source_provenance( source_file_name=source_file_name, source_uri=source_uri, source_ref=source_ref, source_commit=source_commit, source_sha=source_sha, ) if not isinstance(source_text, str) or not 1 <= len(source_text) <= _MAX_SOURCE_TEXT_CHARS: raise ValueError("text source is invalid") try: source_bytes = source_text.encode("utf-8") except UnicodeEncodeError as exc: raise ValueError("text source is invalid") from exc git_blob = f"blob {len(source_bytes)}\0".encode("ascii") + source_bytes actual_sha = ( hashlib.sha1(git_blob, usedforsecurity=False).hexdigest() if len(provenance["sourceSha"]) == 40 else hashlib.sha256(git_blob).hexdigest() ) if actual_sha != provenance["sourceSha"]: raise ValueError("source text does not match the supplied Git blob SHA") normalized_manifest = self._validate_manifest(manifest) normalized_publication = self._validate_publication(publication) source = await self.text_source_scanner( source_text, provenance["fileName"], policy=self.file_policy, ) if not isinstance(source, ProcessedSource): raise FileIngestionError("scan_failed") provenance_envelope = { **provenance, "providedByCaller": True, "verified": False, "suppliedTextMatchesSha": True, } manifest_envelope = json.dumps( { **self._proposal_manifest(project_id, normalized_manifest), "sourceProvenance": [provenance_envelope], }, ensure_ascii=False, separators=(",", ":"), ) proposal_key = self._text_proposal_idempotency_key( actor=actor, project_id=project_id, manifest=normalized_manifest, publication=normalized_publication, source=source, provenance=provenance_envelope, ) created = await self.client.request_json( "POST", "/api/v1/ingestion-proposals", actor=actor, json_body={ "manifest": manifest_envelope, "correlationId": actor.correlation_id, "publication": normalized_publication, }, etag=None, idempotency_key=proposal_key, project_id=project_id, ) proposal_id = self._response_id(created.data) current = await self.client.request_json( "GET", f"/api/v1/ingestion-proposals/{proposal_id}", actor=actor, json_body=None, etag=None, idempotency_key=None, project_id=project_id, ) self._require_proposal_project(current.data, project_id) status = self._status(current.data.get("status")) if status in {"ReadyForReview", "Approved", "Publishing", "Published", "Rejected"}: return self._status_projection(current.data, current.correlation_id, actor) if status not in {"Received", "ChangesRequested", "Analyzing"}: raise IngestionWorkflowError( "proposal_unavailable", "This proposal cannot accept a text source in its current state.", action="refresh", proposal_id=proposal_id, ) if not current.etag and not created.etag: raise IngestionWorkflowError( "version_unavailable", "The proposal version could not be confirmed. Refresh the proposal and retry.", action="refresh", proposal_id=proposal_id, ) current_etag = current.etag or created.etag existing_digests = self._existing_source_digests(current.data) uploaded: list[dict[str, Any]] = [] if source.sha256 not in existing_digests: upload = await self.client.upload_source( proposal_id, SourceFile(source.file_name, source.mime_type, source.content), actor=actor, etag=current_etag, idempotency_key=self._source_idempotency_key(proposal_key, source.sha256), project_id=project_id, ) if not upload.etag: raise IngestionWorkflowError( "version_unavailable", "A source was accepted but its current proposal version " "could not be confirmed.", action="refresh", proposal_id=proposal_id, ) uploaded.append(self._source_projection(source)) existing_digests.add(source.sha256) return { "proposalId": proposal_id, "status": status, "sourceCount": len(existing_digests), "uploadedSources": uploaded, "sourceProvenance": provenance_envelope, "nextAction": "submit_ingestion_proposal", "correlationId": self._correlation(current.correlation_id, actor), } async def submit_ingestion_proposal( self, *, project_id: str | None, proposal_id: str, analysis: str, actor: ActorContext, ) -> dict[str, Any]: project_id = self._resolve_project(actor, "ingestion:write", project_id) normalized_analysis = analysis.strip() if isinstance(analysis, str) else "" if not 1 <= len(normalized_analysis) <= _MAX_ANALYSIS_CHARS: raise ValueError("proposal analysis is invalid") proposal_id = self._validate_uuid(proposal_id, "proposal identifier") response = await self.client.request_json( "GET", f"/api/v1/ingestion-proposals/{proposal_id}", actor=actor, json_body=None, etag=None, idempotency_key=None, project_id=project_id, ) self._require_proposal_project(response.data, project_id) status = self._status(response.data.get("status")) if status == "ReadyForReview": return self._status_projection(response.data, response.correlation_id, actor) if status not in _MUTABLE_ANALYSIS_STATUSES: raise IngestionWorkflowError( "proposal_not_ready", "This proposal cannot be submitted for review in its current state.", action="refresh", proposal_id=proposal_id, ) if not self._has_publication(response.data): raise IngestionWorkflowError( "proposal_details_required", "A complete create or update proposal is required before review.", action="fix_input", proposal_id=proposal_id, ) if not self._existing_source_digests(response.data): raise IngestionWorkflowError( "source_required", "At least one clean source is required before review.", action="fix_input", proposal_id=proposal_id, ) if not response.etag: raise IngestionWorkflowError( "version_unavailable", "The proposal version could not be confirmed. Refresh the proposal and retry.", action="refresh", proposal_id=proposal_id, ) patched = await self.client.request_json( "PATCH", f"/api/v1/ingestion-proposals/{proposal_id}", actor=actor, json_body={"analysis": normalized_analysis}, etag=response.etag, idempotency_key=None, project_id=project_id, ) if self._status(patched.data.get("status")) != "ReadyForReview": raise IngestionWorkflowError( "review_state_unconfirmed", "Knowledge Hub did not confirm that this proposal is ready for human review.", action="refresh", proposal_id=proposal_id, ) return self._status_projection(patched.data, patched.correlation_id, actor) async def get_ingestion_status( self, *, project_id: str | None, proposal_id: str, actor: ActorContext, ) -> dict[str, Any]: project_id = self._resolve_project(actor, "ingestion:write", project_id) proposal_id = self._validate_uuid(proposal_id, "proposal identifier") response = await self.client.request_json( "GET", f"/api/v1/ingestion-proposals/{proposal_id}", actor=actor, json_body=None, etag=None, idempotency_key=None, project_id=project_id, ) self._require_proposal_project(response.data, project_id) return self._status_projection(response.data, response.correlation_id, actor) async def retry_failed_proposal( self, *, project_id: str, proposal_id: str, etag: str, idempotency_key: str, actor: ActorContext, ) -> dict[str, Any]: """Retry a failed proposal internally; this operation is never an MCP tool.""" project_id = self._validate_uuid(project_id, "project identifier") self._authorize(actor, "ingestion:write", project_id) proposal_id = self._validate_uuid(proposal_id, "proposal identifier") if not isinstance(etag, str) or not etag.strip() or len(etag) > 256: raise ValueError("proposal version is invalid") if not isinstance(idempotency_key, str) or not re.fullmatch( r"[A-Za-z0-9._:-]{8,128}", idempotency_key ): raise ValueError("retry key is invalid") current = await self.client.request_json( "GET", f"/api/v1/ingestion-proposals/{proposal_id}", actor=actor, json_body=None, etag=None, idempotency_key=None, project_id=project_id, ) self._require_proposal_project(current.data, project_id) retry_count = current.data.get("retryCount") max_retries = current.data.get("maxPublishRetries") if ( self._status(current.data.get("status")) != "Failed" or current.data.get("isRetryEligible") is not True or not isinstance(retry_count, int) or isinstance(retry_count, bool) or not isinstance(max_retries, int) or isinstance(max_retries, bool) or retry_count < 0 or retry_count >= max_retries ): raise ValueError("proposal is not eligible for retry") if not current.etag or current.etag != etag: raise IngestionWorkflowError( "stale_version", "The proposal changed. Refresh its status before retrying.", action="refresh", proposal_id=proposal_id, ) retried = await self.client.request_json( "POST", f"/api/v1/ingestion-proposals/{proposal_id}/retry", actor=actor, json_body={}, etag=current.etag, idempotency_key=idempotency_key, project_id=project_id, ) return self._status_projection(retried.data, retried.correlation_id, actor) def _authorize( self, actor: ActorContext, capability: Literal["knowledge:read", "ingestion:write"], project_id: str | None, ) -> None: self.actor_authorizer(actor, capability=capability, project=project_id) def _resolve_project( self, actor: ActorContext, capability: Literal["knowledge:read", "ingestion:write"], project_id: str | None, ) -> str | None: if project_id is None: if not self.workspace_read_enabled: raise PermissionError("workspace access is disabled") authorize_workspace_actor(actor) self._authorize(actor, capability, None) return None selected = self._validate_uuid(project_id, "project identifier") self._authorize(actor, capability, selected) return selected @staticmethod def _proposal_manifest(project_id: str | None, user_manifest: str) -> dict[str, Any]: if project_id is None: return {"workspace": "knowledge-hub", "userManifest": user_manifest} return {"projectId": project_id, "userManifest": user_manifest} def _authorize_workspace_read(self, actor: ActorContext) -> None: if not self.workspace_read_enabled: raise PermissionError("workspace page reads are disabled") authorize_workspace_actor(actor) self._authorize(actor, "knowledge:read", None) @staticmethod def _validate_tree_page(limit: int, cursor: str | None) -> None: if ( isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= _MAX_TREE_PAGE_SIZE ): raise ValueError("page list limit is invalid") if cursor is not None and (not isinstance(cursor, str) or len(cursor) > 500): raise ValueError("page list cursor is invalid") @staticmethod def _workspace_tree_projection( data: Mapping[str, Any], correlation_id: str, actor: ActorContext ) -> dict[str, Any]: raw_items = data.get("items") next_cursor = data.get("nextCursor") if ( not isinstance(raw_items, list) or len(raw_items) > _MAX_TREE_PAGE_SIZE or ( next_cursor is not None and (not isinstance(next_cursor, str) or len(next_cursor) > 500) ) ): raise IngestionWorkflowError( "page_list_unavailable", "Knowledge Hub returned an invalid page list.", action="refresh", ) items: list[dict[str, Any]] = [] for item in raw_items: if not isinstance(item, Mapping): raise IngestionWorkflowError( "page_list_unavailable", "Knowledge Hub returned an invalid page list.", action="refresh", ) page_id = IngestionWorkflow._validate_uuid(item.get("id"), "page identifier") parent_id = item.get("parentPageId") if parent_id is not None: parent_id = IngestionWorkflow._validate_uuid(parent_id, "parent page identifier") has_children = item.get("hasChildren") if not isinstance(has_children, bool): raise IngestionWorkflowError( "page_list_unavailable", "Knowledge Hub returned an invalid page list.", action="refresh", ) items.append( { "id": page_id, "parentPageId": parent_id, "title": IngestionWorkflow._string(item.get("title"), max_length=300) or "Untitled page", "slug": IngestionWorkflow._string(item.get("slug"), max_length=200), "documentType": IngestionWorkflow._string( item.get("documentType"), max_length=64 ), "documentStatus": IngestionWorkflow._string( item.get("documentStatus"), max_length=64 ), "hasChildren": has_children, "updatedAt": IngestionWorkflow._string(item.get("updatedAt"), max_length=64), } ) return { "items": items, "nextCursor": next_cursor, "truncated": next_cursor is not None, "correlationId": IngestionWorkflow._correlation(correlation_id, actor), } @staticmethod def _extraction_file_set_digest(processed: Sequence[ProcessedSource]) -> str: material = json.dumps( [ {"fileId": source.file_id, "sha256": source.sha256, "fileName": source.file_name} for source in processed ], ensure_ascii=False, separators=(",", ":"), ) return hashlib.sha256(material.encode("utf-8")).hexdigest() def _cursor_mac(self, payload: bytes, *, actor: ActorContext, project_id: str | None) -> bytes: if self.cursor_signing_key is None: raise ValueError("attachment cursor signing key is unavailable") context = json.dumps( { "subject": actor.subject, "connectorProject": actor.connector_project, "connectorEnvironment": actor.connector_environment, "projectId": project_id, }, sort_keys=True, ensure_ascii=False, separators=(",", ":"), ).encode("utf-8") return hmac.new( self.cursor_signing_key, b"itworx-intake-extract-cursor-v2\0" + context + b"\0" + payload, hashlib.sha256, ).digest() def _encode_extraction_cursor( self, file_set_digest: str, file_index: int, offset: int, *, actor: ActorContext, project_id: str | None, ) -> str: payload = json.dumps( {"digest": file_set_digest, "index": file_index, "offset": offset}, sort_keys=True, separators=(",", ":"), ).encode("ascii") tag = self._cursor_mac(payload, actor=actor, project_id=project_id) return "v2:" + urlsafe_b64encode(payload + tag).decode("ascii").rstrip("=") def _decode_extraction_cursor( self, cursor: str | None, expected_file_set_digest: str, *, actor: ActorContext, project_id: str | None, ) -> tuple[int, int]: if cursor is None: return 0, 0 if not cursor.startswith("v2:") or len(cursor) > 256: raise ValueError("attachment page cursor is invalid") encoded = cursor[3:] try: raw = b64decode(encoded + "=" * (-len(encoded) % 4), altchars=b"-_", validate=True) payload, tag = raw[:-32], raw[-32:] decoded = json.loads(payload) except (Base64Error, UnicodeDecodeError, json.JSONDecodeError, ValueError): raise ValueError("attachment page cursor is invalid") from None if ( len(raw) < 33 or not hmac.compare_digest( tag, self._cursor_mac(payload, actor=actor, project_id=project_id) ) or not isinstance(decoded, dict) or set(decoded) != {"digest", "index", "offset"} or decoded["digest"] != expected_file_set_digest or isinstance(decoded["index"], bool) or not isinstance(decoded["index"], int) or isinstance(decoded["offset"], bool) or not isinstance(decoded["offset"], int) ): raise ValueError("attachment page cursor is invalid") return decoded["index"], decoded["offset"] @staticmethod def _validate_source_provenance( *, source_file_name: str, source_uri: str, source_ref: str, source_commit: str, source_sha: str, ) -> dict[str, str]: if ( not isinstance(source_file_name, str) or not _SOURCE_FILE_NAME.fullmatch(source_file_name) or source_file_name in {".", ".."} or not isinstance(source_uri, str) or len(source_uri) > 2048 or not isinstance(source_ref, str) or not _GIT_REF.fullmatch(source_ref) or any(part == ".." for part in source_ref.split("/")) or not isinstance(source_commit, str) or not _GIT_SHA.fullmatch(source_commit) or not isinstance(source_sha, str) or not _GIT_BLOB_SHA.fullmatch(source_sha) ): raise ValueError("source provenance is invalid") try: parsed = urlsplit(source_uri) host = parsed.hostname port = parsed.port except ValueError as exc: raise ValueError("source provenance is invalid") from exc if ( parsed.scheme != "itworx-git" or host is None or not _GITEA_SLUG.fullmatch(host) or parsed.username is not None or parsed.password is not None or port is not None or parsed.query or parsed.fragment ): raise ValueError("source provenance is invalid") segments = [unquote(part) for part in parsed.path.split("/") if part] if len(segments) < 2 or any( part in {".", ".."} or "/" in part or "\\" in part or any(ord(character) < 0x20 for character in part) for part in segments ): raise ValueError("source provenance is invalid") if segments[-1] != source_file_name: raise ValueError("source filename does not match its repository URI") return { "fileName": source_file_name, "sourceUri": source_uri, "sourceRef": source_ref, "sourceCommit": source_commit.lower(), "sourceSha": source_sha.lower(), } @staticmethod def _project_context_path(project_id: str) -> str: project_id = IngestionWorkflow._validate_uuid(project_id, "project identifier") return f"/api/v1/projects/{project_id}/context?updates=10&decisions=10&documents=20" @staticmethod def _validate_uuid(value: object, label: str) -> str: if not isinstance(value, str) or not _UUID.fullmatch(value): raise ValueError(f"{label} is invalid") return value.lower() @staticmethod def _validate_manifest(manifest: str) -> str: value = manifest.strip() if isinstance(manifest, str) else "" if not value or len(value) > _MAX_MANIFEST_CHARS: raise ValueError("proposal manifest is invalid") return value @staticmethod def _validate_file_refs(files: Any) -> None: if not isinstance(files, Sequence) or isinstance(files, str | bytes) or not files: raise ValueError("one or more file references are required") for item in files: if not isinstance(item, Mapping): raise ValueError("file reference is invalid") keys = set(item) if not {"download_url", "file_id"}.issubset(keys) or keys - { "download_url", "file_id", "mime_type", "file_name", }: raise ValueError("file reference is invalid") if not isinstance(item.get("download_url"), str) or not isinstance( item.get("file_id"), str ): raise ValueError("file reference is invalid") if any( not isinstance(item[key], str) for key in ("mime_type", "file_name") if key in item ): raise ValueError("file reference is invalid") @staticmethod def _validate_publication(publication: Mapping[str, Any]) -> dict[str, Any]: if not isinstance(publication, Mapping): raise ValueError("proposal publication is invalid") required = {"title", "markdown", "document_type", "document_status", "mutation_kind"} allowed = required | {"parent_page_id", "target_page_id", "expected_target_etag"} if set(publication) - allowed or not required.issubset(publication): raise ValueError("proposal publication is invalid") title = publication.get("title") markdown = publication.get("markdown") doc_type = publication.get("document_type") doc_status = publication.get("document_status") mutation = publication.get("mutation_kind") if not isinstance(title, str) or not title.strip() or len(title) > 300: raise ValueError("proposal title is invalid") if ( not isinstance(markdown, str) or not markdown.strip() or len(markdown) > _MAX_MARKDOWN_CHARS ): raise ValueError("proposal content is invalid") if not isinstance(doc_type, str) or not doc_type.strip() or len(doc_type) > 64: raise ValueError("proposal document type is invalid") if not isinstance(doc_status, str) or not doc_status.strip() or len(doc_status) > 64: raise ValueError("proposal document status is invalid") if mutation not in {"CreatePage", "UpdatePage"}: raise ValueError("proposal mutation kind is invalid") result: dict[str, Any] = { "title": title.strip(), "markdown": markdown, "documentType": doc_type.strip(), "documentStatus": doc_status.strip(), "mutationKind": mutation, } parent_id = publication.get("parent_page_id") target_id = publication.get("target_page_id") target_etag = publication.get("expected_target_etag") if parent_id is not None: result["parentPageId"] = IngestionWorkflow._validate_uuid( parent_id, "parent page identifier" ) if mutation == "CreatePage": if "parent_page_id" not in publication: raise ValueError("create proposals require an explicit parent or root placement") if target_id is not None or target_etag is not None: raise ValueError("create proposals cannot target an existing page") else: if ( target_id is None or not isinstance(target_etag, str) or not _WORKSPACE_ETAG.fullmatch(target_etag) ): raise ValueError("update proposals require a target page and current revision") result["targetPageId"] = IngestionWorkflow._validate_uuid( target_id, "target page identifier" ) result["expectedTargetETag"] = target_etag return result @staticmethod def _proposal_idempotency_key( *, actor: ActorContext, project_id: str | None, files: Sequence[Mapping[str, Any]], manifest: str, publication: Mapping[str, Any], ) -> str: material = json.dumps( { "subject": actor.subject, "projectId": project_id, "fileIds": [item.get("file_id") for item in files], "manifest": manifest, "publication": publication, }, sort_keys=True, ensure_ascii=False, separators=(",", ":"), ) return hashlib.sha256(material.encode("utf-8")).hexdigest() @staticmethod def _text_proposal_idempotency_key( *, actor: ActorContext, project_id: str | None, manifest: str, publication: Mapping[str, Any], source: ProcessedSource, provenance: Mapping[str, Any], ) -> str: material = json.dumps( { "subject": actor.subject, "projectId": project_id, "manifest": manifest, "publication": publication, "sourceSha256": source.sha256, "provenance": provenance, }, sort_keys=True, ensure_ascii=False, separators=(",", ":"), ) return hashlib.sha256(material.encode("utf-8")).hexdigest() @staticmethod def _source_idempotency_key(proposal_key: str, digest: str) -> str: return hashlib.sha256(f"{proposal_key}:{digest}".encode("ascii")).hexdigest() @staticmethod def _response_id(data: Mapping[str, Any]) -> str: value = data.get("id") return IngestionWorkflow._validate_uuid(str(value), "proposal identifier") @staticmethod def _project_citations(data: Mapping[str, Any]) -> dict[str, dict[str, Any]]: raw = data.get("citations") if not isinstance(raw, list): return {} citations: dict[str, dict[str, Any]] = {} for item in raw: if not isinstance(item, Mapping) or str(item.get("kind", "")).casefold() != "page": continue page_id = IngestionWorkflow._string(item.get("id"), max_length=64) if not page_id: continue citation: dict[str, Any] = {"pageId": page_id} revision = item.get("revision") if isinstance(revision, int) and not isinstance(revision, bool) and revision >= 0: citation["revision"] = revision canonical_path = IngestionWorkflow._string(item.get("canonicalPath"), max_length=512) title = IngestionWorkflow._string(item.get("title"), max_length=300) if canonical_path: citation["canonicalPath"] = canonical_path if title: citation["title"] = title citations[page_id] = citation return citations @staticmethod def _project_context_projection( data: Mapping[str, Any], correlation_id: str, actor: ActorContext ) -> dict[str, Any]: raw_project = data.get("project") project: Mapping[str, Any] = raw_project if isinstance(raw_project, Mapping) else {} fields = ( "recentUpdates", "activeMilestones", "decisions", "runbooks", "documents", "citations", ) projection: dict[str, Any] = { "project": { "id": IngestionWorkflow._string(project.get("id"), max_length=64), "name": IngestionWorkflow._string(project.get("name"), max_length=300), "status": IngestionWorkflow._string(project.get("status"), max_length=64), "summary": IngestionWorkflow._string( project.get("summary"), max_length=_MAX_CONTEXT_TEXT ), "currentStateMarkdown": IngestionWorkflow._string( project.get("currentStateMarkdown"), max_length=_MAX_CONTEXT_TEXT ), "updatedAt": IngestionWorkflow._string(project.get("updatedAt"), max_length=64), }, "truncation": data.get("truncation") if isinstance(data.get("truncation"), Mapping) else {}, "correlationId": IngestionWorkflow._correlation(correlation_id, actor), } for field in fields: value = data.get(field) if isinstance(value, list): projection[field] = [ IngestionWorkflow._bounded_record(item) for item in value[:20] if isinstance(item, Mapping) ] else: projection[field] = [] return projection @staticmethod def _bounded_record(item: Mapping[str, Any]) -> dict[str, Any]: allowed_limits = { "id": 64, "pageId": 64, "title": 300, "name": 300, "summary": _MAX_CONTEXT_TEXT, "summaryMarkdown": _MAX_CONTEXT_TEXT, "canonicalPath": 512, "pageTitle": 300, "pageSlug": 200, "pageDocumentType": 64, "documentType": 64, "role": 64, "kind": 64, "status": 64, "state": 64, "category": 64, "revision": 64, "updatedAt": 64, "occurredAt": 64, "targetDate": 32, "completionDate": 32, } record: dict[str, Any] = {} for key, limit in allowed_limits.items(): value = item.get(key) if isinstance(value, str): record[key] = value[:limit] elif isinstance(value, int) and not isinstance(value, bool): record[key] = value return record @staticmethod def _existing_source_digests(data: Mapping[str, Any]) -> set[str]: sources = data.get("sources") if not isinstance(sources, list): return set() result: set[str] = set() for source in sources: if isinstance(source, Mapping): digest = source.get("digestSha256") or source.get("sha256") if isinstance(digest, str) and re.fullmatch(r"[0-9a-f]{64}", digest, re.I): result.add(digest.lower()) return result @staticmethod def _source_projection(source: ProcessedSource) -> dict[str, Any]: return { "fileName": source.file_name[:255], "mimeType": source.mime_type[:128], "byteSize": source.byte_size, "sha256": source.sha256, "scanResult": "Clean", } @staticmethod def _status(value: Any) -> str: if isinstance(value, str) and value in KNOWN_STATUSES: return value return "Unknown" @staticmethod def _status_projection( data: Mapping[str, Any], correlation_id: str, actor: ActorContext ) -> dict[str, Any]: raw_status = data.get("status") status = IngestionWorkflow._status(raw_status) retry_eligible = status == "Failed" and data.get("isRetryEligible") is True raw_citations = data.get("citations") citations = ( [ IngestionWorkflow._bounded_record(citation) for citation in raw_citations[:20] if isinstance(citation, Mapping) ] if isinstance(raw_citations, list) else [] ) source_records = data.get("sources") sources = [ { "fileName": IngestionWorkflow._string(source.get("fileName"), max_length=255), "mimeType": IngestionWorkflow._string(source.get("mimeType"), max_length=128), "byteSize": source.get("byteSize") if isinstance(source.get("byteSize"), int) else None, "digestSha256": IngestionWorkflow._string( source.get("digestSha256"), max_length=64 ), "scanResult": IngestionWorkflow._string(source.get("scanResult"), max_length=32), } for source in (source_records[:20] if isinstance(source_records, list) else []) if isinstance(source, Mapping) ] final_citation: dict[str, Any] | None = None page_id = IngestionWorkflow._string(data.get("publishedPageId"), max_length=64) revision = data.get("publishedPageRevisionNumber") if ( status == "Published" and page_id and isinstance(revision, int) and not isinstance(revision, bool) ): final_citation = {"pageId": page_id, "revisionNumber": revision} result: dict[str, Any] = { "proposalId": IngestionWorkflow._string(data.get("id"), max_length=64), "status": status, "terminal": status in _TERMINAL_STATUSES, "retryEligible": retry_eligible, "retryCount": data.get("retryCount") if isinstance(data.get("retryCount"), int) else 0, "maxRetries": data.get("maxPublishRetries") if isinstance(data.get("maxPublishRetries"), int) else 0, "reviewerFeedback": IngestionWorkflow._string( data.get("reviewerComment"), max_length=_MAX_REVIEWER_FEEDBACK ), "sources": sources, "citations": citations, "finalCitation": final_citation, "correlationId": IngestionWorkflow._correlation( IngestionWorkflow._string(data.get("correlationId"), max_length=128) or correlation_id, actor, ), } if status == "Unknown": result["status"] = "Unknown" result["retryEligible"] = False result["terminal"] = False result["finalCitation"] = None return result @staticmethod def _require_proposal_project(data: Mapping[str, Any], project_id: str | None) -> None: manifest_value = data.get("manifest") try: manifest = json.loads(manifest_value) if isinstance(manifest_value, str) else None except (json.JSONDecodeError, TypeError): manifest = None if not isinstance(manifest, Mapping) or ( ( project_id is None and (manifest.get("workspace") != "knowledge-hub" or "projectId" in manifest) ) or (project_id is not None and manifest.get("projectId") != project_id) ): # Avoid disclosing whether a foreign proposal exists. raise IngestionWorkflowError( "proposal_not_found", "The proposal was not found in the requested project.", action="return", ) @staticmethod def _has_publication(data: Mapping[str, Any]) -> bool: destination = data.get("destination") metadata = data.get("metadata") return ( isinstance(destination, Mapping) and bool(destination.get("title")) and isinstance(metadata, Mapping) ) @staticmethod def _correlation(value: Any, actor: ActorContext) -> str: candidate = value if isinstance(value, str) else actor.correlation_id if _SAFE_CORRELATION.fullmatch(candidate): return candidate return str(actor.correlation_id) @staticmethod def _string(value: Any, *, max_length: int) -> str | None: if not isinstance(value, str): return None return value[:max_length] def safe_error_response(error: Exception, correlation_id: str) -> dict[str, Any]: """Map arbitrary exceptions to stable, content-free caller guidance.""" if _SAFE_CORRELATION.fullmatch(correlation_id or "") is None: correlation_id = "unavailable" status = getattr(error, "status_code", None) code = "service_unavailable" message = "Knowledge Hub is temporarily unavailable. Retry later." action = "retry_later" if isinstance(error, IngestionWorkflowError): code = error.code if _SAFE_CODE.fullmatch(error.code) else "request_failed" message = error.safe_message action = error.action elif isinstance(error, PermissionError): code, message, action = ( "forbidden", "You do not have access to this project or action.", "contact_admin", ) elif isinstance(error, FileIngestionError) and error.code == "scan_unavailable": code, message, action = ( "scanner_unavailable", "File security scanning is temporarily unavailable. Retry later.", "retry_later", ) elif isinstance(error, FileIngestionError) and error.code in { "download_failed", "download_timeout", }: code, message, action = ( "source_unavailable", "The uploaded source could not be retrieved. Reattach the file and retry.", "reattach_file", ) elif isinstance(error, ValueError | TypeError | FileIngestionError): code, message, action = ( "invalid_input", "The request or source files could not be processed.", "fix_input", ) elif isinstance(status, int): if status == 400: code, message, action = ( "invalid_input", "The request could not be processed. Check the proposal details.", "fix_input", ) elif status == 401: code, message, action = ( "unauthenticated", "Reconnect to Knowledge Hub to continue.", "reconnect", ) elif status in {403, 404}: code, message, action = ( "not_available", "The requested project or proposal is not available.", "contact_admin", ) elif status == 409: code, message, action = ( "proposal_conflict", "The proposal changed. Refresh its status before continuing.", "refresh", ) elif status == 412: code, message, action = ( "stale_version", "The proposal changed. Refresh its status before continuing.", "refresh", ) elif status == 422: code, message, action = ( "invalid_input", "The proposal could not be validated. Review its details.", "fix_input", ) elif status == 429: code, message, action = ( "rate_limited", "Knowledge Hub is processing too many requests. Retry later.", "retry_later", ) elif status >= 500: code, message, action = ( "service_unavailable", "Knowledge Hub is temporarily unavailable. Retry later.", "retry_later", ) elif isinstance(error, TimeoutError) or type(error).__name__ in { "TimeoutException", "ReadTimeout", "ConnectTimeout", }: code, message, action = ( "timeout", "The operation timed out. Check status before retrying.", "refresh", ) result: dict[str, Any] = { "ok": False, "error": {"code": code, "message": message, "action": action}, "correlationId": correlation_id, } proposal_id = getattr(error, "proposal_id", None) if isinstance(proposal_id, str) and _UUID.fullmatch(proposal_id): result["proposalId"] = proposal_id.lower() retry_after = getattr(error, "retry_after_seconds", None) if ( code == "rate_limited" and isinstance(retry_after, int | float) and not isinstance(retry_after, bool) ): result["error"]["retryAfterSeconds"] = min(max(int(retry_after), 1), 60) return result