1659 lines
65 KiB
Python
1659 lines
65 KiB
Python
"""Scoped Knowledge Hub lookup and review-only ingestion workflows."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import re
|
|
from base64 import b64decode, urlsafe_b64encode
|
|
from binascii import Error as Base64Error
|
|
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
from typing import Any, Literal
|
|
from urllib.parse import unquote, urlencode, urlsplit
|
|
|
|
from .ingestion_auth import ActorContext, authorize_actor, authorize_workspace_actor
|
|
from .ingestion_files import (
|
|
FileIngestionError,
|
|
FilePolicy,
|
|
ProcessedSource,
|
|
acquire_scan_extract,
|
|
scan_text_source,
|
|
)
|
|
from .ingestion_hub_client import HubClient, SourceFile
|
|
|
|
KNOWN_STATUSES = frozenset(
|
|
{
|
|
"Received",
|
|
"Analyzing",
|
|
"ReadyForReview",
|
|
"ChangesRequested",
|
|
"Approved",
|
|
"Publishing",
|
|
"Published",
|
|
"Rejected",
|
|
"Failed",
|
|
}
|
|
)
|
|
_TERMINAL_STATUSES = frozenset({"Published", "Rejected"})
|
|
_MUTABLE_ANALYSIS_STATUSES = frozenset({"Received", "ChangesRequested"})
|
|
_UUID = re.compile(
|
|
r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$", re.I
|
|
)
|
|
_SAFE_CORRELATION = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
|
_SAFE_CODE = re.compile(r"^[a-z][a-z0-9_]{0,47}$")
|
|
_MAX_ANALYSIS_CHARS = 32_000
|
|
_MAX_MANIFEST_CHARS = 10_000
|
|
_MAX_MARKDOWN_CHARS = 32_000
|
|
_MAX_SOURCE_TEXT_CHARS = 100_000
|
|
_MAX_SEARCH_RESULTS = 20
|
|
_MAX_CONTEXT_TEXT = 4_000
|
|
_MAX_REVIEWER_FEEDBACK = 4_000
|
|
_MAX_EXTRACT_PAGE_CHARS = 12_000
|
|
_MAX_PAGE_READ_CHARS = 20_000
|
|
_MAX_KNOWLEDGE_PAGE_CHARS = 1_000_000
|
|
_MAX_TREE_PAGE_SIZE = 100
|
|
_WORKSPACE_ETAG = re.compile(r'^"[\x21\x23-\x7e]{1,254}"$')
|
|
_GITEA_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$")
|
|
_GIT_REF = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,299}$")
|
|
_GIT_SHA = re.compile(r"^[0-9a-fA-F]{7,64}$")
|
|
_GIT_BLOB_SHA = re.compile(r"^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$")
|
|
_SOURCE_FILE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._ -]{0,127}$")
|
|
|
|
ActorAuthorizer = Callable[..., None]
|
|
FileAcquirer = Callable[..., Awaitable[tuple[ProcessedSource, ...]]]
|
|
TextSourceScanner = Callable[..., Awaitable[ProcessedSource]]
|
|
|
|
|
|
class IngestionWorkflowError(Exception):
|
|
"""A stable workflow failure with no raw upstream or source data."""
|
|
|
|
def __init__(
|
|
self,
|
|
code: str,
|
|
message: str,
|
|
*,
|
|
action: str,
|
|
proposal_id: str | None = None,
|
|
retry_after_seconds: int | None = None,
|
|
) -> None:
|
|
super().__init__(message)
|
|
self.code = code
|
|
self.safe_message = message
|
|
self.action = action
|
|
self.proposal_id = proposal_id
|
|
self.retry_after_seconds = retry_after_seconds
|
|
|
|
|
|
class IngestionWorkflow:
|
|
"""Orchestrate bounded MCP operations over the public Hub API only."""
|
|
|
|
def __init__(
|
|
self,
|
|
client: HubClient,
|
|
*,
|
|
file_policy: FilePolicy | None = None,
|
|
file_acquirer: FileAcquirer = acquire_scan_extract,
|
|
text_source_scanner: TextSourceScanner = scan_text_source,
|
|
workspace_read_enabled: bool = False,
|
|
cursor_signing_key: bytes | None = None,
|
|
actor_authorizer: ActorAuthorizer = authorize_actor,
|
|
) -> None:
|
|
self.client = client
|
|
self.file_policy = file_policy if file_policy is not None else FilePolicy.from_environment()
|
|
self.file_acquirer = file_acquirer
|
|
self.text_source_scanner = text_source_scanner
|
|
if not isinstance(workspace_read_enabled, bool):
|
|
raise ValueError("workspace read configuration is invalid")
|
|
self.workspace_read_enabled = workspace_read_enabled
|
|
if cursor_signing_key is not None and (
|
|
not isinstance(cursor_signing_key, bytes) or len(cursor_signing_key) < 32
|
|
):
|
|
raise ValueError("attachment cursor signing key is invalid")
|
|
self.cursor_signing_key = cursor_signing_key
|
|
self.actor_authorizer = actor_authorizer
|
|
|
|
async def search_knowledge(
|
|
self,
|
|
*,
|
|
project_id: str | None,
|
|
query: str,
|
|
actor: ActorContext,
|
|
limit: int = 10,
|
|
cursor: str | None = None,
|
|
) -> dict[str, Any]:
|
|
project_id = self._resolve_project(actor, "knowledge:read", project_id)
|
|
normalized_query = query.strip() if isinstance(query, str) else ""
|
|
if not 1 <= len(normalized_query) <= 500:
|
|
raise ValueError("search query is invalid")
|
|
if (
|
|
isinstance(limit, bool)
|
|
or not isinstance(limit, int)
|
|
or not 1 <= limit <= _MAX_SEARCH_RESULTS
|
|
):
|
|
raise ValueError("search limit is invalid")
|
|
if cursor is not None and (not isinstance(cursor, str) or len(cursor) > 500):
|
|
raise ValueError("search cursor is invalid")
|
|
|
|
params: dict[str, str] = {"q": normalized_query, "limit": str(limit)}
|
|
if cursor:
|
|
params["cursor"] = cursor
|
|
query_path = f"/api/v1/search?{urlencode(params)}"
|
|
search_response = await self.client.request_json(
|
|
"GET",
|
|
query_path,
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
project_id=project_id,
|
|
)
|
|
allowed_citations: dict[str, dict[str, Any]] | None = None
|
|
if project_id is not None:
|
|
context_response = await self.client.request_json(
|
|
"GET",
|
|
self._project_context_path(project_id),
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
project_id=project_id,
|
|
)
|
|
allowed_citations = self._project_citations(context_response.data)
|
|
raw_items = search_response.data.get("items")
|
|
items = raw_items if isinstance(raw_items, list) else []
|
|
results: list[dict[str, Any]] = []
|
|
for item in items:
|
|
if not isinstance(item, Mapping):
|
|
continue
|
|
page_id = self._string(item.get("id"), max_length=64)
|
|
if not page_id:
|
|
continue
|
|
citation = (
|
|
allowed_citations.get(page_id)
|
|
if allowed_citations is not None
|
|
else {"pageId": page_id}
|
|
)
|
|
if citation is None:
|
|
continue
|
|
results.append(
|
|
{
|
|
"id": page_id,
|
|
"title": self._string(item.get("title"), max_length=300) or "Untitled page",
|
|
"slug": self._string(item.get("slug"), max_length=200),
|
|
"documentType": self._string(item.get("documentType"), max_length=64),
|
|
"documentStatus": self._string(item.get("documentStatus"), max_length=64),
|
|
"snippet": self._string(item.get("snippet"), max_length=1_000),
|
|
"updatedAt": self._string(item.get("updatedAt"), max_length=64),
|
|
"citation": citation,
|
|
}
|
|
)
|
|
if len(results) >= limit:
|
|
break
|
|
|
|
next_cursor = self._string(search_response.data.get("nextCursor"), max_length=500)
|
|
return {
|
|
"results": results,
|
|
"nextCursor": next_cursor,
|
|
"projectId": project_id,
|
|
"correlationId": self._correlation(search_response.correlation_id, actor),
|
|
}
|
|
|
|
async def get_knowledge_context(
|
|
self,
|
|
*,
|
|
project_id: str,
|
|
actor: ActorContext,
|
|
) -> dict[str, Any]:
|
|
project_id = self._validate_uuid(project_id, "project identifier")
|
|
self._authorize(actor, "knowledge:read", project_id)
|
|
response = await self.client.request_json(
|
|
"GET",
|
|
self._project_context_path(project_id),
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
project_id=project_id,
|
|
)
|
|
return self._project_context_projection(response.data, response.correlation_id, actor)
|
|
|
|
async def extract_knowledge_files(
|
|
self,
|
|
*,
|
|
project_id: str | None,
|
|
files: Sequence[Mapping[str, Any]],
|
|
actor: ActorContext,
|
|
page_cursor: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Return one scanned text page from the attachments without creating a proposal."""
|
|
|
|
project_id = self._resolve_project(actor, "ingestion:write", project_id)
|
|
if self.cursor_signing_key is None:
|
|
raise ValueError("attachment cursor signing key is unavailable")
|
|
self._validate_file_refs(files)
|
|
if page_cursor is not None and (not isinstance(page_cursor, str) or len(page_cursor) > 256):
|
|
raise ValueError("attachment page cursor is invalid")
|
|
processed = await self.file_acquirer(files, actor=actor, policy=self.file_policy)
|
|
if not processed or len(processed) != len(files):
|
|
raise FileIngestionError("source_limit")
|
|
if any(not isinstance(source.text, str) or not source.text.strip() for source in processed):
|
|
raise FileIngestionError("extraction_failed")
|
|
requested_ids = [item.get("file_id") for item in files]
|
|
if any(
|
|
source.file_id != expected
|
|
for source, expected in zip(processed, requested_ids, strict=True)
|
|
):
|
|
raise FileIngestionError("invalid_input", input_reason="acquired_file_id_mismatch")
|
|
|
|
file_set_digest = self._extraction_file_set_digest(processed)
|
|
file_index, start_offset = self._decode_extraction_cursor(
|
|
page_cursor, file_set_digest, actor=actor, project_id=project_id
|
|
)
|
|
if not 0 <= file_index < len(processed):
|
|
raise ValueError("attachment page cursor is invalid")
|
|
selected = processed[file_index]
|
|
if not isinstance(selected.text, str) or not 0 <= start_offset < max(len(selected.text), 1):
|
|
raise ValueError("attachment page cursor is invalid")
|
|
end_offset = min(start_offset + _MAX_EXTRACT_PAGE_CHARS, len(selected.text))
|
|
next_index, next_offset = file_index, end_offset
|
|
if end_offset >= len(selected.text):
|
|
next_index, next_offset = file_index + 1, 0
|
|
next_cursor = (
|
|
self._encode_extraction_cursor(
|
|
file_set_digest, next_index, next_offset, actor=actor, project_id=project_id
|
|
)
|
|
if next_index < len(processed)
|
|
else None
|
|
)
|
|
|
|
coverage: list[dict[str, Any]] = []
|
|
for index, source in enumerate(processed):
|
|
total_chars = len(source.text)
|
|
covered_chars = (
|
|
total_chars
|
|
if index < file_index
|
|
else min(end_offset, total_chars)
|
|
if index == file_index
|
|
else 0
|
|
)
|
|
coverage.append(
|
|
{
|
|
"fileId": source.file_id,
|
|
"fileName": source.file_name,
|
|
"sha256": source.sha256,
|
|
"totalChars": total_chars,
|
|
"coveredChars": covered_chars,
|
|
"complete": covered_chars == total_chars,
|
|
}
|
|
)
|
|
page = {
|
|
"fileId": selected.file_id,
|
|
"fileName": selected.file_name,
|
|
"mimeType": selected.mime_type,
|
|
"byteSize": selected.byte_size,
|
|
"sha256": selected.sha256,
|
|
"startOffset": start_offset,
|
|
"endOffset": end_offset,
|
|
"totalChars": len(selected.text),
|
|
"text": selected.text[start_offset:end_offset],
|
|
"complete": end_offset >= len(selected.text),
|
|
}
|
|
return {
|
|
"page": page,
|
|
"fileCoverage": coverage,
|
|
"nextCursor": next_cursor,
|
|
"allFilesComplete": next_cursor is None,
|
|
"proposalCreated": False,
|
|
"correlationId": actor.correlation_id,
|
|
}
|
|
|
|
async def get_workspace_roots(
|
|
self,
|
|
*,
|
|
actor: ActorContext,
|
|
limit: int = 50,
|
|
cursor: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Read one bounded page of roots from the token-scoped Knowledge Hub workspace."""
|
|
|
|
self._authorize_workspace_read(actor)
|
|
self._validate_tree_page(limit, cursor)
|
|
params: dict[str, str] = {"limit": str(limit)}
|
|
if cursor:
|
|
params["cursor"] = cursor
|
|
response = await self.client.request_json(
|
|
"GET",
|
|
f"/api/v1/pages/roots?{urlencode(params)}",
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
)
|
|
return self._workspace_tree_projection(response.data, response.correlation_id, actor)
|
|
|
|
async def get_workspace_children(
|
|
self,
|
|
*,
|
|
parent_page_id: str,
|
|
actor: ActorContext,
|
|
limit: int = 50,
|
|
cursor: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Read one bounded child-page slice beneath an explicitly selected page."""
|
|
|
|
self._authorize_workspace_read(actor)
|
|
parent_page_id = self._validate_uuid(parent_page_id, "parent page identifier")
|
|
self._validate_tree_page(limit, cursor)
|
|
params: dict[str, str] = {"limit": str(limit)}
|
|
if cursor:
|
|
params["cursor"] = cursor
|
|
response = await self.client.request_json(
|
|
"GET",
|
|
f"/api/v1/pages/{parent_page_id}/children?{urlencode(params)}",
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
)
|
|
result = self._workspace_tree_projection(response.data, response.correlation_id, actor)
|
|
result["parentPageId"] = parent_page_id
|
|
return result
|
|
|
|
async def get_workspace_page(
|
|
self,
|
|
*,
|
|
page_id: str,
|
|
actor: ActorContext,
|
|
offset_chars: int = 0,
|
|
max_chars: int = _MAX_PAGE_READ_CHARS,
|
|
expected_etag: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Read a version-checked Markdown slice and its current destination ETag."""
|
|
|
|
self._authorize_workspace_read(actor)
|
|
page_id = self._validate_uuid(page_id, "page identifier")
|
|
if (
|
|
isinstance(offset_chars, bool)
|
|
or not isinstance(offset_chars, int)
|
|
or not 0 <= offset_chars <= _MAX_KNOWLEDGE_PAGE_CHARS
|
|
or isinstance(max_chars, bool)
|
|
or not isinstance(max_chars, int)
|
|
or not 1 <= max_chars <= _MAX_PAGE_READ_CHARS
|
|
):
|
|
raise ValueError("page content range is invalid")
|
|
if offset_chars > 0 and expected_etag is None:
|
|
raise ValueError("page pagination requires the current ETag")
|
|
if expected_etag is not None and (
|
|
not isinstance(expected_etag, str) or not _WORKSPACE_ETAG.fullmatch(expected_etag)
|
|
):
|
|
raise ValueError("page ETag is invalid")
|
|
response = await self.client.request_json(
|
|
"GET",
|
|
f"/api/v1/pages/{page_id}",
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
)
|
|
if expected_etag is not None and response.etag != expected_etag:
|
|
from .ingestion_hub_client import HubPreconditionFailed
|
|
|
|
raise HubPreconditionFailed(response.correlation_id)
|
|
if response.data.get("id") != page_id:
|
|
raise IngestionWorkflowError(
|
|
"page_unavailable",
|
|
"The requested Knowledge Hub page could not be confirmed.",
|
|
action="refresh",
|
|
)
|
|
markdown = response.data.get("markdown")
|
|
if not isinstance(markdown, str) or len(markdown) > _MAX_KNOWLEDGE_PAGE_CHARS:
|
|
raise IngestionWorkflowError(
|
|
"page_too_large",
|
|
"This page exceeds the comparison limit and cannot be used for an update proposal.",
|
|
action="return",
|
|
)
|
|
if offset_chars > len(markdown):
|
|
raise ValueError("page content offset is outside the page")
|
|
end_offset = min(offset_chars + max_chars, len(markdown))
|
|
breadcrumbs = response.data.get("breadcrumb", response.data.get("breadcrumbs"))
|
|
if not isinstance(breadcrumbs, list) or len(breadcrumbs) > 50:
|
|
raise IngestionWorkflowError(
|
|
"page_unavailable",
|
|
"The requested page breadcrumbs could not be confirmed.",
|
|
action="refresh",
|
|
)
|
|
page_breadcrumbs = [
|
|
{
|
|
"id": self._validate_uuid(item.get("id"), "breadcrumb page identifier"),
|
|
"title": self._string(item.get("title"), max_length=300) or "Untitled page",
|
|
}
|
|
for item in breadcrumbs
|
|
if isinstance(item, Mapping)
|
|
]
|
|
if len(page_breadcrumbs) != len(breadcrumbs):
|
|
raise IngestionWorkflowError(
|
|
"page_unavailable",
|
|
"The requested page breadcrumbs could not be confirmed.",
|
|
action="refresh",
|
|
)
|
|
return {
|
|
"pageId": page_id,
|
|
"title": self._string(response.data.get("title"), max_length=300) or "Untitled page",
|
|
"slug": self._string(response.data.get("slug"), max_length=200),
|
|
"documentType": self._string(response.data.get("documentType"), max_length=64),
|
|
"documentStatus": self._string(response.data.get("documentStatus"), max_length=64),
|
|
"parentPageId": self._string(response.data.get("parentPageId"), max_length=64),
|
|
"currentRevisionNumber": response.data.get("currentRevisionNumber"),
|
|
"breadcrumbs": page_breadcrumbs,
|
|
"etag": response.etag,
|
|
"offsetChars": offset_chars,
|
|
"nextOffset": end_offset if end_offset < len(markdown) else None,
|
|
"totalChars": len(markdown),
|
|
"markdown": markdown[offset_chars:end_offset],
|
|
"complete": end_offset >= len(markdown),
|
|
"truncated": end_offset < len(markdown),
|
|
"correlationId": self._correlation(response.correlation_id, actor),
|
|
}
|
|
|
|
async def analyze_knowledge_files(
|
|
self,
|
|
*,
|
|
project_id: str | None,
|
|
files: Sequence[Mapping[str, Any]],
|
|
manifest: str,
|
|
publication: Mapping[str, Any],
|
|
actor: ActorContext,
|
|
) -> dict[str, Any]:
|
|
project_id = self._resolve_project(actor, "ingestion:write", project_id)
|
|
normalized_manifest = self._validate_manifest(manifest)
|
|
normalized_publication = self._validate_publication(publication)
|
|
self._validate_file_refs(files)
|
|
# Reject any file that cannot be fully validated, scanned and extracted
|
|
# before a durable proposal shell exists. The file service returns only
|
|
# bounded bytes after its temporary copies have been securely removed.
|
|
processed = await self.file_acquirer(files, actor=actor, policy=self.file_policy)
|
|
if not processed:
|
|
raise FileIngestionError("source_limit")
|
|
if any(not isinstance(source.text, str) or not source.text.strip() for source in processed):
|
|
raise FileIngestionError("extraction_failed")
|
|
|
|
idempotency_key = self._proposal_idempotency_key(
|
|
actor=actor,
|
|
project_id=project_id,
|
|
files=files,
|
|
manifest=normalized_manifest,
|
|
publication=normalized_publication,
|
|
)
|
|
# The public API's manifest is the only persisted field available for a
|
|
# project id on a machine-created shell. Keep it structured and exclude
|
|
# ChatGPT's short-lived URLs and file bodies entirely.
|
|
manifest_envelope = json.dumps(
|
|
self._proposal_manifest(project_id, normalized_manifest),
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
)
|
|
shell = await self.client.request_json(
|
|
"POST",
|
|
"/api/v1/ingestion-proposals",
|
|
actor=actor,
|
|
json_body={
|
|
"manifest": manifest_envelope,
|
|
"correlationId": actor.correlation_id,
|
|
"publication": normalized_publication,
|
|
},
|
|
etag=None,
|
|
idempotency_key=idempotency_key,
|
|
project_id=project_id,
|
|
)
|
|
proposal_id = self._response_id(shell.data)
|
|
|
|
current = await self.client.request_json(
|
|
"GET",
|
|
f"/api/v1/ingestion-proposals/{proposal_id}",
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
project_id=project_id,
|
|
)
|
|
self._require_proposal_project(current.data, project_id)
|
|
current_status = self._status(current.data.get("status"))
|
|
if current_status == "ReadyForReview" or current_status in {
|
|
"Approved",
|
|
"Publishing",
|
|
"Published",
|
|
"Rejected",
|
|
}:
|
|
return self._status_projection(current.data, current.correlation_id, actor)
|
|
if current_status not in {"Received", "ChangesRequested", "Analyzing"}:
|
|
raise IngestionWorkflowError(
|
|
"proposal_unavailable",
|
|
"This proposal cannot accept additional sources in its current state.",
|
|
action="refresh",
|
|
proposal_id=proposal_id,
|
|
)
|
|
|
|
current_etag = current.etag or shell.etag
|
|
if not current_etag:
|
|
raise IngestionWorkflowError(
|
|
"version_unavailable",
|
|
"The proposal version could not be confirmed. Refresh the proposal and retry.",
|
|
action="refresh",
|
|
proposal_id=proposal_id,
|
|
)
|
|
existing_digests = self._existing_source_digests(current.data)
|
|
uploaded: list[dict[str, Any]] = []
|
|
for source in processed:
|
|
if source.sha256 in existing_digests:
|
|
continue
|
|
upload_response = await self.client.upload_source(
|
|
proposal_id,
|
|
SourceFile(source.file_name, source.mime_type, source.content),
|
|
actor=actor,
|
|
etag=current_etag,
|
|
idempotency_key=self._source_idempotency_key(idempotency_key, source.sha256),
|
|
project_id=project_id,
|
|
)
|
|
current_etag = upload_response.etag or ""
|
|
if not current_etag:
|
|
raise IngestionWorkflowError(
|
|
"version_unavailable",
|
|
(
|
|
"A source was accepted but its current proposal version "
|
|
"could not be confirmed."
|
|
),
|
|
action="refresh",
|
|
proposal_id=proposal_id,
|
|
)
|
|
existing_digests.add(source.sha256)
|
|
uploaded.append(self._source_projection(source))
|
|
|
|
return {
|
|
"proposalId": proposal_id,
|
|
"status": current_status if current_status in KNOWN_STATUSES else "Unknown",
|
|
"sourceCount": len(existing_digests),
|
|
"uploadedSources": uploaded,
|
|
"nextAction": "submit_ingestion_proposal",
|
|
"correlationId": self._correlation(current.correlation_id, actor),
|
|
}
|
|
|
|
async def create_text_source_proposal(
|
|
self,
|
|
*,
|
|
project_id: str | None,
|
|
source_file_name: str,
|
|
source_text: str,
|
|
source_complete: bool,
|
|
source_uri: str,
|
|
source_ref: str,
|
|
source_commit: str,
|
|
source_sha: str,
|
|
manifest: str,
|
|
publication: Mapping[str, Any],
|
|
actor: ActorContext,
|
|
) -> dict[str, Any]:
|
|
"""Create a complete review proposal from text already read through Gitea MCP."""
|
|
|
|
project_id = self._resolve_project(actor, "ingestion:write", project_id)
|
|
if source_complete is not True:
|
|
raise ValueError("text source must be complete before proposal creation")
|
|
provenance = self._validate_source_provenance(
|
|
source_file_name=source_file_name,
|
|
source_uri=source_uri,
|
|
source_ref=source_ref,
|
|
source_commit=source_commit,
|
|
source_sha=source_sha,
|
|
)
|
|
if not isinstance(source_text, str) or not 1 <= len(source_text) <= _MAX_SOURCE_TEXT_CHARS:
|
|
raise ValueError("text source is invalid")
|
|
try:
|
|
source_bytes = source_text.encode("utf-8")
|
|
except UnicodeEncodeError as exc:
|
|
raise ValueError("text source is invalid") from exc
|
|
git_blob = f"blob {len(source_bytes)}\0".encode("ascii") + source_bytes
|
|
actual_sha = (
|
|
hashlib.sha1(git_blob, usedforsecurity=False).hexdigest()
|
|
if len(provenance["sourceSha"]) == 40
|
|
else hashlib.sha256(git_blob).hexdigest()
|
|
)
|
|
if actual_sha != provenance["sourceSha"]:
|
|
raise ValueError("source text does not match the supplied Git blob SHA")
|
|
normalized_manifest = self._validate_manifest(manifest)
|
|
normalized_publication = self._validate_publication(publication)
|
|
source = await self.text_source_scanner(
|
|
source_text,
|
|
provenance["fileName"],
|
|
policy=self.file_policy,
|
|
)
|
|
if not isinstance(source, ProcessedSource):
|
|
raise FileIngestionError("scan_failed")
|
|
|
|
provenance_envelope = {
|
|
**provenance,
|
|
"providedByCaller": True,
|
|
"verified": False,
|
|
"suppliedTextMatchesSha": True,
|
|
}
|
|
manifest_envelope = json.dumps(
|
|
{
|
|
**self._proposal_manifest(project_id, normalized_manifest),
|
|
"sourceProvenance": [provenance_envelope],
|
|
},
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
)
|
|
proposal_key = self._text_proposal_idempotency_key(
|
|
actor=actor,
|
|
project_id=project_id,
|
|
manifest=normalized_manifest,
|
|
publication=normalized_publication,
|
|
source=source,
|
|
provenance=provenance_envelope,
|
|
)
|
|
created = await self.client.request_json(
|
|
"POST",
|
|
"/api/v1/ingestion-proposals",
|
|
actor=actor,
|
|
json_body={
|
|
"manifest": manifest_envelope,
|
|
"correlationId": actor.correlation_id,
|
|
"publication": normalized_publication,
|
|
},
|
|
etag=None,
|
|
idempotency_key=proposal_key,
|
|
project_id=project_id,
|
|
)
|
|
proposal_id = self._response_id(created.data)
|
|
current = await self.client.request_json(
|
|
"GET",
|
|
f"/api/v1/ingestion-proposals/{proposal_id}",
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
project_id=project_id,
|
|
)
|
|
self._require_proposal_project(current.data, project_id)
|
|
status = self._status(current.data.get("status"))
|
|
if status in {"ReadyForReview", "Approved", "Publishing", "Published", "Rejected"}:
|
|
return self._status_projection(current.data, current.correlation_id, actor)
|
|
if status not in {"Received", "ChangesRequested", "Analyzing"}:
|
|
raise IngestionWorkflowError(
|
|
"proposal_unavailable",
|
|
"This proposal cannot accept a text source in its current state.",
|
|
action="refresh",
|
|
proposal_id=proposal_id,
|
|
)
|
|
if not current.etag and not created.etag:
|
|
raise IngestionWorkflowError(
|
|
"version_unavailable",
|
|
"The proposal version could not be confirmed. Refresh the proposal and retry.",
|
|
action="refresh",
|
|
proposal_id=proposal_id,
|
|
)
|
|
current_etag = current.etag or created.etag
|
|
existing_digests = self._existing_source_digests(current.data)
|
|
uploaded: list[dict[str, Any]] = []
|
|
if source.sha256 not in existing_digests:
|
|
upload = await self.client.upload_source(
|
|
proposal_id,
|
|
SourceFile(source.file_name, source.mime_type, source.content),
|
|
actor=actor,
|
|
etag=current_etag,
|
|
idempotency_key=self._source_idempotency_key(proposal_key, source.sha256),
|
|
project_id=project_id,
|
|
)
|
|
if not upload.etag:
|
|
raise IngestionWorkflowError(
|
|
"version_unavailable",
|
|
"A source was accepted but its current proposal version "
|
|
"could not be confirmed.",
|
|
action="refresh",
|
|
proposal_id=proposal_id,
|
|
)
|
|
uploaded.append(self._source_projection(source))
|
|
existing_digests.add(source.sha256)
|
|
|
|
return {
|
|
"proposalId": proposal_id,
|
|
"status": status,
|
|
"sourceCount": len(existing_digests),
|
|
"uploadedSources": uploaded,
|
|
"sourceProvenance": provenance_envelope,
|
|
"nextAction": "submit_ingestion_proposal",
|
|
"correlationId": self._correlation(current.correlation_id, actor),
|
|
}
|
|
|
|
async def submit_ingestion_proposal(
|
|
self,
|
|
*,
|
|
project_id: str | None,
|
|
proposal_id: str,
|
|
analysis: str,
|
|
actor: ActorContext,
|
|
) -> dict[str, Any]:
|
|
project_id = self._resolve_project(actor, "ingestion:write", project_id)
|
|
normalized_analysis = analysis.strip() if isinstance(analysis, str) else ""
|
|
if not 1 <= len(normalized_analysis) <= _MAX_ANALYSIS_CHARS:
|
|
raise ValueError("proposal analysis is invalid")
|
|
proposal_id = self._validate_uuid(proposal_id, "proposal identifier")
|
|
response = await self.client.request_json(
|
|
"GET",
|
|
f"/api/v1/ingestion-proposals/{proposal_id}",
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
project_id=project_id,
|
|
)
|
|
self._require_proposal_project(response.data, project_id)
|
|
status = self._status(response.data.get("status"))
|
|
if status == "ReadyForReview":
|
|
return self._status_projection(response.data, response.correlation_id, actor)
|
|
if status not in _MUTABLE_ANALYSIS_STATUSES:
|
|
raise IngestionWorkflowError(
|
|
"proposal_not_ready",
|
|
"This proposal cannot be submitted for review in its current state.",
|
|
action="refresh",
|
|
proposal_id=proposal_id,
|
|
)
|
|
if not self._has_publication(response.data):
|
|
raise IngestionWorkflowError(
|
|
"proposal_details_required",
|
|
"A complete create or update proposal is required before review.",
|
|
action="fix_input",
|
|
proposal_id=proposal_id,
|
|
)
|
|
if not self._existing_source_digests(response.data):
|
|
raise IngestionWorkflowError(
|
|
"source_required",
|
|
"At least one clean source is required before review.",
|
|
action="fix_input",
|
|
proposal_id=proposal_id,
|
|
)
|
|
if not response.etag:
|
|
raise IngestionWorkflowError(
|
|
"version_unavailable",
|
|
"The proposal version could not be confirmed. Refresh the proposal and retry.",
|
|
action="refresh",
|
|
proposal_id=proposal_id,
|
|
)
|
|
patched = await self.client.request_json(
|
|
"PATCH",
|
|
f"/api/v1/ingestion-proposals/{proposal_id}",
|
|
actor=actor,
|
|
json_body={"analysis": normalized_analysis},
|
|
etag=response.etag,
|
|
idempotency_key=None,
|
|
project_id=project_id,
|
|
)
|
|
if self._status(patched.data.get("status")) != "ReadyForReview":
|
|
raise IngestionWorkflowError(
|
|
"review_state_unconfirmed",
|
|
"Knowledge Hub did not confirm that this proposal is ready for human review.",
|
|
action="refresh",
|
|
proposal_id=proposal_id,
|
|
)
|
|
return self._status_projection(patched.data, patched.correlation_id, actor)
|
|
|
|
async def get_ingestion_status(
|
|
self,
|
|
*,
|
|
project_id: str | None,
|
|
proposal_id: str,
|
|
actor: ActorContext,
|
|
) -> dict[str, Any]:
|
|
project_id = self._resolve_project(actor, "ingestion:write", project_id)
|
|
proposal_id = self._validate_uuid(proposal_id, "proposal identifier")
|
|
response = await self.client.request_json(
|
|
"GET",
|
|
f"/api/v1/ingestion-proposals/{proposal_id}",
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
project_id=project_id,
|
|
)
|
|
self._require_proposal_project(response.data, project_id)
|
|
return self._status_projection(response.data, response.correlation_id, actor)
|
|
|
|
async def retry_failed_proposal(
|
|
self,
|
|
*,
|
|
project_id: str,
|
|
proposal_id: str,
|
|
etag: str,
|
|
idempotency_key: str,
|
|
actor: ActorContext,
|
|
) -> dict[str, Any]:
|
|
"""Retry a failed proposal internally; this operation is never an MCP tool."""
|
|
|
|
project_id = self._validate_uuid(project_id, "project identifier")
|
|
self._authorize(actor, "ingestion:write", project_id)
|
|
proposal_id = self._validate_uuid(proposal_id, "proposal identifier")
|
|
if not isinstance(etag, str) or not etag.strip() or len(etag) > 256:
|
|
raise ValueError("proposal version is invalid")
|
|
if not isinstance(idempotency_key, str) or not re.fullmatch(
|
|
r"[A-Za-z0-9._:-]{8,128}", idempotency_key
|
|
):
|
|
raise ValueError("retry key is invalid")
|
|
|
|
current = await self.client.request_json(
|
|
"GET",
|
|
f"/api/v1/ingestion-proposals/{proposal_id}",
|
|
actor=actor,
|
|
json_body=None,
|
|
etag=None,
|
|
idempotency_key=None,
|
|
project_id=project_id,
|
|
)
|
|
self._require_proposal_project(current.data, project_id)
|
|
retry_count = current.data.get("retryCount")
|
|
max_retries = current.data.get("maxPublishRetries")
|
|
if (
|
|
self._status(current.data.get("status")) != "Failed"
|
|
or current.data.get("isRetryEligible") is not True
|
|
or not isinstance(retry_count, int)
|
|
or isinstance(retry_count, bool)
|
|
or not isinstance(max_retries, int)
|
|
or isinstance(max_retries, bool)
|
|
or retry_count < 0
|
|
or retry_count >= max_retries
|
|
):
|
|
raise ValueError("proposal is not eligible for retry")
|
|
if not current.etag or current.etag != etag:
|
|
raise IngestionWorkflowError(
|
|
"stale_version",
|
|
"The proposal changed. Refresh its status before retrying.",
|
|
action="refresh",
|
|
proposal_id=proposal_id,
|
|
)
|
|
|
|
retried = await self.client.request_json(
|
|
"POST",
|
|
f"/api/v1/ingestion-proposals/{proposal_id}/retry",
|
|
actor=actor,
|
|
json_body={},
|
|
etag=current.etag,
|
|
idempotency_key=idempotency_key,
|
|
project_id=project_id,
|
|
)
|
|
return self._status_projection(retried.data, retried.correlation_id, actor)
|
|
|
|
def _authorize(
|
|
self,
|
|
actor: ActorContext,
|
|
capability: Literal["knowledge:read", "ingestion:write"],
|
|
project_id: str | None,
|
|
) -> None:
|
|
self.actor_authorizer(actor, capability=capability, project=project_id)
|
|
|
|
def _resolve_project(
|
|
self,
|
|
actor: ActorContext,
|
|
capability: Literal["knowledge:read", "ingestion:write"],
|
|
project_id: str | None,
|
|
) -> str | None:
|
|
if project_id is None:
|
|
if not self.workspace_read_enabled:
|
|
raise PermissionError("workspace access is disabled")
|
|
authorize_workspace_actor(actor)
|
|
self._authorize(actor, capability, None)
|
|
return None
|
|
selected = self._validate_uuid(project_id, "project identifier")
|
|
self._authorize(actor, capability, selected)
|
|
return selected
|
|
|
|
@staticmethod
|
|
def _proposal_manifest(project_id: str | None, user_manifest: str) -> dict[str, Any]:
|
|
if project_id is None:
|
|
return {"workspace": "knowledge-hub", "userManifest": user_manifest}
|
|
return {"projectId": project_id, "userManifest": user_manifest}
|
|
|
|
def _authorize_workspace_read(self, actor: ActorContext) -> None:
|
|
if not self.workspace_read_enabled:
|
|
raise PermissionError("workspace page reads are disabled")
|
|
authorize_workspace_actor(actor)
|
|
self._authorize(actor, "knowledge:read", None)
|
|
|
|
@staticmethod
|
|
def _validate_tree_page(limit: int, cursor: str | None) -> None:
|
|
if (
|
|
isinstance(limit, bool)
|
|
or not isinstance(limit, int)
|
|
or not 1 <= limit <= _MAX_TREE_PAGE_SIZE
|
|
):
|
|
raise ValueError("page list limit is invalid")
|
|
if cursor is not None and (not isinstance(cursor, str) or len(cursor) > 500):
|
|
raise ValueError("page list cursor is invalid")
|
|
|
|
@staticmethod
|
|
def _workspace_tree_projection(
|
|
data: Mapping[str, Any], correlation_id: str, actor: ActorContext
|
|
) -> dict[str, Any]:
|
|
raw_items = data.get("items")
|
|
next_cursor = data.get("nextCursor")
|
|
if (
|
|
not isinstance(raw_items, list)
|
|
or len(raw_items) > _MAX_TREE_PAGE_SIZE
|
|
or (
|
|
next_cursor is not None
|
|
and (not isinstance(next_cursor, str) or len(next_cursor) > 500)
|
|
)
|
|
):
|
|
raise IngestionWorkflowError(
|
|
"page_list_unavailable",
|
|
"Knowledge Hub returned an invalid page list.",
|
|
action="refresh",
|
|
)
|
|
items: list[dict[str, Any]] = []
|
|
for item in raw_items:
|
|
if not isinstance(item, Mapping):
|
|
raise IngestionWorkflowError(
|
|
"page_list_unavailable",
|
|
"Knowledge Hub returned an invalid page list.",
|
|
action="refresh",
|
|
)
|
|
page_id = IngestionWorkflow._validate_uuid(item.get("id"), "page identifier")
|
|
parent_id = item.get("parentPageId")
|
|
if parent_id is not None:
|
|
parent_id = IngestionWorkflow._validate_uuid(parent_id, "parent page identifier")
|
|
has_children = item.get("hasChildren")
|
|
if not isinstance(has_children, bool):
|
|
raise IngestionWorkflowError(
|
|
"page_list_unavailable",
|
|
"Knowledge Hub returned an invalid page list.",
|
|
action="refresh",
|
|
)
|
|
items.append(
|
|
{
|
|
"id": page_id,
|
|
"parentPageId": parent_id,
|
|
"title": IngestionWorkflow._string(item.get("title"), max_length=300)
|
|
or "Untitled page",
|
|
"slug": IngestionWorkflow._string(item.get("slug"), max_length=200),
|
|
"documentType": IngestionWorkflow._string(
|
|
item.get("documentType"), max_length=64
|
|
),
|
|
"documentStatus": IngestionWorkflow._string(
|
|
item.get("documentStatus"), max_length=64
|
|
),
|
|
"hasChildren": has_children,
|
|
"updatedAt": IngestionWorkflow._string(item.get("updatedAt"), max_length=64),
|
|
}
|
|
)
|
|
return {
|
|
"items": items,
|
|
"nextCursor": next_cursor,
|
|
"truncated": next_cursor is not None,
|
|
"correlationId": IngestionWorkflow._correlation(correlation_id, actor),
|
|
}
|
|
|
|
@staticmethod
|
|
def _extraction_file_set_digest(processed: Sequence[ProcessedSource]) -> str:
|
|
material = json.dumps(
|
|
[
|
|
{"fileId": source.file_id, "sha256": source.sha256, "fileName": source.file_name}
|
|
for source in processed
|
|
],
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
)
|
|
return hashlib.sha256(material.encode("utf-8")).hexdigest()
|
|
|
|
def _cursor_mac(self, payload: bytes, *, actor: ActorContext, project_id: str | None) -> bytes:
|
|
if self.cursor_signing_key is None:
|
|
raise ValueError("attachment cursor signing key is unavailable")
|
|
context = json.dumps(
|
|
{
|
|
"subject": actor.subject,
|
|
"connectorProject": actor.connector_project,
|
|
"connectorEnvironment": actor.connector_environment,
|
|
"projectId": project_id,
|
|
},
|
|
sort_keys=True,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
return hmac.new(
|
|
self.cursor_signing_key,
|
|
b"itworx-intake-extract-cursor-v2\0" + context + b"\0" + payload,
|
|
hashlib.sha256,
|
|
).digest()
|
|
|
|
def _encode_extraction_cursor(
|
|
self,
|
|
file_set_digest: str,
|
|
file_index: int,
|
|
offset: int,
|
|
*,
|
|
actor: ActorContext,
|
|
project_id: str | None,
|
|
) -> str:
|
|
payload = json.dumps(
|
|
{"digest": file_set_digest, "index": file_index, "offset": offset},
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("ascii")
|
|
tag = self._cursor_mac(payload, actor=actor, project_id=project_id)
|
|
return "v2:" + urlsafe_b64encode(payload + tag).decode("ascii").rstrip("=")
|
|
|
|
def _decode_extraction_cursor(
|
|
self,
|
|
cursor: str | None,
|
|
expected_file_set_digest: str,
|
|
*,
|
|
actor: ActorContext,
|
|
project_id: str | None,
|
|
) -> tuple[int, int]:
|
|
if cursor is None:
|
|
return 0, 0
|
|
if not cursor.startswith("v2:") or len(cursor) > 256:
|
|
raise ValueError("attachment page cursor is invalid")
|
|
encoded = cursor[3:]
|
|
try:
|
|
raw = b64decode(encoded + "=" * (-len(encoded) % 4), altchars=b"-_", validate=True)
|
|
payload, tag = raw[:-32], raw[-32:]
|
|
decoded = json.loads(payload)
|
|
except (Base64Error, UnicodeDecodeError, json.JSONDecodeError, ValueError):
|
|
raise ValueError("attachment page cursor is invalid") from None
|
|
if (
|
|
len(raw) < 33
|
|
or not hmac.compare_digest(
|
|
tag, self._cursor_mac(payload, actor=actor, project_id=project_id)
|
|
)
|
|
or not isinstance(decoded, dict)
|
|
or set(decoded) != {"digest", "index", "offset"}
|
|
or decoded["digest"] != expected_file_set_digest
|
|
or isinstance(decoded["index"], bool)
|
|
or not isinstance(decoded["index"], int)
|
|
or isinstance(decoded["offset"], bool)
|
|
or not isinstance(decoded["offset"], int)
|
|
):
|
|
raise ValueError("attachment page cursor is invalid")
|
|
return decoded["index"], decoded["offset"]
|
|
|
|
@staticmethod
|
|
def _validate_source_provenance(
|
|
*,
|
|
source_file_name: str,
|
|
source_uri: str,
|
|
source_ref: str,
|
|
source_commit: str,
|
|
source_sha: str,
|
|
) -> dict[str, str]:
|
|
if (
|
|
not isinstance(source_file_name, str)
|
|
or not _SOURCE_FILE_NAME.fullmatch(source_file_name)
|
|
or source_file_name in {".", ".."}
|
|
or not isinstance(source_uri, str)
|
|
or len(source_uri) > 2048
|
|
or not isinstance(source_ref, str)
|
|
or not _GIT_REF.fullmatch(source_ref)
|
|
or any(part == ".." for part in source_ref.split("/"))
|
|
or not isinstance(source_commit, str)
|
|
or not _GIT_SHA.fullmatch(source_commit)
|
|
or not isinstance(source_sha, str)
|
|
or not _GIT_BLOB_SHA.fullmatch(source_sha)
|
|
):
|
|
raise ValueError("source provenance is invalid")
|
|
try:
|
|
parsed = urlsplit(source_uri)
|
|
host = parsed.hostname
|
|
port = parsed.port
|
|
except ValueError as exc:
|
|
raise ValueError("source provenance is invalid") from exc
|
|
if (
|
|
parsed.scheme != "itworx-git"
|
|
or host is None
|
|
or not _GITEA_SLUG.fullmatch(host)
|
|
or parsed.username is not None
|
|
or parsed.password is not None
|
|
or port is not None
|
|
or parsed.query
|
|
or parsed.fragment
|
|
):
|
|
raise ValueError("source provenance is invalid")
|
|
segments = [unquote(part) for part in parsed.path.split("/") if part]
|
|
if len(segments) < 2 or any(
|
|
part in {".", ".."}
|
|
or "/" in part
|
|
or "\\" in part
|
|
or any(ord(character) < 0x20 for character in part)
|
|
for part in segments
|
|
):
|
|
raise ValueError("source provenance is invalid")
|
|
if segments[-1] != source_file_name:
|
|
raise ValueError("source filename does not match its repository URI")
|
|
return {
|
|
"fileName": source_file_name,
|
|
"sourceUri": source_uri,
|
|
"sourceRef": source_ref,
|
|
"sourceCommit": source_commit.lower(),
|
|
"sourceSha": source_sha.lower(),
|
|
}
|
|
|
|
@staticmethod
|
|
def _project_context_path(project_id: str) -> str:
|
|
project_id = IngestionWorkflow._validate_uuid(project_id, "project identifier")
|
|
return f"/api/v1/projects/{project_id}/context?updates=10&decisions=10&documents=20"
|
|
|
|
@staticmethod
|
|
def _validate_uuid(value: object, label: str) -> str:
|
|
if not isinstance(value, str) or not _UUID.fullmatch(value):
|
|
raise ValueError(f"{label} is invalid")
|
|
return value.lower()
|
|
|
|
@staticmethod
|
|
def _validate_manifest(manifest: str) -> str:
|
|
value = manifest.strip() if isinstance(manifest, str) else ""
|
|
if not value or len(value) > _MAX_MANIFEST_CHARS:
|
|
raise ValueError("proposal manifest is invalid")
|
|
return value
|
|
|
|
@staticmethod
|
|
def _validate_file_refs(files: Any) -> None:
|
|
if not isinstance(files, Sequence) or isinstance(files, str | bytes) or not files:
|
|
raise ValueError("one or more file references are required")
|
|
for item in files:
|
|
if not isinstance(item, Mapping):
|
|
raise ValueError("file reference is invalid")
|
|
keys = set(item)
|
|
if not {"download_url", "file_id"}.issubset(keys) or keys - {
|
|
"download_url",
|
|
"file_id",
|
|
"mime_type",
|
|
"file_name",
|
|
}:
|
|
raise ValueError("file reference is invalid")
|
|
if not isinstance(item.get("download_url"), str) or not isinstance(
|
|
item.get("file_id"), str
|
|
):
|
|
raise ValueError("file reference is invalid")
|
|
if any(
|
|
not isinstance(item[key], str) for key in ("mime_type", "file_name") if key in item
|
|
):
|
|
raise ValueError("file reference is invalid")
|
|
|
|
@staticmethod
|
|
def _validate_publication(publication: Mapping[str, Any]) -> dict[str, Any]:
|
|
if not isinstance(publication, Mapping):
|
|
raise ValueError("proposal publication is invalid")
|
|
required = {"title", "markdown", "document_type", "document_status", "mutation_kind"}
|
|
allowed = required | {"parent_page_id", "target_page_id", "expected_target_etag"}
|
|
if set(publication) - allowed or not required.issubset(publication):
|
|
raise ValueError("proposal publication is invalid")
|
|
title = publication.get("title")
|
|
markdown = publication.get("markdown")
|
|
doc_type = publication.get("document_type")
|
|
doc_status = publication.get("document_status")
|
|
mutation = publication.get("mutation_kind")
|
|
if not isinstance(title, str) or not title.strip() or len(title) > 300:
|
|
raise ValueError("proposal title is invalid")
|
|
if (
|
|
not isinstance(markdown, str)
|
|
or not markdown.strip()
|
|
or len(markdown) > _MAX_MARKDOWN_CHARS
|
|
):
|
|
raise ValueError("proposal content is invalid")
|
|
if not isinstance(doc_type, str) or not doc_type.strip() or len(doc_type) > 64:
|
|
raise ValueError("proposal document type is invalid")
|
|
if not isinstance(doc_status, str) or not doc_status.strip() or len(doc_status) > 64:
|
|
raise ValueError("proposal document status is invalid")
|
|
if mutation not in {"CreatePage", "UpdatePage"}:
|
|
raise ValueError("proposal mutation kind is invalid")
|
|
result: dict[str, Any] = {
|
|
"title": title.strip(),
|
|
"markdown": markdown,
|
|
"documentType": doc_type.strip(),
|
|
"documentStatus": doc_status.strip(),
|
|
"mutationKind": mutation,
|
|
}
|
|
parent_id = publication.get("parent_page_id")
|
|
target_id = publication.get("target_page_id")
|
|
target_etag = publication.get("expected_target_etag")
|
|
if parent_id is not None:
|
|
result["parentPageId"] = IngestionWorkflow._validate_uuid(
|
|
parent_id, "parent page identifier"
|
|
)
|
|
if mutation == "CreatePage":
|
|
if "parent_page_id" not in publication:
|
|
raise ValueError("create proposals require an explicit parent or root placement")
|
|
if target_id is not None or target_etag is not None:
|
|
raise ValueError("create proposals cannot target an existing page")
|
|
else:
|
|
if (
|
|
target_id is None
|
|
or not isinstance(target_etag, str)
|
|
or not _WORKSPACE_ETAG.fullmatch(target_etag)
|
|
):
|
|
raise ValueError("update proposals require a target page and current revision")
|
|
result["targetPageId"] = IngestionWorkflow._validate_uuid(
|
|
target_id, "target page identifier"
|
|
)
|
|
result["expectedTargetETag"] = target_etag
|
|
return result
|
|
|
|
@staticmethod
|
|
def _proposal_idempotency_key(
|
|
*,
|
|
actor: ActorContext,
|
|
project_id: str | None,
|
|
files: Sequence[Mapping[str, Any]],
|
|
manifest: str,
|
|
publication: Mapping[str, Any],
|
|
) -> str:
|
|
material = json.dumps(
|
|
{
|
|
"subject": actor.subject,
|
|
"projectId": project_id,
|
|
"fileIds": [item.get("file_id") for item in files],
|
|
"manifest": manifest,
|
|
"publication": publication,
|
|
},
|
|
sort_keys=True,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
)
|
|
return hashlib.sha256(material.encode("utf-8")).hexdigest()
|
|
|
|
@staticmethod
|
|
def _text_proposal_idempotency_key(
|
|
*,
|
|
actor: ActorContext,
|
|
project_id: str | None,
|
|
manifest: str,
|
|
publication: Mapping[str, Any],
|
|
source: ProcessedSource,
|
|
provenance: Mapping[str, Any],
|
|
) -> str:
|
|
material = json.dumps(
|
|
{
|
|
"subject": actor.subject,
|
|
"projectId": project_id,
|
|
"manifest": manifest,
|
|
"publication": publication,
|
|
"sourceSha256": source.sha256,
|
|
"provenance": provenance,
|
|
},
|
|
sort_keys=True,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
)
|
|
return hashlib.sha256(material.encode("utf-8")).hexdigest()
|
|
|
|
@staticmethod
|
|
def _source_idempotency_key(proposal_key: str, digest: str) -> str:
|
|
return hashlib.sha256(f"{proposal_key}:{digest}".encode("ascii")).hexdigest()
|
|
|
|
@staticmethod
|
|
def _response_id(data: Mapping[str, Any]) -> str:
|
|
value = data.get("id")
|
|
return IngestionWorkflow._validate_uuid(str(value), "proposal identifier")
|
|
|
|
@staticmethod
|
|
def _project_citations(data: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
|
|
raw = data.get("citations")
|
|
if not isinstance(raw, list):
|
|
return {}
|
|
citations: dict[str, dict[str, Any]] = {}
|
|
for item in raw:
|
|
if not isinstance(item, Mapping) or str(item.get("kind", "")).casefold() != "page":
|
|
continue
|
|
page_id = IngestionWorkflow._string(item.get("id"), max_length=64)
|
|
if not page_id:
|
|
continue
|
|
citation: dict[str, Any] = {"pageId": page_id}
|
|
revision = item.get("revision")
|
|
if isinstance(revision, int) and not isinstance(revision, bool) and revision >= 0:
|
|
citation["revision"] = revision
|
|
canonical_path = IngestionWorkflow._string(item.get("canonicalPath"), max_length=512)
|
|
title = IngestionWorkflow._string(item.get("title"), max_length=300)
|
|
if canonical_path:
|
|
citation["canonicalPath"] = canonical_path
|
|
if title:
|
|
citation["title"] = title
|
|
citations[page_id] = citation
|
|
return citations
|
|
|
|
@staticmethod
|
|
def _project_context_projection(
|
|
data: Mapping[str, Any], correlation_id: str, actor: ActorContext
|
|
) -> dict[str, Any]:
|
|
raw_project = data.get("project")
|
|
project: Mapping[str, Any] = raw_project if isinstance(raw_project, Mapping) else {}
|
|
fields = (
|
|
"recentUpdates",
|
|
"activeMilestones",
|
|
"decisions",
|
|
"runbooks",
|
|
"documents",
|
|
"citations",
|
|
)
|
|
projection: dict[str, Any] = {
|
|
"project": {
|
|
"id": IngestionWorkflow._string(project.get("id"), max_length=64),
|
|
"name": IngestionWorkflow._string(project.get("name"), max_length=300),
|
|
"status": IngestionWorkflow._string(project.get("status"), max_length=64),
|
|
"summary": IngestionWorkflow._string(
|
|
project.get("summary"), max_length=_MAX_CONTEXT_TEXT
|
|
),
|
|
"currentStateMarkdown": IngestionWorkflow._string(
|
|
project.get("currentStateMarkdown"), max_length=_MAX_CONTEXT_TEXT
|
|
),
|
|
"updatedAt": IngestionWorkflow._string(project.get("updatedAt"), max_length=64),
|
|
},
|
|
"truncation": data.get("truncation")
|
|
if isinstance(data.get("truncation"), Mapping)
|
|
else {},
|
|
"correlationId": IngestionWorkflow._correlation(correlation_id, actor),
|
|
}
|
|
for field in fields:
|
|
value = data.get(field)
|
|
if isinstance(value, list):
|
|
projection[field] = [
|
|
IngestionWorkflow._bounded_record(item)
|
|
for item in value[:20]
|
|
if isinstance(item, Mapping)
|
|
]
|
|
else:
|
|
projection[field] = []
|
|
return projection
|
|
|
|
@staticmethod
|
|
def _bounded_record(item: Mapping[str, Any]) -> dict[str, Any]:
|
|
allowed_limits = {
|
|
"id": 64,
|
|
"pageId": 64,
|
|
"title": 300,
|
|
"name": 300,
|
|
"summary": _MAX_CONTEXT_TEXT,
|
|
"summaryMarkdown": _MAX_CONTEXT_TEXT,
|
|
"canonicalPath": 512,
|
|
"pageTitle": 300,
|
|
"pageSlug": 200,
|
|
"pageDocumentType": 64,
|
|
"documentType": 64,
|
|
"role": 64,
|
|
"kind": 64,
|
|
"status": 64,
|
|
"state": 64,
|
|
"category": 64,
|
|
"revision": 64,
|
|
"updatedAt": 64,
|
|
"occurredAt": 64,
|
|
"targetDate": 32,
|
|
"completionDate": 32,
|
|
}
|
|
record: dict[str, Any] = {}
|
|
for key, limit in allowed_limits.items():
|
|
value = item.get(key)
|
|
if isinstance(value, str):
|
|
record[key] = value[:limit]
|
|
elif isinstance(value, int) and not isinstance(value, bool):
|
|
record[key] = value
|
|
return record
|
|
|
|
@staticmethod
|
|
def _existing_source_digests(data: Mapping[str, Any]) -> set[str]:
|
|
sources = data.get("sources")
|
|
if not isinstance(sources, list):
|
|
return set()
|
|
result: set[str] = set()
|
|
for source in sources:
|
|
if isinstance(source, Mapping):
|
|
digest = source.get("digestSha256") or source.get("sha256")
|
|
if isinstance(digest, str) and re.fullmatch(r"[0-9a-f]{64}", digest, re.I):
|
|
result.add(digest.lower())
|
|
return result
|
|
|
|
@staticmethod
|
|
def _source_projection(source: ProcessedSource) -> dict[str, Any]:
|
|
return {
|
|
"fileName": source.file_name[:255],
|
|
"mimeType": source.mime_type[:128],
|
|
"byteSize": source.byte_size,
|
|
"sha256": source.sha256,
|
|
"scanResult": "Clean",
|
|
}
|
|
|
|
@staticmethod
|
|
def _status(value: Any) -> str:
|
|
if isinstance(value, str) and value in KNOWN_STATUSES:
|
|
return value
|
|
return "Unknown"
|
|
|
|
@staticmethod
|
|
def _status_projection(
|
|
data: Mapping[str, Any], correlation_id: str, actor: ActorContext
|
|
) -> dict[str, Any]:
|
|
raw_status = data.get("status")
|
|
status = IngestionWorkflow._status(raw_status)
|
|
retry_eligible = status == "Failed" and data.get("isRetryEligible") is True
|
|
raw_citations = data.get("citations")
|
|
citations = (
|
|
[
|
|
IngestionWorkflow._bounded_record(citation)
|
|
for citation in raw_citations[:20]
|
|
if isinstance(citation, Mapping)
|
|
]
|
|
if isinstance(raw_citations, list)
|
|
else []
|
|
)
|
|
source_records = data.get("sources")
|
|
sources = [
|
|
{
|
|
"fileName": IngestionWorkflow._string(source.get("fileName"), max_length=255),
|
|
"mimeType": IngestionWorkflow._string(source.get("mimeType"), max_length=128),
|
|
"byteSize": source.get("byteSize")
|
|
if isinstance(source.get("byteSize"), int)
|
|
else None,
|
|
"digestSha256": IngestionWorkflow._string(
|
|
source.get("digestSha256"), max_length=64
|
|
),
|
|
"scanResult": IngestionWorkflow._string(source.get("scanResult"), max_length=32),
|
|
}
|
|
for source in (source_records[:20] if isinstance(source_records, list) else [])
|
|
if isinstance(source, Mapping)
|
|
]
|
|
final_citation: dict[str, Any] | None = None
|
|
page_id = IngestionWorkflow._string(data.get("publishedPageId"), max_length=64)
|
|
revision = data.get("publishedPageRevisionNumber")
|
|
if (
|
|
status == "Published"
|
|
and page_id
|
|
and isinstance(revision, int)
|
|
and not isinstance(revision, bool)
|
|
):
|
|
final_citation = {"pageId": page_id, "revisionNumber": revision}
|
|
result: dict[str, Any] = {
|
|
"proposalId": IngestionWorkflow._string(data.get("id"), max_length=64),
|
|
"status": status,
|
|
"terminal": status in _TERMINAL_STATUSES,
|
|
"retryEligible": retry_eligible,
|
|
"retryCount": data.get("retryCount") if isinstance(data.get("retryCount"), int) else 0,
|
|
"maxRetries": data.get("maxPublishRetries")
|
|
if isinstance(data.get("maxPublishRetries"), int)
|
|
else 0,
|
|
"reviewerFeedback": IngestionWorkflow._string(
|
|
data.get("reviewerComment"), max_length=_MAX_REVIEWER_FEEDBACK
|
|
),
|
|
"sources": sources,
|
|
"citations": citations,
|
|
"finalCitation": final_citation,
|
|
"correlationId": IngestionWorkflow._correlation(
|
|
IngestionWorkflow._string(data.get("correlationId"), max_length=128)
|
|
or correlation_id,
|
|
actor,
|
|
),
|
|
}
|
|
if status == "Unknown":
|
|
result["status"] = "Unknown"
|
|
result["retryEligible"] = False
|
|
result["terminal"] = False
|
|
result["finalCitation"] = None
|
|
return result
|
|
|
|
@staticmethod
|
|
def _require_proposal_project(data: Mapping[str, Any], project_id: str | None) -> None:
|
|
manifest_value = data.get("manifest")
|
|
try:
|
|
manifest = json.loads(manifest_value) if isinstance(manifest_value, str) else None
|
|
except (json.JSONDecodeError, TypeError):
|
|
manifest = None
|
|
if not isinstance(manifest, Mapping) or (
|
|
(
|
|
project_id is None
|
|
and (manifest.get("workspace") != "knowledge-hub" or "projectId" in manifest)
|
|
)
|
|
or (project_id is not None and manifest.get("projectId") != project_id)
|
|
):
|
|
# Avoid disclosing whether a foreign proposal exists.
|
|
raise IngestionWorkflowError(
|
|
"proposal_not_found",
|
|
"The proposal was not found in the requested project.",
|
|
action="return",
|
|
)
|
|
|
|
@staticmethod
|
|
def _has_publication(data: Mapping[str, Any]) -> bool:
|
|
destination = data.get("destination")
|
|
metadata = data.get("metadata")
|
|
return (
|
|
isinstance(destination, Mapping)
|
|
and bool(destination.get("title"))
|
|
and isinstance(metadata, Mapping)
|
|
)
|
|
|
|
@staticmethod
|
|
def _correlation(value: Any, actor: ActorContext) -> str:
|
|
candidate = value if isinstance(value, str) else actor.correlation_id
|
|
if _SAFE_CORRELATION.fullmatch(candidate):
|
|
return candidate
|
|
return str(actor.correlation_id)
|
|
|
|
@staticmethod
|
|
def _string(value: Any, *, max_length: int) -> str | None:
|
|
if not isinstance(value, str):
|
|
return None
|
|
return value[:max_length]
|
|
|
|
|
|
def safe_error_response(error: Exception, correlation_id: str) -> dict[str, Any]:
|
|
"""Map arbitrary exceptions to stable, content-free caller guidance."""
|
|
|
|
if _SAFE_CORRELATION.fullmatch(correlation_id or "") is None:
|
|
correlation_id = "unavailable"
|
|
status = getattr(error, "status_code", None)
|
|
code = "service_unavailable"
|
|
message = "Knowledge Hub is temporarily unavailable. Retry later."
|
|
action = "retry_later"
|
|
if isinstance(error, IngestionWorkflowError):
|
|
code = error.code if _SAFE_CODE.fullmatch(error.code) else "request_failed"
|
|
message = error.safe_message
|
|
action = error.action
|
|
elif isinstance(error, PermissionError):
|
|
code, message, action = (
|
|
"forbidden",
|
|
"You do not have access to this project or action.",
|
|
"contact_admin",
|
|
)
|
|
elif isinstance(error, FileIngestionError) and error.code == "scan_unavailable":
|
|
code, message, action = (
|
|
"scanner_unavailable",
|
|
"File security scanning is temporarily unavailable. Retry later.",
|
|
"retry_later",
|
|
)
|
|
elif isinstance(error, FileIngestionError) and error.code in {
|
|
"download_failed",
|
|
"download_timeout",
|
|
}:
|
|
code, message, action = (
|
|
"source_unavailable",
|
|
"The uploaded source could not be retrieved. Reattach the file and retry.",
|
|
"reattach_file",
|
|
)
|
|
elif isinstance(error, ValueError | TypeError | FileIngestionError):
|
|
code, message, action = (
|
|
"invalid_input",
|
|
"The request or source files could not be processed.",
|
|
"fix_input",
|
|
)
|
|
elif isinstance(status, int):
|
|
if status == 400:
|
|
code, message, action = (
|
|
"invalid_input",
|
|
"The request could not be processed. Check the proposal details.",
|
|
"fix_input",
|
|
)
|
|
elif status == 401:
|
|
code, message, action = (
|
|
"unauthenticated",
|
|
"Reconnect to Knowledge Hub to continue.",
|
|
"reconnect",
|
|
)
|
|
elif status in {403, 404}:
|
|
code, message, action = (
|
|
"not_available",
|
|
"The requested project or proposal is not available.",
|
|
"contact_admin",
|
|
)
|
|
elif status == 409:
|
|
code, message, action = (
|
|
"proposal_conflict",
|
|
"The proposal changed. Refresh its status before continuing.",
|
|
"refresh",
|
|
)
|
|
elif status == 412:
|
|
code, message, action = (
|
|
"stale_version",
|
|
"The proposal changed. Refresh its status before continuing.",
|
|
"refresh",
|
|
)
|
|
elif status == 422:
|
|
code, message, action = (
|
|
"invalid_input",
|
|
"The proposal could not be validated. Review its details.",
|
|
"fix_input",
|
|
)
|
|
elif status == 429:
|
|
code, message, action = (
|
|
"rate_limited",
|
|
"Knowledge Hub is processing too many requests. Retry later.",
|
|
"retry_later",
|
|
)
|
|
elif status >= 500:
|
|
code, message, action = (
|
|
"service_unavailable",
|
|
"Knowledge Hub is temporarily unavailable. Retry later.",
|
|
"retry_later",
|
|
)
|
|
elif isinstance(error, TimeoutError) or type(error).__name__ in {
|
|
"TimeoutException",
|
|
"ReadTimeout",
|
|
"ConnectTimeout",
|
|
}:
|
|
code, message, action = (
|
|
"timeout",
|
|
"The operation timed out. Check status before retrying.",
|
|
"refresh",
|
|
)
|
|
|
|
result: dict[str, Any] = {
|
|
"ok": False,
|
|
"error": {"code": code, "message": message, "action": action},
|
|
"correlationId": correlation_id,
|
|
}
|
|
proposal_id = getattr(error, "proposal_id", None)
|
|
if isinstance(proposal_id, str) and _UUID.fullmatch(proposal_id):
|
|
result["proposalId"] = proposal_id.lower()
|
|
retry_after = getattr(error, "retry_after_seconds", None)
|
|
if (
|
|
code == "rate_limited"
|
|
and isinstance(retry_after, int | float)
|
|
and not isinstance(retry_after, bool)
|
|
):
|
|
result["error"]["retryAfterSeconds"] = min(max(int(retry_after), 1), 60)
|
|
return result
|