Files
ITWorx_MCP_Hub-Public/tests/test_ingestion_hub_client.py
T

677 lines
25 KiB
Python

from __future__ import annotations
import asyncio
import json
import tempfile
import unittest
from pathlib import Path
from uuid import uuid4
import httpx
from services.runtime.ingestion_auth import ActorContext
from services.runtime.ingestion_hub_client import (
HubClient,
HubConflictError,
HubPreconditionFailed,
HubRateLimitedError,
HubResponseTooLarge,
SourceFile,
)
class IngestionHubClientTests(unittest.TestCase):
def setUp(self) -> None:
self.project_id = str(uuid4())
self.proposal_id = str(uuid4())
self.actor = ActorContext(
subject="external-actor-42",
projects=frozenset({self.project_id}),
scopes=frozenset({"knowledge:read", "ingestion:write"}),
correlation_id="trace-2026-01",
)
def client(self, handler, **kwargs) -> HubClient:
return HubClient(
"https://knowledge.example.test",
"hub-service-token-secret",
transport=httpx.MockTransport(handler),
**kwargs,
)
def test_create_proposal_uses_service_token_idempotency_and_trace_headers(self) -> None:
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(
201,
json={"id": self.proposal_id, "status": "Received"},
headers={"ETag": '"rev-1"'},
request=request,
)
client = self.client(handler)
response = asyncio.run(
client.request_json(
"POST",
"/api/v1/ingestion-proposals",
actor=self.actor,
json_body={
"manifest": json.dumps({"projectId": self.project_id}),
"correlationId": self.actor.correlation_id,
},
etag=None,
idempotency_key="create-42",
project_id=self.project_id,
)
)
self.assertEqual(len(requests), 1)
request = requests[0]
self.assertEqual(request.headers["authorization"], "Bearer hub-service-token-secret")
self.assertEqual(request.headers["Idempotency-Key"], "create-42")
self.assertEqual(request.headers["X-Correlation-ID"], "trace-2026-01")
self.assertNotIn("external-actor-42", request.headers.get("authorization", ""))
self.assertEqual(response.id, self.proposal_id)
self.assertEqual(response.etag, '"rev-1"')
def test_publication_markdown_utf8_boundary_matches_typed_body_limit(self) -> None:
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(
201,
json={"id": self.proposal_id},
headers={"ETag": '"rev-1"'},
request=request,
)
client = self.client(handler)
manifest = json.dumps(
{"projectId": self.project_id, "userManifest": "é" * 10_000}, ensure_ascii=False
)
publication = {
"title": "Course",
"markdown": "🙂" * 32_000,
"documentType": "Course",
"documentStatus": "Draft",
"mutationKind": "CreatePage",
}
body = {"manifest": manifest, "publication": publication}
asyncio.run(
client.request_json(
"POST",
"/api/v1/ingestion-proposals",
actor=self.actor,
json_body=body,
etag=None,
idempotency_key="max-publication",
project_id=self.project_id,
)
)
self.assertEqual(len(requests), 1)
self.assertLessEqual(len(requests[0].content), 256 * 1024)
publication["markdown"] += "🙂"
with self.assertRaises(ValueError):
asyncio.run(
client.request_json(
"POST",
"/api/v1/ingestion-proposals",
actor=self.actor,
json_body=body,
etag=None,
idempotency_key="oversized-publication",
project_id=self.project_id,
)
)
self.assertEqual(len(requests), 1)
def test_source_upload_uses_multipart_current_etag_and_idempotency(self) -> None:
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(
200,
json={"id": self.proposal_id, "status": "Analyzing"},
headers={"ETag": '"rev-2"'},
request=request,
)
client = self.client(handler)
response = asyncio.run(
client.upload_source(
self.proposal_id,
SourceFile("notes.txt", "text/plain", b"safe source bytes"),
actor=self.actor,
etag='"rev-1"',
idempotency_key="upload-42",
project_id=self.project_id,
)
)
request = requests[0]
body = request.read().decode("latin-1")
self.assertIn('name="file"; filename="notes.txt"', body)
self.assertIn("safe source bytes", body)
self.assertEqual(request.headers["If-Match"], '"rev-1"')
self.assertEqual(request.headers["Idempotency-Key"], "upload-42")
self.assertEqual(request.headers["X-Correlation-ID"], "trace-2026-01")
self.assertEqual(response.etag, '"rev-2"')
def test_patch_and_get_allow_only_own_canonical_proposal_routes(self) -> None:
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(
200,
json={"id": self.proposal_id, "status": "ReadyForReview"},
headers={"ETag": '"rev-3"'},
request=request,
)
client = self.client(handler)
patched = asyncio.run(
client.request_json(
"PATCH",
f"/api/v1/ingestion-proposals/{self.proposal_id}",
actor=self.actor,
json_body={"analysis": "review-ready analysis"},
etag='"rev-2"',
idempotency_key=None,
project_id=self.project_id,
)
)
fetched = asyncio.run(
client.request_json(
"GET",
f"/api/v1/ingestion-proposals/{self.proposal_id}",
actor=self.actor,
json_body=None,
etag=None,
idempotency_key=None,
project_id=self.project_id,
)
)
self.assertEqual([request.method for request in requests], ["PATCH", "GET"])
self.assertEqual(requests[0].headers["If-Match"], '"rev-2"')
self.assertEqual(json.loads(requests[0].content), {"analysis": "review-ready analysis"})
self.assertEqual(patched.status_code, 200)
self.assertEqual(fetched.data["status"], "ReadyForReview")
def test_project_denial_and_unregistered_route_happen_before_http(self) -> None:
calls: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(request)
return httpx.Response(200, json={}, request=request)
client = self.client(handler)
with self.assertRaises(PermissionError):
asyncio.run(
client.request_json(
"GET",
f"/api/v1/ingestion-proposals/{self.proposal_id}",
actor=self.actor,
json_body=None,
etag=None,
idempotency_key=None,
project_id=str(uuid4()),
)
)
for method, path in (
("GET", "https://attacker.example/collect"),
("GET", "/api/v1/../admin"),
("POST", f"/api/v1/ingestion-proposals/{self.proposal_id}/submit"),
("POST", f"/api/v1/ingestion-proposals/{self.proposal_id}/publish"),
("GET", "/api/v1/ingestion-proposals"),
):
with self.subTest(method=method, path=path), self.assertRaises(ValueError):
asyncio.run(
client.request_json(
method,
path,
actor=self.actor,
json_body=None,
etag=None,
idempotency_key=None,
project_id=self.project_id,
)
)
self.assertEqual(calls, [])
def test_retry_uses_current_version_and_a_new_idempotency_key(self) -> None:
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(
200,
json={"id": self.proposal_id, "status": "Received"},
headers={"ETag": '"rev-4"'},
request=request,
)
client = self.client(handler)
asyncio.run(
client.request_json(
"POST",
f"/api/v1/ingestion-proposals/{self.proposal_id}/retry",
actor=self.actor,
json_body=None,
etag='"rev-3"',
idempotency_key="retry-operation-42",
project_id=self.project_id,
)
)
self.assertEqual(requests[0].headers["If-Match"], '"rev-3"')
self.assertEqual(requests[0].headers["Idempotency-Key"], "retry-operation-42")
with self.assertRaises(ValueError):
asyncio.run(
client.request_json(
"POST",
f"/api/v1/ingestion-proposals/{self.proposal_id}/retry",
actor=self.actor,
json_body=None,
etag=None,
idempotency_key="retry-operation-43",
project_id=self.project_id,
)
)
self.assertEqual(len(requests), 1)
def test_project_context_and_search_are_bounded_reads(self) -> None:
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(200, json={"items": [], "nextCursor": None}, request=request)
client = self.client(handler)
asyncio.run(
client.request_json(
"GET",
f"/api/v1/projects/{self.project_id}/context?updates=3&decisions=2",
actor=self.actor,
json_body=None,
etag=None,
idempotency_key=None,
project_id=self.project_id,
)
)
asyncio.run(
client.request_json(
"GET",
"/api/v1/search?q=release%20plan&limit=5",
actor=self.actor,
json_body=None,
etag=None,
idempotency_key=None,
project_id=self.project_id,
)
)
self.assertEqual(
[request.url.path for request in requests],
[
f"/api/v1/projects/{self.project_id}/context",
"/api/v1/search",
],
)
def test_workspace_page_reads_require_trusted_identity_and_return_etag(self) -> None:
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
if request.url.path == "/api/v1/pages/roots":
return httpx.Response(
200,
json={"items": [], "nextCursor": "next"},
request=request,
)
return httpx.Response(
200,
json={"id": self.proposal_id, "markdown": "# Current"},
headers={"ETag": '"page-v4"'},
request=request,
)
client = self.client(handler, workspace_enabled=True)
trusted = ActorContext(
subject="external-actor-42",
projects=frozenset({self.project_id}),
scopes=frozenset({"knowledge:read", "ingestion:write"}),
correlation_id="trace-2026-01",
connector_project="knowledge-hub",
connector_environment="prod",
)
roots = asyncio.run(
client.request_json(
"GET",
"/api/v1/pages/roots?cursor=cursor-1&limit=5",
actor=trusted,
json_body=None,
etag=None,
idempotency_key=None,
)
)
children = asyncio.run(
client.request_json(
"GET",
f"/api/v1/pages/{self.proposal_id}/children?limit=5",
actor=trusted,
json_body=None,
etag=None,
idempotency_key=None,
)
)
page = asyncio.run(
client.request_json(
"GET",
f"/api/v1/pages/{self.proposal_id}",
actor=trusted,
json_body=None,
etag=None,
idempotency_key=None,
)
)
self.assertEqual(roots.data["nextCursor"], "next")
self.assertEqual(children.status_code, 200)
self.assertEqual(
str(requests[0].url),
"https://knowledge.example.test/api/v1/pages/roots?cursor=cursor-1&limit=5",
)
self.assertEqual(
str(requests[1].url),
f"https://knowledge.example.test/api/v1/pages/{self.proposal_id}/children?limit=5",
)
self.assertEqual(page.etag, '"page-v4"')
with self.assertRaises(PermissionError):
asyncio.run(
client.request_json(
"GET",
"/api/v1/pages/roots",
actor=self.actor,
json_body=None,
etag=None,
idempotency_key=None,
)
)
with self.assertRaises(ValueError):
asyncio.run(
client.request_json(
"GET",
"/api/v1/pages/roots?limit=101",
actor=trusted,
json_body=None,
etag=None,
idempotency_key=None,
)
)
for invalid_query in (
"cursor=",
"cursor=%0A",
f"cursor={'x' * 501}",
"unexpected=value",
):
with self.subTest(query=invalid_query), self.assertRaises(ValueError):
asyncio.run(
client.request_json(
"GET",
f"/api/v1/pages/roots?{invalid_query}",
actor=trusted,
json_body=None,
etag=None,
idempotency_key=None,
)
)
self.assertEqual(len(requests), 3)
def test_projectless_proposal_routes_require_opt_in_and_exact_workspace_identity(self) -> None:
requests: list[httpx.Request] = []
manifest = json.dumps({"workspace": "knowledge-hub", "userManifest": "Course review"})
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
if request.method == "POST" and request.url.path == "/api/v1/ingestion-proposals":
return httpx.Response(
201, json={"id": self.proposal_id}, headers={"ETag": '"v1"'}, request=request
)
if request.method == "GET" and request.url.path == "/api/v1/search":
return httpx.Response(200, json={"items": []}, request=request)
return httpx.Response(
200,
json={"id": self.proposal_id, "manifest": manifest, "status": "Received"},
headers={"ETag": '"v2"'},
request=request,
)
trusted = ActorContext(
subject="trusted-client",
projects=frozenset({self.project_id}),
scopes=frozenset({"knowledge:read", "ingestion:write"}),
correlation_id="trace-workspace",
connector_project="knowledge-hub",
connector_environment="prod",
)
disabled = self.client(handler)
with self.assertRaises(PermissionError):
asyncio.run(
disabled.request_json(
"GET",
"/api/v1/search?q=course&limit=2",
actor=trusted,
json_body=None,
etag=None,
idempotency_key=None,
)
)
self.assertEqual(requests, [])
enabled = self.client(handler, workspace_enabled=True)
with self.assertRaises(PermissionError):
asyncio.run(
enabled.request_json(
"GET",
"/api/v1/search?q=course&limit=2",
actor=self.actor,
json_body=None,
etag=None,
idempotency_key=None,
)
)
self.assertEqual(requests, [])
asyncio.run(
enabled.request_json(
"GET",
"/api/v1/search?q=course&limit=2",
actor=trusted,
json_body=None,
etag=None,
idempotency_key=None,
)
)
asyncio.run(
enabled.request_json(
"POST",
"/api/v1/ingestion-proposals",
actor=trusted,
json_body={"manifest": manifest, "correlationId": trusted.correlation_id},
etag=None,
idempotency_key="workspace-proposal",
)
)
asyncio.run(
enabled.request_json(
"GET",
f"/api/v1/ingestion-proposals/{self.proposal_id}",
actor=trusted,
json_body=None,
etag=None,
idempotency_key=None,
)
)
asyncio.run(
enabled.request_json(
"PATCH",
f"/api/v1/ingestion-proposals/{self.proposal_id}",
actor=trusted,
json_body={"analysis": "Ready for human review"},
etag='"v2"',
idempotency_key=None,
)
)
asyncio.run(
enabled.upload_source(
self.proposal_id,
SourceFile("guide.md", "text/plain", b"# Guide"),
actor=trusted,
etag='"v2"',
idempotency_key="workspace-source",
)
)
self.assertEqual(
[request.method for request in requests], ["GET", "POST", "GET", "PATCH", "POST"]
)
self.assertFalse(
any("projectId" in request.content.decode(errors="ignore") for request in requests)
)
def test_workspace_page_read_requires_a_response_etag_and_exact_route(self) -> None:
calls: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(request)
return httpx.Response(200, json={"id": self.proposal_id}, request=request)
client = self.client(handler, workspace_enabled=True)
trusted = ActorContext(
subject="external-actor-42",
projects=frozenset({self.project_id}),
scopes=frozenset({"knowledge:read"}),
correlation_id="trace-2026-01",
connector_project="knowledge-hub",
connector_environment="prod",
)
with self.assertRaises(Exception) as caught:
asyncio.run(
client.request_json(
"GET",
f"/api/v1/pages/{self.proposal_id}",
actor=trusted,
json_body=None,
etag=None,
idempotency_key=None,
)
)
self.assertEqual(getattr(caught.exception, "code", None), "hub_missing_etag")
with self.assertRaises(ValueError):
asyncio.run(
client.request_json(
"GET",
f"/api/v1/pages/{self.proposal_id}/revisions",
actor=trusted,
json_body=None,
etag=None,
idempotency_key=None,
)
)
self.assertEqual(len(calls), 1)
def test_typed_conflict_and_precondition_errors_are_redacted(self) -> None:
def handler_for(status: int):
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(
status,
json={"detail": "secret response body must not escape"},
headers={"Retry-After": "15"},
request=request,
)
return handler
for status, error_type in ((409, HubConflictError), (412, HubPreconditionFailed)):
client = self.client(handler_for(status))
with self.subTest(status=status), self.assertRaises(error_type) as caught:
asyncio.run(
client.request_json(
"PATCH",
f"/api/v1/ingestion-proposals/{self.proposal_id}",
actor=self.actor,
json_body={"analysis": "safe"},
etag='"rev-1"',
idempotency_key=None,
project_id=self.project_id,
)
)
self.assertEqual(caught.exception.status_code, status)
self.assertEqual(caught.exception.correlation_id, "trace-2026-01")
self.assertNotIn("secret response body", str(caught.exception))
self.assertNotIn("hub-service-token-secret", str(caught.exception))
def test_oversized_response_is_rejected_before_json_parsing(self) -> None:
client = self.client(
lambda request: httpx.Response(200, content=b"x" * 300, request=request),
max_response_bytes=128,
)
with self.assertRaises(HubResponseTooLarge):
asyncio.run(
client.request_json(
"GET",
"/api/v1/search?q=test&limit=2",
actor=self.actor,
json_body=None,
etag=None,
idempotency_key=None,
project_id=self.project_id,
)
)
def test_rate_limit_delay_is_bounded_and_safe(self) -> None:
client = self.client(
lambda request: httpx.Response(
429,
json={"detail": "do not expose backend message"},
headers={"Retry-After": "3600"},
request=request,
)
)
with self.assertRaises(HubRateLimitedError) as caught:
asyncio.run(
client.request_json(
"GET",
"/api/v1/search?q=test&limit=2",
actor=self.actor,
json_body=None,
etag=None,
idempotency_key=None,
project_id=self.project_id,
)
)
self.assertLessEqual(caught.exception.retry_after_seconds, 60)
self.assertNotIn("backend message", str(caught.exception))
def test_non_https_endpoint_is_rejected_and_environment_loads_secret_internally(self) -> None:
with self.assertRaises(ValueError):
HubClient("http://knowledge.example.test", "token")
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
token_path = root / "ingestion-token"
token_path.write_text("service-secret\n", encoding="utf-8")
token_path.chmod(0o600)
client = HubClient.from_environment(
environ={
"KNOWLEDGE_HUB_ENDPOINT": "https://knowledge.example.test",
"KNOWLEDGE_HUB_INGESTION_TOKEN_FILE": str(token_path),
},
secret_root=root,
transport=httpx.MockTransport(lambda request: httpx.Response(200, json={})),
)
self.assertNotIn("service-secret", repr(client))