Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e
This commit is contained in:
Public Source Publisher committed
commit
7ca726caf2
659 files changed
+98171
No files matched your search
@@ -0,0 +1,389 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import yaml
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from ragcore.api.v1.documents.dependencies import (
|
||||
get_documents_read_principal,
|
||||
get_documents_sync_principal,
|
||||
)
|
||||
from ragcore.application.authorization import AuthorizationService
|
||||
from ragcore.application.control.api import ControlPlaneApplication
|
||||
from ragcore.application.documents import (
|
||||
DocumentRecord,
|
||||
DocumentVersionRecord,
|
||||
DocumentVersionState,
|
||||
)
|
||||
from ragcore.domain.authorization import Principal, PrincipalType
|
||||
from ragcore.domain.control import Role
|
||||
from ragcore.main import create_app
|
||||
|
||||
|
||||
class FakeDocumentRepository:
|
||||
def __init__(self) -> None:
|
||||
self.documents: dict[UUID, DocumentRecord] = {}
|
||||
self.versions: dict[UUID, list[DocumentVersionRecord]] = {}
|
||||
self.retired: list[UUID] = []
|
||||
|
||||
def seed(self, document: DocumentRecord, *versions: DocumentVersionRecord) -> None:
|
||||
self.documents[document.id] = document
|
||||
self.versions[document.id] = list(versions)
|
||||
|
||||
async def get_document(self, document_id: UUID) -> DocumentRecord | None:
|
||||
return self.documents.get(document_id)
|
||||
|
||||
async def get_document_by_source(
|
||||
self, workspace_id: UUID, source_key: str, external_id: str
|
||||
) -> DocumentRecord | None:
|
||||
return next(
|
||||
(
|
||||
document
|
||||
for document in self.documents.values()
|
||||
if document.workspace_id == workspace_id
|
||||
and document.source_key == source_key
|
||||
and document.external_id == external_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
async def list_documents_by_source(
|
||||
self, workspace_id: UUID, source_id: UUID
|
||||
) -> tuple[DocumentRecord, ...]:
|
||||
return tuple(
|
||||
document
|
||||
for document in self.documents.values()
|
||||
if document.workspace_id == workspace_id and document.source_id == source_id
|
||||
)
|
||||
|
||||
async def list_versions(self, document_id: UUID) -> tuple[DocumentVersionRecord, ...]:
|
||||
return tuple(self.versions.get(document_id, ()))
|
||||
|
||||
async def list_searchable_versions(
|
||||
self, document_id: UUID
|
||||
) -> tuple[DocumentVersionRecord, ...]:
|
||||
document = self.documents[document_id]
|
||||
if document.source_deleted:
|
||||
return ()
|
||||
return tuple(
|
||||
version
|
||||
for version in self.versions.get(document_id, ())
|
||||
if version.state is DocumentVersionState.PUBLISHED
|
||||
)
|
||||
|
||||
async def mark_source_deleted(self, document_id: UUID) -> None:
|
||||
self.retired.append(document_id)
|
||||
document = self.documents[document_id]
|
||||
self.documents[document_id] = DocumentRecord(
|
||||
id=document.id,
|
||||
workspace_id=document.workspace_id,
|
||||
space_id=document.space_id,
|
||||
source_key=document.source_key,
|
||||
external_id=document.external_id,
|
||||
title=document.title,
|
||||
source_deleted=True,
|
||||
)
|
||||
|
||||
|
||||
def _document(workspace_id: UUID, *, source_key: str | None = None) -> DocumentRecord:
|
||||
return DocumentRecord(
|
||||
id=uuid4(),
|
||||
workspace_id=workspace_id,
|
||||
space_id=uuid4(),
|
||||
source_key=source_key or str(uuid4()),
|
||||
external_id="wiki-page-1",
|
||||
title="Wiki page",
|
||||
)
|
||||
|
||||
|
||||
def _version(document_id: UUID, state: DocumentVersionState) -> DocumentVersionRecord:
|
||||
return DocumentVersionRecord(
|
||||
id=uuid4(),
|
||||
document_id=document_id,
|
||||
source_fingerprint="a" * 64,
|
||||
content_hash="b" * 64,
|
||||
parser_version="text-v1",
|
||||
chunk_profile_version="prose-v1",
|
||||
state=state,
|
||||
)
|
||||
|
||||
|
||||
def _fixture():
|
||||
workspace_id = uuid4()
|
||||
principal = Principal(
|
||||
uuid4(), workspace_id, PrincipalType.SERVICE_ACCOUNT, frozenset({Role.EDITOR})
|
||||
)
|
||||
repository = FakeDocumentRepository()
|
||||
service = ControlPlaneApplication(
|
||||
repository=None, # type: ignore[arg-type]
|
||||
authorization=AuthorizationService(),
|
||||
document_repository=repository, # type: ignore[arg-type]
|
||||
)
|
||||
app = create_app()
|
||||
from ragcore.api.v1.control.dependencies import get_control_application
|
||||
|
||||
app.dependency_overrides[get_control_application] = lambda: service
|
||||
app.dependency_overrides[get_documents_read_principal] = lambda: principal
|
||||
app.dependency_overrides[get_documents_sync_principal] = lambda: principal
|
||||
return app, repository, principal
|
||||
|
||||
|
||||
def test_openapi_and_runtime_expose_same_document_and_job_operations() -> None:
|
||||
app, _, _ = _fixture()
|
||||
document = yaml.safe_load(
|
||||
(Path(__file__).parents[4] / "docs" / "contracts" / "openapi.yaml").read_text()
|
||||
)
|
||||
for path in (
|
||||
"/v1/documents",
|
||||
"/v1/documents/{document_id}",
|
||||
"/v1/documents/{document_id}/versions",
|
||||
"/v1/documents/{document_id}/retire",
|
||||
):
|
||||
assert path in document["paths"], f"{path} missing from openapi.yaml"
|
||||
documented_methods = {method.lower() for method in document["paths"][path]}
|
||||
runtime_methods = {method.lower() for method in app.openapi()["paths"][path]}
|
||||
assert runtime_methods == documented_methods, path
|
||||
|
||||
|
||||
def test_list_documents_requires_a_source_id() -> None:
|
||||
"""source_id alone lists the identity's documents (see
|
||||
test_list_documents_by_source_returns_every_document_of_the_identity);
|
||||
no filter at all is still rejected, and an unknown source is an empty page."""
|
||||
app, _, _ = _fixture()
|
||||
response = TestClient(app).get("/v1/documents")
|
||||
assert response.status_code == 400
|
||||
assert response.json()["code"] == "MISSING_FILTER"
|
||||
unknown = TestClient(app).get(f"/v1/documents?source_id={uuid4()}")
|
||||
assert unknown.status_code == 200
|
||||
assert unknown.json() == {"items": [], "next_cursor": None}
|
||||
|
||||
|
||||
def test_list_documents_empty_when_no_match() -> None:
|
||||
app, _, _ = _fixture()
|
||||
response = TestClient(app).get(f"/v1/documents?source_id={uuid4()}&external_id=missing")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"items": [], "next_cursor": None}
|
||||
|
||||
|
||||
def test_list_documents_returns_the_active_content_sha256() -> None:
|
||||
app, repository, principal = _fixture()
|
||||
source_id = uuid4()
|
||||
document = _document(principal.workspace_id, source_key=str(source_id))
|
||||
published = _version(document.id, DocumentVersionState.PUBLISHED)
|
||||
repository.seed(document, published)
|
||||
|
||||
response = TestClient(app).get(
|
||||
f"/v1/documents?source_id={source_id}&external_id={document.external_id}"
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
[item] = response.json()["items"]
|
||||
assert item["id"] == str(document.id)
|
||||
assert item["source_id"] == str(source_id)
|
||||
assert item["status"] == "active"
|
||||
assert item["active_version"]["content_sha256"] == published.content_hash
|
||||
|
||||
|
||||
def test_list_documents_cross_workspace_is_empty_not_leaked() -> None:
|
||||
app, repository, _ = _fixture()
|
||||
source_id = uuid4()
|
||||
document = _document(uuid4(), source_key=str(source_id))
|
||||
repository.seed(document)
|
||||
|
||||
response = TestClient(app).get(
|
||||
f"/v1/documents?source_id={source_id}&external_id={document.external_id}"
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["items"] == []
|
||||
|
||||
|
||||
def test_get_document_by_id_not_found_is_problem_details() -> None:
|
||||
app, _, _ = _fixture()
|
||||
response = TestClient(app).get(f"/v1/documents/{uuid4()}")
|
||||
assert response.status_code == 404
|
||||
assert response.json()["code"] == "DOCUMENT_NOT_FOUND"
|
||||
|
||||
|
||||
def test_get_document_versions_lists_every_state() -> None:
|
||||
app, repository, principal = _fixture()
|
||||
document = _document(principal.workspace_id)
|
||||
superseded = _version(document.id, DocumentVersionState.SUPERSEDED)
|
||||
published = _version(document.id, DocumentVersionState.PUBLISHED)
|
||||
repository.seed(document, superseded, published)
|
||||
|
||||
response = TestClient(app).get(f"/v1/documents/{document.id}/versions")
|
||||
|
||||
assert response.status_code == 200
|
||||
ids = {item["id"] for item in response.json()["items"]}
|
||||
assert ids == {str(superseded.id), str(published.id)}
|
||||
|
||||
|
||||
def test_retire_document_marks_source_deleted_and_is_idempotent() -> None:
|
||||
app, repository, principal = _fixture()
|
||||
document = _document(principal.workspace_id)
|
||||
repository.seed(document)
|
||||
client = TestClient(app)
|
||||
|
||||
first = client.post(f"/v1/documents/{document.id}/retire")
|
||||
assert first.status_code == 200
|
||||
assert first.json()["status"] == "source_deleted"
|
||||
|
||||
second = client.post(f"/v1/documents/{document.id}/retire")
|
||||
assert second.status_code == 200
|
||||
assert repository.retired == [document.id]
|
||||
|
||||
|
||||
def test_retire_document_does_not_hard_delete() -> None:
|
||||
app, repository, principal = _fixture()
|
||||
document = _document(principal.workspace_id)
|
||||
repository.seed(document)
|
||||
|
||||
TestClient(app).post(f"/v1/documents/{document.id}/retire")
|
||||
|
||||
assert document.id in repository.documents
|
||||
assert repository.documents[document.id].source_deleted is True
|
||||
|
||||
|
||||
def test_missing_authentication_is_problem_details() -> None:
|
||||
app, _, _ = _fixture()
|
||||
app.dependency_overrides.pop(get_documents_read_principal)
|
||||
response = TestClient(app).get(f"/v1/documents?source_id={uuid4()}&external_id=x")
|
||||
assert response.status_code == 401
|
||||
assert response.headers["content-type"].startswith("application/problem+json")
|
||||
assert response.json()["code"] == "AUTHENTICATION_REQUIRED"
|
||||
|
||||
|
||||
def test_insufficient_role_is_forbidden_problem_details() -> None:
|
||||
app, repository, principal = _fixture()
|
||||
document = _document(principal.workspace_id)
|
||||
repository.seed(document)
|
||||
denied = Principal(
|
||||
principal.id,
|
||||
principal.workspace_id,
|
||||
PrincipalType.SERVICE_ACCOUNT,
|
||||
frozenset({Role.AUDITOR}),
|
||||
)
|
||||
app.dependency_overrides[get_documents_read_principal] = lambda: denied
|
||||
|
||||
response = TestClient(app).get(f"/v1/documents/{document.id}")
|
||||
|
||||
assert response.status_code == 403
|
||||
assert response.json()["code"] == "RESOURCE_NOT_FOUND"
|
||||
|
||||
|
||||
def test_reconcile_space_index_runs_the_scoped_reconciler_and_maps_errors() -> None:
|
||||
from ragcore.api.v1.documents.dependencies import get_space_index_reconciler
|
||||
from ragcore.application.indexes.space_reconcile import SpaceReconciliationReport
|
||||
from ragcore.domain.control import KnowledgeSpace
|
||||
|
||||
app, repository, _principal = _fixture()
|
||||
space_id = uuid4()
|
||||
|
||||
class FakeControlRepository:
|
||||
async def list_spaces(self, workspace_id: UUID):
|
||||
return (
|
||||
KnowledgeSpace(
|
||||
id=space_id,
|
||||
workspace_id=workspace_id,
|
||||
name="portfolio-public",
|
||||
slug="portfolio-public",
|
||||
),
|
||||
)
|
||||
|
||||
class FakeReconciler:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[UUID] = []
|
||||
|
||||
async def reconcile(self, space, documents):
|
||||
self.calls.append(space)
|
||||
return SpaceReconciliationReport(
|
||||
space_id=space,
|
||||
documents_total=3,
|
||||
documents_retired=1,
|
||||
searchable_versions=2,
|
||||
points_before=10,
|
||||
deleted_points=4,
|
||||
orphaned_documents=2,
|
||||
points_after=6,
|
||||
)
|
||||
|
||||
from ragcore.api.v1.control.dependencies import get_control_application
|
||||
|
||||
service = ControlPlaneApplication(
|
||||
repository=FakeControlRepository(), # type: ignore[arg-type]
|
||||
authorization=AuthorizationService(),
|
||||
document_repository=repository, # type: ignore[arg-type]
|
||||
)
|
||||
app.dependency_overrides[get_control_application] = lambda: service
|
||||
reconciler = FakeReconciler()
|
||||
app.dependency_overrides[get_space_index_reconciler] = lambda: reconciler
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.post(f"/v1/spaces/{space_id}/index/reconcile")
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {
|
||||
"space_id": str(space_id),
|
||||
"documents_total": 3,
|
||||
"documents_retired": 1,
|
||||
"searchable_versions": 2,
|
||||
"points_before": 10,
|
||||
"deleted_points": 4,
|
||||
"orphaned_documents": 2,
|
||||
"points_after": 6,
|
||||
}
|
||||
assert reconciler.calls == [space_id]
|
||||
|
||||
missing = client.post(f"/v1/spaces/{uuid4()}/index/reconcile")
|
||||
assert missing.status_code == 404
|
||||
assert missing.json()["code"] == "SPACE_NOT_FOUND"
|
||||
|
||||
app.dependency_overrides[get_space_index_reconciler] = lambda: None
|
||||
unavailable = client.post(f"/v1/spaces/{space_id}/index/reconcile")
|
||||
assert unavailable.status_code == 503
|
||||
assert unavailable.json()["code"] == "INDEX_UNAVAILABLE"
|
||||
|
||||
|
||||
def test_openapi_documents_the_reconcile_endpoint() -> None:
|
||||
app, _, _ = _fixture()
|
||||
document = yaml.safe_load(
|
||||
(Path(__file__).parents[4] / "docs" / "contracts" / "openapi.yaml").read_text()
|
||||
)
|
||||
path = "/v1/spaces/{space_id}/index/reconcile"
|
||||
assert path in document["paths"]
|
||||
assert {m.lower() for m in app.openapi()["paths"][path]} == {"post"}
|
||||
|
||||
|
||||
def test_list_documents_by_source_returns_every_document_of_the_identity() -> None:
|
||||
app, repository, principal = _fixture()
|
||||
source_id = uuid4()
|
||||
first = _document(principal.workspace_id, source_key=str(source_id))
|
||||
second = DocumentRecord(
|
||||
id=uuid4(),
|
||||
workspace_id=principal.workspace_id,
|
||||
space_id=first.space_id,
|
||||
source_key=str(source_id),
|
||||
external_id="wiki-page-2",
|
||||
title="Second page",
|
||||
source_deleted=True,
|
||||
)
|
||||
other = _document(principal.workspace_id) # another source identity, must not appear
|
||||
repository.seed(first, _version(first.id, DocumentVersionState.PUBLISHED))
|
||||
repository.seed(second, _version(second.id, DocumentVersionState.PUBLISHED))
|
||||
repository.seed(other, _version(other.id, DocumentVersionState.PUBLISHED))
|
||||
|
||||
response = TestClient(app).get(f"/v1/documents?source_id={source_id}")
|
||||
assert response.status_code == 200, response.text
|
||||
items = response.json()["items"]
|
||||
assert {item["external_id"] for item in items} == {"wiki-page-1", "wiki-page-2"}
|
||||
by_id = {item["external_id"]: item for item in items}
|
||||
assert by_id["wiki-page-1"]["status"] == "active"
|
||||
assert by_id["wiki-page-1"]["active_version"]["source_fingerprint"] == "a" * 64
|
||||
assert by_id["wiki-page-2"]["status"] == "source_deleted"
|
||||
assert by_id["wiki-page-2"]["active_version"] is None
|
||||
|
||||
missing = TestClient(app).get("/v1/documents")
|
||||
assert missing.status_code == 400
|
||||
assert missing.json()["code"] == "MISSING_FILTER"
|
||||
Reference in new issue
Block a user