Initial curated ITWorx_MCP_Hub source from 368fba691ea8d23403b4d56b53b5ec670d7c5020
This commit is contained in:
Public Source Publisher committed
commit
cb5be898ea
663 files changed
+82674
No files matched your search
@@ -0,0 +1,105 @@
|
||||
"""Shared connector SDK primitives."""
|
||||
|
||||
from .contracts import (
|
||||
ConnectorContext,
|
||||
HealthSnapshot,
|
||||
PageRequest,
|
||||
ResourcePublication,
|
||||
ResultEnvelope,
|
||||
ResultMeta,
|
||||
ToolPublication,
|
||||
)
|
||||
from .gitea import (
|
||||
FileTokenProvider,
|
||||
GiteaError,
|
||||
GiteaRepositoryRegistry,
|
||||
GiteaSettings,
|
||||
RegisteredRepository,
|
||||
StaticTokenProvider,
|
||||
decode_file_payload,
|
||||
)
|
||||
from .gitea_resilient import GiteaClient
|
||||
from .mobilityops import (
|
||||
DEFAULT_KNOWLEDGE_LOCALE,
|
||||
KNOWLEDGE_LOCALES,
|
||||
MobilityOpsClient,
|
||||
MobilityOpsError,
|
||||
MobilityOpsSettings,
|
||||
)
|
||||
from .mobilityops import FileTokenProvider as MobilityOpsFileTokenProvider
|
||||
from .mobilityops import StaticTokenProvider as MobilityOpsStaticTokenProvider
|
||||
from .pagination import Cursor, InvalidCursor, decode_cursor, encode_cursor
|
||||
from .redaction import (
|
||||
is_secret_like,
|
||||
normalize_text,
|
||||
redact_sensitive_text,
|
||||
safe_content,
|
||||
safe_snippet,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ConnectorContext",
|
||||
"SourceType",
|
||||
"SourceStatus",
|
||||
"SearchPage",
|
||||
"SearchHit",
|
||||
"RegisteredSource",
|
||||
"PathEscapeError",
|
||||
"ParserRegistry",
|
||||
"KnowledgeRefreshJob",
|
||||
"InMemoryKnowledgeIndex",
|
||||
"IndexStatus",
|
||||
"IndexReport",
|
||||
"DocumentRecord",
|
||||
"DocumentNotFound",
|
||||
"DeterministicChunker",
|
||||
"ChunkRecord",
|
||||
"Cursor",
|
||||
"HealthSnapshot",
|
||||
"InvalidCursor",
|
||||
"PageRequest",
|
||||
"ResourcePublication",
|
||||
"ResultEnvelope",
|
||||
"ResultMeta",
|
||||
"ToolPublication",
|
||||
"decode_cursor",
|
||||
"encode_cursor",
|
||||
"is_secret_like",
|
||||
"normalize_text",
|
||||
"redact_sensitive_text",
|
||||
"safe_content",
|
||||
"safe_snippet",
|
||||
"FileTokenProvider",
|
||||
"GiteaClient",
|
||||
"GiteaError",
|
||||
"GiteaRepositoryRegistry",
|
||||
"GiteaSettings",
|
||||
"RegisteredRepository",
|
||||
"StaticTokenProvider",
|
||||
"decode_file_payload",
|
||||
"MobilityOpsClient",
|
||||
"MobilityOpsError",
|
||||
"MobilityOpsFileTokenProvider",
|
||||
"MobilityOpsSettings",
|
||||
"MobilityOpsStaticTokenProvider",
|
||||
"KNOWLEDGE_LOCALES",
|
||||
"DEFAULT_KNOWLEDGE_LOCALE",
|
||||
]
|
||||
|
||||
from .knowledge import (
|
||||
ChunkRecord,
|
||||
DeterministicChunker,
|
||||
DocumentNotFound,
|
||||
DocumentRecord,
|
||||
IndexReport,
|
||||
IndexStatus,
|
||||
InMemoryKnowledgeIndex,
|
||||
KnowledgeRefreshJob,
|
||||
ParserRegistry,
|
||||
PathEscapeError,
|
||||
RegisteredSource,
|
||||
SearchHit,
|
||||
SearchPage,
|
||||
SourceStatus,
|
||||
SourceType,
|
||||
)
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Shared bounded connector contracts used by all first-party connectors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class ConnectorContext(BaseModel):
|
||||
"""Validated invocation scope supplied by the Hub, never by target content."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
workspace_id: UUID
|
||||
project_id: UUID
|
||||
project_slug: str = Field(pattern=r"^[a-z0-9][a-z0-9-]{0,62}$")
|
||||
environment: str = Field(pattern=r"^[a-z0-9][a-z0-9-]{0,31}$")
|
||||
client_id: UUID | None = None
|
||||
correlation_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
|
||||
class PageRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
limit: int = Field(default=20, ge=1, le=100)
|
||||
cursor: str | None = Field(default=None, max_length=500)
|
||||
|
||||
|
||||
class ResultMeta(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
project: str
|
||||
environment: str
|
||||
source: str
|
||||
truncated: bool = False
|
||||
next_cursor: str | None = None
|
||||
generated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ResultEnvelope[T](BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
data: T
|
||||
meta: ResultMeta
|
||||
warnings: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class HealthSnapshot(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
status: str = Field(pattern=r"^(healthy|ready|degraded|failed)$")
|
||||
component: str = Field(min_length=1, max_length=100)
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ToolPublication(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
canonical_name: str = Field(pattern=r"^[a-z0-9]+(?:\.[a-z0-9-]+){2,}$")
|
||||
version: str = Field(pattern=r"^\d+\.\d+\.\d+$")
|
||||
connector_type: str = Field(min_length=1, max_length=64)
|
||||
read_only: bool = True
|
||||
risk_class: str = Field(pattern=r"^R[0-9]+$")
|
||||
input_schema: dict[str, Any]
|
||||
output_schema: dict[str, Any]
|
||||
required_scopes: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class ResourcePublication(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
uri_template: str = Field(min_length=1, max_length=2048)
|
||||
connector_type: str = Field(min_length=1, max_length=64)
|
||||
risk_class: str = Field(pattern=r"^R[0-9]+$")
|
||||
mime_types: tuple[str, ...] = ()
|
||||
max_bytes: int = Field(ge=1, le=4 * 1024 * 1024)
|
||||
required_scopes: tuple[str, ...] = ()
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,114 @@
|
||||
"""Compatibility hardening for Gitea validation discovery and request latency.
|
||||
|
||||
Some Gitea versions expose pull-request check results as commit statuses while
|
||||
returning an empty branch-filtered Actions run wrapper. The base client treats
|
||||
collections strictly; this subclass preserves that strictness while translating
|
||||
a fully-green exact-head status set into one synthetic completed validation run.
|
||||
|
||||
Large repositories can require longer than the historical 15-second client
|
||||
budget to commit files, apply changesets, or materialize a server-side squash
|
||||
merge. The resilient client therefore uses the already allowlisted 60-second
|
||||
maximum from ``GiteaSettings``. All HTTP method allowlisting, redirect handling,
|
||||
response budgets, access checks and non-forcing merge behavior remain in the
|
||||
base client and are not duplicated here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from .gitea import GiteaClient as BaseGiteaClient
|
||||
from .gitea import GiteaError, GiteaSettings, RegisteredRepository, TokenProvider
|
||||
|
||||
GITEA_REQUEST_TIMEOUT_SECONDS = 60.0
|
||||
|
||||
|
||||
class GiteaClient(BaseGiteaClient):
|
||||
"""Gitea client with exact-head validation fallback and bounded latency."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings: GiteaSettings,
|
||||
token_provider: TokenProvider,
|
||||
*,
|
||||
transport: httpx.BaseTransport | None = None,
|
||||
) -> None:
|
||||
resilient_settings = settings.model_copy(
|
||||
update={"timeout_seconds": GITEA_REQUEST_TIMEOUT_SECONDS}
|
||||
)
|
||||
super().__init__(resilient_settings, token_provider, transport=transport)
|
||||
|
||||
def action_runs(
|
||||
self,
|
||||
repository: RegisteredRepository,
|
||||
*,
|
||||
branch: str,
|
||||
page: int,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
payload = self._request(
|
||||
self._path("repos", repository.owner, repository.name, "actions", "runs"),
|
||||
{"branch": branch, "page": page, "limit": limit},
|
||||
)
|
||||
if isinstance(payload, dict):
|
||||
if "workflow_runs" in payload:
|
||||
payload = payload.get("workflow_runs")
|
||||
elif "runs" in payload:
|
||||
payload = payload.get("runs")
|
||||
else:
|
||||
payload = None
|
||||
runs = self._as_list(payload)
|
||||
if runs:
|
||||
return runs
|
||||
|
||||
# Only an actually empty Actions result can fall back. Malformed
|
||||
# collections still fail above, preserving strict response validation.
|
||||
try:
|
||||
branch_payload = self.branch(repository, branch)
|
||||
except GiteaError:
|
||||
return []
|
||||
commit = branch_payload.get("commit")
|
||||
if not isinstance(commit, dict):
|
||||
return []
|
||||
head_sha = str(commit.get("id") or commit.get("sha") or "")
|
||||
if len(head_sha) != 40 or any(
|
||||
character not in "0123456789abcdefABCDEF" for character in head_sha
|
||||
):
|
||||
return []
|
||||
|
||||
statuses = self.commit_statuses(repository, head_sha, 1, 100)
|
||||
if not statuses:
|
||||
return []
|
||||
latest_by_context: dict[str, dict[str, Any]] = {}
|
||||
for item in statuses:
|
||||
status_context = str(item.get("context") or "__default__")
|
||||
previous = latest_by_context.get(status_context)
|
||||
if previous is None or str(item.get("created_at") or "") > str(
|
||||
previous.get("created_at") or ""
|
||||
):
|
||||
latest_by_context[status_context] = item
|
||||
latest_states = {
|
||||
str(item.get("status") or "").casefold() for item in latest_by_context.values()
|
||||
}
|
||||
# Gitea can omit ``head_branch`` for pull_request runs, causing the
|
||||
# branch-filtered run list above to be empty even though the exact-head
|
||||
# commit statuses are available. Skipped jobs are neutral evidence;
|
||||
# require at least one success and reject every non-terminal state.
|
||||
if "success" not in latest_states or not latest_states <= {"success", "skipped"}:
|
||||
return []
|
||||
|
||||
return [
|
||||
{
|
||||
"id": 0,
|
||||
"name": "Exact-head commit status validation",
|
||||
"status": "completed",
|
||||
"conclusion": "success",
|
||||
"event": "commit_status_fallback",
|
||||
"head_branch": branch,
|
||||
"head_sha": head_sha,
|
||||
"html_url": "",
|
||||
"validation_evidence": "commit-statuses",
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,580 @@
|
||||
"""Registered-root project knowledge index with deterministic lexical search."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import mimetypes
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from urllib.parse import unquote, urlsplit
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from .pagination import decode_cursor, encode_cursor
|
||||
from .redaction import safe_content, safe_snippet
|
||||
|
||||
|
||||
class SourceType(StrEnum):
|
||||
DOCUMENTATION = "documentation"
|
||||
ADR = "adr"
|
||||
RUNBOOK = "runbook"
|
||||
README = "readme"
|
||||
CURRENT_STATE = "current-state"
|
||||
RELEASE_NOTE = "release-note"
|
||||
|
||||
|
||||
class PathEscapeError(ValueError):
|
||||
"""A registered root or symlink would escape its configured containment."""
|
||||
|
||||
|
||||
class DocumentNotFound(LookupError):
|
||||
"""The requested source-backed document is not indexed for this project."""
|
||||
|
||||
|
||||
class RegisteredSource(BaseModel):
|
||||
"""A control-plane registered root; MCP callers never provide this path."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, arbitrary_types_allowed=True)
|
||||
|
||||
source_id: UUID
|
||||
project_id: UUID
|
||||
project_slug: str = Field(pattern=r"^[a-z0-9][a-z0-9-]{0,62}$")
|
||||
source_ref: str = Field(pattern=r"^[a-z0-9][a-z0-9-]{0,127}$")
|
||||
source_type: SourceType
|
||||
root: Path
|
||||
repository_ref: str | None = Field(default=None, max_length=200)
|
||||
repository_commit: str | None = Field(default=None, max_length=200)
|
||||
allowed_extensions: frozenset[str] = frozenset(
|
||||
{".md", ".markdown", ".txt", ".rst", ".adoc", ".json", ".yaml", ".yml"}
|
||||
)
|
||||
excluded_dirs: frozenset[str] = frozenset(
|
||||
{
|
||||
".git",
|
||||
".hg",
|
||||
".svn",
|
||||
"node_modules",
|
||||
"dist",
|
||||
"build",
|
||||
"coverage",
|
||||
"__pycache__",
|
||||
".venv",
|
||||
"venv",
|
||||
"backups",
|
||||
"secrets",
|
||||
}
|
||||
)
|
||||
max_file_bytes: int = Field(default=1_048_576, ge=1024, le=4 * 1024 * 1024)
|
||||
enabled: bool = True
|
||||
revision: str = Field(default="registered", max_length=200)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_root(self) -> RegisteredSource:
|
||||
if not self.root.is_absolute():
|
||||
raise ValueError("registered source roots must be absolute")
|
||||
if any(not extension.startswith(".") for extension in self.allowed_extensions):
|
||||
raise ValueError("allowed extensions must start with a dot")
|
||||
return self
|
||||
|
||||
@property
|
||||
def uri_prefix(self) -> str:
|
||||
return f"itworx-docs://{self.project_slug}/{self.source_ref}"
|
||||
|
||||
def canonical_path(self, relative_path: str | Path) -> Path:
|
||||
relative = Path(relative_path)
|
||||
if relative.is_absolute() or ".." in relative.parts:
|
||||
raise PathEscapeError("source path must remain relative to the registered root")
|
||||
root = self.root.resolve(strict=True)
|
||||
candidate = (root / relative).resolve(strict=False)
|
||||
if candidate != root and root not in candidate.parents:
|
||||
raise PathEscapeError("source path escapes the registered root")
|
||||
return candidate
|
||||
|
||||
def allows(self, relative_path: Path) -> bool:
|
||||
if not relative_path.parts or any(
|
||||
part in self.excluded_dirs for part in relative_path.parts
|
||||
):
|
||||
return False
|
||||
lowered = relative_path.name.lower()
|
||||
if lowered in {
|
||||
".env",
|
||||
".env.local",
|
||||
".env.production",
|
||||
"id_rsa",
|
||||
"id_ed25519",
|
||||
"credentials.json",
|
||||
"secrets.yaml",
|
||||
}:
|
||||
return False
|
||||
if any(token in lowered for token in ("secret", "password", "token", "private-key")):
|
||||
return False
|
||||
return relative_path.suffix.lower() in self.allowed_extensions
|
||||
|
||||
|
||||
class ChunkRecord(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
index: int = Field(ge=0)
|
||||
text: str = Field(min_length=1, max_length=200_000)
|
||||
content_hash: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
class DocumentRecord(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
document_id: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
source_id: UUID
|
||||
project_id: UUID
|
||||
project_slug: str
|
||||
source_ref: str
|
||||
source_type: SourceType
|
||||
external_ref: str = Field(min_length=1, max_length=1024)
|
||||
title: str = Field(min_length=1, max_length=500)
|
||||
uri: str = Field(min_length=1, max_length=2048)
|
||||
mime_type: str = Field(min_length=1, max_length=100)
|
||||
content_hash: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||
repository_ref: str | None = None
|
||||
repository_commit: str | None = None
|
||||
source_revision: str = Field(max_length=200)
|
||||
modified_at: datetime
|
||||
content_bytes: int = Field(ge=0, le=4 * 1024 * 1024)
|
||||
version: int = Field(ge=1)
|
||||
state: Literal["registered", "deleted"] = "registered"
|
||||
content: str = ""
|
||||
chunks: tuple[ChunkRecord, ...] = ()
|
||||
|
||||
|
||||
class SearchHit(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
uri: str
|
||||
title: str
|
||||
path: str
|
||||
source_type: SourceType
|
||||
source_revision: str
|
||||
repository_ref: str | None = None
|
||||
repository_commit: str | None = None
|
||||
content_hash: str
|
||||
modified_at: datetime
|
||||
snippet: str
|
||||
score: float = Field(ge=0)
|
||||
truncated: bool = False
|
||||
|
||||
|
||||
class SearchPage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
items: tuple[SearchHit, ...]
|
||||
next_cursor: str | None
|
||||
truncated: bool
|
||||
|
||||
|
||||
class SourceStatus(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
source_ref: str
|
||||
source_type: SourceType
|
||||
uri: str
|
||||
status: Literal["ready", "indexing", "stale", "failed", "empty", "disabled"]
|
||||
source_revision: str | None
|
||||
last_indexed_at: datetime | None
|
||||
document_count: int = Field(ge=0)
|
||||
|
||||
|
||||
class IndexStatus(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
status: Literal["ready", "indexing", "stale", "failed", "empty"]
|
||||
source_count: int = Field(ge=0)
|
||||
document_count: int = Field(ge=0)
|
||||
last_completed_at: datetime | None
|
||||
last_error: str | None = Field(default=None, max_length=1000)
|
||||
|
||||
|
||||
class IndexReport(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
source_ref: str
|
||||
indexed: int = Field(ge=0)
|
||||
skipped: int = Field(ge=0)
|
||||
tombstoned: int = Field(ge=0)
|
||||
completed_at: datetime
|
||||
|
||||
|
||||
class ParserRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._extensions: dict[str, str] = {}
|
||||
|
||||
def register(self, extension: str, parser_name: str) -> None:
|
||||
normalized = extension.lower()
|
||||
if not normalized.startswith(".") or not parser_name:
|
||||
raise ValueError("parser registration is invalid")
|
||||
self._extensions[normalized] = parser_name
|
||||
|
||||
def supports(self, path: Path) -> bool:
|
||||
return path.suffix.lower() in self._extensions
|
||||
|
||||
def title(self, path: Path, content: str) -> str:
|
||||
for line in content.splitlines()[:20]:
|
||||
match = re.match(r"^#{1,6}\s+(.+?)\s*#*$", line.strip())
|
||||
if match:
|
||||
return match.group(1)[:500]
|
||||
return path.stem.replace("_", " ").replace("-", " ").strip()[:500] or path.name
|
||||
|
||||
@classmethod
|
||||
def default(cls) -> ParserRegistry:
|
||||
registry = cls()
|
||||
for extension in (".md", ".markdown", ".txt", ".rst", ".adoc", ".json", ".yaml", ".yml"):
|
||||
registry.register(extension, "plain-text")
|
||||
return registry
|
||||
|
||||
|
||||
class DeterministicChunker:
|
||||
def __init__(self, *, max_chars: int = 4000, overlap_chars: int = 200) -> None:
|
||||
if not 256 <= max_chars <= 20_000 or not 0 <= overlap_chars < max_chars // 2:
|
||||
raise ValueError("chunk bounds are invalid")
|
||||
self.max_chars = max_chars
|
||||
self.overlap_chars = overlap_chars
|
||||
|
||||
def chunk(self, content: str) -> tuple[ChunkRecord, ...]:
|
||||
if not content:
|
||||
return ()
|
||||
chunks: list[ChunkRecord] = []
|
||||
start = 0
|
||||
index = 0
|
||||
while start < len(content):
|
||||
end = min(len(content), start + self.max_chars)
|
||||
if end < len(content):
|
||||
boundary = content.rfind("\n", start + self.max_chars // 2, end)
|
||||
if boundary > start:
|
||||
end = boundary
|
||||
text = content[start:end].strip()
|
||||
if text:
|
||||
chunks.append(
|
||||
ChunkRecord(
|
||||
index=index,
|
||||
text=text,
|
||||
content_hash=hashlib.sha256(text.encode("utf-8")).hexdigest(),
|
||||
)
|
||||
)
|
||||
index += 1
|
||||
if end >= len(content):
|
||||
break
|
||||
start = max(end - self.overlap_chars, start + 1)
|
||||
return tuple(chunks)
|
||||
|
||||
|
||||
def _document_key(source: RegisteredSource, relative: str) -> str:
|
||||
return hashlib.sha256(f"{source.source_id}:{relative}".encode()).hexdigest()
|
||||
|
||||
|
||||
def _content_hash(content: str) -> str:
|
||||
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _score_text(query: str, title: str, content: str) -> float:
|
||||
terms = [term for term in re.findall(r"[\w-]{2,}", query.casefold())]
|
||||
if not terms:
|
||||
return 0.0
|
||||
folded_title = title.casefold()
|
||||
folded_content = content.casefold()
|
||||
score = sum(folded_title.count(term) * 4 for term in terms)
|
||||
score += sum(folded_content.count(term) for term in terms)
|
||||
if query.casefold() in folded_content:
|
||||
score += 2
|
||||
return float(score)
|
||||
|
||||
|
||||
class InMemoryKnowledgeIndex:
|
||||
"""Deterministic index used by the connector and as a fake target for tests."""
|
||||
|
||||
def __init__(
|
||||
self, *, parser: ParserRegistry | None = None, chunker: DeterministicChunker | None = None
|
||||
) -> None:
|
||||
self.parser = parser or ParserRegistry.default()
|
||||
self.chunker = chunker or DeterministicChunker()
|
||||
self.sources: dict[UUID, RegisteredSource] = {}
|
||||
self.documents: dict[str, DocumentRecord] = {}
|
||||
self._last_completed_at: datetime | None = None
|
||||
self._last_error: str | None = None
|
||||
self._indexing = False
|
||||
|
||||
def register_source(self, source: RegisteredSource) -> None:
|
||||
if not source.enabled:
|
||||
self.sources[source.source_id] = source
|
||||
return
|
||||
canonical_root = source.root.resolve(strict=True)
|
||||
if not canonical_root.is_dir():
|
||||
raise ValueError("registered source root must be a directory")
|
||||
self.sources[source.source_id] = source.model_copy(update={"root": canonical_root})
|
||||
|
||||
def index_source(self, source_id: UUID) -> IndexReport:
|
||||
source = self.sources.get(source_id)
|
||||
if source is None:
|
||||
raise KeyError("source is not registered")
|
||||
if not source.enabled:
|
||||
return IndexReport(
|
||||
source_ref=source.source_ref,
|
||||
indexed=0,
|
||||
skipped=0,
|
||||
tombstoned=0,
|
||||
completed_at=datetime.now(UTC),
|
||||
)
|
||||
self._indexing = True
|
||||
self._last_error = None
|
||||
previous = {
|
||||
doc.external_ref: doc
|
||||
for doc in self.documents.values()
|
||||
if doc.source_id == source.source_id and doc.state == "registered"
|
||||
}
|
||||
seen: set[str] = set()
|
||||
indexed = 0
|
||||
skipped = 0
|
||||
try:
|
||||
for path in self._iter_files(source):
|
||||
relative = path.relative_to(source.root.resolve(strict=True)).as_posix()
|
||||
seen.add(relative)
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if len(text.encode("utf-8")) > source.max_file_bytes:
|
||||
skipped += 1
|
||||
continue
|
||||
parsed = safe_content(text)
|
||||
if parsed is None or not self.parser.supports(path):
|
||||
skipped += 1
|
||||
continue
|
||||
key = _document_key(source, relative)
|
||||
old = previous.get(relative)
|
||||
content_hash = _content_hash(parsed)
|
||||
if old is not None and old.content_hash == content_hash:
|
||||
continue
|
||||
modified = datetime.fromtimestamp(path.stat().st_mtime, tz=UTC)
|
||||
document = DocumentRecord(
|
||||
document_id=key,
|
||||
source_id=source.source_id,
|
||||
project_id=source.project_id,
|
||||
project_slug=source.project_slug,
|
||||
source_ref=source.source_ref,
|
||||
source_type=source.source_type,
|
||||
external_ref=relative,
|
||||
title=self.parser.title(path, parsed),
|
||||
uri=f"{source.uri_prefix}/{key}",
|
||||
mime_type=mimetypes.guess_type(path.name)[0] or "text/plain",
|
||||
content_hash=content_hash,
|
||||
repository_ref=source.repository_ref,
|
||||
repository_commit=source.repository_commit,
|
||||
source_revision=source.revision,
|
||||
modified_at=modified,
|
||||
content_bytes=len(parsed.encode("utf-8")),
|
||||
version=(old.version + 1 if old is not None else 1),
|
||||
content=parsed,
|
||||
chunks=self.chunker.chunk(parsed),
|
||||
)
|
||||
self.documents[key] = document
|
||||
indexed += 1
|
||||
except (OSError, UnicodeDecodeError):
|
||||
skipped += 1
|
||||
tombstoned = 0
|
||||
for relative, old in previous.items():
|
||||
if relative not in seen:
|
||||
self.documents[old.document_id] = old.model_copy(
|
||||
update={
|
||||
"state": "deleted",
|
||||
"content": "",
|
||||
"chunks": (),
|
||||
"content_bytes": 0,
|
||||
"version": old.version + 1,
|
||||
}
|
||||
)
|
||||
tombstoned += 1
|
||||
self._last_completed_at = datetime.now(UTC)
|
||||
return IndexReport(
|
||||
source_ref=source.source_ref,
|
||||
indexed=indexed,
|
||||
skipped=skipped,
|
||||
tombstoned=tombstoned,
|
||||
completed_at=self._last_completed_at,
|
||||
)
|
||||
except Exception as exc:
|
||||
self._last_error = type(exc).__name__
|
||||
raise
|
||||
finally:
|
||||
self._indexing = False
|
||||
|
||||
def _iter_files(self, source: RegisteredSource) -> Iterable[Path]:
|
||||
root = source.root.resolve(strict=True)
|
||||
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix().casefold()):
|
||||
if not path.is_file():
|
||||
continue
|
||||
relative = path.relative_to(root)
|
||||
if not source.allows(relative):
|
||||
continue
|
||||
canonical = source.canonical_path(relative)
|
||||
if canonical != path.resolve(strict=False) and not path.is_symlink():
|
||||
continue
|
||||
if not canonical.is_file():
|
||||
continue
|
||||
yield canonical
|
||||
|
||||
def get_document(self, project_id: UUID, uri: str) -> DocumentRecord:
|
||||
parts = urlsplit(uri)
|
||||
if parts.scheme != "itworx-docs" or parts.query or parts.fragment:
|
||||
raise DocumentNotFound("document URI is invalid")
|
||||
project = unquote(parts.netloc)
|
||||
path_parts = [unquote(part) for part in parts.path.split("/") if part]
|
||||
if len(path_parts) != 2 or any(
|
||||
part in {".", ".."} or "/" in part or "\\" in part for part in path_parts
|
||||
):
|
||||
raise DocumentNotFound("document URI is invalid")
|
||||
if project != next(
|
||||
(
|
||||
source.project_slug
|
||||
for source in self.sources.values()
|
||||
if source.project_id == project_id
|
||||
),
|
||||
"",
|
||||
):
|
||||
raise DocumentNotFound("document is outside the caller project")
|
||||
document = self.documents.get(path_parts[1])
|
||||
if (
|
||||
document is None
|
||||
or document.project_id != project_id
|
||||
or document.state != "registered"
|
||||
or document.source_ref != path_parts[0]
|
||||
):
|
||||
raise DocumentNotFound("document is not indexed")
|
||||
return document
|
||||
|
||||
def search(
|
||||
self,
|
||||
project_id: UUID,
|
||||
query: str,
|
||||
*,
|
||||
source_types: Iterable[SourceType] = (),
|
||||
limit: int = 20,
|
||||
cursor: str | None = None,
|
||||
) -> SearchPage:
|
||||
if not query.strip() or len(query) > 1000:
|
||||
raise ValueError("query must be between 1 and 1000 characters")
|
||||
if not 1 <= limit <= 20:
|
||||
raise ValueError("limit must be between 1 and 20")
|
||||
selected = tuple(sorted(str(value) for value in source_types))
|
||||
page_cursor = decode_cursor(cursor, str(project_id), query, *selected)
|
||||
hits: list[SearchHit] = []
|
||||
for document in self.documents.values():
|
||||
if (
|
||||
document.project_id != project_id
|
||||
or document.state != "registered"
|
||||
or (selected and str(document.source_type) not in selected)
|
||||
):
|
||||
continue
|
||||
for chunk in document.chunks:
|
||||
score = _score_text(query, document.title, chunk.text)
|
||||
if score <= 0:
|
||||
continue
|
||||
hits.append(
|
||||
SearchHit(
|
||||
uri=document.uri,
|
||||
title=document.title,
|
||||
path=document.external_ref,
|
||||
source_type=document.source_type,
|
||||
source_revision=document.source_revision,
|
||||
repository_ref=document.repository_ref,
|
||||
repository_commit=document.repository_commit,
|
||||
content_hash=document.content_hash,
|
||||
modified_at=document.modified_at,
|
||||
snippet=safe_snippet(chunk.text),
|
||||
score=score,
|
||||
truncated=len(chunk.text.encode("utf-8")) > 2000,
|
||||
)
|
||||
)
|
||||
hits.sort(key=lambda hit: (-hit.score, hit.uri, hit.path))
|
||||
page = hits[page_cursor.offset : page_cursor.offset + limit]
|
||||
next_offset = page_cursor.offset + len(page)
|
||||
next_cursor = (
|
||||
encode_cursor(next_offset, str(project_id), query, *selected)
|
||||
if next_offset < len(hits)
|
||||
else None
|
||||
)
|
||||
return SearchPage(
|
||||
items=tuple(page), next_cursor=next_cursor, truncated=next_cursor is not None
|
||||
)
|
||||
|
||||
def list_sources(self, project_id: UUID) -> tuple[SourceStatus, ...]:
|
||||
return tuple(
|
||||
SourceStatus(
|
||||
source_ref=source.source_ref,
|
||||
source_type=source.source_type,
|
||||
uri=source.uri_prefix,
|
||||
status="disabled"
|
||||
if not source.enabled
|
||||
else (
|
||||
"ready"
|
||||
if any(
|
||||
doc.project_id == project_id
|
||||
and doc.source_id == source.source_id
|
||||
and doc.state == "registered"
|
||||
for doc in self.documents.values()
|
||||
)
|
||||
else "empty"
|
||||
),
|
||||
source_revision=source.revision,
|
||||
last_indexed_at=self._last_completed_at,
|
||||
document_count=sum(
|
||||
1
|
||||
for doc in self.documents.values()
|
||||
if doc.project_id == project_id
|
||||
and doc.source_id == source.source_id
|
||||
and doc.state == "registered"
|
||||
),
|
||||
)
|
||||
for source in sorted(self.sources.values(), key=lambda item: item.source_ref)
|
||||
if source.project_id == project_id
|
||||
)
|
||||
|
||||
def status(self, project_id: UUID) -> IndexStatus:
|
||||
sources = self.list_sources(project_id)
|
||||
count = sum(source.document_count for source in sources)
|
||||
status: Literal["ready", "indexing", "stale", "failed", "empty"] = (
|
||||
"indexing"
|
||||
if self._indexing
|
||||
else ("failed" if self._last_error else ("empty" if count == 0 else "ready"))
|
||||
)
|
||||
return IndexStatus(
|
||||
status=status,
|
||||
source_count=len(sources),
|
||||
document_count=count,
|
||||
last_completed_at=self._last_completed_at,
|
||||
last_error=self._last_error,
|
||||
)
|
||||
|
||||
def documents_for_source(
|
||||
self, project_id: UUID, source_type: SourceType | None = None
|
||||
) -> tuple[DocumentRecord, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
(
|
||||
doc
|
||||
for doc in self.documents.values()
|
||||
if doc.project_id == project_id
|
||||
and doc.state == "registered"
|
||||
and (source_type is None or doc.source_type == source_type)
|
||||
),
|
||||
key=lambda doc: (doc.external_ref, doc.document_id),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeRefreshJob:
|
||||
"""Single bounded refresh operation; scheduling is owned by the control plane."""
|
||||
|
||||
def __init__(self, index: InMemoryKnowledgeIndex, source_ids: Iterable[UUID]) -> None:
|
||||
self.index = index
|
||||
self.source_ids = tuple(source_ids)
|
||||
|
||||
def run_once(self) -> tuple[IndexReport, ...]:
|
||||
return tuple(self.index.index_source(source_id) for source_id in self.source_ids)
|
||||
@@ -0,0 +1,356 @@
|
||||
"""Typed, read-only MobilityOps (Fleet Ops) API client primitives.
|
||||
|
||||
MobilityOps has exactly one configured target per environment (unlike Gitea's
|
||||
per-project repository registry) -- shape follows the simpler `unraid` broker
|
||||
client pattern, with Gitea's stricter settings/transport validation reused
|
||||
since this, like Gitea, is an external HTTPS origin authenticated by a
|
||||
long-lived shared secret rather than an internally-issued short-lived token.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol, cast
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
KNOWLEDGE_LOCALES: tuple[str, ...] = ("nl-BE", "en-GB", "fr-BE")
|
||||
"""Locales MobilityOps (Fleet Ops) accepts on `POST /search-knowledge`."""
|
||||
|
||||
DEFAULT_KNOWLEDGE_LOCALE = "en-GB"
|
||||
"""Compatible default for older clients that send no locale at all."""
|
||||
|
||||
CORRELATION_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
|
||||
class MobilityOpsError(RuntimeError):
|
||||
"""Stable connector error that never contains credentials or raw headers."""
|
||||
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
self.code = code
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class MobilityOpsSettings(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
base_url: str = Field(min_length=1, max_length=2048)
|
||||
token_reference: str = Field(min_length=1, max_length=512)
|
||||
timeout_seconds: float = Field(default=15.0, gt=0, le=60)
|
||||
max_response_bytes: int = Field(default=262144, ge=1024, le=4 * 1024 * 1024)
|
||||
allow_insecure_http_for_tests: bool = False
|
||||
|
||||
@field_validator("base_url")
|
||||
@classmethod
|
||||
def validate_base_url(cls, value: str) -> str:
|
||||
parsed = urlparse(value.strip().rstrip("/"))
|
||||
if parsed.scheme not in {"https", "http"} or not parsed.hostname:
|
||||
raise ValueError("MobilityOps base_url must be an absolute HTTP(S) URL")
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise ValueError("MobilityOps base_url must not contain credentials or query data")
|
||||
hostname = (parsed.hostname or "").casefold()
|
||||
if hostname in {"localhost", "metadata", "metadata.google.internal"}:
|
||||
raise ValueError("MobilityOps base_url host is not allowed")
|
||||
try:
|
||||
address = ipaddress.ip_address(hostname)
|
||||
except ValueError:
|
||||
address = None
|
||||
if address is not None and (
|
||||
address.is_private
|
||||
or address.is_loopback
|
||||
or address.is_link_local
|
||||
or address.is_reserved
|
||||
or address.is_unspecified
|
||||
or address.is_multicast
|
||||
):
|
||||
raise ValueError("MobilityOps base_url host is not allowed")
|
||||
return value.strip().rstrip("/")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_tls(self) -> MobilityOpsSettings:
|
||||
if urlparse(self.base_url).scheme != "https" and not self.allow_insecure_http_for_tests:
|
||||
raise ValueError("MobilityOps base_url must use HTTPS")
|
||||
if not self.token_reference.startswith(("docker-secret://", "/run/secrets/")):
|
||||
raise ValueError("token_reference must name a mounted secret, not contain a token")
|
||||
return self
|
||||
|
||||
|
||||
class TokenProvider(Protocol):
|
||||
def get_token(self, reference: str) -> str: ...
|
||||
|
||||
|
||||
class FileTokenProvider:
|
||||
"""Read-only provider for Docker secrets mounted beneath /run/secrets."""
|
||||
|
||||
def get_token(self, reference: str) -> str:
|
||||
if reference.startswith("docker-secret://"):
|
||||
name = reference.removeprefix("docker-secret://")
|
||||
path = Path("/run/secrets") / name
|
||||
else:
|
||||
path = Path(reference)
|
||||
root = Path("/run/secrets").resolve()
|
||||
try:
|
||||
resolved = path.resolve()
|
||||
resolved.relative_to(root)
|
||||
except (OSError, ValueError) as exc:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_SECRET_REFERENCE_INVALID", "secret reference is outside /run/secrets"
|
||||
) from exc
|
||||
try:
|
||||
token = resolved.read_text(encoding="utf-8").strip()
|
||||
except OSError as exc:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_SECRET_UNAVAILABLE", "configured MobilityOps secret is unavailable"
|
||||
) from exc
|
||||
if not token or len(token) > 4096 or any(ord(char) < 0x20 for char in token):
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_SECRET_INVALID", "configured MobilityOps secret is invalid"
|
||||
)
|
||||
return token
|
||||
|
||||
|
||||
class StaticTokenProvider:
|
||||
"""Deterministic fake-only provider; never serializes the token."""
|
||||
|
||||
def __init__(self, token: str) -> None:
|
||||
self._token = token
|
||||
|
||||
def get_token(self, reference: str) -> str:
|
||||
if not reference.startswith(("docker-secret://", "/run/secrets/")):
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_SECRET_REFERENCE_INVALID", "invalid secret reference"
|
||||
)
|
||||
return self._token
|
||||
|
||||
|
||||
class MobilityOpsClient:
|
||||
"""Small MobilityOps API client with same-origin redirect enforcement.
|
||||
|
||||
Auth is a shared-secret `X-Service-Token` header (MobilityOps's own
|
||||
`require_mcp_service_token` dependency), not a per-repo bearer token --
|
||||
there is exactly one configured MobilityOps target per environment.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings: MobilityOpsSettings,
|
||||
token_provider: TokenProvider,
|
||||
*,
|
||||
transport: httpx.BaseTransport | None = None,
|
||||
) -> None:
|
||||
self.settings = settings
|
||||
self._token_provider = token_provider
|
||||
self._client = httpx.Client(
|
||||
base_url=f"{settings.base_url}/api/v1/integrations/mcp",
|
||||
timeout=settings.timeout_seconds,
|
||||
follow_redirects=False,
|
||||
transport=transport,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
def _request(
|
||||
self,
|
||||
client_id: str,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
params: Mapping[str, str | int] | None = None,
|
||||
json_body: Mapping[str, Any] | None = None,
|
||||
correlation_id: str | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> Any:
|
||||
token = self._token_provider.get_token(self.settings.token_reference)
|
||||
headers = {
|
||||
"X-Service-Token": token,
|
||||
"X-Client-Id": client_id,
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if correlation_id is not None:
|
||||
# The Hub already has a trace id for this invocation; MobilityOps preserves
|
||||
# whatever valid correlation id it receives, so never mint a second one here.
|
||||
if not CORRELATION_ID.fullmatch(correlation_id):
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_CORRELATION_ID_INVALID",
|
||||
"inbound correlation id is not a safe header value",
|
||||
)
|
||||
headers["X-Correlation-Id"] = correlation_id
|
||||
request_timeout = self.settings.timeout_seconds if timeout is None else timeout
|
||||
try:
|
||||
response = self._client.request(
|
||||
method,
|
||||
path,
|
||||
params=params,
|
||||
json=json_body,
|
||||
headers=headers,
|
||||
timeout=request_timeout,
|
||||
)
|
||||
if 300 <= response.status_code < 400:
|
||||
location = response.headers.get("location")
|
||||
if not location:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_REDIRECT_DENIED",
|
||||
"MobilityOps redirect did not provide a location",
|
||||
)
|
||||
target = urlparse(urljoin(str(response.url), location))
|
||||
base = urlparse(self.settings.base_url)
|
||||
if (target.scheme, target.hostname, target.port) != (
|
||||
base.scheme,
|
||||
base.hostname,
|
||||
base.port,
|
||||
):
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_REDIRECT_DENIED",
|
||||
"MobilityOps redirect leaves the configured origin",
|
||||
)
|
||||
response = self._client.request(
|
||||
method, target.geturl(), headers=headers, timeout=request_timeout
|
||||
)
|
||||
if 300 <= response.status_code < 400:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_REDIRECT_DENIED", "MobilityOps returned multiple redirects"
|
||||
)
|
||||
except MobilityOpsError:
|
||||
raise
|
||||
except (httpx.TimeoutException, httpx.NetworkError) as exc:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_UNAVAILABLE", "MobilityOps target is unavailable"
|
||||
) from exc
|
||||
if len(response.content) > self.settings.max_response_bytes:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_RESPONSE_TOO_LARGE",
|
||||
"MobilityOps response exceeds the connector budget",
|
||||
)
|
||||
if response.status_code == 404:
|
||||
raise MobilityOpsError("MOBILITYOPS_NOT_FOUND", "MobilityOps resource was not found")
|
||||
if response.status_code in {401, 403}:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_ACCESS_DENIED", "MobilityOps denied the read request"
|
||||
)
|
||||
if response.status_code == 422:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_REQUEST_INVALID", "MobilityOps rejected the request shape"
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_REQUEST_FAILED", "MobilityOps returned an unsuccessful response"
|
||||
)
|
||||
try:
|
||||
return response.json()
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_INVALID_RESPONSE", "MobilityOps returned invalid JSON"
|
||||
) from exc
|
||||
|
||||
def operations_summary(
|
||||
self, client_id: str, *, correlation_id: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
return cast(
|
||||
dict[str, Any],
|
||||
self._request(client_id, "GET", "/operations-summary", correlation_id=correlation_id),
|
||||
)
|
||||
|
||||
READINESS_TIMEOUT_SECONDS = 5.0
|
||||
|
||||
def readiness_probe(self, client_id: str, *, correlation_id: str | None = None) -> None:
|
||||
"""Short, bounded, authenticated read used as the connector's readiness signal.
|
||||
|
||||
MobilityOps exposes no authenticated readiness route on this origin -- its only
|
||||
health route (`GET /health`) is unauthenticated and lives outside the
|
||||
`/api/v1/integrations/mcp` prefix this client is bound to -- so the narrowest
|
||||
authenticated read, `operations-summary`, is the readiness probe instead.
|
||||
Raises `MobilityOpsError` with a stable code; never leaks the token or a raw
|
||||
transport exception.
|
||||
"""
|
||||
payload = self._request(
|
||||
client_id,
|
||||
"GET",
|
||||
"/operations-summary",
|
||||
correlation_id=correlation_id,
|
||||
timeout=min(self.settings.timeout_seconds, self.READINESS_TIMEOUT_SECONDS),
|
||||
)
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("metrics"), dict):
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_INVALID_RESPONSE",
|
||||
"MobilityOps readiness response did not carry operational metrics",
|
||||
)
|
||||
|
||||
def attention_vehicles(
|
||||
self,
|
||||
client_id: str,
|
||||
*,
|
||||
minimum_severity: str,
|
||||
date: str | None,
|
||||
limit: int,
|
||||
correlation_id: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
params: dict[str, str | int] = {"minimum_severity": minimum_severity, "limit": limit}
|
||||
if date:
|
||||
params["date"] = date
|
||||
value = self._request(
|
||||
client_id,
|
||||
"GET",
|
||||
"/attention-vehicles",
|
||||
params=params,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_INVALID_RESPONSE", "MobilityOps returned an unexpected collection"
|
||||
)
|
||||
return value
|
||||
|
||||
_VEHICLE_REF = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{0,63}$")
|
||||
|
||||
def vehicle_details(
|
||||
self, client_id: str, vehicle_ref: str, *, correlation_id: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
if not self._VEHICLE_REF.fullmatch(vehicle_ref):
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_VEHICLE_REF_INVALID", "vehicle reference is invalid"
|
||||
)
|
||||
return cast(
|
||||
dict[str, Any],
|
||||
self._request(
|
||||
client_id,
|
||||
"GET",
|
||||
f"/vehicles/{vehicle_ref}",
|
||||
correlation_id=correlation_id,
|
||||
),
|
||||
)
|
||||
|
||||
def search_knowledge(
|
||||
self,
|
||||
client_id: str,
|
||||
question: str,
|
||||
*,
|
||||
max_sources: int,
|
||||
locale: str = DEFAULT_KNOWLEDGE_LOCALE,
|
||||
correlation_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if locale not in KNOWLEDGE_LOCALES:
|
||||
raise MobilityOpsError(
|
||||
"MOBILITYOPS_LOCALE_INVALID",
|
||||
"locale must be one of nl-BE, en-GB, fr-BE",
|
||||
)
|
||||
return cast(
|
||||
dict[str, Any],
|
||||
self._request(
|
||||
client_id,
|
||||
"POST",
|
||||
"/search-knowledge",
|
||||
json_body={
|
||||
"question": question,
|
||||
"max_sources": max_sources,
|
||||
"locale": locale,
|
||||
},
|
||||
correlation_id=correlation_id,
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Opaque, scope-bound pagination cursors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
|
||||
|
||||
class InvalidCursor(ValueError):
|
||||
"""Raised for malformed, cross-query, or out-of-range cursors."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Cursor:
|
||||
offset: int
|
||||
fingerprint: str
|
||||
|
||||
|
||||
def _fingerprint(*parts: str) -> str:
|
||||
return sha256("\x1f".join(parts).encode("utf-8")).hexdigest()[:32]
|
||||
|
||||
|
||||
def encode_cursor(offset: int, *scope: str) -> str:
|
||||
if offset < 0:
|
||||
raise ValueError("cursor offset must be non-negative")
|
||||
payload = {"o": offset, "f": _fingerprint(*scope)}
|
||||
raw = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("ascii")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(value: str | None, *scope: str) -> Cursor:
|
||||
if value is None:
|
||||
return Cursor(0, _fingerprint(*scope))
|
||||
if not 1 <= len(value) <= 500:
|
||||
raise InvalidCursor("cursor length is invalid")
|
||||
try:
|
||||
padded = value + "=" * (-len(value) % 4)
|
||||
payload = json.loads(base64.urlsafe_b64decode(padded).decode("ascii"))
|
||||
offset = payload["o"]
|
||||
fingerprint = payload["f"]
|
||||
except (ValueError, KeyError, TypeError, json.JSONDecodeError) as exc:
|
||||
raise InvalidCursor("cursor is invalid") from exc
|
||||
if type(offset) is not int or offset < 0 or fingerprint != _fingerprint(*scope):
|
||||
raise InvalidCursor("cursor does not match this query")
|
||||
return Cursor(offset, fingerprint)
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Deterministic secret exclusion and safe text normalization for untrusted content."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
_SECRET_PATTERNS = (
|
||||
re.compile(r"-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----"),
|
||||
re.compile(
|
||||
r"(?i)\b(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[:=]\s*[^\s,;]{8,}"
|
||||
),
|
||||
re.compile(r"(?i)\b(?:password|passwd|secret)\s*[:=]\s*['\"]?[^\s'\",;]{8,}"),
|
||||
re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
|
||||
re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
|
||||
)
|
||||
_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
||||
_INLINE_SECRET_PATTERNS = (
|
||||
re.compile(
|
||||
r"(?i)(\b(?:authorization|bearer|api[_-]?key|access[_-]?token|refresh[_-]?token|"
|
||||
r"password|passwd|secret|credential|private[_-]?key)\s*[:=]\s*['\"]?)"
|
||||
r"[^\s'\",;]{4,}"
|
||||
),
|
||||
re.compile(
|
||||
r"(?i)(\b[A-Z][A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASSWD|API_KEY|CREDENTIAL)"
|
||||
r"\s*=\s*['\"]?)[^\s'\",;]{4,}"
|
||||
),
|
||||
re.compile(
|
||||
r"\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|gitea_[A-Za-z0-9_-]{20,})\b"
|
||||
),
|
||||
re.compile(r"(?i)(https?://[^\s/:@]+:)[^\s/@]+@"),
|
||||
)
|
||||
|
||||
|
||||
def normalize_text(value: str) -> str:
|
||||
"""Normalize Unicode and controls without interpreting content."""
|
||||
|
||||
normalized = unicodedata.normalize("NFC", value).replace("\r\n", "\n").replace("\r", "\n")
|
||||
return _CONTROL_RE.sub("", normalized)
|
||||
|
||||
|
||||
def is_secret_like(value: str) -> bool:
|
||||
return any(pattern.search(value) is not None for pattern in _SECRET_PATTERNS)
|
||||
|
||||
|
||||
def safe_content(value: str) -> str | None:
|
||||
"""Return normalized content or None when the whole document must be excluded."""
|
||||
|
||||
normalized = normalize_text(value)
|
||||
return None if is_secret_like(normalized) else normalized
|
||||
|
||||
|
||||
def safe_snippet(value: str, *, max_bytes: int = 2000) -> str:
|
||||
normalized = normalize_text(value)
|
||||
encoded = normalized.encode("utf-8")
|
||||
if len(encoded) <= max_bytes:
|
||||
return normalized
|
||||
return encoded[: max_bytes - 3].decode("utf-8", errors="ignore") + "..."
|
||||
|
||||
|
||||
def redact_sensitive_text(value: str, *, max_bytes: int = 2000) -> str:
|
||||
"""Normalize, truncate and redact credential-shaped inline values."""
|
||||
|
||||
result = safe_snippet(value, max_bytes=max_bytes)
|
||||
for pattern in _INLINE_SECRET_PATTERNS:
|
||||
result = pattern.sub(
|
||||
lambda match: f"{match.group(1)}<redacted>" if match.lastindex else "<redacted>",
|
||||
result,
|
||||
)
|
||||
return result
|
||||
Reference in new issue
Block a user