397 lines
17 KiB
Python
397 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
from collections import Counter
|
|
from datetime import UTC, datetime
|
|
import hashlib
|
|
import math
|
|
from pathlib import Path
|
|
import re
|
|
from threading import RLock
|
|
from typing import Any, Iterable
|
|
|
|
from sqlalchemy import delete, distinct, func
|
|
from sqlmodel import Session, select
|
|
|
|
from ..core.redaction import REDACTION_PATTERNS, redact_message
|
|
from ..core.time import utc_now
|
|
from ..db.models import App, AppSetting, AuditEvent, KnowledgeChunk, NotificationEvent, Policy
|
|
|
|
|
|
_WORD = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{1,}")
|
|
_HEADING = re.compile(r"^(#{1,4})\s+(.+?)\s*$", re.MULTILINE)
|
|
_BLOCKED_FILES: set[str] = set()
|
|
_INDEX_MANIFEST_KEY = "knowledge:index_manifest"
|
|
_MANIFEST_CACHE: dict[str, Any] = {}
|
|
_PERSISTED_CACHE: dict[str, Any] = {}
|
|
_INDEX_LOCK = RLock()
|
|
_STOPWORDS = {
|
|
"aan", "als", "bij", "dat", "de", "den", "der", "die", "dit", "een", "en", "for", "het",
|
|
"hoe", "in", "is", "met", "naar", "of", "om", "op", "te", "the", "to", "van", "voor", "wat",
|
|
}
|
|
|
|
|
|
def _tokens(value: str) -> list[str]:
|
|
return [token.casefold() for token in _WORD.findall(value) if token.casefold() not in _STOPWORDS]
|
|
|
|
|
|
def _semantic_features(value: str) -> set[str]:
|
|
normalized = " ".join(_tokens(value))
|
|
words = normalized.split()
|
|
features = {f"word:{word}" for word in words}
|
|
features.update(f"pair:{words[index]}_{words[index + 1]}" for index in range(len(words) - 1))
|
|
for word in words:
|
|
if len(word) >= 5:
|
|
features.update(f"tri:{word[index:index + 3]}" for index in range(len(word) - 2))
|
|
return features
|
|
|
|
|
|
def _expanded_query(value: str) -> str:
|
|
normalized = value.casefold()
|
|
expansions: list[str] = []
|
|
if "authentic" in normalized:
|
|
expansions.append("operator authentication private deployment security beveiliging")
|
|
if "destruct" in normalized or "geblokkeerd" in normalized:
|
|
expansions.append("destructive blocked policy guardrail blokkeren acties")
|
|
if ("secret" in normalized or "geheim" in normalized) and ("encrypt" in normalized or "opgeslagen" in normalized):
|
|
expansions.append("secret storage encrypted-at-rest Fernet SQLite")
|
|
return " ".join([value, *expansions])
|
|
|
|
|
|
def _safe_text(value: str) -> str:
|
|
value = re.sub(r"<script\b[^>]*>.*?</script>", " ", value, flags=re.IGNORECASE | re.DOTALL)
|
|
value = redact_message(value)
|
|
return re.sub(r"\s+", " ", value).strip()
|
|
|
|
|
|
def _split_markdown(source_uri: str, raw: str) -> Iterable[tuple[str, str, str]]:
|
|
matches = list(_HEADING.finditer(raw))
|
|
title = _safe_text(Path(source_uri).stem.replace("_", " ").replace("-", " ").title())[:200]
|
|
if matches:
|
|
title = _safe_text(matches[0].group(2))[:200]
|
|
boundaries = matches or [re.match(r"", raw)]
|
|
for index, match in enumerate(boundaries):
|
|
start = match.end() if match else 0
|
|
end = boundaries[index + 1].start() if index + 1 < len(boundaries) else len(raw)
|
|
section = _safe_text(match.group(2))[:200] if match and match.lastindex else title
|
|
body = _safe_text(raw[start:end])
|
|
if not body:
|
|
continue
|
|
for offset in range(0, len(body), 1800):
|
|
content = body[offset:offset + 2200].strip()
|
|
if len(content) >= 40:
|
|
yield title, section, content
|
|
|
|
|
|
def _knowledge_files(root: Path) -> list[Path]:
|
|
candidates = [root / "README.md", root / "CHANGELOG.md"]
|
|
docs = root / "docs"
|
|
if docs.is_dir():
|
|
candidates.extend(sorted(docs.glob("*.md")))
|
|
return [
|
|
path for path in candidates[:160]
|
|
if path.is_file() and path.name.casefold() not in _BLOCKED_FILES and path.stat().st_size <= 1_000_000
|
|
]
|
|
|
|
|
|
def _source_manifest(root: Path) -> dict[str, Any]:
|
|
files = _knowledge_files(root)
|
|
stat_signature = tuple(
|
|
(path.relative_to(root).as_posix(), path.stat().st_size, path.stat().st_mtime_ns)
|
|
for path in files
|
|
)
|
|
cached = _MANIFEST_CACHE.get(str(root))
|
|
if cached and cached.get("statSignature") == stat_signature:
|
|
return dict(cached["manifest"])
|
|
digest = hashlib.sha256()
|
|
for path in files:
|
|
digest.update(path.relative_to(root).as_posix().encode("utf-8"))
|
|
digest.update(b"\0")
|
|
digest.update(path.read_bytes())
|
|
digest.update(b"\0")
|
|
manifest = {"hash": digest.hexdigest(), "sourceCount": len(files)}
|
|
_MANIFEST_CACHE[str(root)] = {"statSignature": stat_signature, "manifest": manifest}
|
|
return dict(manifest)
|
|
|
|
|
|
def _redaction_match_count(value: str) -> int:
|
|
return sum(len(pattern.findall(value)) for pattern in REDACTION_PATTERNS)
|
|
|
|
|
|
def _compact_excerpt(value: str, limit: int = 360) -> str:
|
|
sentences = re.split(r"(?<=[.!?])\s+", value.strip())
|
|
summary = " ".join(sentences[:2]).strip()
|
|
candidate = summary or value.strip()
|
|
return candidate if len(candidate) <= limit else f"{candidate[:limit - 1].rstrip()}…"
|
|
|
|
|
|
def _rebuild_index(session: Session, root: Path | None = None) -> dict[str, Any]:
|
|
root = (root or Path.cwd()).resolve()
|
|
manifest = _source_manifest(root)
|
|
rows: list[KnowledgeChunk] = []
|
|
sources: set[str] = set()
|
|
redacted_matches = 0
|
|
for path in _knowledge_files(root):
|
|
relative = path.relative_to(root).as_posix()
|
|
source_uri = f"appops-doc://{relative}"
|
|
raw = path.read_text(encoding="utf-8", errors="replace")
|
|
redacted_matches += _redaction_match_count(raw)
|
|
source_has_chunks = False
|
|
for ordinal, (title, section, content) in enumerate(_split_markdown(relative, raw)):
|
|
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
rows.append(KnowledgeChunk(
|
|
id=f"kc-{hashlib.sha256(f'{source_uri}:{ordinal}:{digest}'.encode()).hexdigest()[:20]}",
|
|
source_uri=source_uri,
|
|
source_type="documentation",
|
|
title=title,
|
|
section=section,
|
|
content=content,
|
|
content_hash=digest,
|
|
))
|
|
source_has_chunks = True
|
|
if source_has_chunks:
|
|
sources.add(source_uri)
|
|
session.exec(delete(KnowledgeChunk))
|
|
session.add_all(rows)
|
|
setting = session.get(AppSetting, _INDEX_MANIFEST_KEY)
|
|
if not setting:
|
|
setting = AppSetting(key=_INDEX_MANIFEST_KEY, is_secret=False)
|
|
generated_at = datetime.now(UTC).isoformat()
|
|
setting.value_json = {
|
|
"manifestHash": manifest["hash"],
|
|
"sourceCount": len(sources),
|
|
"chunkCount": len(rows),
|
|
"generatedAt": generated_at,
|
|
"redactedMatches": redacted_matches,
|
|
"secretScanStatus": "passed_with_redactions" if redacted_matches else "passed",
|
|
}
|
|
setting.updated_at = utc_now()
|
|
session.add(setting)
|
|
session.commit()
|
|
_PERSISTED_CACHE.clear()
|
|
return {
|
|
"status": "ready",
|
|
"engine": "local_hybrid_lexical_semantic",
|
|
"chunks": len(rows),
|
|
"sources": len(sources),
|
|
"generatedAt": generated_at,
|
|
"manifestHash": manifest["hash"],
|
|
"redactedMatches": redacted_matches,
|
|
"secretScanStatus": setting.value_json["secretScanStatus"],
|
|
}
|
|
|
|
|
|
def rebuild_index(session: Session, root: Path | None = None) -> dict[str, Any]:
|
|
with _INDEX_LOCK:
|
|
return _rebuild_index(session, root)
|
|
|
|
|
|
def index_status(session: Session) -> dict[str, Any]:
|
|
chunks, sources, last_indexed = session.exec(
|
|
select(
|
|
func.count(KnowledgeChunk.id),
|
|
func.count(distinct(KnowledgeChunk.source_uri)),
|
|
func.max(KnowledgeChunk.updated_at),
|
|
)
|
|
).one()
|
|
manifest = session.get(AppSetting, _INDEX_MANIFEST_KEY)
|
|
metadata = manifest.value_json if manifest and isinstance(manifest.value_json, dict) else {}
|
|
return {
|
|
"status": "ready" if chunks else "empty",
|
|
"engine": "local_hybrid_lexical_semantic",
|
|
"chunks": chunks,
|
|
"sources": sources,
|
|
"lastIndexedAt": metadata.get("generatedAt") or last_indexed,
|
|
"secretsIndexed": False,
|
|
"secretScanStatus": metadata.get("secretScanStatus", "unknown"),
|
|
"redactedMatches": int(metadata.get("redactedMatches", 0)),
|
|
"manifestHash": metadata.get("manifestHash"),
|
|
"generationMode": "extractive_grounded",
|
|
}
|
|
|
|
|
|
def ensure_index_current(session: Session, root: Path | None = None) -> dict[str, Any]:
|
|
with _INDEX_LOCK:
|
|
root = (root or Path.cwd()).resolve()
|
|
current = _source_manifest(root)
|
|
stored = session.get(AppSetting, _INDEX_MANIFEST_KEY)
|
|
stored_hash = stored.value_json.get("manifestHash") if stored and isinstance(stored.value_json, dict) else None
|
|
has_chunks = session.exec(select(KnowledgeChunk.id).limit(1)).first() is not None
|
|
if not has_chunks or stored_hash != current["hash"]:
|
|
return _rebuild_index(session, root)
|
|
return index_status(session)
|
|
|
|
|
|
def _runtime_chunks(session: Session) -> list[dict[str, str]]:
|
|
chunks: list[dict[str, str]] = []
|
|
for app in session.exec(select(App).order_by(App.name)).all():
|
|
chunks.append({
|
|
"id": f"runtime-app-{app.id}", "source_uri": f"appops://apps/{app.id}", "source_type": "live_app",
|
|
"title": app.name, "section": "Actuele appstatus",
|
|
"content": _safe_text(
|
|
f"App {app.name}; categorie {app.category}; status {app.status}; criticality {app.criticality}; "
|
|
f"connector {app.connector_status}; container {app.container_status}; protected {app.protected}; review {app.needs_review}."
|
|
),
|
|
})
|
|
for policy in session.exec(select(Policy).order_by(Policy.action)).all():
|
|
chunks.append({
|
|
"id": f"runtime-policy-{policy.id}", "source_uri": f"appops://policies/{policy.id}", "source_type": "live_policy",
|
|
"title": policy.action, "section": "Actuele policy",
|
|
"content": _safe_text(
|
|
f"Actie {policy.action}; target {policy.target}; manual {policy.manual}; voice {policy.voice}; "
|
|
f"risk {policy.risk}; reden {policy.reason or 'geen'}"
|
|
),
|
|
})
|
|
for event in session.exec(select(AuditEvent).order_by(AuditEvent.timestamp.desc()).limit(80)).all():
|
|
chunks.append({
|
|
"id": f"runtime-audit-{event.id}", "source_uri": f"appops://audit/{event.id}", "source_type": "live_audit",
|
|
"title": event.action_id, "section": "Recent audit-event",
|
|
"content": _safe_text(
|
|
f"{event.timestamp.isoformat()} actie {event.action_id}; target {event.target}; bron {event.source}; "
|
|
f"policy {event.policy_decision}; resultaat {event.result}; reden {event.reason or 'geen'}"
|
|
),
|
|
})
|
|
for event in session.exec(select(NotificationEvent).order_by(NotificationEvent.created_at.desc()).limit(40)).all():
|
|
chunks.append({
|
|
"id": f"runtime-notification-{event.id}", "source_uri": f"appops://notifications/{event.id}", "source_type": "live_notification",
|
|
"title": _safe_text(event.title), "section": "Recente notificatie",
|
|
"content": _safe_text(f"{event.severity} {event.source} {event.category or ''}: {event.message}; status {event.status}"),
|
|
})
|
|
return chunks
|
|
|
|
|
|
def query(session: Session, question: str, *, limit: int = 6, root: Path | None = None) -> dict[str, Any]:
|
|
question = _safe_text(question)[:500]
|
|
if len(question) < 2:
|
|
raise ValueError("question_required")
|
|
status = ensure_index_current(session, root)
|
|
cache_key = str(status.get("manifestHash") or status.get("lastIndexedAt") or "empty")
|
|
if _PERSISTED_CACHE.get("key") != cache_key:
|
|
persisted = [
|
|
{
|
|
"id": row.id, "source_uri": row.source_uri, "source_type": row.source_type,
|
|
"title": row.title, "section": row.section or row.title, "content": row.content,
|
|
}
|
|
for row in session.exec(select(KnowledgeChunk)).all()
|
|
]
|
|
_PERSISTED_CACHE.update({
|
|
"key": cache_key,
|
|
"items": persisted,
|
|
"terms": [Counter(_tokens(f"{item['title']} {item['section']} {item['content']}")) for item in persisted],
|
|
"features": [_semantic_features(f"{item['title']} {item['section']} {item['content']}") for item in persisted],
|
|
})
|
|
persisted = list(_PERSISTED_CACHE.get("items", []))
|
|
persisted_terms = list(_PERSISTED_CACHE.get("terms", []))
|
|
persisted_features = list(_PERSISTED_CACHE.get("features", []))
|
|
candidates = persisted + _runtime_chunks(session)
|
|
expanded_question = _expanded_query(question)
|
|
query_terms = Counter(_tokens(expanded_question))
|
|
if not query_terms:
|
|
raise ValueError("question_has_no_search_terms")
|
|
document_frequency: Counter[str] = Counter()
|
|
tokenized: list[tuple[dict[str, str], Counter[str], set[str]]] = []
|
|
for index, item in enumerate(candidates):
|
|
terms = persisted_terms[index] if index < len(persisted_terms) else Counter(
|
|
_tokens(f"{item['title']} {item['section']} {item['content']}")
|
|
)
|
|
features = persisted_features[index] if index < len(persisted_features) else _semantic_features(
|
|
f"{item['title']} {item['section']} {item['content']}"
|
|
)
|
|
tokenized.append((item, terms, features))
|
|
document_frequency.update(terms.keys())
|
|
scored: list[tuple[float, dict[str, str]]] = []
|
|
total = max(1, len(tokenized))
|
|
query_features = _semantic_features(expanded_question)
|
|
for item, terms, features in tokenized:
|
|
score = 0.0
|
|
title_terms = set(_tokens(f"{item['title']} {item['section']}"))
|
|
for term, query_count in query_terms.items():
|
|
if not terms[term]:
|
|
continue
|
|
idf = math.log((total + 1) / (document_frequency[term] + 1)) + 1
|
|
score += query_count * (1 + math.log(terms[term])) * idf
|
|
if term in title_terms:
|
|
score += 1.5 * idf
|
|
overlap = len(query_features.intersection(features))
|
|
semantic_score = overlap / max(1, math.sqrt(len(query_features) * len(features)))
|
|
score += semantic_score * 4.0
|
|
if score > 0.15:
|
|
scored.append((score, item))
|
|
selected: list[tuple[float, dict[str, str]]] = []
|
|
source_counts: Counter[str] = Counter()
|
|
requested_limit = max(1, min(limit, 10))
|
|
for score, item in sorted(scored, key=lambda value: value[0], reverse=True):
|
|
source_uri = item["source_uri"]
|
|
if source_counts[source_uri] >= 2:
|
|
continue
|
|
selected.append((score, item))
|
|
source_counts[source_uri] += 1
|
|
if len(selected) >= requested_limit:
|
|
break
|
|
results = []
|
|
for rank, (score, item) in enumerate(selected, start=1):
|
|
results.append({
|
|
"id": item["id"], "rank": rank, "title": item["title"], "section": item["section"],
|
|
"uri": item["source_uri"], "sourceType": item["source_type"], "score": round(score, 3),
|
|
"excerpt": item["content"][:700],
|
|
})
|
|
if not results:
|
|
answer = "Er is geen voldoende relevante, brongebonden context gevonden. Er wordt geen antwoord gegokt."
|
|
else:
|
|
evidence = "\n".join(
|
|
f"[{result['rank']}] {result['title']}: {_compact_excerpt(result['excerpt'])}"
|
|
for result in results[:3]
|
|
)
|
|
answer = f"Gevonden broncontext:\n{evidence}"
|
|
return {
|
|
"question": question,
|
|
"answer": answer,
|
|
"confidence": "high" if len(results) >= 3 else "medium" if results else "none",
|
|
"citations": results,
|
|
"guardrails": {
|
|
"readOnly": True,
|
|
"secretsExcluded": True,
|
|
"instructionsFromSourcesIgnored": True,
|
|
"executionViaPolicyOnly": True,
|
|
},
|
|
"retrieval": {
|
|
"engine": "local_hybrid_lexical_semantic",
|
|
"vectorSpace": "normalized_binary_word_bigram_char_trigram",
|
|
"externalEmbeddingProvider": False,
|
|
},
|
|
}
|
|
|
|
|
|
_RETRIEVAL_EVALUATION_CASES = (
|
|
("Hoe werkt private operator authenticatie?", ("PRIVATE_DEPLOYMENT_SECURITY", "API_CONTRACT")),
|
|
("Hoe controleer en herstel ik een SQLite backup?", ("BACKUP_RESTORE", "RELEASE_CHECKLIST")),
|
|
("Hoe zijn destructieve acties geblokkeerd?", ("API_CONTRACT", "HARDENING")),
|
|
("Hoe koppelen MCP, RAG en n8n?", ("MCP_RAG_N8N_INTEGRATION",)),
|
|
("Hoe worden connector secrets encrypted-at-rest opgeslagen?", ("SECRET_STORAGE",)),
|
|
)
|
|
|
|
|
|
def evaluate_retrieval(session: Session) -> dict[str, Any]:
|
|
cases = []
|
|
for question, expected_sources in _RETRIEVAL_EVALUATION_CASES:
|
|
result = query(session, question, limit=5)
|
|
uris = [str(item.get("uri") or "") for item in result.get("citations", [])]
|
|
passed = any(expected.lower() in uri.lower() for expected in expected_sources for uri in uris)
|
|
cases.append({
|
|
"question": question,
|
|
"passed": passed,
|
|
"expectedSourceHints": list(expected_sources),
|
|
"topSourceUris": uris[:3],
|
|
"citationCount": len(uris),
|
|
})
|
|
passed_count = sum(1 for case in cases if case["passed"])
|
|
return {
|
|
"status": "passed" if passed_count == len(cases) else "needs_tuning",
|
|
"evaluationVersion": 2,
|
|
"engine": "local_hybrid_lexical_semantic",
|
|
"vectorSpace": "normalized_binary_word_bigram_char_trigram",
|
|
"passed": passed_count,
|
|
"total": len(cases),
|
|
"passRate": round(passed_count / max(1, len(cases)), 3),
|
|
"cases": cases,
|
|
}
|