Initial curated Unraid-AppOps-Gateway source from 3bd6f6fee310125bfbb4d4650389a817f498724d
This commit is contained in:
Public Source Publisher committed
commit
1dfbca08a1
528 files changed
+131054
No files matched your search
@@ -0,0 +1,396 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
from threading import RLock
|
||||
from typing import Any, Iterable
|
||||
|
||||
from sqlalchemy import delete, distinct, func
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ..core.redaction import REDACTION_PATTERNS, redact_message
|
||||
from ..core.time import utc_now
|
||||
from ..db.models import App, AppSetting, AuditEvent, KnowledgeChunk, NotificationEvent, Policy
|
||||
|
||||
|
||||
_WORD = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{1,}")
|
||||
_HEADING = re.compile(r"^(#{1,4})\s+(.+?)\s*$", re.MULTILINE)
|
||||
_BLOCKED_FILES: set[str] = set()
|
||||
_INDEX_MANIFEST_KEY = "knowledge:index_manifest"
|
||||
_MANIFEST_CACHE: dict[str, Any] = {}
|
||||
_PERSISTED_CACHE: dict[str, Any] = {}
|
||||
_INDEX_LOCK = RLock()
|
||||
_STOPWORDS = {
|
||||
"aan", "als", "bij", "dat", "de", "den", "der", "die", "dit", "een", "en", "for", "het",
|
||||
"hoe", "in", "is", "met", "naar", "of", "om", "op", "te", "the", "to", "van", "voor", "wat",
|
||||
}
|
||||
|
||||
|
||||
def _tokens(value: str) -> list[str]:
|
||||
return [token.casefold() for token in _WORD.findall(value) if token.casefold() not in _STOPWORDS]
|
||||
|
||||
|
||||
def _semantic_features(value: str) -> set[str]:
|
||||
normalized = " ".join(_tokens(value))
|
||||
words = normalized.split()
|
||||
features = {f"word:{word}" for word in words}
|
||||
features.update(f"pair:{words[index]}_{words[index + 1]}" for index in range(len(words) - 1))
|
||||
for word in words:
|
||||
if len(word) >= 5:
|
||||
features.update(f"tri:{word[index:index + 3]}" for index in range(len(word) - 2))
|
||||
return features
|
||||
|
||||
|
||||
def _expanded_query(value: str) -> str:
|
||||
normalized = value.casefold()
|
||||
expansions: list[str] = []
|
||||
if "authentic" in normalized:
|
||||
expansions.append("operator authentication private deployment security beveiliging")
|
||||
if "destruct" in normalized or "geblokkeerd" in normalized:
|
||||
expansions.append("destructive blocked policy guardrail blokkeren acties")
|
||||
if ("secret" in normalized or "geheim" in normalized) and ("encrypt" in normalized or "opgeslagen" in normalized):
|
||||
expansions.append("secret storage encrypted-at-rest Fernet SQLite")
|
||||
return " ".join([value, *expansions])
|
||||
|
||||
|
||||
def _safe_text(value: str) -> str:
|
||||
value = re.sub(r"<script\b[^>]*>.*?</script>", " ", value, flags=re.IGNORECASE | re.DOTALL)
|
||||
value = redact_message(value)
|
||||
return re.sub(r"\s+", " ", value).strip()
|
||||
|
||||
|
||||
def _split_markdown(source_uri: str, raw: str) -> Iterable[tuple[str, str, str]]:
|
||||
matches = list(_HEADING.finditer(raw))
|
||||
title = _safe_text(Path(source_uri).stem.replace("_", " ").replace("-", " ").title())[:200]
|
||||
if matches:
|
||||
title = _safe_text(matches[0].group(2))[:200]
|
||||
boundaries = matches or [re.match(r"", raw)]
|
||||
for index, match in enumerate(boundaries):
|
||||
start = match.end() if match else 0
|
||||
end = boundaries[index + 1].start() if index + 1 < len(boundaries) else len(raw)
|
||||
section = _safe_text(match.group(2))[:200] if match and match.lastindex else title
|
||||
body = _safe_text(raw[start:end])
|
||||
if not body:
|
||||
continue
|
||||
for offset in range(0, len(body), 1800):
|
||||
content = body[offset:offset + 2200].strip()
|
||||
if len(content) >= 40:
|
||||
yield title, section, content
|
||||
|
||||
|
||||
def _knowledge_files(root: Path) -> list[Path]:
|
||||
candidates = [root / "README.md", root / "CHANGELOG.md"]
|
||||
docs = root / "docs"
|
||||
if docs.is_dir():
|
||||
candidates.extend(sorted(docs.glob("*.md")))
|
||||
return [
|
||||
path for path in candidates[:160]
|
||||
if path.is_file() and path.name.casefold() not in _BLOCKED_FILES and path.stat().st_size <= 1_000_000
|
||||
]
|
||||
|
||||
|
||||
def _source_manifest(root: Path) -> dict[str, Any]:
|
||||
files = _knowledge_files(root)
|
||||
stat_signature = tuple(
|
||||
(path.relative_to(root).as_posix(), path.stat().st_size, path.stat().st_mtime_ns)
|
||||
for path in files
|
||||
)
|
||||
cached = _MANIFEST_CACHE.get(str(root))
|
||||
if cached and cached.get("statSignature") == stat_signature:
|
||||
return dict(cached["manifest"])
|
||||
digest = hashlib.sha256()
|
||||
for path in files:
|
||||
digest.update(path.relative_to(root).as_posix().encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(path.read_bytes())
|
||||
digest.update(b"\0")
|
||||
manifest = {"hash": digest.hexdigest(), "sourceCount": len(files)}
|
||||
_MANIFEST_CACHE[str(root)] = {"statSignature": stat_signature, "manifest": manifest}
|
||||
return dict(manifest)
|
||||
|
||||
|
||||
def _redaction_match_count(value: str) -> int:
|
||||
return sum(len(pattern.findall(value)) for pattern in REDACTION_PATTERNS)
|
||||
|
||||
|
||||
def _compact_excerpt(value: str, limit: int = 360) -> str:
|
||||
sentences = re.split(r"(?<=[.!?])\s+", value.strip())
|
||||
summary = " ".join(sentences[:2]).strip()
|
||||
candidate = summary or value.strip()
|
||||
return candidate if len(candidate) <= limit else f"{candidate[:limit - 1].rstrip()}…"
|
||||
|
||||
|
||||
def _rebuild_index(session: Session, root: Path | None = None) -> dict[str, Any]:
|
||||
root = (root or Path.cwd()).resolve()
|
||||
manifest = _source_manifest(root)
|
||||
rows: list[KnowledgeChunk] = []
|
||||
sources: set[str] = set()
|
||||
redacted_matches = 0
|
||||
for path in _knowledge_files(root):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
source_uri = f"appops-doc://{relative}"
|
||||
raw = path.read_text(encoding="utf-8", errors="replace")
|
||||
redacted_matches += _redaction_match_count(raw)
|
||||
source_has_chunks = False
|
||||
for ordinal, (title, section, content) in enumerate(_split_markdown(relative, raw)):
|
||||
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
rows.append(KnowledgeChunk(
|
||||
id=f"kc-{hashlib.sha256(f'{source_uri}:{ordinal}:{digest}'.encode()).hexdigest()[:20]}",
|
||||
source_uri=source_uri,
|
||||
source_type="documentation",
|
||||
title=title,
|
||||
section=section,
|
||||
content=content,
|
||||
content_hash=digest,
|
||||
))
|
||||
source_has_chunks = True
|
||||
if source_has_chunks:
|
||||
sources.add(source_uri)
|
||||
session.exec(delete(KnowledgeChunk))
|
||||
session.add_all(rows)
|
||||
setting = session.get(AppSetting, _INDEX_MANIFEST_KEY)
|
||||
if not setting:
|
||||
setting = AppSetting(key=_INDEX_MANIFEST_KEY, is_secret=False)
|
||||
generated_at = datetime.now(UTC).isoformat()
|
||||
setting.value_json = {
|
||||
"manifestHash": manifest["hash"],
|
||||
"sourceCount": len(sources),
|
||||
"chunkCount": len(rows),
|
||||
"generatedAt": generated_at,
|
||||
"redactedMatches": redacted_matches,
|
||||
"secretScanStatus": "passed_with_redactions" if redacted_matches else "passed",
|
||||
}
|
||||
setting.updated_at = utc_now()
|
||||
session.add(setting)
|
||||
session.commit()
|
||||
_PERSISTED_CACHE.clear()
|
||||
return {
|
||||
"status": "ready",
|
||||
"engine": "local_hybrid_lexical_semantic",
|
||||
"chunks": len(rows),
|
||||
"sources": len(sources),
|
||||
"generatedAt": generated_at,
|
||||
"manifestHash": manifest["hash"],
|
||||
"redactedMatches": redacted_matches,
|
||||
"secretScanStatus": setting.value_json["secretScanStatus"],
|
||||
}
|
||||
|
||||
|
||||
def rebuild_index(session: Session, root: Path | None = None) -> dict[str, Any]:
|
||||
with _INDEX_LOCK:
|
||||
return _rebuild_index(session, root)
|
||||
|
||||
|
||||
def index_status(session: Session) -> dict[str, Any]:
|
||||
chunks, sources, last_indexed = session.exec(
|
||||
select(
|
||||
func.count(KnowledgeChunk.id),
|
||||
func.count(distinct(KnowledgeChunk.source_uri)),
|
||||
func.max(KnowledgeChunk.updated_at),
|
||||
)
|
||||
).one()
|
||||
manifest = session.get(AppSetting, _INDEX_MANIFEST_KEY)
|
||||
metadata = manifest.value_json if manifest and isinstance(manifest.value_json, dict) else {}
|
||||
return {
|
||||
"status": "ready" if chunks else "empty",
|
||||
"engine": "local_hybrid_lexical_semantic",
|
||||
"chunks": chunks,
|
||||
"sources": sources,
|
||||
"lastIndexedAt": metadata.get("generatedAt") or last_indexed,
|
||||
"secretsIndexed": False,
|
||||
"secretScanStatus": metadata.get("secretScanStatus", "unknown"),
|
||||
"redactedMatches": int(metadata.get("redactedMatches", 0)),
|
||||
"manifestHash": metadata.get("manifestHash"),
|
||||
"generationMode": "extractive_grounded",
|
||||
}
|
||||
|
||||
|
||||
def ensure_index_current(session: Session, root: Path | None = None) -> dict[str, Any]:
|
||||
with _INDEX_LOCK:
|
||||
root = (root or Path.cwd()).resolve()
|
||||
current = _source_manifest(root)
|
||||
stored = session.get(AppSetting, _INDEX_MANIFEST_KEY)
|
||||
stored_hash = stored.value_json.get("manifestHash") if stored and isinstance(stored.value_json, dict) else None
|
||||
has_chunks = session.exec(select(KnowledgeChunk.id).limit(1)).first() is not None
|
||||
if not has_chunks or stored_hash != current["hash"]:
|
||||
return _rebuild_index(session, root)
|
||||
return index_status(session)
|
||||
|
||||
|
||||
def _runtime_chunks(session: Session) -> list[dict[str, str]]:
|
||||
chunks: list[dict[str, str]] = []
|
||||
for app in session.exec(select(App).order_by(App.name)).all():
|
||||
chunks.append({
|
||||
"id": f"runtime-app-{app.id}", "source_uri": f"appops://apps/{app.id}", "source_type": "live_app",
|
||||
"title": app.name, "section": "Actuele appstatus",
|
||||
"content": _safe_text(
|
||||
f"App {app.name}; categorie {app.category}; status {app.status}; criticality {app.criticality}; "
|
||||
f"connector {app.connector_status}; container {app.container_status}; protected {app.protected}; review {app.needs_review}."
|
||||
),
|
||||
})
|
||||
for policy in session.exec(select(Policy).order_by(Policy.action)).all():
|
||||
chunks.append({
|
||||
"id": f"runtime-policy-{policy.id}", "source_uri": f"appops://policies/{policy.id}", "source_type": "live_policy",
|
||||
"title": policy.action, "section": "Actuele policy",
|
||||
"content": _safe_text(
|
||||
f"Actie {policy.action}; target {policy.target}; manual {policy.manual}; voice {policy.voice}; "
|
||||
f"risk {policy.risk}; reden {policy.reason or 'geen'}"
|
||||
),
|
||||
})
|
||||
for event in session.exec(select(AuditEvent).order_by(AuditEvent.timestamp.desc()).limit(80)).all():
|
||||
chunks.append({
|
||||
"id": f"runtime-audit-{event.id}", "source_uri": f"appops://audit/{event.id}", "source_type": "live_audit",
|
||||
"title": event.action_id, "section": "Recent audit-event",
|
||||
"content": _safe_text(
|
||||
f"{event.timestamp.isoformat()} actie {event.action_id}; target {event.target}; bron {event.source}; "
|
||||
f"policy {event.policy_decision}; resultaat {event.result}; reden {event.reason or 'geen'}"
|
||||
),
|
||||
})
|
||||
for event in session.exec(select(NotificationEvent).order_by(NotificationEvent.created_at.desc()).limit(40)).all():
|
||||
chunks.append({
|
||||
"id": f"runtime-notification-{event.id}", "source_uri": f"appops://notifications/{event.id}", "source_type": "live_notification",
|
||||
"title": _safe_text(event.title), "section": "Recente notificatie",
|
||||
"content": _safe_text(f"{event.severity} {event.source} {event.category or ''}: {event.message}; status {event.status}"),
|
||||
})
|
||||
return chunks
|
||||
|
||||
|
||||
def query(session: Session, question: str, *, limit: int = 6, root: Path | None = None) -> dict[str, Any]:
|
||||
question = _safe_text(question)[:500]
|
||||
if len(question) < 2:
|
||||
raise ValueError("question_required")
|
||||
status = ensure_index_current(session, root)
|
||||
cache_key = str(status.get("manifestHash") or status.get("lastIndexedAt") or "empty")
|
||||
if _PERSISTED_CACHE.get("key") != cache_key:
|
||||
persisted = [
|
||||
{
|
||||
"id": row.id, "source_uri": row.source_uri, "source_type": row.source_type,
|
||||
"title": row.title, "section": row.section or row.title, "content": row.content,
|
||||
}
|
||||
for row in session.exec(select(KnowledgeChunk)).all()
|
||||
]
|
||||
_PERSISTED_CACHE.update({
|
||||
"key": cache_key,
|
||||
"items": persisted,
|
||||
"terms": [Counter(_tokens(f"{item['title']} {item['section']} {item['content']}")) for item in persisted],
|
||||
"features": [_semantic_features(f"{item['title']} {item['section']} {item['content']}") for item in persisted],
|
||||
})
|
||||
persisted = list(_PERSISTED_CACHE.get("items", []))
|
||||
persisted_terms = list(_PERSISTED_CACHE.get("terms", []))
|
||||
persisted_features = list(_PERSISTED_CACHE.get("features", []))
|
||||
candidates = persisted + _runtime_chunks(session)
|
||||
expanded_question = _expanded_query(question)
|
||||
query_terms = Counter(_tokens(expanded_question))
|
||||
if not query_terms:
|
||||
raise ValueError("question_has_no_search_terms")
|
||||
document_frequency: Counter[str] = Counter()
|
||||
tokenized: list[tuple[dict[str, str], Counter[str], set[str]]] = []
|
||||
for index, item in enumerate(candidates):
|
||||
terms = persisted_terms[index] if index < len(persisted_terms) else Counter(
|
||||
_tokens(f"{item['title']} {item['section']} {item['content']}")
|
||||
)
|
||||
features = persisted_features[index] if index < len(persisted_features) else _semantic_features(
|
||||
f"{item['title']} {item['section']} {item['content']}"
|
||||
)
|
||||
tokenized.append((item, terms, features))
|
||||
document_frequency.update(terms.keys())
|
||||
scored: list[tuple[float, dict[str, str]]] = []
|
||||
total = max(1, len(tokenized))
|
||||
query_features = _semantic_features(expanded_question)
|
||||
for item, terms, features in tokenized:
|
||||
score = 0.0
|
||||
title_terms = set(_tokens(f"{item['title']} {item['section']}"))
|
||||
for term, query_count in query_terms.items():
|
||||
if not terms[term]:
|
||||
continue
|
||||
idf = math.log((total + 1) / (document_frequency[term] + 1)) + 1
|
||||
score += query_count * (1 + math.log(terms[term])) * idf
|
||||
if term in title_terms:
|
||||
score += 1.5 * idf
|
||||
overlap = len(query_features.intersection(features))
|
||||
semantic_score = overlap / max(1, math.sqrt(len(query_features) * len(features)))
|
||||
score += semantic_score * 4.0
|
||||
if score > 0.15:
|
||||
scored.append((score, item))
|
||||
selected: list[tuple[float, dict[str, str]]] = []
|
||||
source_counts: Counter[str] = Counter()
|
||||
requested_limit = max(1, min(limit, 10))
|
||||
for score, item in sorted(scored, key=lambda value: value[0], reverse=True):
|
||||
source_uri = item["source_uri"]
|
||||
if source_counts[source_uri] >= 2:
|
||||
continue
|
||||
selected.append((score, item))
|
||||
source_counts[source_uri] += 1
|
||||
if len(selected) >= requested_limit:
|
||||
break
|
||||
results = []
|
||||
for rank, (score, item) in enumerate(selected, start=1):
|
||||
results.append({
|
||||
"id": item["id"], "rank": rank, "title": item["title"], "section": item["section"],
|
||||
"uri": item["source_uri"], "sourceType": item["source_type"], "score": round(score, 3),
|
||||
"excerpt": item["content"][:700],
|
||||
})
|
||||
if not results:
|
||||
answer = "Er is geen voldoende relevante, brongebonden context gevonden. Er wordt geen antwoord gegokt."
|
||||
else:
|
||||
evidence = "\n".join(
|
||||
f"[{result['rank']}] {result['title']}: {_compact_excerpt(result['excerpt'])}"
|
||||
for result in results[:3]
|
||||
)
|
||||
answer = f"Gevonden broncontext:\n{evidence}"
|
||||
return {
|
||||
"question": question,
|
||||
"answer": answer,
|
||||
"confidence": "high" if len(results) >= 3 else "medium" if results else "none",
|
||||
"citations": results,
|
||||
"guardrails": {
|
||||
"readOnly": True,
|
||||
"secretsExcluded": True,
|
||||
"instructionsFromSourcesIgnored": True,
|
||||
"executionViaPolicyOnly": True,
|
||||
},
|
||||
"retrieval": {
|
||||
"engine": "local_hybrid_lexical_semantic",
|
||||
"vectorSpace": "normalized_binary_word_bigram_char_trigram",
|
||||
"externalEmbeddingProvider": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
_RETRIEVAL_EVALUATION_CASES = (
|
||||
("Hoe werkt private operator authenticatie?", ("PRIVATE_DEPLOYMENT_SECURITY", "API_CONTRACT")),
|
||||
("Hoe controleer en herstel ik een SQLite backup?", ("BACKUP_RESTORE", "RELEASE_CHECKLIST")),
|
||||
("Hoe zijn destructieve acties geblokkeerd?", ("API_CONTRACT", "HARDENING")),
|
||||
("Hoe koppelen MCP, RAG en n8n?", ("MCP_RAG_N8N_INTEGRATION",)),
|
||||
("Hoe worden connector secrets encrypted-at-rest opgeslagen?", ("SECRET_STORAGE",)),
|
||||
)
|
||||
|
||||
|
||||
def evaluate_retrieval(session: Session) -> dict[str, Any]:
|
||||
cases = []
|
||||
for question, expected_sources in _RETRIEVAL_EVALUATION_CASES:
|
||||
result = query(session, question, limit=5)
|
||||
uris = [str(item.get("uri") or "") for item in result.get("citations", [])]
|
||||
passed = any(expected.lower() in uri.lower() for expected in expected_sources for uri in uris)
|
||||
cases.append({
|
||||
"question": question,
|
||||
"passed": passed,
|
||||
"expectedSourceHints": list(expected_sources),
|
||||
"topSourceUris": uris[:3],
|
||||
"citationCount": len(uris),
|
||||
})
|
||||
passed_count = sum(1 for case in cases if case["passed"])
|
||||
return {
|
||||
"status": "passed" if passed_count == len(cases) else "needs_tuning",
|
||||
"evaluationVersion": 2,
|
||||
"engine": "local_hybrid_lexical_semantic",
|
||||
"vectorSpace": "normalized_binary_word_bigram_char_trigram",
|
||||
"passed": passed_count,
|
||||
"total": len(cases),
|
||||
"passRate": round(passed_count / max(1, len(cases)), 3),
|
||||
"cases": cases,
|
||||
}
|
||||
Reference in new issue
Block a user