from __future__ import annotations

from collections import Counter
from datetime import UTC, datetime
import hashlib
import math
from pathlib import Path
import re
from threading import RLock
from typing import Any, Iterable

from sqlalchemy import delete, distinct, func
from sqlmodel import Session, select

from ..core.redaction import REDACTION_PATTERNS, redact_message
from ..core.time import utc_now
from ..db.models import App, AppSetting, AuditEvent, KnowledgeChunk, NotificationEvent, Policy


_WORD = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{1,}")
_HEADING = re.compile(r"^(#{1,4})\s+(.+?)\s*$", re.MULTILINE)
_BLOCKED_FILES: set[str] = set()
_INDEX_MANIFEST_KEY = "knowledge:index_manifest"
_MANIFEST_CACHE: dict[str, Any] = {}
_PERSISTED_CACHE: dict[str, Any] = {}
_INDEX_LOCK = RLock()
_STOPWORDS = {
    "aan", "als", "bij", "dat", "de", "den", "der", "die", "dit", "een", "en", "for", "het",
    "hoe", "in", "is", "met", "naar", "of", "om", "op", "te", "the", "to", "van", "voor", "wat",
}


def _tokens(value: str) -> list[str]:
    return [token.casefold() for token in _WORD.findall(value) if token.casefold() not in _STOPWORDS]


def _semantic_features(value: str) -> set[str]:
    normalized = " ".join(_tokens(value))
    words = normalized.split()
    features = {f"word:{word}" for word in words}
    features.update(f"pair:{words[index]}_{words[index + 1]}" for index in range(len(words) - 1))
    for word in words:
        if len(word) >= 5:
            features.update(f"tri:{word[index:index + 3]}" for index in range(len(word) - 2))
    return features


def _expanded_query(value: str) -> str:
    normalized = value.casefold()
    expansions: list[str] = []
    if "authentic" in normalized:
        expansions.append("operator authentication private deployment security beveiliging")
    if "destruct" in normalized or "geblokkeerd" in normalized:
        expansions.append("destructive blocked policy guardrail blokkeren acties")
    if ("secret" in normalized or "geheim" in normalized) and ("encrypt" in normalized or "opgeslagen" in normalized):
        expansions.append("secret storage encrypted-at-rest Fernet SQLite")
    return " ".join([value, *expansions])


def _safe_text(value: str) -> str:
    value = re.sub(r"<script\b[^>]*>.*?</script>", " ", value, flags=re.IGNORECASE | re.DOTALL)
    value = redact_message(value)
    return re.sub(r"\s+", " ", value).strip()


def _split_markdown(source_uri: str, raw: str) -> Iterable[tuple[str, str, str]]:
    matches = list(_HEADING.finditer(raw))
    title = _safe_text(Path(source_uri).stem.replace("_", " ").replace("-", " ").title())[:200]
    if matches:
        title = _safe_text(matches[0].group(2))[:200]
    boundaries = matches or [re.match(r"", raw)]
    for index, match in enumerate(boundaries):
        start = match.end() if match else 0
        end = boundaries[index + 1].start() if index + 1 < len(boundaries) else len(raw)
        section = _safe_text(match.group(2))[:200] if match and match.lastindex else title
        body = _safe_text(raw[start:end])
        if not body:
            continue
        for offset in range(0, len(body), 1800):
            content = body[offset:offset + 2200].strip()
            if len(content) >= 40:
                yield title, section, content


def _knowledge_files(root: Path) -> list[Path]:
    candidates = [root / "README.md", root / "CHANGELOG.md"]
    docs = root / "docs"
    if docs.is_dir():
        candidates.extend(sorted(docs.glob("*.md")))
    return [
        path for path in candidates[:160]
        if path.is_file() and path.name.casefold() not in _BLOCKED_FILES and path.stat().st_size <= 1_000_000
    ]


def _source_manifest(root: Path) -> dict[str, Any]:
    files = _knowledge_files(root)
    stat_signature = tuple(
        (path.relative_to(root).as_posix(), path.stat().st_size, path.stat().st_mtime_ns)
        for path in files
    )
    cached = _MANIFEST_CACHE.get(str(root))
    if cached and cached.get("statSignature") == stat_signature:
        return dict(cached["manifest"])
    digest = hashlib.sha256()
    for path in files:
        digest.update(path.relative_to(root).as_posix().encode("utf-8"))
        digest.update(b"\0")
        digest.update(path.read_bytes())
        digest.update(b"\0")
    manifest = {"hash": digest.hexdigest(), "sourceCount": len(files)}
    _MANIFEST_CACHE[str(root)] = {"statSignature": stat_signature, "manifest": manifest}
    return dict(manifest)


def _redaction_match_count(value: str) -> int:
    return sum(len(pattern.findall(value)) for pattern in REDACTION_PATTERNS)


def _compact_excerpt(value: str, limit: int = 360) -> str:
    sentences = re.split(r"(?<=[.!?])\s+", value.strip())
    summary = " ".join(sentences[:2]).strip()
    candidate = summary or value.strip()
    return candidate if len(candidate) <= limit else f"{candidate[:limit - 1].rstrip()}…"


def _rebuild_index(session: Session, root: Path | None = None) -> dict[str, Any]:
    root = (root or Path.cwd()).resolve()
    manifest = _source_manifest(root)
    rows: list[KnowledgeChunk] = []
    sources: set[str] = set()
    redacted_matches = 0
    for path in _knowledge_files(root):
        relative = path.relative_to(root).as_posix()
        source_uri = f"appops-doc://{relative}"
        raw = path.read_text(encoding="utf-8", errors="replace")
        redacted_matches += _redaction_match_count(raw)
        source_has_chunks = False
        for ordinal, (title, section, content) in enumerate(_split_markdown(relative, raw)):
            digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
            rows.append(KnowledgeChunk(
                id=f"kc-{hashlib.sha256(f'{source_uri}:{ordinal}:{digest}'.encode()).hexdigest()[:20]}",
                source_uri=source_uri,
                source_type="documentation",
                title=title,
                section=section,
                content=content,
                content_hash=digest,
            ))
            source_has_chunks = True
        if source_has_chunks:
            sources.add(source_uri)
    session.exec(delete(KnowledgeChunk))
    session.add_all(rows)
    setting = session.get(AppSetting, _INDEX_MANIFEST_KEY)
    if not setting:
        setting = AppSetting(key=_INDEX_MANIFEST_KEY, is_secret=False)
    generated_at = datetime.now(UTC).isoformat()
    setting.value_json = {
        "manifestHash": manifest["hash"],
        "sourceCount": len(sources),
        "chunkCount": len(rows),
        "generatedAt": generated_at,
        "redactedMatches": redacted_matches,
        "secretScanStatus": "passed_with_redactions" if redacted_matches else "passed",
    }
    setting.updated_at = utc_now()
    session.add(setting)
    session.commit()
    _PERSISTED_CACHE.clear()
    return {
        "status": "ready",
        "engine": "local_hybrid_lexical_semantic",
        "chunks": len(rows),
        "sources": len(sources),
        "generatedAt": generated_at,
        "manifestHash": manifest["hash"],
        "redactedMatches": redacted_matches,
        "secretScanStatus": setting.value_json["secretScanStatus"],
    }


def rebuild_index(session: Session, root: Path | None = None) -> dict[str, Any]:
    with _INDEX_LOCK:
        return _rebuild_index(session, root)


def index_status(session: Session) -> dict[str, Any]:
    chunks, sources, last_indexed = session.exec(
        select(
            func.count(KnowledgeChunk.id),
            func.count(distinct(KnowledgeChunk.source_uri)),
            func.max(KnowledgeChunk.updated_at),
        )
    ).one()
    manifest = session.get(AppSetting, _INDEX_MANIFEST_KEY)
    metadata = manifest.value_json if manifest and isinstance(manifest.value_json, dict) else {}
    return {
        "status": "ready" if chunks else "empty",
        "engine": "local_hybrid_lexical_semantic",
        "chunks": chunks,
        "sources": sources,
        "lastIndexedAt": metadata.get("generatedAt") or last_indexed,
        "secretsIndexed": False,
        "secretScanStatus": metadata.get("secretScanStatus", "unknown"),
        "redactedMatches": int(metadata.get("redactedMatches", 0)),
        "manifestHash": metadata.get("manifestHash"),
        "generationMode": "extractive_grounded",
    }


def ensure_index_current(session: Session, root: Path | None = None) -> dict[str, Any]:
    with _INDEX_LOCK:
        root = (root or Path.cwd()).resolve()
        current = _source_manifest(root)
        stored = session.get(AppSetting, _INDEX_MANIFEST_KEY)
        stored_hash = stored.value_json.get("manifestHash") if stored and isinstance(stored.value_json, dict) else None
        has_chunks = session.exec(select(KnowledgeChunk.id).limit(1)).first() is not None
        if not has_chunks or stored_hash != current["hash"]:
            return _rebuild_index(session, root)
        return index_status(session)


def _runtime_chunks(session: Session) -> list[dict[str, str]]:
    chunks: list[dict[str, str]] = []
    for app in session.exec(select(App).order_by(App.name)).all():
        chunks.append({
            "id": f"runtime-app-{app.id}", "source_uri": f"appops://apps/{app.id}", "source_type": "live_app",
            "title": app.name, "section": "Actuele appstatus",
            "content": _safe_text(
                f"App {app.name}; categorie {app.category}; status {app.status}; criticality {app.criticality}; "
                f"connector {app.connector_status}; container {app.container_status}; protected {app.protected}; review {app.needs_review}."
            ),
        })
    for policy in session.exec(select(Policy).order_by(Policy.action)).all():
        chunks.append({
            "id": f"runtime-policy-{policy.id}", "source_uri": f"appops://policies/{policy.id}", "source_type": "live_policy",
            "title": policy.action, "section": "Actuele policy",
            "content": _safe_text(
                f"Actie {policy.action}; target {policy.target}; manual {policy.manual}; voice {policy.voice}; "
                f"risk {policy.risk}; reden {policy.reason or 'geen'}"
            ),
        })
    for event in session.exec(select(AuditEvent).order_by(AuditEvent.timestamp.desc()).limit(80)).all():
        chunks.append({
            "id": f"runtime-audit-{event.id}", "source_uri": f"appops://audit/{event.id}", "source_type": "live_audit",
            "title": event.action_id, "section": "Recent audit-event",
            "content": _safe_text(
                f"{event.timestamp.isoformat()} actie {event.action_id}; target {event.target}; bron {event.source}; "
                f"policy {event.policy_decision}; resultaat {event.result}; reden {event.reason or 'geen'}"
            ),
        })
    for event in session.exec(select(NotificationEvent).order_by(NotificationEvent.created_at.desc()).limit(40)).all():
        chunks.append({
            "id": f"runtime-notification-{event.id}", "source_uri": f"appops://notifications/{event.id}", "source_type": "live_notification",
            "title": _safe_text(event.title), "section": "Recente notificatie",
            "content": _safe_text(f"{event.severity} {event.source} {event.category or ''}: {event.message}; status {event.status}"),
        })
    return chunks


def query(session: Session, question: str, *, limit: int = 6, root: Path | None = None) -> dict[str, Any]:
    question = _safe_text(question)[:500]
    if len(question) < 2:
        raise ValueError("question_required")
    status = ensure_index_current(session, root)
    cache_key = str(status.get("manifestHash") or status.get("lastIndexedAt") or "empty")
    if _PERSISTED_CACHE.get("key") != cache_key:
        persisted = [
            {
                "id": row.id, "source_uri": row.source_uri, "source_type": row.source_type,
                "title": row.title, "section": row.section or row.title, "content": row.content,
            }
            for row in session.exec(select(KnowledgeChunk)).all()
        ]
        _PERSISTED_CACHE.update({
            "key": cache_key,
            "items": persisted,
            "terms": [Counter(_tokens(f"{item['title']} {item['section']} {item['content']}")) for item in persisted],
            "features": [_semantic_features(f"{item['title']} {item['section']} {item['content']}") for item in persisted],
        })
    persisted = list(_PERSISTED_CACHE.get("items", []))
    persisted_terms = list(_PERSISTED_CACHE.get("terms", []))
    persisted_features = list(_PERSISTED_CACHE.get("features", []))
    candidates = persisted + _runtime_chunks(session)
    expanded_question = _expanded_query(question)
    query_terms = Counter(_tokens(expanded_question))
    if not query_terms:
        raise ValueError("question_has_no_search_terms")
    document_frequency: Counter[str] = Counter()
    tokenized: list[tuple[dict[str, str], Counter[str], set[str]]] = []
    for index, item in enumerate(candidates):
        terms = persisted_terms[index] if index < len(persisted_terms) else Counter(
            _tokens(f"{item['title']} {item['section']} {item['content']}")
        )
        features = persisted_features[index] if index < len(persisted_features) else _semantic_features(
            f"{item['title']} {item['section']} {item['content']}"
        )
        tokenized.append((item, terms, features))
        document_frequency.update(terms.keys())
    scored: list[tuple[float, dict[str, str]]] = []
    total = max(1, len(tokenized))
    query_features = _semantic_features(expanded_question)
    for item, terms, features in tokenized:
        score = 0.0
        title_terms = set(_tokens(f"{item['title']} {item['section']}"))
        for term, query_count in query_terms.items():
            if not terms[term]:
                continue
            idf = math.log((total + 1) / (document_frequency[term] + 1)) + 1
            score += query_count * (1 + math.log(terms[term])) * idf
            if term in title_terms:
                score += 1.5 * idf
        overlap = len(query_features.intersection(features))
        semantic_score = overlap / max(1, math.sqrt(len(query_features) * len(features)))
        score += semantic_score * 4.0
        if score > 0.15:
            scored.append((score, item))
    selected: list[tuple[float, dict[str, str]]] = []
    source_counts: Counter[str] = Counter()
    requested_limit = max(1, min(limit, 10))
    for score, item in sorted(scored, key=lambda value: value[0], reverse=True):
        source_uri = item["source_uri"]
        if source_counts[source_uri] >= 2:
            continue
        selected.append((score, item))
        source_counts[source_uri] += 1
        if len(selected) >= requested_limit:
            break
    results = []
    for rank, (score, item) in enumerate(selected, start=1):
        results.append({
            "id": item["id"], "rank": rank, "title": item["title"], "section": item["section"],
            "uri": item["source_uri"], "sourceType": item["source_type"], "score": round(score, 3),
            "excerpt": item["content"][:700],
        })
    if not results:
        answer = "Er is geen voldoende relevante, brongebonden context gevonden. Er wordt geen antwoord gegokt."
    else:
        evidence = "\n".join(
            f"[{result['rank']}] {result['title']}: {_compact_excerpt(result['excerpt'])}"
            for result in results[:3]
        )
        answer = f"Gevonden broncontext:\n{evidence}"
    return {
        "question": question,
        "answer": answer,
        "confidence": "high" if len(results) >= 3 else "medium" if results else "none",
        "citations": results,
        "guardrails": {
            "readOnly": True,
            "secretsExcluded": True,
            "instructionsFromSourcesIgnored": True,
            "executionViaPolicyOnly": True,
        },
        "retrieval": {
            "engine": "local_hybrid_lexical_semantic",
            "vectorSpace": "normalized_binary_word_bigram_char_trigram",
            "externalEmbeddingProvider": False,
        },
    }


_RETRIEVAL_EVALUATION_CASES = (
    ("Hoe werkt private operator authenticatie?", ("PRIVATE_DEPLOYMENT_SECURITY", "API_CONTRACT")),
    ("Hoe controleer en herstel ik een SQLite backup?", ("BACKUP_RESTORE", "RELEASE_CHECKLIST")),
    ("Hoe zijn destructieve acties geblokkeerd?", ("API_CONTRACT", "HARDENING")),
    ("Hoe koppelen MCP, RAG en n8n?", ("MCP_RAG_N8N_INTEGRATION",)),
    ("Hoe worden connector secrets encrypted-at-rest opgeslagen?", ("SECRET_STORAGE",)),
)


def evaluate_retrieval(session: Session) -> dict[str, Any]:
    cases = []
    for question, expected_sources in _RETRIEVAL_EVALUATION_CASES:
        result = query(session, question, limit=5)
        uris = [str(item.get("uri") or "") for item in result.get("citations", [])]
        passed = any(expected.lower() in uri.lower() for expected in expected_sources for uri in uris)
        cases.append({
            "question": question,
            "passed": passed,
            "expectedSourceHints": list(expected_sources),
            "topSourceUris": uris[:3],
            "citationCount": len(uris),
        })
    passed_count = sum(1 for case in cases if case["passed"])
    return {
        "status": "passed" if passed_count == len(cases) else "needs_tuning",
        "evaluationVersion": 2,
        "engine": "local_hybrid_lexical_semantic",
        "vectorSpace": "normalized_binary_word_bigram_char_trigram",
        "passed": passed_count,
        "total": len(cases),
        "passRate": round(passed_count / max(1, len(cases)), 3),
        "cases": cases,
    }