Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/unit/ingestion/test_language.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from ragcore.ingestion.language import (
+    detect_language,
+    normalize_language_tag,
+    resolve_document_language,
+)
+
+_DUTCH = (
+    "Dit is een handleiding voor het herstarten van de dienst. Controleer eerst "
+    "of de gateway niet bereikbaar is en wacht daarna tien seconden voordat de "
+    "schakelaar opnieuw wordt omgezet. Het systeem moet daarna weer normaal werken."
+)
+_ENGLISH = (
+    "This is the runbook for restarting the service. First confirm that the "
+    "gateway is not reachable and then wait ten seconds before you cycle the "
+    "breaker again. The system should be healthy after that and the alert clears."
+)
+_FRENCH = (
+    "Ce document décrit la procédure pour redémarrer le service. Vérifiez d'abord "
+    "que la passerelle n'est pas joignable et attendez ensuite dix secondes avant "
+    "de basculer le disjoncteur. Le système doit être de nouveau disponible."
+)
+_GERMAN = (
+    "Dieses Dokument beschreibt das Neustarten des Dienstes. Prüfen Sie zuerst, "
+    "ob das Gateway nicht erreichbar ist, und warten Sie danach zehn Sekunden, "
+    "bevor der Schalter wieder umgelegt wird. Das System muss danach laufen."
+)
+
+
+def test_detects_each_supported_language() -> None:
+    assert detect_language(_DUTCH) == "nl"
+    assert detect_language(_ENGLISH) == "en"
+    assert detect_language(_FRENCH) == "fr"
+    assert detect_language(_GERMAN) == "de"
+
+
+def test_short_or_codeish_content_stays_undetected() -> None:
+    assert detect_language("restart") is None
+    assert detect_language("SELECT id FROM documents WHERE space_id = $1;") is None
+
+
+def test_normalize_language_tag_reduces_locales() -> None:
+    assert normalize_language_tag("nl-BE") == "nl"
+    assert normalize_language_tag("EN_us") == "en"
+    assert normalize_language_tag("not a tag!") is None
+
+
+def test_resolution_priority_is_explicit_then_detected_then_fallback() -> None:
+    assert resolve_document_language(explicit_locale="fr-FR", text=_DUTCH) == "fr"
+    assert resolve_document_language(explicit_locale=None, text=_DUTCH) == "nl"
+    assert resolve_document_language(explicit_locale="??", text="x") == "en"