Publish curated VacatureRadar source snapshot

apps/core/services/__init__.py

@@ -0,0 +1 @@
+"""Read services for the application cockpit."""

apps/core/services/automation.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import Any
+
+from django.conf import settings
+from django.core.cache import cache
+from django.db.models import Count, Q, Sum
+from django.utils import timezone
+
+from apps.jobs.models import JobPosting, ScoreRun
+from apps.jobs.services.relevance import it_relevance_query
+from apps.notifications.models import DigestOutbox, ReminderOutbox
+from apps.profiles.models import SearchProfile
+from apps.sources.models import Source, SourceRun
+
+from ..health import readiness
+from ..tasks import RUNTIME_HEARTBEAT_KEY
+
+
+def _runtime_heartbeat_state() -> dict[str, Any]:
+    if settings.CELERY_TASK_ALWAYS_EAGER:
+        return {"label": "Eager/lokaal", "healthy": True, "observed_at": None}
+
+    raw_timestamp = cache.get(RUNTIME_HEARTBEAT_KEY)
+    if not raw_timestamp:
+        return {"label": "Geen recente heartbeat", "healthy": False, "observed_at": None}
+
+    try:
+        observed_at = timezone.datetime.fromisoformat(str(raw_timestamp))
+        if timezone.is_naive(observed_at):
+            observed_at = timezone.make_aware(observed_at)
+    except (TypeError, ValueError):
+        return {"label": "Ongeldige heartbeat", "healthy": False, "observed_at": None}
+
+    age_seconds = max(0, int((timezone.now() - observed_at).total_seconds()))
+    return {
+        "label": f"Actief · {age_seconds}s geleden"
+        if age_seconds <= 150
+        else "Heartbeat verouderd",
+        "healthy": age_seconds <= 150,
+        "observed_at": observed_at,
+    }
+
+
+def build_automation_cockpit() -> dict[str, Any]:
+    """Build an honest operational read model without probing unsafe hidden endpoints."""
+
+    since = timezone.now() - timedelta(hours=24)
+    recent_runs = SourceRun.objects.filter(started_at__gte=since)
+    run_metrics = recent_runs.aggregate(
+        discovered=Sum("discovered_count"),
+        extracted=Sum("extracted_count"),
+        created=Sum("created_count"),
+        updated=Sum("updated_count"),
+        duplicates=Sum("duplicate_count"),
+        failures=Count("id", filter=Q(status=SourceRun.Status.FAILED)),
+    )
+    notification_counts = {
+        "pending": DigestOutbox.objects.filter(status=DigestOutbox.Status.PENDING).count()
+        + ReminderOutbox.objects.filter(status=ReminderOutbox.Status.PENDING).count(),
+        "failed": DigestOutbox.objects.filter(status=DigestOutbox.Status.FAILED).count()
+        + ReminderOutbox.objects.filter(status=ReminderOutbox.Status.FAILED).count(),
+        "sent_24h": DigestOutbox.objects.filter(
+            status=DigestOutbox.Status.SENT, sent_at__gte=since
+        ).count()
+        + ReminderOutbox.objects.filter(
+            status=ReminderOutbox.Status.SENT, sent_at__gte=since
+        ).count(),
+    }
+    source_total = Source.objects.count()
+    source_active = Source.objects.filter(status=Source.Status.ACTIVE).count()
+    active_jobs = JobPosting.objects.filter(status=JobPosting.Status.ACTIVE)
+    phase_rows = [
+        {"key": "discover", "label": "Ontdekken", "value": source_total, "unit": "bronnen"},
+        {"key": "fetch", "label": "Ophalen", "value": recent_runs.count(), "unit": "runs / 24u"},
+        {
+            "key": "extract",
+            "label": "Extraheren",
+            "value": run_metrics["extracted"] or 0,
+            "unit": "items",
+        },
+        {
+            "key": "normalize",
+            "label": "Normaliseren",
+            "value": (run_metrics["created"] or 0) + (run_metrics["updated"] or 0),
+            "unit": "records",
+        },
+        {
+            "key": "dedupe",
+            "label": "Dedupliceren",
+            "value": run_metrics["duplicates"] or 0,
+            "unit": "herkend",
+        },
+        {
+            "key": "filter",
+            "label": "IT-filter",
+            "value": active_jobs.exclude(it_relevance_query()).count(),
+            "unit": "uit beeld",
+        },
+        {
+            "key": "analyze",
+            "label": "Analyseren",
+            "value": ScoreRun.objects.filter(created_at__gte=since).count(),
+            "unit": "scores / 24u",
+        },
+        {
+            "key": "notify",
+            "label": "Notificeren",
+            "value": notification_counts["sent_24h"],
+            "unit": "verzonden / 24u",
+        },
+    ]
+    runtime_heartbeat = _runtime_heartbeat_state()
+    return {
+        "health": readiness(),
+        "source_total": source_total,
+        "source_active": source_active,
+        "run_metrics": run_metrics,
+        "run_count": recent_runs.count(),
+        "active_jobs": active_jobs.filter(it_relevance_query()).count(),
+        "review_jobs": ScoreRun.objects.filter(
+            recommendation__in=[ScoreRun.Recommendation.STRONG, ScoreRun.Recommendation.POSSIBLE],
+            job__status=JobPosting.Status.ACTIVE,
+        ).count(),
+        "notification_counts": notification_counts,
+        "phase_rows": phase_rows,
+        "recent_runs": list(
+            SourceRun.objects.select_related("source").order_by("-started_at")[:10]
+        ),
+        "source_rows": Source.objects.annotate(run_count=Count("runs")).order_by("name")[:12],
+        "queue_state": "Eager/lokaal"
+        if settings.CELERY_TASK_ALWAYS_EAGER
+        else "Diepte niet beschikbaar",
+        "runtime_heartbeat": runtime_heartbeat,
+        "ai_state": "Geconfigureerd"
+        if settings.OLLAMA_ENABLED and settings.OLLAMA_MODEL
+        else "Niet geconfigureerd",
+        "ai_profiles": SearchProfile.objects.filter(ai_scoring_enabled=True).count(),
+        "browser_state": "Geen aparte browserextractor geconfigureerd",
+    }

apps/core/services/microsoft365.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import base64
+
+from django.conf import settings
+
+
+class Microsoft365OAuthError(RuntimeError):
+    """Safe OAuth boundary error that never includes tokens or client secrets."""
+
+
+def acquire_exchange_access_token(*, client_factory=None) -> str:
+    tenant_id = settings.M365_TENANT_ID.strip()
+    client_id = settings.M365_CLIENT_ID.strip()
+    client_secret = settings.M365_CLIENT_SECRET.strip()
+    if not all((tenant_id, client_id, client_secret)):
+        raise Microsoft365OAuthError("Microsoft 365 OAuth is niet volledig geconfigureerd.")
+
+    if client_factory is None:
+        import msal
+
+        client_factory = msal.ConfidentialClientApplication
+    factory = client_factory
+    client = factory(
+        client_id,
+        authority=f"https://login.microsoftonline.com/{tenant_id}",
+        client_credential=client_secret,
+    )
+    result = client.acquire_token_for_client(scopes=[settings.M365_EXCHANGE_SCOPE])
+    token = result.get("access_token") if isinstance(result, dict) else None
+    if not isinstance(token, str) or not token:
+        category = "unknown"
+        if isinstance(result, dict):
+            category = str(result.get("error") or "unknown")[:80]
+        raise Microsoft365OAuthError(
+            f"Microsoft 365 OAuth-token kon niet worden verkregen ({category})."
+        )
+    return token
+
+
+def xoauth2_bytes(username: str, access_token: str) -> bytes:
+    if not username or "\x00" in username or "\r" in username or "\n" in username:
+        raise Microsoft365OAuthError("Ongeldig Microsoft 365-mailboxadres.")
+    return f"user={username}\x01auth=Bearer {access_token}\x01\x01".encode()
+
+
+def xoauth2_b64(username: str, access_token: str) -> str:
+    return base64.b64encode(xoauth2_bytes(username, access_token)).decode("ascii")