Publish curated VacatureRadar source snapshot

apps/sources/management/commands/discover_sources.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from django.core.management.base import BaseCommand, CommandError
+
+from apps.sources.services.discovery import (
+    discover_from_email,
+    discover_from_feed,
+    discover_from_html,
+    discover_from_sitemap,
+    persist_discovery_candidates,
+)
+
+
+class Command(BaseCommand):
+    help = (
+        "Voert bronontdekking uit op lokaal aangeleverde content "
+        "en slaat alleen kandidaatregels op met policy review."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "mode",
+            choices=["html", "sitemap", "feed", "email"],
+            help="Bronmodus voor ontdekking",
+        )
+        parser.add_argument(
+            "--input",
+            required=True,
+            dest="input_path",
+            help="Pad naar inputbestand",
+        )
+        parser.add_argument(
+            "--base-url",
+            dest="base_url",
+            help="Base URL voor relativiteitsresolutie",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Geen database-write; toon alleen ontdekte kandidaten",
+        )
+
+    def handle(self, *args, **options):
+        mode = options["mode"]
+        input_path = Path(options["input_path"]).resolve()
+        base_url = (options["base_url"] or "").strip()
+        if mode in {"html", "sitemap", "feed"} and not base_url:
+            raise CommandError("--base-url is verplicht voor html/sitemap/feed")
+
+        if not input_path.is_file():
+            raise CommandError(f"Inputbestand niet gevonden: {input_path}")
+
+        if mode in {"html", "sitemap", "feed"}:
+            content = input_path.read_text(encoding="utf-8")
+        else:
+            content = input_path.read_bytes()
+
+        if mode == "html":
+            candidates = discover_from_html(content, base_url=base_url)
+        elif mode == "sitemap":
+            candidates = discover_from_sitemap(content, base_url=base_url)
+        elif mode == "feed":
+            candidates = discover_from_feed(content, base_url=base_url)
+        else:
+            candidates = discover_from_email(content)
+
+        self.stdout.write(f"Ontdekt {len(candidates)} kandidaten (mode={mode}).")
+        if not candidates:
+            self.stdout.write(self.style.WARNING("Geen bruikbare kandidaten gevonden."))
+            return
+
+        if options["dry_run"]:
+            self.stdout.write(self.style.WARNING("Dry-run modus: er wordt niet geschreven."))
+            for candidate in candidates:
+                self.stdout.write(f"- {candidate.source_type} {candidate.url} [{candidate.reason}]")
+            return
+
+        created, updated, skipped = persist_discovery_candidates(candidates)
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Persist result: {created} nieuw, {updated} bijgewerkt, {skipped} ongewijzigd."
+            )
+        )

apps/sources/management/commands/import_job_fixture.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+
+from apps.jobs.services.pipeline import process_raw_document
+from apps.sources.models import RawDocument, Source, SourceRun
+
+
+class Command(BaseCommand):
+    help = "Importeert een lokale HTML/XML-fixture via dezelfde pipeline als een live bron."
+
+    def add_arguments(self, parser):
+        parser.add_argument("path")
+        parser.add_argument("--url", required=True)
+        parser.add_argument("--source-name", default="Lokale fixture")
+        parser.add_argument("--source-type", default=Source.Type.EMPLOYER)
+
+    def handle(self, *args, **options):
+        path = Path(options["path"]).resolve()
+        if not path.is_file():
+            raise CommandError(f"Bestand bestaat niet: {path}")
+        content = path.read_text(encoding="utf-8")
+        hostname = (urlsplit(options["url"]).hostname or "").lower()
+        if not hostname:
+            raise CommandError("--url bevat geen geldig domein")
+        source, _ = Source.objects.get_or_create(
+            base_url=options["url"],
+            source_type=options["source_type"],
+            defaults={
+                "domain": hostname,
+                "name": options["source_name"],
+                "status": Source.Status.ACTIVE,
+                "policy": Source.Policy.ALLOW,
+                "parser_key": "auto",
+            },
+        )
+        run = SourceRun.objects.create(source=source)
+        suffix = path.suffix.lower()
+        kind = RawDocument.Kind.XML if suffix in {".xml", ".rss"} else RawDocument.Kind.HTML
+        document = RawDocument.objects.create(
+            source=source,
+            source_run=run,
+            url=options["url"],
+            final_url=options["url"],
+            kind=kind,
+            content_type="application/xml" if kind == RawDocument.Kind.XML else "text/html",
+            content_hash=hashlib.sha256(content.encode()).hexdigest(),
+            body_text=content,
+            byte_length=len(content.encode()),
+            retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
+        )
+        metrics = process_raw_document(document)
+        run.finish(
+            SourceRun.Status.SUCCESS,
+            extracted_count=int(metrics["extracted"]),
+            created_count=int(metrics["created"]),
+            updated_count=int(metrics["updated"]),
+            duplicate_count=int(metrics["duplicates"]),
+            metrics=metrics,
+        )
+        self.stdout.write(self.style.SUCCESS(str(metrics)))

apps/sources/management/commands/rotate_mailbox_credentials.py

@@ -0,0 +1,17 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from apps.sources.services.mailbox_connections import (
+    MailboxCredentialError,
+    rotate_mailbox_credentials,
+)
+
+
+class Command(BaseCommand):
+    help = "Versleutel alle mailbox-app-wachtwoorden opnieuw met de eerste ingestelde sleutel."
+
+    def handle(self, *args, **options):
+        try:
+            count = rotate_mailbox_credentials()
+        except MailboxCredentialError as exc:
+            raise CommandError(str(exc)) from exc
+        self.stdout.write(self.style.SUCCESS(f"{count} mailboxcredential(s) geroteerd."))

apps/sources/management/commands/seed_sources.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import yaml
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.dateparse import parse_datetime
+
+from apps.sources.models import Source, SourcePolicyReview
+from apps.sources.services.policy import is_denied_domain
+
+
+class Command(BaseCommand):
+    help = "Laadt gecontroleerde bronseeds uit YAML."
+
+    def add_arguments(self, parser):
+        parser.add_argument("path", nargs="?", default="config-data/seed_sources.yaml")
+
+    def handle(self, *args, **options):
+        path = Path(options["path"])
+        if not path.is_file():
+            raise CommandError(f"Seedbestand ontbreekt: {path}")
+        payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        created = updated = reviews_created = 0
+        if payload.get("disable_demo_sources") is True:
+            demo_sources = Source.objects.filter(domain="jobs.example.org")
+            for demo_source in demo_sources:
+                demo_source.status = Source.Status.DISABLED
+                demo_source.policy = Source.Policy.DENY
+                demo_source.policy_reason = "Demobron verborgen na installatie van livebronnen"
+                demo_source.metadata = {**demo_source.metadata, "demo": True}
+                demo_source.save(
+                    update_fields=[
+                        "status",
+                        "policy",
+                        "policy_reason",
+                        "metadata",
+                        "updated_at",
+                    ]
+                )
+        retired = 0
+        for retired_url in payload.get("retire_source_urls", []):
+            for retired_source in Source.objects.filter(base_url=str(retired_url).strip()):
+                retired_source.status = Source.Status.DISABLED
+                retired_source.next_run_at = None
+                retired_source.metadata = {
+                    **retired_source.metadata,
+                    "retired": True,
+                    "retired_reason": "Buiten de ingestelde regionale dekking",
+                    "hidden_from_source_list": True,
+                }
+                retired_source.save(
+                    update_fields=["status", "next_run_at", "metadata", "updated_at"]
+                )
+                retired += 1
+        for item in payload.get("sources", []):
+            url = str(item.get("url") or "").strip()
+            domain = (urlsplit(url).hostname or "").lower()
+            if not domain:
+                self.stderr.write(f"Overgeslagen zonder domein: {item}")
+                continue
+            deny = is_denied_domain(domain)
+            source, was_created = Source.objects.update_or_create(
+                base_url=url,
+                source_type=item.get("type", Source.Type.EMPLOYER),
+                defaults={
+                    "domain": domain,
+                    "name": item.get("name") or domain,
+                    "status": Source.Status.PAUSED
+                    if deny
+                    else item.get("status", Source.Status.TRIAL),
+                    "policy": Source.Policy.DENY
+                    if deny
+                    else item.get("policy", Source.Policy.REVIEW),
+                    "policy_reason": "Standaard platformdenylist"
+                    if deny
+                    else item.get("policy_reason", ""),
+                    "parser_key": item.get("parser", "auto"),
+                    "crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)),
+                    "minimum_interval_seconds": int(item.get("minimum_interval_seconds", 30)),
+                    "allow_public_endpoint": bool(item.get("allow_public_endpoint", False)),
+                    "metadata": item.get("metadata") or {},
+                },
+            )
+            review = item.get("review") or {}
+            if review:
+                expires_at = parse_datetime(str(review.get("expires_at") or ""))
+                if expires_at is None:
+                    raise CommandError(
+                        f"Bron {source.name}: review.expires_at moet een ISO-datetime zijn"
+                    )
+                decision = str(review.get("decision") or "")
+                if decision not in SourcePolicyReview.Decision.values:
+                    raise CommandError(f"Bron {source.name}: ongeldige reviewbeslissing")
+                _, review_created = SourcePolicyReview.objects.get_or_create(
+                    source=source,
+                    decision=decision,
+                    evidence_link=str(review.get("evidence_link") or ""),
+                    expires_at=expires_at,
+                    defaults={
+                        "scope": SourcePolicyReview.Scope.SOURCE,
+                        "reason": str(review.get("reason") or "Bronseedreview"),
+                        "notes": str(review.get("notes") or ""),
+                        "metadata": review.get("metadata") or {},
+                    },
+                )
+                reviews_created += int(review_created)
+            created += int(was_created)
+            updated += int(not was_created)
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Bronnen: {created} aangemaakt, {updated} bijgewerkt, "
+                f"{reviews_created} reviews geregistreerd, {retired} buiten regio uitgeschakeld"
+            )
+        )