Publish curated VacatureRadar source snapshot

apps/sources/adapters/__init__.py

@@ -0,0 +1,30 @@
+from .ats import (
+    GreenhouseAdapter,
+    LeverAdapter,
+    RecruiteeAdapter,
+    SmartRecruitersAdapter,
+    WorkableAdapter,
+)
+from .base import ExtractedJob, ExtractionResult
+from .generic_html import GenericHtmlAdapter
+from .jsonld import JsonLdJobPostingAdapter
+from .kempen import KempenEmployerAdapter
+from .mol_region import MolRegionEmployerAdapter
+from .regional import LocalEmployerListingAdapter
+from .rss import RssAdapter
+
+__all__ = [
+    "ExtractedJob",
+    "ExtractionResult",
+    "GenericHtmlAdapter",
+    "GreenhouseAdapter",
+    "JsonLdJobPostingAdapter",
+    "KempenEmployerAdapter",
+    "LeverAdapter",
+    "LocalEmployerListingAdapter",
+    "MolRegionEmployerAdapter",
+    "RecruiteeAdapter",
+    "RssAdapter",
+    "SmartRecruitersAdapter",
+    "WorkableAdapter",
+]

apps/sources/adapters/ats.py

@@ -0,0 +1,591 @@
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+CLOSED_STATUSES = {
+    "closed",
+    "inactive",
+    "withdrawn",
+    "removed",
+    "filled",
+    "expired",
+    "archived",
+}
+
+
+def _to_text(value):
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value.strip()
+    if isinstance(value, bool):
+        return str(value).lower()
+    if isinstance(value, int | float):
+        return str(value)
+    return ""
+
+
+def _find_nested(data, path: str):
+    if data is None:
+        return None
+    node = data
+    for part in path.split("."):
+        if not isinstance(node, dict) or part not in node:
+            return None
+        node = node[part]
+    return node
+
+
+def _first_text(data, *candidates):
+    for candidate in candidates:
+        if "." in candidate:
+            value = _find_nested(data, candidate)
+        else:
+            value = data.get(candidate) if isinstance(data, dict) else None
+        if value is None:
+            continue
+        if isinstance(value, list | tuple):
+            for item in value:
+                text = _to_text(item)
+                if text:
+                    return text
+            continue
+        text = _to_text(value)
+        if text:
+            return text
+        if isinstance(value, dict):
+            nested = value.values()
+            for nested_value in nested:
+                nested_text = _to_text(nested_value)
+                if nested_text:
+                    return nested_text
+    return ""
+
+
+def _collect_texts(value) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value.strip()] if value.strip() else []
+    if isinstance(value, list):
+        out: list[str] = []
+        for item in value:
+            for inner in _collect_texts(item):
+                if inner:
+                    out.append(inner)
+        return out
+    if isinstance(value, dict):
+        out: list[str] = []
+        for key in ("name", "city", "location", "address", "raw", "region", "country"):
+            if key in value:
+                for inner in _collect_texts(value[key]):
+                    if inner:
+                        out.append(inner)
+        return out
+    if isinstance(value, bool | int | float):
+        return [_to_text(value)]
+    return []
+
+
+def _to_list(value) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [item.strip() for item in (_to_text(v) for v in value) if item.strip()]
+    text = _to_text(value)
+    if not text:
+        return []
+    return [part.strip() for part in text.replace(";", ",").split(",") if part.strip()]
+
+
+def _as_text(value) -> str:
+    text = _to_text(value)
+    if not text:
+        return ""
+    return BeautifulSoup(text, "lxml").get_text(" ", strip=True)
+
+
+def _extract_payload(content: str):
+    try:
+        return json.loads(content.lstrip("\ufeff"))
+    except json.JSONDecodeError:
+        pass
+    soup = BeautifulSoup(content, "lxml")
+    for script in soup.find_all("script", attrs={"type": "application/json"}):
+        script_text = script.string or script.get_text("", strip=True)
+        if not script_text:
+            continue
+        try:
+            return json.loads(script_text.lstrip("\ufeff"))
+        except json.JSONDecodeError:
+            continue
+    return None
+
+
+class _AtsAdapter(ABC):
+    parser_key = "ats-provider"
+    parser_version = "1.0.0"
+    source_hosts: tuple[str, ...] = ()
+    support_markers: tuple[str, ...] = ()
+    listing_paths: tuple[tuple[str, ...], ...] = ()
+    detail_paths: tuple[tuple[str, ...], ...] = ()
+    closed_statuses = CLOSED_STATUSES
+
+    @abstractmethod
+    def _extract_records(self, payload) -> list[dict[str, object]]: ...
+
+    def _supports_url(self, url: str) -> bool:
+        host = (urlsplit(url).hostname or "").lower()
+        return any(host.endswith(suffix) for suffix in self.source_hosts)
+
+    def _supports_payload(self, payload, content: str) -> bool:
+        if not payload:
+            return False
+        payload_text = str(payload).lower()
+        return any(
+            marker in payload_text or marker in content.lower() for marker in self.support_markers
+        )
+
+    def _job_is_closed(self, record: dict[str, object]) -> bool:
+        status = _first_text(record, "status", "state", "job_status", "data.status").lower()
+        if status in self.closed_statuses:
+            return True
+        active = record.get("active")
+        if isinstance(active, bool):
+            return not active
+        return False
+
+    def _coerce_url(self, raw_url: str, base_url: str) -> str:
+        if raw_url:
+            return urljoin(base_url, raw_url)
+        return base_url
+
+    def _extract_location(self, record: dict[str, object]) -> tuple[str, str, str, str]:
+        location_raw = _first_text(
+            record,
+            "location.city",
+            "location.name",
+            "location.address",
+            "location.raw",
+            "categories.location",
+            "city",
+            "cityName",
+            "place",
+            "office",
+            "address",
+            "data.location",
+            "officeLocation",
+            "locationName",
+            "location",
+        )
+        if not location_raw:
+            location_raw = _first_text(record, "data.city", "data.location")
+            if not location_raw:
+                location_values = []
+                for key in ("city", "address", "location", "region", "office"):
+                    location_values.extend(_collect_texts(record.get(key, "")))
+                location_raw = ", ".join(location_values)
+
+        location = location_raw
+        [part.strip() for part in _to_text(location).split(",") if part.strip()]
+        region = _first_text(
+            record,
+            "region",
+            "data.region",
+            "location.region",
+            "address.region",
+        )
+        postal_code = _first_text(
+            record,
+            "postal_code",
+            "postalCode",
+            "location.postalCode",
+            "zipCode",
+        )
+        country = _first_text(
+            record,
+            "country",
+            "countryName",
+            "location.country",
+            "address.country",
+            "data.country",
+        )
+        return location, region, postal_code, country
+
+    def _extract_workplace(self, record: dict[str, object]) -> str:
+        workplace_raw = _first_text(
+            record,
+            "workplaceType",
+            "workplace",
+            "remoteType",
+            "remote_type",
+            "job_type",
+        ).lower()
+        if not workplace_raw:
+            if record.get("hybrid") is True:
+                return "hybrid"
+            if record.get("remote") is True:
+                return "remote"
+            if record.get("on_site") is True:
+                return "on_site"
+        if "remote" in workplace_raw:
+            return "remote" if "hybrid" not in workplace_raw else "hybrid"
+        if "hybrid" in workplace_raw:
+            return "hybrid"
+        if "onsite" in workplace_raw or "on site" in workplace_raw or "on-site" in workplace_raw:
+            return "on_site"
+        return ""
+
+    def _to_job(self, record: dict[str, object], base_url: str) -> ExtractedJob:
+        external_id = _first_text(
+            record,
+            "id",
+            "jobId",
+            "requisitionId",
+            "postingId",
+            "referenceId",
+            "positionId",
+        )
+        title = _first_text(
+            record,
+            "title",
+            "name",
+            "position",
+            "positionName",
+            "jobTitle",
+            "text",
+            "title.value",
+        )
+        raw_url = _first_text(
+            record,
+            "url",
+            "jobUrl",
+            "job_url",
+            "hostedUrl",
+            "applyUrl",
+            "link",
+            "absoluteUrl",
+            "careers_url",
+            "data.url",
+        )
+        employer_name = _first_text(
+            record,
+            "company_name",
+            "company.name",
+            "company",
+            "employer",
+            "organization",
+            "organizationName",
+            "department",
+            "hiringOrganization",
+        )
+        location_text, region, postal_code, country = self._extract_location(record)
+        description_raw = _first_text(
+            record,
+            "description",
+            "jobDescription",
+            "content",
+            "descriptionHtml",
+            "descriptionText",
+        )
+        description_text = _as_text(description_raw)
+        date_posted = _first_text(
+            record,
+            "createdAt",
+            "created",
+            "datePosted",
+            "publishedAt",
+            "published_at",
+            "created_at",
+            "jobCreated",
+            "releasedDate",
+        )
+        valid_through = _first_text(
+            record,
+            "closeDate",
+            "expiresAt",
+            "validThrough",
+            "valid_until",
+            "expirationDate",
+            "expiryDate",
+        )
+        employment_types = _to_list(
+            _first_text(
+                record,
+                "employmentType",
+                "employment_types",
+                "jobType",
+                "type",
+                "data.employmentType",
+                "employment_type_code",
+                "categories.commitment",
+                "typeOfEmployment.label",
+            )
+        )
+        workplace_type = self._extract_workplace(record)
+
+        evidence = [
+            FieldEvidence("external_id", "ats-id", 0.95, external_id[:240]),
+            FieldEvidence("url", "ats-url", 0.9, raw_url[:240]),
+            FieldEvidence("location_text", "ats-location", 0.88, location_text[:240]),
+            FieldEvidence("date_posted", "ats-date", 0.8, date_posted[:240]),
+            FieldEvidence("valid_through", "ats-date", 0.8, valid_through[:240]),
+            FieldEvidence(
+                "employment_types",
+                "ats-employment",
+                0.9,
+                ", ".join(employment_types)[:240],
+            ),
+        ]
+
+        return ExtractedJob(
+            url=self._coerce_url(raw_url, base_url),
+            title=title,
+            employer_name=employer_name,
+            external_id=external_id,
+            location_text=location_text,
+            region=region,
+            postal_code=postal_code,
+            country=country,
+            description_html=description_raw,
+            description_text=description_text,
+            date_posted=date_posted,
+            valid_through=valid_through,
+            employment_types=employment_types,
+            workplace_type=workplace_type,
+            raw=record.get("__raw__", record),
+            evidence=evidence,
+        )
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        if not self._supports_url(url):
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Onherkenbare ATS-host"],
+            )
+
+        payload = _extract_payload(content)
+        if not payload:
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Geen parseerbare ATS-response"],
+            )
+
+        if not self._supports_payload(payload, content):
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Geen herkenbare ATS-markup voor deze adapter"],
+            )
+
+        jobs: list[ExtractedJob] = []
+        for record in self._extract_records(payload):
+            if not isinstance(record, dict):
+                continue
+            if self._job_is_closed(record):
+                continue
+            job = self._to_job(record, base_url=url)
+            if job.title:
+                jobs.append(job)
+
+        warnings: list[str] = []
+        if not jobs:
+            warnings.append("Geen actieve ATS-vacatures gevonden")
+        return ExtractionResult(
+            jobs, self.parser_key, self.parser_version, 0.9 if jobs else 0.0, warnings
+        )
+
+
+class GreenhouseAdapter(_AtsAdapter):
+    parser_key = "ats-greenhouse"
+    source_hosts = ("greenhouse.io", "boards.greenhouse.io")
+    support_markers = ("greenhouse", "job board", "jobboard")
+    listing_paths = (("jobs",), ("data", "jobs"), ("data", "results"))
+    detail_paths = (("job",), ("data", "job"), ("result", "job"), ("result", "position"))
+    closed_statuses = CLOSED_STATUSES | {"published", "draft", "deleted"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+            if isinstance(value, dict) and "results" in value:
+                possible = value.get("results")
+                if isinstance(possible, list):
+                    return [item for item in possible if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []
+
+
+class LeverAdapter(_AtsAdapter):
+    parser_key = "ats-lever"
+    source_hosts = (
+        "jobs.lever.co",
+        "jobs.eu.lever.co",
+        "api.lever.co",
+        "api.eu.lever.co",
+    )
+    support_markers = ("lever", "requisition", "posting")
+    listing_paths = (("data",), ("jobs",), ("results",))
+    detail_paths = (("data",), ("job",), ("position",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"archived", "deleted"}
+
+    def _extract_records(self, payload):
+        if isinstance(payload, list):
+            return [item for item in payload if isinstance(item, dict)]
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                if path == ("position",):
+                    raw_payload = {
+                        "position": value,
+                        "work_type": _to_text(value.get("workplaceType")),
+                    }
+                    return [{**value, "__raw__": raw_payload}]
+                return [value]
+        return []
+
+
+class RecruiteeAdapter(_AtsAdapter):
+    parser_key = "ats-recruitee"
+    source_hosts = ("recruitee.com",)
+    support_markers = ("recruitee", "career", "vacancy")
+    listing_paths = (("jobs",), ("offers",), ("data", "jobs"), ("vacancies",))
+    detail_paths = (("job",), ("data", "job"), ("vacancy",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"hidden", "paused"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []
+
+
+class SmartRecruitersAdapter(_AtsAdapter):
+    parser_key = "ats-smartrecruiters"
+    parser_version = "1.1.0"
+    source_hosts = ("smartrecruiters.com",)
+    support_markers = ("smartrecruiters", "smart recruiter")
+    listing_paths = (("jobs",), ("data", "jobs"), ("results",), ("content",))
+    detail_paths = (("job",), ("data", "job"), ("posting",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"unpublished", "expired"}
+
+    def _to_job(self, record: dict[str, object], base_url: str) -> ExtractedJob:
+        prepared = dict(record)
+        if not _first_text(prepared, "url", "jobUrl", "hostedUrl", "applyUrl", "link"):
+            company_identifier = _first_text(prepared, "company.identifier")
+            posting_id = _first_text(prepared, "id", "uuid")
+            if company_identifier and posting_id:
+                prepared["url"] = (
+                    f"https://jobs.smartrecruiters.com/{company_identifier}/{posting_id}"
+                )
+        if _find_nested(prepared, "location.remote") is True:
+            prepared["remote"] = True
+        return super()._to_job(prepared, base_url)
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []
+
+
+class WorkableAdapter(_AtsAdapter):
+    parser_key = "ats-workable"
+    source_hosts = ("apply.workable.com",)
+    support_markers = ("workable", "workable job")
+    listing_paths = (("jobs",), ("data", "jobs"), ("results",))
+    detail_paths = (("job",), ("data", "job"), ("position",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"draft", "inactive"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []

apps/sources/adapters/base.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+
+@dataclass(slots=True)
+class FieldEvidence:
+    field_name: str
+    method: str
+    confidence: float
+    evidence: str = ""
+
+
+@dataclass(slots=True)
+class ExtractedJob:
+    url: str
+    title: str
+    employer_name: str = ""
+    external_id: str = ""
+    location_text: str = ""
+    region: str = ""
+    postal_code: str = ""
+    country: str = ""
+    description_html: str = ""
+    description_text: str = ""
+    language: str = ""
+    date_posted: str = ""
+    valid_through: str = ""
+    employment_types: list[str] = field(default_factory=list)
+    workplace_type: str = ""
+    compensation: dict[str, Any] = field(default_factory=dict)
+    skills_required: list[str] = field(default_factory=list)
+    skills_preferred: list[str] = field(default_factory=list)
+    raw: dict[str, Any] = field(default_factory=dict)
+    evidence: list[FieldEvidence] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class ExtractionResult:
+    jobs: list[ExtractedJob]
+    parser_key: str
+    parser_version: str
+    confidence: float
+    warnings: list[str] = field(default_factory=list)
+
+
+class SourceAdapter(Protocol):
+    parser_key: str
+    parser_version: str
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult: ...

apps/sources/adapters/email_alert.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import html
+import re
+from email import policy
+from email.message import Message
+from email.parser import BytesParser
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+
+from apps.sources.platforms import PLATFORM_ALERTS
+from apps.sources.services.canonicalize import canonicalize_url
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I)
+SKIP_TEXT = re.compile(
+    r"unsubscribe|afmelden|uitschrijven|privacy|view in browser|bekijk online|"
+    r"account|aanmelden|inloggen|login|voorkeuren|preferences|voorwaarden|terms|"
+    r"contact|help|hulp|over ons|about us",
+    re.I,
+)
+SKIP_PATH = re.compile(
+    r"/(?:unsubscribe|uitschrijven|afmelden|privacy|account|login|signin|preferences|"
+    r"settings|terms|legal|help|contact)(?:/|$)",
+    re.I,
+)
+
+
+def _matches_domain(hostname: str, domains: tuple[str, ...]) -> bool:
+    return any(hostname == domain or hostname.endswith(f".{domain}") for domain in domains)
+
+
+class EmailAlertAdapter:
+    parser_key = "email-alert"
+    parser_version = "1.1.0"
+
+    @staticmethod
+    def _provider_for_candidates(candidates: list[tuple[str, str]]) -> str:
+        hostnames = {
+            (urlsplit(href).hostname or "").lower()
+            for _, href in candidates
+            if href.lower().startswith(("http://", "https://"))
+        }
+        for provider, alert in PLATFORM_ALERTS.items():
+            if any(_matches_domain(hostname, alert.domains) for hostname in hostnames):
+                return provider
+        return "other"
+
+    @staticmethod
+    def _is_expected_platform_link(*, label: str, href: str, provider: str) -> bool:
+        alert = PLATFORM_ALERTS.get(provider)
+        if alert is None or len(label.strip()) < 4:
+            return False
+        parsed = urlsplit(href)
+        hostname = (parsed.hostname or "").lower()
+        return (
+            parsed.scheme.lower() == "https"
+            and _matches_domain(hostname, alert.domains)
+            and not SKIP_TEXT.search(label)
+            and not SKIP_PATH.search(parsed.path)
+        )
+
+    @staticmethod
+    def _decode_parts(message: Message) -> tuple[str, str]:
+        plain_parts: list[str] = []
+        html_parts: list[str] = []
+        for part in message.walk() if message.is_multipart() else [message]:
+            disposition = str(part.get("Content-Disposition") or "")
+            if "attachment" in disposition.lower():
+                continue
+            content_type = part.get_content_type()
+            try:
+                payload = part.get_content()
+            except Exception:
+                raw = part.get_payload(decode=True) or b""
+                payload = raw.decode(part.get_content_charset() or "utf-8", errors="replace")
+            if content_type == "text/plain":
+                plain_parts.append(str(payload))
+            elif content_type == "text/html":
+                html_parts.append(str(payload))
+        return "\n".join(plain_parts), "\n".join(html_parts)
+
+    def extract_message(
+        self, raw_message: bytes, *, expected_provider: str | None = None
+    ) -> ExtractionResult:
+        message = BytesParser(policy=policy.default).parsebytes(raw_message)
+        plain, html_body = self._decode_parts(message)
+        candidates: list[tuple[str, str]] = []
+        if html_body:
+            soup = BeautifulSoup(html_body, "lxml")
+            for anchor in soup.find_all("a", href=True):
+                label = " ".join(anchor.get_text(" ", strip=True).split())
+                href = html.unescape(str(anchor["href"]).strip())
+                if not href.lower().startswith(("http://", "https://")):
+                    continue
+                if SKIP_TEXT.search(label) or SKIP_TEXT.search(href):
+                    continue
+                candidates.append((label, href))
+        for href in URL_RE.findall(plain):
+            clean_href = href.rstrip(".,);]")
+            if SKIP_TEXT.search(clean_href):
+                continue
+            candidates.append(("", clean_href))
+
+        alert_provider = expected_provider or self._provider_for_candidates(candidates)
+        if expected_provider:
+            candidates = [
+                (label, href)
+                for label, href in candidates
+                if self._is_expected_platform_link(
+                    label=label,
+                    href=href,
+                    provider=expected_provider,
+                )
+            ]
+
+        jobs: list[ExtractedJob] = []
+        seen: set[str] = set()
+        subject = str(message.get("subject") or "Vacature uit e-mail").strip()
+        for label, href in candidates:
+            canonical = canonicalize_url(urljoin("https://invalid.local/", href))
+            if not canonical or canonical in seen:
+                continue
+            seen.add(canonical)
+            hostname = urlsplit(canonical).hostname or ""
+            title = label if len(label) >= 4 else subject
+            if len(title) > 300:
+                title = title[:300]
+            jobs.append(
+                ExtractedJob(
+                    url=canonical,
+                    title=title,
+                    employer_name="",
+                    description_text=plain[:5000]
+                    or BeautifulSoup(html_body, "lxml").get_text("\n", strip=True)[:5000],
+                    raw={
+                        "email_subject": subject,
+                        "email_sender": str(message.get("from") or ""),
+                        "target_domain": hostname,
+                        "alert_provider": alert_provider,
+                    },
+                    evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])],
+                )
+            )
+        return ExtractionResult(
+            jobs,
+            self.parser_key,
+            self.parser_version,
+            0.65 if jobs else 0.0,
+            [] if jobs else ["Geen vacaturelinks in e-mail gevonden"],
+        )
+
+    def extract(self, content: str, *, url: str = "") -> ExtractionResult:
+        return self.extract_message(content.encode("utf-8", errors="replace"))

apps/sources/adapters/generic_html.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+LABEL_PATTERNS = {
+    "location": re.compile(r"^(locatie|location|lieu|plaats|standplaats)\s*:?$", re.I),
+    "date_posted": re.compile(r"^(publicatiedatum|geplaatst|date posted|published)\s*:?$", re.I),
+    "employment_type": re.compile(r"^(dienstverband|contract|employment type)\s*:?$", re.I),
+    "employer": re.compile(r"^(werkgever|employer|company|organisatie|société)\s*:?$", re.I),
+}
+
+
+class GenericHtmlAdapter:
+    parser_key = "generic-html"
+    parser_version = "1.1.0"
+
+    @staticmethod
+    def _meta(soup: BeautifulSoup, *names: str) -> str:
+        for name in names:
+            tag = soup.find("meta", attrs={"property": name}) or soup.find(
+                "meta", attrs={"name": name}
+            )
+            if tag and tag.get("content"):
+                return str(tag["content"]).strip()
+        return ""
+
+    @staticmethod
+    def _label_value(soup: BeautifulSoup, pattern: re.Pattern[str]) -> str:
+        label = soup.find(string=lambda value: bool(value and pattern.match(value.strip())))
+        if not label:
+            return ""
+        parent = label.parent
+        if not parent:
+            return ""
+        sibling = parent.find_next_sibling()
+        if sibling:
+            return sibling.get_text(" ", strip=True)
+        text = parent.get_text(" ", strip=True)
+        return pattern.sub("", text).strip(" :-")
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        soup = BeautifulSoup(content, "lxml")
+        for element in soup(["script", "style", "noscript", "template"]):
+            element.decompose()
+        h1 = soup.find("h1")
+        title = (h1.get_text(" ", strip=True) if h1 else "") or self._meta(
+            soup, "og:title", "twitter:title"
+        )
+        if not title and soup.title:
+            title = soup.title.get_text(" ", strip=True)
+        title = re.sub(r"\s+[|\\u2013\\u2014-]\s+.*$", "", title).strip()
+        if not title:
+            return ExtractionResult([], self.parser_key, self.parser_version, 0.0, ["Geen titel"])
+
+        employer = self._meta(soup, "og:site_name", "application-name") or self._label_value(
+            soup, LABEL_PATTERNS["employer"]
+        )
+        location = self._label_value(soup, LABEL_PATTERNS["location"])
+        date_posted = self._label_value(soup, LABEL_PATTERNS["date_posted"])
+        employment_type = self._label_value(soup, LABEL_PATTERNS["employment_type"])
+        main = soup.find("main") or soup.find("article") or soup.body
+        description_html = str(main) if main else ""
+        description_text = (
+            main.get_text("\n", strip=True) if main else soup.get_text("\n", strip=True)
+        )
+        canonical = soup.find("link", rel=lambda value: value and "canonical" in value)
+        job_url = (
+            urljoin(url, canonical.get("href")) if canonical and canonical.get("href") else url
+        )
+        evidence = [
+            FieldEvidence("title", "html-heading", 0.78, title[:240]),
+            FieldEvidence("employer_name", "html-meta-or-label", 0.65, employer[:240]),
+            FieldEvidence("location_text", "html-label", 0.62, location[:240]),
+            FieldEvidence("description", "html-main", 0.70, description_text[:300]),
+        ]
+        if date_posted:
+            evidence.append(FieldEvidence("date_posted", "html-label", 0.62, date_posted[:40]))
+        if employment_type:
+            evidence.append(
+                FieldEvidence("employment_types", "html-label", 0.62, employment_type[:120])
+            )
+        job = ExtractedJob(
+            url=job_url,
+            title=title,
+            employer_name=employer,
+            location_text=location,
+            description_html=description_html,
+            description_text=description_text,
+            date_posted=date_posted,
+            employment_types=[employment_type] if employment_type else [],
+            raw={"generic_html": True},
+            evidence=evidence,
+        )
+        return ExtractionResult([job], self.parser_key, self.parser_version, 0.70)

apps/sources/adapters/jsonld.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from typing import Any
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class JsonLdJobPostingAdapter:
+    parser_key = "jsonld-jobposting"
+    parser_version = "1.0.0"
+
+    @staticmethod
+    def _is_jobposting(value: Any) -> bool:
+        types = value if isinstance(value, list) else [value]
+        return any(str(item).lower() == "jobposting" for item in types)
+
+    def _walk(self, value: Any) -> Iterable[dict[str, Any]]:
+        if isinstance(value, dict):
+            if self._is_jobposting(value.get("@type")):
+                yield value
+            graph = value.get("@graph")
+            if graph is not None:
+                yield from self._walk(graph)
+            for child in value.values():
+                if isinstance(child, dict | list):
+                    yield from self._walk(child)
+        elif isinstance(value, list):
+            for child in value:
+                yield from self._walk(child)
+
+    @staticmethod
+    def _name(value: Any) -> str:
+        if isinstance(value, str):
+            return value.strip()
+        if isinstance(value, dict):
+            return str(value.get("name") or value.get("legalName") or "").strip()
+        return ""
+
+    @staticmethod
+    def _location(value: Any) -> tuple[str, str, str, str]:
+        locations = value if isinstance(value, list) else [value]
+        parts: list[str] = []
+        region = postal = country = ""
+        for location in locations:
+            if not isinstance(location, dict):
+                continue
+            address = location.get("address", location)
+            if isinstance(address, str):
+                parts.append(address)
+                continue
+            if not isinstance(address, dict):
+                continue
+            locality = str(address.get("addressLocality") or "").strip()
+            region = region or str(address.get("addressRegion") or "").strip()
+            postal = postal or str(address.get("postalCode") or "").strip()
+            country_value = address.get("addressCountry")
+            country = (
+                country
+                or JsonLdJobPostingAdapter._name(country_value)
+                or str(country_value or "").strip()
+            )
+            label = ", ".join(part for part in [locality, region, postal, country] if part)
+            if label:
+                parts.append(label)
+        return " | ".join(dict.fromkeys(parts)), region, postal, country
+
+    @staticmethod
+    def _employment_types(value: Any) -> list[str]:
+        if isinstance(value, list):
+            return [str(item).strip() for item in value if str(item).strip()]
+        if value:
+            return [str(value).strip()]
+        return []
+
+    @staticmethod
+    def _identifier(value: Any) -> str:
+        if isinstance(value, dict):
+            return str(value.get("value") or value.get("name") or "").strip()
+        return str(value or "").strip()
+
+    @staticmethod
+    def _salary(value: Any) -> dict[str, Any]:
+        if not isinstance(value, dict):
+            return {}
+        result: dict[str, Any] = {}
+        currency = value.get("currency")
+        if currency:
+            result["currency"] = currency
+        raw_value = value.get("value")
+        if isinstance(raw_value, dict):
+            for key in ("minValue", "maxValue", "value", "unitText"):
+                if key in raw_value:
+                    result[key] = raw_value[key]
+        elif raw_value is not None:
+            result["value"] = raw_value
+        return result
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        soup = BeautifulSoup(content, "lxml")
+        records: list[dict[str, Any]] = []
+        warnings: list[str] = []
+        for script in soup.find_all(
+            "script", attrs={"type": lambda value: value and "ld+json" in value}
+        ):
+            raw = script.string or script.get_text("", strip=True)
+            if not raw:
+                continue
+            try:
+                payload = json.loads(raw)
+            except json.JSONDecodeError:
+                warnings.append("Ongeldige JSON-LD overgeslagen")
+                continue
+            records.extend(self._walk(payload))
+
+        jobs: list[ExtractedJob] = []
+        for record in records:
+            title = str(record.get("title") or record.get("name") or "").strip()
+            if not title:
+                warnings.append("JobPosting zonder titel overgeslagen")
+                continue
+            description_html = str(record.get("description") or "").strip()
+            description_text = BeautifulSoup(description_html, "lxml").get_text("\n", strip=True)
+            employer = self._name(record.get("hiringOrganization"))
+            location, region, postal, country = self._location(record.get("jobLocation"))
+            workplace_type = str(record.get("jobLocationType") or "").strip()
+            if not location and record.get("applicantLocationRequirements"):
+                location, region, postal, country = self._location(
+                    record.get("applicantLocationRequirements")
+                )
+            job_url = str(record.get("url") or url).strip()
+            job_url = urljoin(url, job_url)
+            evidence = [
+                FieldEvidence("title", "jsonld", 0.98, title[:240]),
+                FieldEvidence("employer_name", "jsonld", 0.95, employer[:240]),
+                FieldEvidence("location_text", "jsonld", 0.92, location[:240]),
+                FieldEvidence("description", "jsonld", 0.95, description_text[:300]),
+            ]
+            jobs.append(
+                ExtractedJob(
+                    url=job_url,
+                    title=title,
+                    employer_name=employer,
+                    external_id=self._identifier(record.get("identifier")),
+                    location_text=location,
+                    region=region,
+                    postal_code=postal,
+                    country=country,
+                    description_html=description_html,
+                    description_text=description_text,
+                    language=str(record.get("inLanguage") or "").strip(),
+                    date_posted=str(record.get("datePosted") or "").strip(),
+                    valid_through=str(record.get("validThrough") or "").strip(),
+                    employment_types=self._employment_types(record.get("employmentType")),
+                    workplace_type=workplace_type,
+                    compensation=self._salary(record.get("baseSalary")),
+                    raw=record,
+                    evidence=evidence,
+                )
+            )
+        confidence = 0.95 if jobs else 0.0
+        return ExtractionResult(jobs, self.parser_key, self.parser_version, confidence, warnings)

apps/sources/adapters/kempen.py

@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import parse_qs, urljoin, urlsplit
+
+from bs4 import BeautifulSoup, Tag
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class KempenEmployerAdapter:
+    """Extract public employer lists around Mol from exact reviewed routes."""
+
+    parser_key = "regional-kempen-employers"
+    parser_version = "1.0.0"
+    supported_routes = {
+        "ziekenhuisgeel.careersite.be": {"/nl/vacatures"},
+        "geel.hro.be": {"/"},
+        "jobs.turnhout.be": {"/"},
+        "jobs.renotec.be": {"/nl/alle-jobs"},
+        "ravago.softgarden.io": {"/en/vacancies"},
+        "jobs.sanofi.com": {"/en/belgium"},
+        "www.daf.com": {"/nl-nl/werken-bij-daf/vacatures"},
+    }
+
+    def _route(self, url: str) -> tuple[str, str] | None:
+        parts = urlsplit(url)
+        host = (parts.hostname or "").lower()
+        path = parts.path.rstrip("/") or "/"
+        if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
+            return None
+        return host, path
+
+    def _supports_url(self, url: str) -> bool:
+        return self._route(url) is not None
+
+    @staticmethod
+    def _same_host_url(base_url: str, href: str) -> str:
+        job_url = urljoin(base_url, href)
+        parts = urlsplit(job_url)
+        if parts.scheme != "https":
+            return ""
+        if (parts.hostname or "").lower() != (urlsplit(base_url).hostname or "").lower():
+            return ""
+        return job_url
+
+    @staticmethod
+    def _job(
+        *,
+        url: str,
+        title: str,
+        employer: str,
+        location: str,
+        postal_code: str,
+        description: str,
+        external_id: str,
+        evidence_source: str,
+    ) -> ExtractedJob:
+        return ExtractedJob(
+            url=url,
+            title=title,
+            employer_name=employer,
+            external_id=external_id,
+            location_text=location,
+            postal_code=postal_code,
+            region="Antwerpen",
+            country="BE",
+            description_text=description,
+            raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
+            evidence=[
+                FieldEvidence("title", evidence_source, 0.94, title[:240]),
+                FieldEvidence("employer_name", "reviewed-source", 0.97, employer),
+                FieldEvidence("location_text", "kempen-location-marker", 0.9, location),
+            ],
+        )
+
+    def _ziekenhuis_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select(".vacature-tegel"):
+            link = card.select_one("a.vacature-tegel__link[href]")
+            title_node = card.select_one(".vacature-tegel__titel")
+            if not isinstance(link, Tag) or not isinstance(title_node, Tag):
+                continue
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            match = re.fullmatch(r"/nl/vacature/(\d+)/[^/]+", urlsplit(job_url).path)
+            title = title_node.get_text(" ", strip=True)
+            if not title or not match:
+                continue
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Ziekenhuis Geel",
+                    location="Geel, Antwerpen",
+                    postal_code="2440",
+                    description=card.get_text(" ", strip=True),
+                    external_id=match.group(1),
+                    evidence_source="ziekenhuis-geel-card",
+                )
+            )
+        return jobs
+
+    def _stad_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select(".vacatureKader[data-href]"):
+            href = str(card.get("data-href") or "")
+            match = re.fullmatch(r"vacature\.php\?id=(\d+)", href)
+            title_node = card.find("h4")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            if not title or not match:
+                continue
+            jobs.append(
+                self._job(
+                    url=self._same_host_url(url, href),
+                    title=title,
+                    employer="Lokaal bestuur Geel",
+                    location="Geel, Antwerpen",
+                    postal_code="2440",
+                    description=card.get_text(" ", strip=True),
+                    external_id=match.group(1),
+                    evidence_source="stad-geel-hro-card",
+                )
+            )
+        return jobs
+
+    def _stad_turnhout(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.find_all("a", href=True):
+            href = str(link.get("href") or "")
+            match = re.fullmatch(r"vacature\.php\?id=(\d+)", href)
+            title_node = link.select_one(".block-update__body__title__inner")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            if not title or not match:
+                continue
+            jobs.append(
+                self._job(
+                    url=self._same_host_url(url, href),
+                    title=title,
+                    employer="Stad Turnhout",
+                    location="Turnhout, Antwerpen",
+                    postal_code="2300",
+                    description=link.get_text(" ", strip=True),
+                    external_id=match.group(1),
+                    evidence_source="stad-turnhout-hro-card",
+                )
+            )
+        return jobs
+
+    def _renotec(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select(".s-tile.s-card"):
+            card_text = card.get_text(" ", strip=True)
+            if "Geel" not in card_text:
+                continue
+            title_node = card.find("h3")
+            link = card.find("a", href=True)
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
+            parts = urlsplit(job_url)
+            query = parse_qs(parts.query)
+            external_id = (query.get("id") or [""])[0]
+            if parts.path != "/nl/detail/" or not external_id.isdigit() or not title:
+                continue
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Group Renotec",
+                    location="Geel, Antwerpen (mogelijk meerdere werfregio's)",
+                    postal_code="2440",
+                    description=card_text,
+                    external_id=external_id,
+                    evidence_source="renotec-geel-card",
+                )
+            )
+        return jobs
+
+    def _ravago(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select(".matchElement"):
+            locations = {
+                node.get_text(" ", strip=True) for node in card.select(".location-view-item")
+            }
+            local_places = locations.intersection({"Arendonk", "Olen"})
+            if not local_places:
+                continue
+            link = card.find("a", href=True)
+            title = link.get_text(" ", strip=True) if link else ""
+            job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
+            match = re.fullmatch(r"/job/(\d+)/[^/]+/?", urlsplit(job_url).path)
+            if not title or not match:
+                continue
+            place = "Arendonk" if "Arendonk" in local_places else "Olen"
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Ravago",
+                    location=f"{place}, Antwerpen",
+                    postal_code="2370" if place == "Arendonk" else "2250",
+                    description=card.get_text(" ", strip=True),
+                    external_id=match.group(1),
+                    evidence_source="ravago-softgarden-card",
+                )
+            )
+        return jobs
+
+    def _sanofi(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.select(".job-list a[data-job-id][href]"):
+            location_node = link.select_one(".job-location")
+            title_node = link.select_one(".job-title")
+            location = location_node.get_text(" ", strip=True) if location_node else ""
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            external_id = str(link.get("data-job-id") or "")
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            if location != "Geel, Belgium" or not title or not external_id.isdigit():
+                continue
+            if not re.fullmatch(r"/en/job/geel/[^/]+/\d+/\d+", urlsplit(job_url).path):
+                continue
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Sanofi",
+                    location="Geel, Antwerpen",
+                    postal_code="2440",
+                    description=link.get_text(" ", strip=True),
+                    external_id=external_id,
+                    evidence_source="sanofi-belgium-geel-card",
+                )
+            )
+        return jobs
+
+    def _daf(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select("li.itemlist__item"):
+            location_node = card.select_one(".js-vac-metalocation")
+            title_link = card.select_one("a.js-vac-title[href]")
+            location = location_node.get_text(" ", strip=True) if location_node else ""
+            title = title_link.get_text(" ", strip=True) if title_link else ""
+            job_url = (
+                self._same_host_url(url, str(title_link.get("href") or "")) if title_link else ""
+            )
+            path = urlsplit(job_url).path
+            prefix = "/nl-nl/werken-bij-daf/vacatures/"
+            if location != "Westerlo" or not title or not path.startswith(prefix):
+                continue
+            slug = path.removeprefix(prefix).strip("/")
+            if not slug or "/" in slug:
+                continue
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="DAF Trucks",
+                    location="Westerlo, Antwerpen",
+                    postal_code="2260",
+                    description=card.get_text(" ", strip=True),
+                    external_id=slug,
+                    evidence_source="daf-westerlo-card",
+                )
+            )
+        return jobs
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        route = self._route(url)
+        if route is None:
+            return ExtractionResult(
+                [], self.parser_key, self.parser_version, 0.0, ["Onherkende Kempen-bron"]
+            )
+        host, _ = route
+        extractors = {
+            "ziekenhuisgeel.careersite.be": self._ziekenhuis_geel,
+            "geel.hro.be": self._stad_geel,
+            "jobs.turnhout.be": self._stad_turnhout,
+            "jobs.renotec.be": self._renotec,
+            "ravago.softgarden.io": self._ravago,
+            "jobs.sanofi.com": self._sanofi,
+            "www.daf.com": self._daf,
+        }
+        jobs = extractors[host](BeautifulSoup(content, "lxml"), url)
+        unique_jobs = list({job.url: job for job in jobs if job.url}.values())
+        return ExtractionResult(
+            unique_jobs,
+            self.parser_key,
+            self.parser_version,
+            0.92 if unique_jobs else 0.0,
+            [] if unique_jobs else ["Geen actuele regionale vacatures gevonden"],
+        )

apps/sources/adapters/mol_region.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import parse_qs, urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class MolRegionEmployerAdapter:
+    """Extract reviewed employer listings around postcode 2400 without detail fetches."""
+
+    parser_key = "regional-mol-employers"
+    parser_version = "1.0.0"
+    thomas_more_company_guid = "eab9ca13-ee10-4504-8a87-a785d0b037ef"
+    supported_routes = {
+        "www.sckcen.be": {"/nl/carriere/vacatures"},
+        "cipalschaubroeck.teamtailor.com": {"/jobs"},
+        "www.vanroey.be": {"/en/job-overview"},
+        "netropolix.recruitee.com": {"/"},
+        "jobpage.cvwarehouse.com": {"/"},
+    }
+
+    def _route(self, url: str) -> tuple[str, str] | None:
+        parts = urlsplit(url)
+        host = (parts.hostname or "").lower()
+        path = parts.path.rstrip("/") or "/"
+        if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
+            return None
+        if host == "jobpage.cvwarehouse.com":
+            query = parse_qs(parts.query)
+            if query.get("companyGuid") != [self.thomas_more_company_guid] or "job" in query:
+                return None
+        return host, path
+
+    def _supports_url(self, url: str) -> bool:
+        return self._route(url) is not None
+
+    @staticmethod
+    def _same_host_url(base_url: str, href: str) -> str:
+        job_url = urljoin(base_url, href)
+        if urlsplit(job_url).scheme != "https":
+            return ""
+        if (urlsplit(job_url).hostname or "").lower() != (
+            urlsplit(base_url).hostname or ""
+        ).lower():
+            return ""
+        return job_url
+
+    @staticmethod
+    def _job(
+        *,
+        url: str,
+        title: str,
+        employer: str,
+        location: str,
+        postal_code: str,
+        description: str,
+        external_id: str,
+        evidence_source: str,
+    ) -> ExtractedJob:
+        return ExtractedJob(
+            url=url,
+            title=title,
+            employer_name=employer,
+            external_id=external_id,
+            location_text=location,
+            postal_code=postal_code,
+            region="Antwerpen",
+            country="BE",
+            description_text=description,
+            raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
+            evidence=[
+                FieldEvidence("title", evidence_source, 0.94, title[:240]),
+                FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
+                FieldEvidence("location_text", "mol-region-review", 0.82, location),
+            ],
+        )
+
+    def _sck_cen(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.select("article a[href]"):
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            path = urlsplit(job_url).path if job_url else ""
+            if not path.startswith("/nl/carriere/vacatures/"):
+                continue
+            title = link.get_text(" ", strip=True)
+            if not title:
+                continue
+            card = link.find_parent("article")
+            description = card.get_text(" ", strip=True) if card else title
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="SCK CEN",
+                    location="Mol, Antwerpen",
+                    postal_code="2400",
+                    description=description,
+                    external_id=path.rstrip("/").rsplit("/", 1)[-1],
+                    evidence_source="sck-cen-vacancy-card",
+                )
+            )
+        return jobs
+
+    def _cipal(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select("li"):
+            card_text = card.get_text(" ", strip=True)
+            if "Westerlo" not in card_text and "Geel" not in card_text:
+                continue
+            link = card.find("a", href=True)
+            if not link:
+                continue
+            title = link.get_text(" ", strip=True)
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            path = urlsplit(job_url).path if job_url else ""
+            match = re.fullmatch(r"/jobs/(\d+)-[^/]+", path.rstrip("/"))
+            if not title or not match:
+                continue
+            location = "Geel, Antwerpen" if "Geel" in card_text else "Westerlo, Antwerpen"
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Cipal Schaubroeck",
+                    location=location,
+                    postal_code="2440" if "Geel" in card_text else "2260",
+                    description=card_text,
+                    external_id=match.group(1),
+                    evidence_source="cipal-teamtailor-card",
+                )
+            )
+        return jobs
+
+    def _vanroey(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for heading in soup.select("h3.elementor-heading-title"):
+            link = heading.find("a", href=True)
+            if not link:
+                continue
+            title = link.get_text(" ", strip=True)
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            path = urlsplit(job_url).path if job_url else ""
+            if not title or not re.fullmatch(r"/en/job/[^/]+/", path):
+                continue
+            slug = path.rstrip("/").rsplit("/", 1)[-1]
+            if "oost-vlaanderen" in slug:
+                continue
+            card = heading.find_parent("section") or heading.parent
+            description = card.get_text(" ", strip=True) if card else title
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="VanRoey",
+                    location="Turnhout/Geel (hybride; controleer vacaturedetail)",
+                    postal_code="",
+                    description=description,
+                    external_id=slug,
+                    evidence_source="vanroey-job-card",
+                )
+            )
+        return jobs
+
+    def _netropolix(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.find_all("a", href=True):
+            title = link.get_text(" ", strip=True)
+            if "Geel" not in title:
+                continue
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            path = urlsplit(job_url).path if job_url else ""
+            if not re.fullmatch(r"/o/[^/]+", path.rstrip("/")):
+                continue
+            card = link.find_parent("div")
+            description = card.get_text(" ", strip=True) if card else title
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="NTX (Netropolix)",
+                    location="Geel, Antwerpen",
+                    postal_code="2440",
+                    description=description,
+                    external_id=path.rstrip("/").rsplit("/", 1)[-1],
+                    evidence_source="netropolix-recruitee-card",
+                )
+            )
+        return jobs
+
+    def _thomas_more(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.select("a.jobLink[data-item='readmore'][data-jobid][href]"):
+            title = link.get_text(" ", strip=True)
+            title_casefold = title.casefold()
+            if not any(place in title_casefold for place in ("geel", "turnhout", "vorselaar")):
+                continue
+            external_id = str(link.get("data-jobid") or "")
+            if not external_id.isdigit():
+                continue
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            query = parse_qs(urlsplit(job_url).query) if job_url else {}
+            if query.get("companyGuid") != [self.thomas_more_company_guid] or query.get("job") != [
+                external_id
+            ]:
+                continue
+            if "turnhout" in title_casefold:
+                location, postal_code = "Turnhout, Antwerpen", "2300"
+            elif "vorselaar" in title_casefold:
+                location, postal_code = "Vorselaar, Antwerpen", "2290"
+            else:
+                location, postal_code = "Geel, Antwerpen", "2440"
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Thomas More",
+                    location=location,
+                    postal_code=postal_code,
+                    description=f"Publieke regionale vacature bij Thomas More: {title}.",
+                    external_id=external_id,
+                    evidence_source="thomas-more-cvwarehouse-card",
+                )
+            )
+        return jobs
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        route = self._route(url)
+        if route is None:
+            return ExtractionResult(
+                [], self.parser_key, self.parser_version, 0.0, ["Onherkende Mol-regiobron"]
+            )
+        host, _ = route
+        soup = BeautifulSoup(content, "lxml")
+        extractors = {
+            "www.sckcen.be": self._sck_cen,
+            "cipalschaubroeck.teamtailor.com": self._cipal,
+            "www.vanroey.be": self._vanroey,
+            "netropolix.recruitee.com": self._netropolix,
+            "jobpage.cvwarehouse.com": self._thomas_more,
+        }
+        jobs = extractors[host](soup, url)
+        unique_jobs = list({job.url: job for job in jobs}.values())
+        warnings = [] if unique_jobs else ["Geen actuele vacatures rond Mol gevonden"]
+        return ExtractionResult(
+            unique_jobs,
+            self.parser_key,
+            self.parser_version,
+            0.9 if unique_jobs else 0.0,
+            warnings,
+        )

apps/sources/adapters/regional.py

@@ -0,0 +1,312 @@
+from __future__ import annotations
+
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class CordaCampusAdapter:
+    """Extract the public job cards intentionally published by Corda Campus."""
+
+    parser_key = "regional-corda-campus"
+    parser_version = "1.0.0"
+    source_hosts = ("cordacampus.com",)
+
+    def _supports_url(self, url: str) -> bool:
+        parts = urlsplit(url)
+        host = (parts.hostname or "").lower()
+        return host.endswith(self.source_hosts) and parts.path.rstrip("/") == "/jobs"
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        if not self._supports_url(url):
+            return ExtractionResult(
+                [], self.parser_key, self.parser_version, 0.0, ["Onherkenbare regiobron"]
+            )
+
+        source_host = (urlsplit(url).hostname or "").lower()
+        soup = BeautifulSoup(content, "lxml")
+        jobs: list[ExtractedJob] = []
+        seen_urls: set[str] = set()
+
+        for card in soup.select('a.event-item[href*="/job/"]'):
+            job_url = urljoin(url, str(card.get("href") or ""))
+            job_host = (urlsplit(job_url).hostname or "").lower()
+            if job_host != source_host or job_url in seen_urls:
+                continue
+            title_node = card.select_one(".job-title")
+            employer_node = card.select_one(".company-title")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            employer = employer_node.get_text(" ", strip=True) if employer_node else ""
+            if not title:
+                continue
+
+            date_node = card.select_one(".bottom-info")
+            date_posted = date_node.get_text(" ", strip=True) if date_node else ""
+            external_id = urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1]
+            description = f"Vacature van {employer or 'een Corda-werkgever'} via Corda Campus."
+            jobs.append(
+                ExtractedJob(
+                    url=job_url,
+                    title=title,
+                    employer_name=employer,
+                    external_id=external_id,
+                    location_text="Hasselt, Limburg",
+                    region="Limburg",
+                    country="BE",
+                    description_text=description,
+                    date_posted=date_posted,
+                    raw={"regional_listing": "corda-campus"},
+                    evidence=[
+                        FieldEvidence("title", "corda-job-card", 0.95, title[:240]),
+                        FieldEvidence("employer_name", "corda-job-card", 0.92, employer[:240]),
+                        FieldEvidence(
+                            "location_text",
+                            "regional-source-scope",
+                            0.72,
+                            "Corda Campus, Hasselt",
+                        ),
+                    ],
+                )
+            )
+            seen_urls.add(job_url)
+
+        warnings = [] if jobs else ["Geen actuele Corda-vacatures gevonden"]
+        return ExtractionResult(
+            jobs,
+            self.parser_key,
+            self.parser_version,
+            0.9 if jobs else 0.0,
+            warnings,
+        )
+
+
+class LocalEmployerListingAdapter:
+    """Extract reviewed public listing pages of employers in the Hasselt-Genk area."""
+
+    parser_key = "regional-local-employers"
+    parser_version = "1.0.0"
+    supported_routes = {
+        "acagroup.be": {"/en/jobs"},
+        "www.xploregroup.be": {"/en/jobs"},
+        "www.uhasselt.be": {"/vacatures"},
+        "ses.pxl.be": {"/"},
+        "ziekenhuis-oost-limburg.cvw.io": {"/"},
+    }
+
+    def _supports_url(self, url: str) -> bool:
+        return self._route(url) is not None
+
+    def _route(self, url: str) -> tuple[str, str] | None:
+        parts = urlsplit(url)
+        host = (parts.hostname or "").lower()
+        path = parts.path.rstrip("/") or "/"
+        if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
+            return None
+        return host, path
+
+    @staticmethod
+    def _job(
+        *,
+        url: str,
+        title: str,
+        employer: str,
+        location: str,
+        description: str,
+        external_id: str,
+        evidence_source: str,
+    ) -> ExtractedJob:
+        return ExtractedJob(
+            url=url,
+            title=title,
+            employer_name=employer,
+            external_id=external_id,
+            location_text=location,
+            region="Limburg",
+            country="BE",
+            description_text=description,
+            raw={"regional_listing": evidence_source},
+            evidence=[
+                FieldEvidence("title", evidence_source, 0.94, title[:240]),
+                FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
+                FieldEvidence("location_text", "regional-source-scope", 0.78, location),
+            ],
+        )
+
+    @staticmethod
+    def _same_host_url(base_url: str, href: str) -> str:
+        job_url = urljoin(base_url, href)
+        if (urlsplit(job_url).hostname or "").lower() != (
+            urlsplit(base_url).hostname or ""
+        ).lower():
+            return ""
+        return job_url
+
+    def _aca(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.find_all("a", href=True):
+            href = str(link.get("href") or "")
+            if not href.startswith("/en/jobs/"):
+                continue
+            title_node = link.find("h3")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            job_url = self._same_host_url(url, href)
+            if not title or not job_url:
+                continue
+            description_node = link.find("p")
+            description = (
+                description_node.get_text(" ", strip=True)
+                if description_node
+                else f"Vacature bij ACA Group: {title}."
+            )
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="ACA Group",
+                    location="Hasselt (hybride; kantoorselectie per vacature)",
+                    description=description,
+                    external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1],
+                    evidence_source="aca-job-card",
+                )
+            )
+        return jobs
+
+    def _xplore(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for heading in soup.find_all("h3"):
+            link = heading.find("a", href=True)
+            if not link or not str(link.get("href") or "").startswith("/en/jobs/"):
+                continue
+            card = heading.parent
+            location_node = next(
+                (
+                    node
+                    for node in card.find_all("p")
+                    if "Hasselt" in node.get_text(" ", strip=True)
+                ),
+                None,
+            )
+            if not location_node:
+                continue
+            title = link.get_text(" ", strip=True)
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            if not title or not job_url:
+                continue
+            location = location_node.get_text(" ", strip=True)
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Xplore Group",
+                    location=location,
+                    description=(
+                        f"Vacature bij Xplore Group met Hasselt als mogelijke werklocatie: {title}."
+                    ),
+                    external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1],
+                    evidence_source="xplore-job-card",
+                )
+            )
+        return jobs
+
+    def _uhasselt(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select("section.vacancy-item"):
+            title_node = card.find("h3")
+            link = card.find("a", href=True)
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
+            if not title or not job_url or "/vacatures/detail/" not in urlsplit(job_url).path:
+                continue
+            external_id = urlsplit(job_url).path.split("/detail/", 1)[-1].split("-", 1)[0]
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Universiteit Hasselt",
+                    location="Hasselt/Diepenbeek, Limburg",
+                    description=card.get_text(" ", strip=True),
+                    external_id=external_id,
+                    evidence_source="uhasselt-vacancy-card",
+                )
+            )
+        return jobs
+
+    def _pxl(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select(".vacature-card[id^='Vacature_']"):
+            external_id = str(card.get("id") or "").removeprefix("Vacature_")
+            title_node = card.select_one(".vacature-card-titel")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            if not title or not external_id.isdigit():
+                continue
+            cells = [
+                node.get_text(" ", strip=True) for node in card.select("td.vacature-card-td-text")
+            ]
+            campus = next((value for value in cells if value.startswith("Campus ")), "Hasselt")
+            job_url = f"{url.rstrip('/')}?vacature_id={external_id}"
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Hogeschool PXL",
+                    location=f"{campus}, Limburg",
+                    description=" · ".join(cells),
+                    external_id=external_id,
+                    evidence_source="pxl-vacancy-card",
+                )
+            )
+        return jobs
+
+    def _zol(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.select("a[data-item='readmore'][data-jobid][href]"):
+            external_id = str(link.get("data-jobid") or "")
+            title_node = link.select_one(".job-title")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            if not title or not external_id.isdigit() or not job_url:
+                continue
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Ziekenhuis Oost-Limburg",
+                    location="Genk/Lanaken/Maaseik, Limburg",
+                    description=f"Publieke vacature van Ziekenhuis Oost-Limburg: {title}.",
+                    external_id=external_id,
+                    evidence_source="zol-cvwarehouse-card",
+                )
+            )
+        return jobs
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        route = self._route(url)
+        if route is None:
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Onherkenbare lokale werkgeversbron"],
+            )
+        host, _ = route
+        soup = BeautifulSoup(content, "lxml")
+        extractors = {
+            "acagroup.be": self._aca,
+            "www.xploregroup.be": self._xplore,
+            "www.uhasselt.be": self._uhasselt,
+            "ses.pxl.be": self._pxl,
+            "ziekenhuis-oost-limburg.cvw.io": self._zol,
+        }
+        jobs = extractors[host](soup, url)
+        unique_jobs = list({job.url: job for job in jobs}.values())
+        warnings = [] if unique_jobs else ["Geen actuele lokale vacatures gevonden"]
+        return ExtractionResult(
+            unique_jobs,
+            self.parser_key,
+            self.parser_version,
+            0.9 if unique_jobs else 0.0,
+            warnings,
+        )

apps/sources/adapters/registry.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from apps.sources.models import RawDocument
+
+from .ats import (
+    GreenhouseAdapter,
+    LeverAdapter,
+    RecruiteeAdapter,
+    SmartRecruitersAdapter,
+    WorkableAdapter,
+)
+from .base import ExtractionResult
+from .generic_html import GenericHtmlAdapter
+from .jsonld import JsonLdJobPostingAdapter
+from .kempen import KempenEmployerAdapter
+from .mol_region import MolRegionEmployerAdapter
+from .regional import CordaCampusAdapter, LocalEmployerListingAdapter
+from .rss import RssAdapter
+
+
+class AdapterRegistry:
+    def __init__(self) -> None:
+        self.greenhouse = GreenhouseAdapter()
+        self.lever = LeverAdapter()
+        self.recruitee = RecruiteeAdapter()
+        self.smartrecruiters = SmartRecruitersAdapter()
+        self.workable = WorkableAdapter()
+        self.corda_campus = CordaCampusAdapter()
+        self.local_employers = LocalEmployerListingAdapter()
+        self.mol_region_employers = MolRegionEmployerAdapter()
+        self.kempen_employers = KempenEmployerAdapter()
+        self.jsonld = JsonLdJobPostingAdapter()
+        self.generic = GenericHtmlAdapter()
+        self.rss = RssAdapter()
+        self.providers = [
+            self.corda_campus,
+            self.local_employers,
+            self.mol_region_employers,
+            self.kempen_employers,
+            self.greenhouse,
+            self.lever,
+            self.recruitee,
+            self.smartrecruiters,
+            self.workable,
+        ]
+
+    def extract(self, document: RawDocument) -> ExtractionResult:
+        content = document.body_text
+        url = document.final_url or document.url
+        content_type = (document.content_type or "").lower()
+        if document.kind == RawDocument.Kind.XML or "rss" in content_type or "atom" in content_type:
+            return self.rss.extract(content, url=url)
+
+        for provider in self.providers:
+            if provider._supports_url(url):
+                result = provider.extract(content, url=url)
+                if any(
+                    msg in result.warnings
+                    for msg in (
+                        "Geen parseerbare ATS-response",
+                        "Geen herkenbare ATS-markup voor deze adapter",
+                    )
+                ):
+                    continue
+                return result
+
+        jsonld_result = self.jsonld.extract(content, url=url)
+        if jsonld_result.jobs:
+            return jsonld_result
+        generic_result = self.generic.extract(content, url=url)
+        generic_result.warnings = jsonld_result.warnings + generic_result.warnings
+        return generic_result
+
+
+registry = AdapterRegistry()

apps/sources/adapters/rss.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from datetime import datetime
+from time import mktime
+
+import feedparser
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class RssAdapter:
+    parser_key = "rss-atom"
+    parser_version = "1.0.0"
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        feed = feedparser.parse(content)
+        jobs: list[ExtractedJob] = []
+        for entry in feed.entries:
+            title = str(entry.get("title") or "").strip()
+            link = str(entry.get("link") or "").strip()
+            if not title or not link:
+                continue
+            summary = str(entry.get("summary") or entry.get("description") or "")
+            text = BeautifulSoup(summary, "lxml").get_text("\n", strip=True)
+            published = ""
+            if entry.get("published_parsed"):
+                published = datetime.fromtimestamp(mktime(entry.published_parsed)).isoformat()
+            jobs.append(
+                ExtractedJob(
+                    url=link,
+                    title=title,
+                    external_id=str(entry.get("id") or ""),
+                    description_html=summary,
+                    description_text=text,
+                    date_posted=published,
+                    raw=dict(entry),
+                    evidence=[
+                        FieldEvidence("title", "rss", 0.88, title[:240]),
+                        FieldEvidence("description", "rss", 0.80, text[:300]),
+                    ],
+                )
+            )
+        warnings: list[str] = []
+        if getattr(feed, "bozo", False):
+            warnings.append(str(getattr(feed, "bozo_exception", "Ongeldige feed")))
+        return ExtractionResult(
+            jobs,
+            self.parser_key,
+            self.parser_version,
+            0.82 if jobs else 0.0,
+            warnings,
+        )

apps/sources/admin.py

@@ -0,0 +1,104 @@
+from django.contrib import admin
+
+from .models import (
+    EmailMessageRecord,
+    MailboxConnection,
+    RawDocument,
+    Source,
+    SourceLease,
+    SourceOriginState,
+    SourcePolicyReview,
+    SourceRobotsCache,
+    SourceRun,
+)
+
+
+@admin.register(MailboxConnection)
+class MailboxConnectionAdmin(admin.ModelAdmin):
+    list_display = (
+        "platform",
+        "provider",
+        "username",
+        "enabled",
+        "poll_interval_minutes",
+        "last_success_at",
+        "next_poll_at",
+    )
+    list_filter = ("platform", "provider", "enabled")
+    search_fields = ("username",)
+    exclude = ("encrypted_password", "lease_token")
+
+
+@admin.register(Source)
+class SourceAdmin(admin.ModelAdmin):
+    list_display = (
+        "name",
+        "domain",
+        "source_type",
+        "status",
+        "policy",
+        "last_success_at",
+        "failure_count",
+        "next_run_at",
+    )
+    list_filter = ("source_type", "status", "policy", "strict_mode")
+    search_fields = ("name", "domain", "base_url")
+
+
+@admin.register(SourceRun)
+class SourceRunAdmin(admin.ModelAdmin):
+    list_display = ("source", "status", "started_at", "finished_at", "extracted_count")
+    list_filter = ("status", "error_category")
+    readonly_fields = [field.name for field in SourceRun._meta.fields]
+
+
+@admin.register(SourcePolicyReview)
+class SourcePolicyReviewAdmin(admin.ModelAdmin):
+    list_display = (
+        "source",
+        "decision",
+        "scope",
+        "actor",
+        "expires_at",
+        "created_at",
+    )
+    list_filter = ("decision", "scope", "expires_at")
+    search_fields = ("source__name", "source__domain", "actor__username", "reason")
+    readonly_fields = [field.name for field in SourcePolicyReview._meta.fields]
+
+
+@admin.register(SourceRobotsCache)
+class SourceRobotsCacheAdmin(admin.ModelAdmin):
+    list_display = ("origin", "expires_at", "byte_length", "updated_at", "error")
+    list_filter = ("error",)
+    search_fields = ("origin",)
+    readonly_fields = [field.name for field in SourceRobotsCache._meta.fields]
+
+
+@admin.register(SourceLease)
+class SourceLeaseAdmin(admin.ModelAdmin):
+    list_display = ("source", "token", "worker_id", "expires_at", "updated_at")
+    search_fields = ("source__name", "source__domain", "worker_id")
+    readonly_fields = [field.name for field in SourceLease._meta.fields]
+
+
+@admin.register(SourceOriginState)
+class SourceOriginStateAdmin(admin.ModelAdmin):
+    list_display = ("domain", "next_allowed_at", "updated_at")
+    search_fields = ("domain",)
+    readonly_fields = [field.name for field in SourceOriginState._meta.fields]
+
+
+@admin.register(RawDocument)
+class RawDocumentAdmin(admin.ModelAdmin):
+    list_display = ("source", "kind", "http_status", "byte_length", "quarantined", "created_at")
+    list_filter = ("kind", "quarantined", "http_status")
+    search_fields = ("url", "final_url", "content_hash")
+    readonly_fields = [field.name for field in RawDocument._meta.fields]
+
+
+@admin.register(EmailMessageRecord)
+class EmailMessageRecordAdmin(admin.ModelAdmin):
+    list_display = ("subject", "sender", "received_at", "processed", "created_at")
+    list_filter = ("processed", "mailbox")
+    search_fields = ("subject", "sender", "message_id")

apps/sources/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class SourcesConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "apps.sources"
+    verbose_name = "Bronnen"

apps/sources/forms.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from django import forms
+
+from .models import MailboxConnection
+
+
+class MailboxConnectionForm(forms.ModelForm):
+    password = forms.CharField(
+        required=False,
+        label="App-wachtwoord",
+        help_text=(
+            "Alleen voor Gmail of een aangepaste provider. Microsoft 365 gebruikt "
+            "de centrale OAuth-configuratie."
+        ),
+        widget=forms.PasswordInput(
+            attrs={"autocomplete": "new-password", "placeholder": "App-wachtwoord"},
+            render_value=False,
+        ),
+    )
+
+    class Meta:
+        model = MailboxConnection
+        fields = (
+            "platform",
+            "provider",
+            "custom_host",
+            "port",
+            "username",
+            "mailbox",
+            "poll_interval_minutes",
+            "enabled",
+        )
+        labels = {
+            "platform": "Vacatureplatform",
+            "provider": "Mailboxprovider",
+            "custom_host": "Aangepaste IMAP-host",
+            "port": "IMAP-poort",
+            "username": "Mailboxaccount",
+            "mailbox": "Map",
+            "poll_interval_minutes": "Controlefrequentie",
+            "enabled": "Automatisch synchroniseren",
+        }
+        widgets = {
+            "username": forms.EmailInput(attrs={"autocomplete": "username"}),
+            "custom_host": forms.TextInput(
+                attrs={"autocomplete": "off", "placeholder": "imap.provider.be"}
+            ),
+            "mailbox": forms.TextInput(attrs={"autocomplete": "off"}),
+        }
+
+    def clean(self):
+        data = super().clean()
+        if data.get("provider") != MailboxConnection.Provider.CUSTOM:
+            data["custom_host"] = ""
+        uses_oauth = data.get("provider") == MailboxConnection.Provider.OUTLOOK
+        if not self.instance.pk and not uses_oauth and not data.get("password"):
+            self.add_error("password", "Een app-wachtwoord is verplicht voor een nieuwe koppeling.")
+        return data
+
+
+class ManualImportForm(forms.Form):
+    source_url = forms.URLField(
+        required=False,
+        label="Vacature-URL",
+        max_length=1000,
+        widget=forms.URLInput(attrs={"autocomplete": "off"}),
+    )
+    pasted_text = forms.CharField(
+        required=False,
+        label="Tekst plakken",
+        widget=forms.Textarea(
+            attrs={
+                "rows": 6,
+                "placeholder": "Plak hier de vacaturetekst of relevante pagina-inhoud.",
+            }
+        ),
+    )
+
+    def clean(self):
+        data = super().clean()
+        source_url = (data.get("source_url") or "").strip()
+        pasted_text = (data.get("pasted_text") or "").strip()
+        if not source_url and not pasted_text:
+            raise forms.ValidationError("Vul een URL of een tekstfragment in.")
+        return {"source_url": source_url, "pasted_text": pasted_text}

apps/sources/management/commands/discover_sources.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from django.core.management.base import BaseCommand, CommandError
+
+from apps.sources.services.discovery import (
+    discover_from_email,
+    discover_from_feed,
+    discover_from_html,
+    discover_from_sitemap,
+    persist_discovery_candidates,
+)
+
+
+class Command(BaseCommand):
+    help = (
+        "Voert bronontdekking uit op lokaal aangeleverde content "
+        "en slaat alleen kandidaatregels op met policy review."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "mode",
+            choices=["html", "sitemap", "feed", "email"],
+            help="Bronmodus voor ontdekking",
+        )
+        parser.add_argument(
+            "--input",
+            required=True,
+            dest="input_path",
+            help="Pad naar inputbestand",
+        )
+        parser.add_argument(
+            "--base-url",
+            dest="base_url",
+            help="Base URL voor relativiteitsresolutie",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Geen database-write; toon alleen ontdekte kandidaten",
+        )
+
+    def handle(self, *args, **options):
+        mode = options["mode"]
+        input_path = Path(options["input_path"]).resolve()
+        base_url = (options["base_url"] or "").strip()
+        if mode in {"html", "sitemap", "feed"} and not base_url:
+            raise CommandError("--base-url is verplicht voor html/sitemap/feed")
+
+        if not input_path.is_file():
+            raise CommandError(f"Inputbestand niet gevonden: {input_path}")
+
+        if mode in {"html", "sitemap", "feed"}:
+            content = input_path.read_text(encoding="utf-8")
+        else:
+            content = input_path.read_bytes()
+
+        if mode == "html":
+            candidates = discover_from_html(content, base_url=base_url)
+        elif mode == "sitemap":
+            candidates = discover_from_sitemap(content, base_url=base_url)
+        elif mode == "feed":
+            candidates = discover_from_feed(content, base_url=base_url)
+        else:
+            candidates = discover_from_email(content)
+
+        self.stdout.write(f"Ontdekt {len(candidates)} kandidaten (mode={mode}).")
+        if not candidates:
+            self.stdout.write(self.style.WARNING("Geen bruikbare kandidaten gevonden."))
+            return
+
+        if options["dry_run"]:
+            self.stdout.write(self.style.WARNING("Dry-run modus: er wordt niet geschreven."))
+            for candidate in candidates:
+                self.stdout.write(f"- {candidate.source_type} {candidate.url} [{candidate.reason}]")
+            return
+
+        created, updated, skipped = persist_discovery_candidates(candidates)
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Persist result: {created} nieuw, {updated} bijgewerkt, {skipped} ongewijzigd."
+            )
+        )

apps/sources/management/commands/import_job_fixture.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+
+from apps.jobs.services.pipeline import process_raw_document
+from apps.sources.models import RawDocument, Source, SourceRun
+
+
+class Command(BaseCommand):
+    help = "Importeert een lokale HTML/XML-fixture via dezelfde pipeline als een live bron."
+
+    def add_arguments(self, parser):
+        parser.add_argument("path")
+        parser.add_argument("--url", required=True)
+        parser.add_argument("--source-name", default="Lokale fixture")
+        parser.add_argument("--source-type", default=Source.Type.EMPLOYER)
+
+    def handle(self, *args, **options):
+        path = Path(options["path"]).resolve()
+        if not path.is_file():
+            raise CommandError(f"Bestand bestaat niet: {path}")
+        content = path.read_text(encoding="utf-8")
+        hostname = (urlsplit(options["url"]).hostname or "").lower()
+        if not hostname:
+            raise CommandError("--url bevat geen geldig domein")
+        source, _ = Source.objects.get_or_create(
+            base_url=options["url"],
+            source_type=options["source_type"],
+            defaults={
+                "domain": hostname,
+                "name": options["source_name"],
+                "status": Source.Status.ACTIVE,
+                "policy": Source.Policy.ALLOW,
+                "parser_key": "auto",
+            },
+        )
+        run = SourceRun.objects.create(source=source)
+        suffix = path.suffix.lower()
+        kind = RawDocument.Kind.XML if suffix in {".xml", ".rss"} else RawDocument.Kind.HTML
+        document = RawDocument.objects.create(
+            source=source,
+            source_run=run,
+            url=options["url"],
+            final_url=options["url"],
+            kind=kind,
+            content_type="application/xml" if kind == RawDocument.Kind.XML else "text/html",
+            content_hash=hashlib.sha256(content.encode()).hexdigest(),
+            body_text=content,
+            byte_length=len(content.encode()),
+            retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
+        )
+        metrics = process_raw_document(document)
+        run.finish(
+            SourceRun.Status.SUCCESS,
+            extracted_count=int(metrics["extracted"]),
+            created_count=int(metrics["created"]),
+            updated_count=int(metrics["updated"]),
+            duplicate_count=int(metrics["duplicates"]),
+            metrics=metrics,
+        )
+        self.stdout.write(self.style.SUCCESS(str(metrics)))

apps/sources/management/commands/rotate_mailbox_credentials.py

@@ -0,0 +1,17 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from apps.sources.services.mailbox_connections import (
+    MailboxCredentialError,
+    rotate_mailbox_credentials,
+)
+
+
+class Command(BaseCommand):
+    help = "Versleutel alle mailbox-app-wachtwoorden opnieuw met de eerste ingestelde sleutel."
+
+    def handle(self, *args, **options):
+        try:
+            count = rotate_mailbox_credentials()
+        except MailboxCredentialError as exc:
+            raise CommandError(str(exc)) from exc
+        self.stdout.write(self.style.SUCCESS(f"{count} mailboxcredential(s) geroteerd."))

apps/sources/management/commands/seed_sources.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import yaml
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.dateparse import parse_datetime
+
+from apps.sources.models import Source, SourcePolicyReview
+from apps.sources.services.policy import is_denied_domain
+
+
+class Command(BaseCommand):
+    help = "Laadt gecontroleerde bronseeds uit YAML."
+
+    def add_arguments(self, parser):
+        parser.add_argument("path", nargs="?", default="config-data/seed_sources.yaml")
+
+    def handle(self, *args, **options):
+        path = Path(options["path"])
+        if not path.is_file():
+            raise CommandError(f"Seedbestand ontbreekt: {path}")
+        payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        created = updated = reviews_created = 0
+        if payload.get("disable_demo_sources") is True:
+            demo_sources = Source.objects.filter(domain="jobs.example.org")
+            for demo_source in demo_sources:
+                demo_source.status = Source.Status.DISABLED
+                demo_source.policy = Source.Policy.DENY
+                demo_source.policy_reason = "Demobron verborgen na installatie van livebronnen"
+                demo_source.metadata = {**demo_source.metadata, "demo": True}
+                demo_source.save(
+                    update_fields=[
+                        "status",
+                        "policy",
+                        "policy_reason",
+                        "metadata",
+                        "updated_at",
+                    ]
+                )
+        retired = 0
+        for retired_url in payload.get("retire_source_urls", []):
+            for retired_source in Source.objects.filter(base_url=str(retired_url).strip()):
+                retired_source.status = Source.Status.DISABLED
+                retired_source.next_run_at = None
+                retired_source.metadata = {
+                    **retired_source.metadata,
+                    "retired": True,
+                    "retired_reason": "Buiten de ingestelde regionale dekking",
+                    "hidden_from_source_list": True,
+                }
+                retired_source.save(
+                    update_fields=["status", "next_run_at", "metadata", "updated_at"]
+                )
+                retired += 1
+        for item in payload.get("sources", []):
+            url = str(item.get("url") or "").strip()
+            domain = (urlsplit(url).hostname or "").lower()
+            if not domain:
+                self.stderr.write(f"Overgeslagen zonder domein: {item}")
+                continue
+            deny = is_denied_domain(domain)
+            source, was_created = Source.objects.update_or_create(
+                base_url=url,
+                source_type=item.get("type", Source.Type.EMPLOYER),
+                defaults={
+                    "domain": domain,
+                    "name": item.get("name") or domain,
+                    "status": Source.Status.PAUSED
+                    if deny
+                    else item.get("status", Source.Status.TRIAL),
+                    "policy": Source.Policy.DENY
+                    if deny
+                    else item.get("policy", Source.Policy.REVIEW),
+                    "policy_reason": "Standaard platformdenylist"
+                    if deny
+                    else item.get("policy_reason", ""),
+                    "parser_key": item.get("parser", "auto"),
+                    "crawl_interval_minutes": int(item.get("crawl_interval_minutes", 720)),
+                    "minimum_interval_seconds": int(item.get("minimum_interval_seconds", 30)),
+                    "allow_public_endpoint": bool(item.get("allow_public_endpoint", False)),
+                    "metadata": item.get("metadata") or {},
+                },
+            )
+            review = item.get("review") or {}
+            if review:
+                expires_at = parse_datetime(str(review.get("expires_at") or ""))
+                if expires_at is None:
+                    raise CommandError(
+                        f"Bron {source.name}: review.expires_at moet een ISO-datetime zijn"
+                    )
+                decision = str(review.get("decision") or "")
+                if decision not in SourcePolicyReview.Decision.values:
+                    raise CommandError(f"Bron {source.name}: ongeldige reviewbeslissing")
+                _, review_created = SourcePolicyReview.objects.get_or_create(
+                    source=source,
+                    decision=decision,
+                    evidence_link=str(review.get("evidence_link") or ""),
+                    expires_at=expires_at,
+                    defaults={
+                        "scope": SourcePolicyReview.Scope.SOURCE,
+                        "reason": str(review.get("reason") or "Bronseedreview"),
+                        "notes": str(review.get("notes") or ""),
+                        "metadata": review.get("metadata") or {},
+                    },
+                )
+                reviews_created += int(review_created)
+            created += int(was_created)
+            updated += int(not was_created)
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Bronnen: {created} aangemaakt, {updated} bijgewerkt, "
+                f"{reviews_created} reviews geregistreerd, {retired} buiten regio uitgeschakeld"
+            )
+        )

apps/sources/migrations/0001_initial.py

@@ -0,0 +1,136 @@
+# Generated by Django 5.2.16 on 2026-07-20 23:57
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RawDocument',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('url', models.URLField(blank=True, max_length=2000)),
+                ('final_url', models.URLField(blank=True, max_length=2000)),
+                ('kind', models.CharField(choices=[('html', 'HTML'), ('xml', 'XML'), ('email', 'E-mail'), ('json', 'JSON'), ('text', 'Tekst')], max_length=16)),
+                ('content_type', models.CharField(blank=True, max_length=200)),
+                ('http_status', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('response_headers', models.JSONField(blank=True, default=dict)),
+                ('content_hash', models.CharField(db_index=True, max_length=64)),
+                ('body_text', models.TextField(blank=True)),
+                ('byte_length', models.PositiveIntegerField(default=0)),
+                ('parser_key', models.CharField(blank=True, max_length=120)),
+                ('parser_version', models.CharField(blank=True, max_length=80)),
+                ('extraction_confidence', models.DecimalField(decimal_places=3, default=0, max_digits=4)),
+                ('retain_until', models.DateTimeField(blank=True, null=True)),
+                ('quarantined', models.BooleanField(default=False)),
+                ('quarantine_reason', models.CharField(blank=True, max_length=500)),
+                ('metadata', models.JSONField(blank=True, default=dict)),
+            ],
+            options={
+                'ordering': ['-created_at'],
+            },
+        ),
+        migrations.CreateModel(
+            name='EmailMessageRecord',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('message_id', models.CharField(max_length=998, unique=True)),
+                ('mailbox', models.CharField(default='INBOX', max_length=255)),
+                ('sender', models.CharField(blank=True, max_length=500)),
+                ('subject', models.CharField(blank=True, max_length=998)),
+                ('received_at', models.DateTimeField(blank=True, null=True)),
+                ('links', models.JSONField(blank=True, default=list)),
+                ('processed', models.BooleanField(default=False)),
+                ('error_message', models.CharField(blank=True, max_length=1000)),
+                ('raw_document', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='email_record', to='sources.rawdocument')),
+            ],
+            options={
+                'ordering': ['-received_at', '-created_at'],
+            },
+        ),
+        migrations.CreateModel(
+            name='Source',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=200)),
+                ('source_type', models.CharField(choices=[('employer', 'Werkgeverspagina'), ('ats', 'Publieke ATS-pagina'), ('rss', 'RSS/Atom'), ('sitemap', 'Sitemap'), ('email', 'Vacaturemail'), ('manual', 'Handmatige import')], max_length=24)),
+                ('base_url', models.URLField(blank=True, max_length=1000)),
+                ('domain', models.CharField(db_index=True, max_length=255)),
+                ('status', models.CharField(choices=[('candidate', 'Kandidaat'), ('trial', 'Proefrun'), ('active', 'Actief'), ('quarantined', 'Quarantaine'), ('paused', 'Gepauzeerd'), ('disabled', 'Uitgeschakeld')], default='candidate', max_length=24)),
+                ('policy', models.CharField(choices=[('allow', 'Toestaan'), ('review', 'Te beoordelen'), ('deny', 'Blokkeren')], default='review', max_length=16)),
+                ('policy_reason', models.CharField(blank=True, max_length=500)),
+                ('parser_key', models.CharField(default='auto', max_length=120)),
+                ('strict_mode', models.BooleanField(default=True)),
+                ('allow_public_endpoint', models.BooleanField(default=False)),
+                ('honor_robots', models.BooleanField(default=True)),
+                ('crawl_interval_minutes', models.PositiveIntegerField(default=720)),
+                ('minimum_interval_seconds', models.PositiveIntegerField(default=30)),
+                ('max_concurrency', models.PositiveSmallIntegerField(default=1)),
+                ('next_run_at', models.DateTimeField(blank=True, db_index=True, null=True)),
+                ('last_success_at', models.DateTimeField(blank=True, null=True)),
+                ('last_failure_at', models.DateTimeField(blank=True, null=True)),
+                ('failure_count', models.PositiveIntegerField(default=0)),
+                ('etag', models.CharField(blank=True, max_length=500)),
+                ('last_modified', models.CharField(blank=True, max_length=500)),
+                ('robots_checked_at', models.DateTimeField(blank=True, null=True)),
+                ('terms_checked_at', models.DateTimeField(blank=True, null=True)),
+                ('metadata', models.JSONField(blank=True, default=dict)),
+            ],
+            options={
+                'ordering': ['name'],
+                'constraints': [models.UniqueConstraint(fields=('domain', 'source_type'), name='unique_domain_source_type')],
+            },
+        ),
+        migrations.AddField(
+            model_name='rawdocument',
+            name='source',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='sources.source'),
+        ),
+        migrations.CreateModel(
+            name='SourceRun',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('status', models.CharField(choices=[('running', 'Bezig'), ('success', 'Geslaagd'), ('partial', 'Gedeeltelijk'), ('failed', 'Mislukt'), ('skipped', 'Overgeslagen')], default='running', max_length=16)),
+                ('started_at', models.DateTimeField(default=django.utils.timezone.now)),
+                ('finished_at', models.DateTimeField(blank=True, null=True)),
+                ('http_status', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('discovered_count', models.PositiveIntegerField(default=0)),
+                ('extracted_count', models.PositiveIntegerField(default=0)),
+                ('created_count', models.PositiveIntegerField(default=0)),
+                ('updated_count', models.PositiveIntegerField(default=0)),
+                ('duplicate_count', models.PositiveIntegerField(default=0)),
+                ('error_category', models.CharField(blank=True, max_length=80)),
+                ('error_message', models.CharField(blank=True, max_length=1000)),
+                ('metrics', models.JSONField(blank=True, default=dict)),
+                ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='runs', to='sources.source')),
+            ],
+            options={
+                'ordering': ['-started_at'],
+            },
+        ),
+        migrations.AddField(
+            model_name='rawdocument',
+            name='source_run',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='sources.sourcerun'),
+        ),
+        migrations.AddIndex(
+            model_name='rawdocument',
+            index=models.Index(fields=['source', 'content_hash'], name='sources_raw_source__4a7b2a_idx'),
+        ),
+    ]

apps/sources/migrations/0002_policy_review_and_robots_cache.py

@@ -0,0 +1,51 @@
+# Generated by Django 5.2.16 on 2026-07-21 for VR-102
+
+import django.conf
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sources", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SourcePolicyReview",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("scope", models.CharField(choices=[("source", "Bronspecifiek"), ("domain", "Domeinbrede controle")], default="source", max_length=20)),
+                ("decision", models.CharField(choices=[("allow", "Toestaan"), ("trial", "Proefrun"), ("pause", "Pauzeren"), ("deny", "Blokkeren"),], max_length=16)),
+                ("reason", models.CharField(max_length=500)),
+                ("evidence_link", models.URLField(blank=True, max_length=500)),
+                ("notes", models.TextField(blank=True)),
+                ("expires_at", models.DateTimeField(blank=True, null=True)),
+                ("metadata", models.JSONField(blank=True, default=dict)),
+                ("actor", models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="policy_reviews", to=django.conf.settings.AUTH_USER_MODEL)),
+                ("source", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name="policy_reviews", to="sources.source")),
+            ],
+            options={"ordering": ["-created_at"]},
+        ),
+        migrations.CreateModel(
+            name="SourceRobotsCache",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("origin", models.CharField(max_length=255, unique=True)),
+                ("expires_at", models.DateTimeField()),
+                ("etag", models.CharField(blank=True, max_length=255)),
+                ("last_modified", models.CharField(blank=True, max_length=255)),
+                ("allow_rules", models.JSONField(blank=True, default=dict)),
+                ("disallow_rules", models.JSONField(blank=True, default=dict)),
+                ("error", models.CharField(blank=True, max_length=500)),
+                ("byte_length", models.PositiveIntegerField(default=0)),
+            ],
+            options={"ordering": ["origin"]},
+        ),
+    ]

apps/sources/migrations/0003_source_lease_and_origin_state.py

@@ -0,0 +1,45 @@
+# Generated by Django 5.2.16 on 2026-07-21 for VR-103
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("sources", "0002_policy_review_and_robots_cache"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="SourceLease",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("token", models.CharField(default="", max_length=64)),
+                ("worker_id", models.CharField(blank=True, max_length=128)),
+                ("expires_at", models.DateTimeField(db_index=True)),
+                (
+                    "source",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="lease",
+                        to="sources.source",
+                    ),
+                ),
+            ],
+            options={"ordering": ["source"]},
+        ),
+        migrations.CreateModel(
+            name="SourceOriginState",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("domain", models.CharField(max_length=255, unique=True)),
+                ("next_allowed_at", models.DateTimeField()),
+            ],
+            options={"ordering": ["domain"]},
+        ),
+    ]

apps/sources/migrations/0004_remove_source_unique_domain_source_type_and_more.py

@@ -0,0 +1,21 @@
+# Generated by Django 5.2.16 on 2026-07-21 22:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sources', '0003_source_lease_and_origin_state'),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name='source',
+            name='unique_domain_source_type',
+        ),
+        migrations.AddConstraint(
+            model_name='source',
+            constraint=models.UniqueConstraint(condition=models.Q(('base_url', ''), _negated=True), fields=('source_type', 'base_url'), name='unique_source_type_base_url_nonempty'),
+        ),
+    ]

apps/sources/migrations/0005_alter_emailmessagerecord_message_id_and_more.py

@@ -0,0 +1,67 @@
+# Generated by Django 5.2.16 on 2026-07-22 00:12
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sources', '0004_remove_source_unique_domain_source_type_and_more'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='emailmessagerecord',
+            name='message_id',
+            field=models.CharField(max_length=998),
+        ),
+        migrations.CreateModel(
+            name='MailboxConnection',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('platform', models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed')], max_length=24)),
+                ('provider', models.CharField(choices=[('gmail', 'Gmail'), ('outlook', 'Outlook / Microsoft 365'), ('custom', 'Andere IMAP-provider')], max_length=24)),
+                ('custom_host', models.CharField(blank=True, max_length=255)),
+                ('port', models.PositiveIntegerField(default=993)),
+                ('username', models.CharField(max_length=320)),
+                ('encrypted_password', models.TextField()),
+                ('mailbox', models.CharField(default='INBOX', max_length=255)),
+                ('enabled', models.BooleanField(default=True)),
+                ('poll_interval_minutes', models.PositiveIntegerField(choices=[(5, 'Elke 5 minuten'), (15, 'Elke 15 minuten'), (30, 'Elke 30 minuten'), (60, 'Elk uur'), (180, 'Elke 3 uur'), (360, 'Elke 6 uur')], default=30)),
+                ('next_poll_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                ('last_polled_at', models.DateTimeField(blank=True, null=True)),
+                ('last_success_at', models.DateTimeField(blank=True, null=True)),
+                ('last_error_category', models.CharField(blank=True, max_length=80)),
+                ('last_error_message', models.CharField(blank=True, max_length=500)),
+                ('lease_token', models.CharField(blank=True, max_length=64)),
+                ('lease_expires_at', models.DateTimeField(blank=True, db_index=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mailbox_connections', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ['platform'],
+            },
+        ),
+        migrations.AddField(
+            model_name='emailmessagerecord',
+            name='mailbox_connection',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='sources.mailboxconnection'),
+        ),
+        migrations.AddConstraint(
+            model_name='emailmessagerecord',
+            constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', False)), fields=('mailbox_connection', 'message_id'), name='unique_message_per_mailbox_connection'),
+        ),
+        migrations.AddConstraint(
+            model_name='emailmessagerecord',
+            constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', True)), fields=('message_id',), name='unique_legacy_email_message_id'),
+        ),
+        migrations.AddConstraint(
+            model_name='mailboxconnection',
+            constraint=models.UniqueConstraint(fields=('user', 'platform'), name='unique_mailbox_platform_per_user'),
+        ),
+    ]

apps/sources/migrations/0006_alter_mailboxconnection_platform.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.16 on 2026-07-22 00:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sources', '0005_alter_emailmessagerecord_message_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='mailboxconnection',
+            name='platform',
+            field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('eurobrussels', 'EuroBrussels'), ('brusselsjobs', 'BrusselsJobs')], max_length=24),
+        ),
+    ]

apps/sources/migrations/0007_alter_mailboxconnection_platform.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.2.16 on 2026-07-22 01:25
+
+from django.db import migrations, models
+
+
+def disable_retired_brussels_mailboxes(apps, schema_editor):
+    mailbox_connection = apps.get_model("sources", "MailboxConnection")
+    mailbox_connection.objects.filter(
+        platform__in=("eurobrussels", "brusselsjobs")
+    ).update(enabled=False)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sources', '0006_alter_mailboxconnection_platform'),
+    ]
+
+    operations = [
+        migrations.RunPython(disable_retired_brussels_mailboxes, migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name='mailboxconnection',
+            name='platform',
+            field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('careerjet', 'Careerjet'), ('randstad', 'Randstad'), ('roberthalf', 'Robert Half')], max_length=24),
+        ),
+    ]

apps/sources/migrations/0008_alter_mailboxconnection_encrypted_password.py

@@ -0,0 +1,13 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [("sources", "0007_alter_mailboxconnection_platform")]
+
+    operations = [
+        migrations.AlterField(
+            model_name="mailboxconnection",
+            name="encrypted_password",
+            field=models.TextField(blank=True),
+        )
+    ]

apps/sources/models.py

@@ -0,0 +1,442 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from urllib.parse import urlparse
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.utils import timezone
+
+from apps.core.models import TimeStampedModel
+
+
+class Source(TimeStampedModel):
+    class Type(models.TextChoices):
+        EMPLOYER = "employer", "Werkgeverspagina"
+        ATS = "ats", "Publieke ATS-pagina"
+        RSS = "rss", "RSS/Atom"
+        SITEMAP = "sitemap", "Sitemap"
+        EMAIL = "email", "Vacaturemail"
+        MANUAL = "manual", "Handmatige import"
+
+    class Status(models.TextChoices):
+        CANDIDATE = "candidate", "Kandidaat"
+        TRIAL = "trial", "Proefrun"
+        ACTIVE = "active", "Actief"
+        QUARANTINED = "quarantined", "Quarantaine"
+        PAUSED = "paused", "Gepauzeerd"
+        DISABLED = "disabled", "Uitgeschakeld"
+
+    class Policy(models.TextChoices):
+        ALLOW = "allow", "Toestaan"
+        REVIEW = "review", "Te beoordelen"
+        DENY = "deny", "Blokkeren"
+
+    name = models.CharField(max_length=200)
+    source_type = models.CharField(max_length=24, choices=Type.choices)
+    base_url = models.URLField(max_length=1000, blank=True)
+    domain = models.CharField(max_length=255, db_index=True)
+    status = models.CharField(max_length=24, choices=Status.choices, default=Status.CANDIDATE)
+    policy = models.CharField(max_length=16, choices=Policy.choices, default=Policy.REVIEW)
+    policy_reason = models.CharField(max_length=500, blank=True)
+    parser_key = models.CharField(max_length=120, default="auto")
+    strict_mode = models.BooleanField(default=True)
+    allow_public_endpoint = models.BooleanField(default=False)
+    honor_robots = models.BooleanField(default=True)
+
+    crawl_interval_minutes = models.PositiveIntegerField(default=720)
+    minimum_interval_seconds = models.PositiveIntegerField(default=30)
+    max_concurrency = models.PositiveSmallIntegerField(default=1)
+    next_run_at = models.DateTimeField(null=True, blank=True, db_index=True)
+    last_success_at = models.DateTimeField(null=True, blank=True)
+    last_failure_at = models.DateTimeField(null=True, blank=True)
+    failure_count = models.PositiveIntegerField(default=0)
+    etag = models.CharField(max_length=500, blank=True)
+    last_modified = models.CharField(max_length=500, blank=True)
+    robots_checked_at = models.DateTimeField(null=True, blank=True)
+    terms_checked_at = models.DateTimeField(null=True, blank=True)
+    metadata = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["name"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["source_type", "base_url"],
+                condition=~models.Q(base_url=""),
+                name="unique_source_type_base_url_nonempty",
+            )
+        ]
+
+    def __str__(self) -> str:
+        return self.name
+
+    def clean(self) -> None:
+        if self.base_url:
+            parsed = urlparse(self.base_url)
+            if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+                raise ValidationError({"base_url": "Alleen geldige http(s)-URLs zijn toegestaan."})
+            if self.domain and parsed.hostname.lower().rstrip(".") != self.domain.lower().rstrip(
+                "."
+            ):
+                raise ValidationError({"domain": "Domein moet overeenkomen met de base URL."})
+        if self.policy == self.Policy.DENY and self.status == self.Status.ACTIVE:
+            raise ValidationError("Een geblokkeerde bron kan niet actief zijn.")
+
+    @property
+    def discovery_evidence(self) -> list[dict[str, str | float]]:
+        raw = self.metadata if isinstance(self.metadata, dict) else {}
+        entries = raw.get("discovery", [])
+        if not isinstance(entries, list):
+            return []
+        return [entry for entry in entries if isinstance(entry, dict)]
+
+    @property
+    def latest_policy_review(self) -> SourcePolicyReview | None:
+        return self.policy_reviews.order_by("-created_at").first()
+
+    @property
+    def policy_review_state(self) -> str:
+        review = self.latest_policy_review
+        if not review:
+            return "missing"
+        if review.is_expired:
+            return "expired"
+        return "active"
+
+    @property
+    def policy_review_summary(self) -> str:
+        review = self.latest_policy_review
+        if not review:
+            return "Geen review geregistreerd"
+        if review.is_expired:
+            return (
+                f"Review verlopen op {review.expires_at:%Y-%m-%d}"
+                if review.expires_at
+                else "Review verlopen"
+            )
+        if review.expires_at:
+            return f"{review.get_decision_display()} geldig tot {review.expires_at:%Y-%m-%d}"
+        return f"{review.get_decision_display()} zonder vervaldatum"
+
+    @property
+    def requires_terms_review(self) -> bool:
+        return self.policy == Source.Policy.ALLOW and self.policy_review_state in {
+            "missing",
+            "expired",
+        }
+
+    def schedule_after_success(self, *, now=None, jitter_seconds: int = 0) -> None:
+        now = now or timezone.now()
+        self.last_success_at = now
+        self.failure_count = 0
+        self.next_run_at = (
+            now
+            + timedelta(minutes=self.crawl_interval_minutes)
+            + timedelta(seconds=max(0, jitter_seconds))
+        )
+        self.save(update_fields=["last_success_at", "failure_count", "next_run_at", "updated_at"])
+
+    def schedule_after_failure(
+        self,
+        *,
+        now=None,
+        backoff_minutes: int | None = None,
+        backoff_seconds: int | None = None,
+    ) -> None:
+        now = now or timezone.now()
+        self.last_failure_at = now
+        self.failure_count += 1
+        if backoff_seconds is None:
+            backoff = backoff_minutes or min(24 * 60, max(15, 2 ** min(self.failure_count, 10)))
+            backoff_seconds = backoff * 60
+        self.next_run_at = now + timedelta(seconds=max(0, backoff_seconds))
+        self.save(
+            update_fields=[
+                "last_failure_at",
+                "failure_count",
+                "next_run_at",
+                "updated_at",
+            ]
+        )
+
+
+class SourceRun(TimeStampedModel):
+    class Status(models.TextChoices):
+        RUNNING = "running", "Bezig"
+        SUCCESS = "success", "Geslaagd"
+        PARTIAL = "partial", "Gedeeltelijk"
+        FAILED = "failed", "Mislukt"
+        SKIPPED = "skipped", "Overgeslagen"
+
+    source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="runs")
+    status = models.CharField(max_length=16, choices=Status.choices, default=Status.RUNNING)
+    started_at = models.DateTimeField(default=timezone.now)
+    finished_at = models.DateTimeField(null=True, blank=True)
+    http_status = models.PositiveSmallIntegerField(null=True, blank=True)
+    discovered_count = models.PositiveIntegerField(default=0)
+    extracted_count = models.PositiveIntegerField(default=0)
+    created_count = models.PositiveIntegerField(default=0)
+    updated_count = models.PositiveIntegerField(default=0)
+    duplicate_count = models.PositiveIntegerField(default=0)
+    error_category = models.CharField(max_length=80, blank=True)
+    error_message = models.CharField(max_length=1000, blank=True)
+    metrics = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["-started_at"]
+
+    def finish(self, status: str, **metrics) -> None:
+        self.status = status
+        self.finished_at = timezone.now()
+        for key, value in metrics.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+        self.save()
+
+
+class SourceLease(TimeStampedModel):
+    source = models.OneToOneField(Source, on_delete=models.CASCADE, related_name="lease")
+    token = models.CharField(max_length=64, default="")
+    worker_id = models.CharField(max_length=128, blank=True)
+    expires_at = models.DateTimeField(db_index=True)
+
+    class Meta:
+        ordering = ["source"]
+
+    def __str__(self) -> str:
+        return f"{self.source_id}:{self.token}"
+
+    @property
+    def is_expired(self) -> bool:
+        return self.expires_at <= timezone.now()
+
+
+class SourceOriginState(TimeStampedModel):
+    domain = models.CharField(max_length=255, unique=True)
+    next_allowed_at = models.DateTimeField()
+
+    class Meta:
+        ordering = ["domain"]
+
+    def __str__(self) -> str:
+        return self.domain
+
+
+class SourcePolicyReview(TimeStampedModel):
+    class Scope(models.TextChoices):
+        SOURCE = "source", "Bronspecifiek"
+        DOMAIN = "domain", "Domeinbrede controle"
+
+    class Decision(models.TextChoices):
+        ALLOW = "allow", "Toestaan"
+        TRIAL = "trial", "Proefrun"
+        PAUSE = "pause", "Pauzeren"
+        DENY = "deny", "Blokkeren"
+
+    source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="policy_reviews")
+    actor = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.SET_NULL,
+        null=True,
+        blank=True,
+        related_name="policy_reviews",
+    )
+    scope = models.CharField(max_length=20, choices=Scope.choices, default=Scope.SOURCE)
+    decision = models.CharField(max_length=16, choices=Decision.choices)
+    reason = models.CharField(max_length=500)
+    evidence_link = models.URLField(max_length=500, blank=True)
+    notes = models.TextField(blank=True)
+    expires_at = models.DateTimeField(null=True, blank=True)
+    metadata = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["-created_at"]
+
+    @property
+    def is_expired(self) -> bool:
+        if not self.expires_at:
+            return False
+        return self.expires_at <= timezone.now()
+
+
+class SourceRobotsCache(TimeStampedModel):
+    origin = models.CharField(max_length=255, unique=True)
+    expires_at = models.DateTimeField()
+    etag = models.CharField(max_length=255, blank=True)
+    last_modified = models.CharField(max_length=255, blank=True)
+    allow_rules = models.JSONField(default=dict, blank=True)
+    disallow_rules = models.JSONField(default=dict, blank=True)
+    error = models.CharField(max_length=500, blank=True)
+    byte_length = models.PositiveIntegerField(default=0)
+
+    class Meta:
+        ordering = ["origin"]
+
+    @property
+    def is_fresh(self) -> bool:
+        return self.expires_at > timezone.now()
+
+
+class RawDocument(TimeStampedModel):
+    class Kind(models.TextChoices):
+        HTML = "html", "HTML"
+        XML = "xml", "XML"
+        EMAIL = "email", "E-mail"
+        JSON = "json", "JSON"
+        TEXT = "text", "Tekst"
+
+    source = models.ForeignKey(
+        Source, on_delete=models.SET_NULL, null=True, related_name="documents"
+    )
+    source_run = models.ForeignKey(
+        SourceRun, on_delete=models.SET_NULL, null=True, blank=True, related_name="documents"
+    )
+    url = models.URLField(max_length=2000, blank=True)
+    final_url = models.URLField(max_length=2000, blank=True)
+    kind = models.CharField(max_length=16, choices=Kind.choices)
+    content_type = models.CharField(max_length=200, blank=True)
+    http_status = models.PositiveSmallIntegerField(null=True, blank=True)
+    response_headers = models.JSONField(default=dict, blank=True)
+    content_hash = models.CharField(max_length=64, db_index=True)
+    body_text = models.TextField(blank=True)
+    byte_length = models.PositiveIntegerField(default=0)
+    parser_key = models.CharField(max_length=120, blank=True)
+    parser_version = models.CharField(max_length=80, blank=True)
+    extraction_confidence = models.DecimalField(max_digits=4, decimal_places=3, default=0)
+    retain_until = models.DateTimeField(null=True, blank=True)
+    quarantined = models.BooleanField(default=False)
+    quarantine_reason = models.CharField(max_length=500, blank=True)
+    metadata = models.JSONField(default=dict, blank=True)
+
+    class Meta:
+        ordering = ["-created_at"]
+        indexes = [models.Index(fields=["source", "content_hash"])]
+
+    def __str__(self) -> str:
+        return self.final_url or self.url or f"Document {self.pk}"
+
+
+class MailboxConnection(TimeStampedModel):
+    class Platform(models.TextChoices):
+        VDAB = "vdab", "VDAB"
+        INDEED = "indeed", "Indeed"
+        LINKEDIN = "linkedin", "LinkedIn"
+        ICTJOB = "ictjob", "ictjob.be"
+        JOBAT = "jobat", "Jobat"
+        STEPSTONE = "stepstone", "StepStone"
+        CAREERJET = "careerjet", "Careerjet"
+        RANDSTAD = "randstad", "Randstad"
+        ROBERTHALF = "roberthalf", "Robert Half"
+
+    class Provider(models.TextChoices):
+        GMAIL = "gmail", "Gmail"
+        OUTLOOK = "outlook", "Outlook / Microsoft 365"
+        CUSTOM = "custom", "Andere IMAP-provider"
+
+    class PollInterval(models.IntegerChoices):
+        FIVE_MINUTES = 5, "Elke 5 minuten"
+        FIFTEEN_MINUTES = 15, "Elke 15 minuten"
+        THIRTY_MINUTES = 30, "Elke 30 minuten"
+        HOURLY = 60, "Elk uur"
+        THREE_HOURS = 180, "Elke 3 uur"
+        SIX_HOURS = 360, "Elke 6 uur"
+
+    PROVIDER_HOSTS = {
+        Provider.GMAIL: "imap.gmail.com",
+        Provider.OUTLOOK: "outlook.office365.com",
+    }
+
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        related_name="mailbox_connections",
+    )
+    platform = models.CharField(max_length=24, choices=Platform.choices)
+    provider = models.CharField(max_length=24, choices=Provider.choices)
+    custom_host = models.CharField(max_length=255, blank=True)
+    port = models.PositiveIntegerField(default=993)
+    username = models.CharField(max_length=320)
+    encrypted_password = models.TextField(blank=True)
+    mailbox = models.CharField(max_length=255, default="INBOX")
+    enabled = models.BooleanField(default=True)
+    poll_interval_minutes = models.PositiveIntegerField(
+        choices=PollInterval.choices,
+        default=PollInterval.THIRTY_MINUTES,
+    )
+    next_poll_at = models.DateTimeField(default=timezone.now, db_index=True)
+    last_polled_at = models.DateTimeField(null=True, blank=True)
+    last_success_at = models.DateTimeField(null=True, blank=True)
+    last_error_category = models.CharField(max_length=80, blank=True)
+    last_error_message = models.CharField(max_length=500, blank=True)
+    lease_token = models.CharField(max_length=64, blank=True)
+    lease_expires_at = models.DateTimeField(null=True, blank=True, db_index=True)
+
+    class Meta:
+        ordering = ["platform"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["user", "platform"], name="unique_mailbox_platform_per_user"
+            )
+        ]
+
+    def __str__(self) -> str:
+        return f"{self.get_platform_display()} · {self.username}"
+
+    @property
+    def imap_host(self) -> str:
+        return self.PROVIDER_HOSTS.get(self.provider, self.custom_host).strip().lower()
+
+    def clean(self) -> None:
+        host = self.imap_host
+        if (
+            not host
+            or "://" in host
+            or "/" in host
+            or "@" in host
+            or any(c.isspace() for c in host)
+        ):
+            raise ValidationError({"custom_host": "Vul een geldige IMAP-hostnaam in."})
+        if not 1 <= self.port <= 65535:
+            raise ValidationError({"port": "De IMAP-poort moet tussen 1 en 65535 liggen."})
+        if "\n" in self.mailbox or "\r" in self.mailbox:
+            raise ValidationError({"mailbox": "De mailboxnaam bevat ongeldige tekens."})
+
+
+class EmailMessageRecord(TimeStampedModel):
+    mailbox_connection = models.ForeignKey(
+        MailboxConnection,
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+        related_name="messages",
+    )
+    message_id = models.CharField(max_length=998)
+    mailbox = models.CharField(max_length=255, default="INBOX")
+    sender = models.CharField(max_length=500, blank=True)
+    subject = models.CharField(max_length=998, blank=True)
+    received_at = models.DateTimeField(null=True, blank=True)
+    raw_document = models.OneToOneField(
+        RawDocument, on_delete=models.SET_NULL, null=True, blank=True, related_name="email_record"
+    )
+    links = models.JSONField(default=list, blank=True)
+    processed = models.BooleanField(default=False)
+    error_message = models.CharField(max_length=1000, blank=True)
+
+    class Meta:
+        ordering = ["-received_at", "-created_at"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["mailbox_connection", "message_id"],
+                condition=models.Q(mailbox_connection__isnull=False),
+                name="unique_message_per_mailbox_connection",
+            ),
+            models.UniqueConstraint(
+                fields=["message_id"],
+                condition=models.Q(mailbox_connection__isnull=True),
+                name="unique_legacy_email_message_id",
+            ),
+        ]
+
+    def __str__(self) -> str:
+        return self.subject or self.message_id

apps/sources/platforms.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class PlatformAlert:
+    key: str
+    label: str
+    domains: tuple[str, ...]
+    alert_url: str
+    help_text: str
+
+
+PLATFORM_ALERTS = {
+    alert.key: alert
+    for alert in (
+        PlatformAlert(
+            "vdab",
+            "VDAB",
+            ("vdab.be",),
+            "https://www.vdab.be/jobs/job-alerts",
+            "Brede Vlaamse vacaturealerts.",
+        ),
+        PlatformAlert(
+            "indeed",
+            "Indeed",
+            ("indeed.com", "indeed.be"),
+            "https://support.indeed.com/hc/nl/articles/204488890-"
+            "Vacature-alerts-activeren-stopzetten-en-beheren",
+            "Algemene en regionale vacaturealerts.",
+        ),
+        PlatformAlert(
+            "linkedin",
+            "LinkedIn",
+            ("linkedin.com",),
+            "https://www.linkedin.com/help/linkedin/answer/a511279",
+            "Dagelijkse of wekelijkse alerts, maximaal twintig per LinkedIn-account.",
+        ),
+        PlatformAlert(
+            "ictjob",
+            "ictjob.be",
+            ("ictjob.be",),
+            "https://www.ictjob.be/nl/it-job-alert",
+            "Gespecialiseerde Belgische IT- en telecomvacatures.",
+        ),
+        PlatformAlert(
+            "jobat",
+            "Jobat",
+            ("jobat.be",),
+            "https://www.jobat.be/nl/jobalert",
+            "Belgische jobs per functie, regio en contracttype.",
+        ),
+        PlatformAlert(
+            "stepstone",
+            "StepStone",
+            ("stepstone.be",),
+            "https://www.stepstone.be/vacatures",
+            "Dagelijkse vergelijkbare vacatures vanuit een zoekopdracht.",
+        ),
+        PlatformAlert(
+            "careerjet",
+            "Careerjet",
+            ("careerjet.be",),
+            "https://www.careerjet.be/",
+            "Brede Belgische zoekresultaten met regio- en postcodegebonden e-mailalerts.",
+        ),
+        PlatformAlert(
+            "randstad",
+            "Randstad",
+            ("randstad.be",),
+            "https://www.randstad.be/jobalert-aanmaken/",
+            "Belgische jobs met instelbare specialisatie, locatie, contract en frequentie.",
+        ),
+        PlatformAlert(
+            "roberthalf",
+            "Robert Half",
+            ("roberthalf.com",),
+            "https://www.roberthalf.com/be/en/find-jobs/job-alerts",
+            "Belgische IT- en digitaljobs, waaronder systeem-, netwerk- en supportfuncties.",
+        ),
+    )
+}
+
+
+def platform_label(key: str) -> str:
+    alert = PLATFORM_ALERTS.get(key)
+    return alert.label if alert else key

apps/sources/services/canonicalize.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import posixpath
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+TRACKING_PARAMETERS = {
+    "fbclid",
+    "gclid",
+    "mc_cid",
+    "mc_eid",
+    "ref",
+    "referrer",
+    "source",
+    "trk",
+    "trackingid",
+}
+TRACKING_PREFIXES = ("utm_", "pk_")
+
+
+def canonicalize_url(url: str) -> str:
+    value = (url or "").strip()
+    if not value:
+        return ""
+    parts = urlsplit(value)
+    scheme = parts.scheme.lower()
+    host = (parts.hostname or "").lower().rstrip(".")
+    if not scheme or not host:
+        return value
+    port = parts.port
+    if port and not ((scheme == "http" and port == 80) or (scheme == "https" and port == 443)):
+        netloc = f"{host}:{port}"
+    else:
+        netloc = host
+    path = parts.path or "/"
+    normalized_path = posixpath.normpath(path)
+    if path.endswith("/") and not normalized_path.endswith("/"):
+        normalized_path += "/"
+    if not normalized_path.startswith("/"):
+        normalized_path = "/" + normalized_path
+    query_pairs = []
+    for key, value in parse_qsl(parts.query, keep_blank_values=True):
+        lower = key.lower()
+        if lower in TRACKING_PARAMETERS or any(
+            lower.startswith(prefix) for prefix in TRACKING_PREFIXES
+        ):
+            continue
+        query_pairs.append((key, value))
+    query_pairs.sort()
+    return urlunsplit((scheme, netloc, normalized_path, urlencode(query_pairs, doseq=True), ""))
+
+
+def domain_matches(hostname: str, domain: str) -> bool:
+    host = hostname.lower().rstrip(".")
+    target = domain.lower().rstrip(".")
+    return host == target or host.endswith("." + target)

apps/sources/services/discovery.py

@@ -0,0 +1,400 @@
+from __future__ import annotations
+
+import ipaddress
+import json
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+from defusedxml import ElementTree
+from django.db import transaction
+
+from apps.sources.adapters.email_alert import EmailAlertAdapter
+from apps.sources.adapters.rss import RssAdapter
+from apps.sources.models import Source
+
+from .canonicalize import canonicalize_url, domain_matches
+from .policy import is_denied_domain
+
+CAREER_PATTERN = re.compile(
+    r"\b(career|careers|jobs|job|vacature|vacatures|werken-bij|werken bij|emploi|emplois|offres)\b",
+    re.I,
+)
+FEED_TYPE_PATTERN = re.compile(r"application/(?:atom|rss)\+xml|text/xml|application/xml", re.I)
+
+
+@dataclass(frozen=True)
+class SourceCandidate:
+    url: str
+    domain: str
+    source_type: str
+    label: str
+    confidence: float
+    reason: str
+    discovered_from: str
+
+
+def discover_career_links(html: str, *, base_url: str) -> list[SourceCandidate]:
+    return _discover_html(
+        html,
+        base_url=base_url,
+        include_jsonld=False,
+        include_feed_links=False,
+    )
+
+
+def discover_from_html(html: str, *, base_url: str) -> list[SourceCandidate]:
+    return _discover_html(
+        html,
+        base_url=base_url,
+        include_jsonld=True,
+        include_feed_links=True,
+    )
+
+
+def discover_from_feed(feed_content: str, *, base_url: str) -> list[SourceCandidate]:
+    adapter = RssAdapter()
+    result = adapter.extract(feed_content, url=base_url)
+    base_domain = _hostname(base_url)
+    candidates: list[SourceCandidate] = []
+    for extracted in result.jobs:
+        candidate = _build_candidate(
+            raw_url=extracted.url,
+            base_domain=base_domain,
+            source_type=Source.Type.RSS,
+            label=extracted.title[:120],
+            reason="rss",
+            discovered_from="feed",
+            confidence=0.82,
+            allow_off_domain=False,
+        )
+        if candidate:
+            candidates.append(candidate)
+    return _dedupe(candidates)
+
+
+def discover_from_sitemap(xml_content: str, *, base_url: str) -> list[SourceCandidate]:
+    try:
+        root = ElementTree.fromstring(xml_content)
+    except ElementTree.ParseError:
+        return []
+
+    root_name = _local_tag(root.tag)
+    if root_name == "sitemapindex":
+        discovered_from = "sitemap-index"
+    elif root_name == "urlset":
+        discovered_from = "sitemap-urlset"
+    else:
+        return []
+
+    base_domain = _hostname(base_url)
+    candidates: list[SourceCandidate] = []
+    for location in root.findall(".//{*}loc"):
+        raw = (location.text or "").strip()
+        if not raw:
+            continue
+        candidate = _build_candidate(
+            raw_url=urljoin(base_url, raw),
+            base_domain=base_domain,
+            source_type=Source.Type.SITEMAP,
+            label="Sitemaplocatie",
+            reason=discovered_from,
+            discovered_from=discovered_from,
+            confidence=0.86 if discovered_from == "sitemap-urlset" else 0.75,
+            allow_off_domain=False,
+        )
+        if candidate:
+            candidates.append(candidate)
+
+    return _dedupe(candidates)
+
+
+def discover_from_email(raw_message: bytes) -> list[SourceCandidate]:
+    adapter = EmailAlertAdapter()
+    result = adapter.extract_message(raw_message)
+    candidates: list[SourceCandidate] = []
+    for extracted in result.jobs:
+        candidate_url = _to_domain_root_url(extracted.url)
+        candidate = _build_candidate(
+            raw_url=candidate_url,
+            base_domain=_hostname(extracted.url),
+            source_type=Source.Type.EMPLOYER,
+            label=extracted.title[:120],
+            reason="email",
+            discovered_from="email",
+            confidence=0.62,
+            allow_off_domain=True,
+        )
+        if candidate:
+            candidates.append(candidate)
+
+    return _dedupe(candidates)
+
+
+def persist_discovery_candidates(candidates: list[SourceCandidate]) -> tuple[int, int, int]:
+    created = updated = skipped = 0
+    ordered = _dedupe(candidates)
+    with transaction.atomic():
+        for candidate in ordered:
+            provenance = _provenance_entry(candidate)
+            now = _utc_now()
+            source = (
+                Source.objects.filter(
+                    domain=candidate.domain,
+                    source_type=candidate.source_type,
+                )
+                .order_by("id")
+                .first()
+            )
+            was_created = source is None
+            if source is None:
+                source = Source.objects.create(
+                    domain=candidate.domain,
+                    source_type=candidate.source_type,
+                    name=_candidate_name(candidate),
+                    base_url=candidate.url,
+                    status=Source.Status.CANDIDATE,
+                    policy=Source.Policy.REVIEW,
+                    policy_reason="Automatisch ontdekt",
+                    parser_key="auto",
+                    strict_mode=True,
+                    metadata={"discovery": [provenance], "discovered_at": now},
+                )
+            if was_created:
+                created += 1
+                continue
+
+            updated_fields: list[str] = ["updated_at"]
+            metadata = source.metadata if isinstance(source.metadata, dict) else {}
+            evidence = metadata.get("discovery", [])
+            if not isinstance(evidence, list):
+                evidence = []
+            if not any(
+                item.get("url") == candidate.url for item in evidence if isinstance(item, dict)
+            ):
+                evidence.append(provenance)
+                metadata["discovery"] = evidence[-20:]
+                metadata["discovered_at"] = now
+                source.metadata = metadata
+                updated_fields.append("metadata")
+            if not source.name:
+                source.name = _candidate_name(candidate)
+                updated_fields.append("name")
+            if not source.base_url:
+                source.base_url = candidate.url
+                updated_fields.append("base_url")
+
+            if len(updated_fields) > 1:
+                source.save(update_fields=sorted(set(updated_fields)))
+                updated += 1
+            else:
+                skipped += 1
+
+    return (created, updated, skipped)
+
+
+def _candidate_name(candidate: SourceCandidate) -> str:
+    label = candidate.label.strip()
+    if label:
+        return label
+    return candidate.domain
+
+
+def _provenance_entry(candidate: SourceCandidate) -> dict[str, str | float]:
+    return {
+        "url": candidate.url,
+        "source_type": candidate.source_type,
+        "discovered_from": candidate.discovered_from,
+        "confidence": candidate.confidence,
+        "reason": candidate.reason,
+        "label": candidate.label[:240],
+        "seen_at": _utc_now(),
+    }
+
+
+def _discover_html(
+    html: str,
+    *,
+    base_url: str,
+    include_jsonld: bool,
+    include_feed_links: bool,
+) -> list[SourceCandidate]:
+    soup = BeautifulSoup(html, "lxml")
+    base_domain = _hostname(base_url)
+    candidates: list[SourceCandidate] = []
+
+    for anchor in soup.find_all("a", href=True):
+        label = " ".join(anchor.get_text(" ", strip=True).split())
+        raw_url = urljoin(base_url, str(anchor["href"]))
+        candidate_host = _hostname(raw_url)
+        same_domain = bool(base_domain and domain_matches(candidate_host, base_domain))
+        candidate = _build_candidate(
+            raw_url=raw_url,
+            base_domain=base_domain,
+            source_type=Source.Type.EMPLOYER,
+            label=label,
+            reason="career-link",
+            discovered_from="html",
+            confidence=0.9 if same_domain else 0.7,
+            allow_off_domain=True,
+        )
+        if not candidate:
+            continue
+        if not CAREER_PATTERN.search(label) and not CAREER_PATTERN.search(
+            urlsplit(candidate.url).path
+        ):
+            continue
+        candidates.append(candidate)
+
+    if include_jsonld:
+        for js in soup.find_all("script", type=re.compile(r"application/ld\+json", re.I)):
+            raw = js.get_text("", strip=True)
+            if not raw:
+                continue
+            try:
+                parsed = json.loads(raw)
+            except json.JSONDecodeError:
+                continue
+            for url in _collect_jsonld_urls(parsed):
+                candidate = _build_candidate(
+                    raw_url=urljoin(base_url, url),
+                    base_domain=base_domain,
+                    source_type=Source.Type.EMPLOYER,
+                    label="JSON-LD job",
+                    reason="jsonld",
+                    discovered_from="html-jsonld",
+                    confidence=0.95,
+                    allow_off_domain=False,
+                )
+                if candidate:
+                    candidates.append(candidate)
+
+    if include_feed_links:
+        for link in soup.find_all("link", href=True):
+            rel = {str(item).lower() for item in (link.get("rel") or [])}
+            if "alternate" not in rel:
+                continue
+            link_type = str(link.get("type") or "")
+            if not FEED_TYPE_PATTERN.search(link_type):
+                continue
+            candidate = _build_candidate(
+                raw_url=urljoin(base_url, str(link["href"])),
+                base_domain=base_domain,
+                source_type=Source.Type.RSS,
+                label=(str(link.get("title") or "Feedlink")).strip()[:120],
+                reason="feed",
+                discovered_from="html",
+                confidence=0.86,
+                allow_off_domain=False,
+            )
+            if candidate:
+                candidates.append(candidate)
+
+    return _dedupe(candidates)
+
+
+def _collect_jsonld_urls(payload) -> list[str]:
+    urls: list[str] = []
+
+    def walk(node):
+        if isinstance(node, dict):
+            candidate_type = str(node.get("@type", "")).lower()
+            if candidate_type == "jobposting":
+                for key in ("url", "applyUrl", "application", "applicationurl"):
+                    value = node.get(key)
+                    if isinstance(value, str) and value:
+                        urls.append(value)
+            for key in ("@graph", "itemListElement", "item", "jobs", "jobPosting"):
+                nested = node.get(key)
+                if nested is not None:
+                    walk(nested)
+        elif isinstance(node, list):
+            for item in node:
+                walk(item)
+
+    walk(payload)
+    return urls
+
+
+def _build_candidate(
+    *,
+    raw_url: str,
+    base_domain: str,
+    source_type: str,
+    label: str,
+    reason: str,
+    discovered_from: str,
+    confidence: float,
+    allow_off_domain: bool,
+) -> SourceCandidate | None:
+    canonical = canonicalize_url(raw_url)
+    if not canonical:
+        return None
+    if not _url_is_http(canonical):
+        return None
+    hostname = _hostname(canonical)
+    if not hostname:
+        return None
+    if is_denied_domain(hostname):
+        return None
+    if _is_private_host(hostname):
+        return None
+    if (not allow_off_domain) and base_domain and not domain_matches(hostname, base_domain):
+        return None
+    return SourceCandidate(
+        url=canonical,
+        domain=hostname,
+        source_type=source_type,
+        label=label,
+        confidence=confidence,
+        reason=reason,
+        discovered_from=discovered_from,
+    )
+
+
+def _url_is_http(url: str) -> bool:
+    return urlsplit(url).scheme.lower() in {"http", "https"}
+
+
+def _hostname(value: str) -> str:
+    return (urlsplit(value).hostname or "").lower()
+
+
+def _is_private_host(hostname: str) -> bool:
+    host = hostname.lower().rstrip(".")
+    if host in {"localhost", "127.0.0.1", "::1"} or host.endswith((".local", ".localhost")):
+        return True
+    try:
+        ip = ipaddress.ip_address(host)
+    except ValueError:
+        return False
+    return not ip.is_global
+
+
+def _dedupe(candidates: list[SourceCandidate]) -> list[SourceCandidate]:
+    seen: set[tuple[str, str]] = set()
+    result: list[SourceCandidate] = []
+    for candidate in sorted(candidates, key=lambda item: item.confidence, reverse=True):
+        key = (candidate.url, candidate.source_type)
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(candidate)
+    return result
+
+
+def _local_tag(tag_name: str) -> str:
+    return tag_name.rsplit("}", 1)[-1].lower()
+
+
+def _to_domain_root_url(url: str) -> str:
+    parsed = urlsplit(url)
+    if not parsed.scheme or not parsed.hostname:
+        return url
+    return f"{parsed.scheme}://{parsed.hostname}"
+
+
+def _utc_now() -> str:
+    return datetime.now(UTC).isoformat()

apps/sources/services/email_import.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from email import policy
+from email.parser import BytesParser
+from email.utils import parsedate_to_datetime
+
+from django.conf import settings
+from django.db import transaction
+from django.utils import timezone
+
+from apps.jobs.services.normalization import normalize_extracted_job
+from apps.jobs.services.pipeline import persist_draft
+from apps.sources.adapters.email_alert import EmailAlertAdapter
+from apps.sources.models import EmailMessageRecord, MailboxConnection, RawDocument, Source
+from apps.sources.platforms import platform_label
+
+
+def message_identity(raw_message: bytes) -> str:
+    """Return the RFC Message-ID or a deterministic hash when it is absent."""
+    parsed = BytesParser(policy=policy.default).parsebytes(raw_message, headersonly=True)
+    fallback_id = hashlib.sha256(raw_message).hexdigest()
+    return str(parsed.get("message-id") or f"sha256:{fallback_id}").strip()
+
+
+@transaction.atomic
+def ingest_email(
+    raw_message: bytes,
+    *,
+    mailbox: str = "INBOX",
+    mailbox_connection: MailboxConnection | None = None,
+    platform: str | None = None,
+) -> EmailMessageRecord:
+    parsed = BytesParser(policy=policy.default).parsebytes(raw_message)
+    message_id = message_identity(raw_message)
+    existing = EmailMessageRecord.objects.filter(
+        message_id=message_id, mailbox_connection=mailbox_connection
+    ).first()
+    if existing:
+        return existing
+
+    source_domain = f"{platform}.mailbox.local" if platform else "mailbox.local"
+    source_name = f"{platform_label(platform)} vacaturemailbox" if platform else "Vacaturemailbox"
+    source, _ = Source.objects.get_or_create(
+        domain=source_domain,
+        source_type=Source.Type.EMAIL,
+        defaults={
+            "name": source_name,
+            "status": Source.Status.ACTIVE,
+            "policy": Source.Policy.ALLOW,
+            "parser_key": "email-alert",
+            "crawl_interval_minutes": 1440,
+        },
+    )
+    content_hash = hashlib.sha256(raw_message).hexdigest()
+    retain_until = timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS)
+    document = RawDocument.objects.create(
+        source=source,
+        kind=RawDocument.Kind.EMAIL,
+        content_type="message/rfc822",
+        content_hash=content_hash,
+        body_text=raw_message.decode("utf-8", errors="replace"),
+        byte_length=len(raw_message),
+        retain_until=retain_until,
+        metadata={
+            "mailbox": mailbox,
+            "mailbox_connection_id": mailbox_connection.pk if mailbox_connection else None,
+            "alert_provider": platform or "unknown",
+        },
+    )
+    received_at = None
+    if parsed.get("date"):
+        try:
+            received_at = parsedate_to_datetime(str(parsed.get("date")))
+            if timezone.is_naive(received_at):
+                received_at = timezone.make_aware(received_at, timezone.get_current_timezone())
+        except (TypeError, ValueError, OverflowError):
+            received_at = None
+
+    adapter = EmailAlertAdapter()
+    result = adapter.extract_message(raw_message, expected_provider=platform)
+    detected_provider = next(
+        (
+            str(extracted.raw.get("alert_provider"))
+            for extracted in result.jobs
+            if extracted.raw.get("alert_provider")
+        ),
+        "unknown",
+    )
+    alert_provider = platform or detected_provider
+    document.parser_key = result.parser_key
+    document.parser_version = result.parser_version
+    document.extraction_confidence = result.confidence
+    document.metadata = {**document.metadata, "alert_provider": alert_provider}
+    document.save(
+        update_fields=[
+            "parser_key",
+            "parser_version",
+            "extraction_confidence",
+            "metadata",
+            "updated_at",
+        ]
+    )
+    links: list[str] = []
+    errors: list[str] = []
+    for extracted in result.jobs:
+        links.append(extracted.url)
+        try:
+            draft = normalize_extracted_job(extracted)
+            persist_draft(
+                draft,
+                document=document,
+                parser_key=result.parser_key,
+                parser_version=result.parser_version,
+                extraction_confidence=result.confidence,
+            )
+        except Exception as exc:
+            errors.append(f"{exc.__class__.__name__}: {exc}")
+    return EmailMessageRecord.objects.create(
+        mailbox_connection=mailbox_connection,
+        message_id=message_id,
+        mailbox=mailbox,
+        sender=str(parsed.get("from") or "")[:500],
+        subject=str(parsed.get("subject") or "")[:998],
+        received_at=received_at,
+        raw_document=document,
+        links=links,
+        processed=not errors,
+        error_message="; ".join(errors)[:1000],
+    )

apps/sources/services/fetcher.py

@@ -0,0 +1,219 @@
+from __future__ import annotations
+
+import hashlib
+import ipaddress
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import datetime
+from datetime import timezone as utc
+from email.utils import parsedate_to_datetime
+from urllib.parse import urljoin
+
+import httpx
+from django.conf import settings
+
+from apps.sources.models import Source
+
+from .policy import assess_url
+from .tls import trusted_tls_context
+from .url_security import ValidatedUrl, validate_public_url
+
+ALLOWED_CONTENT_TYPES = (
+    "text/html",
+    "application/xhtml+xml",
+    "application/xml",
+    "text/xml",
+    "application/json",
+    "application/ld+json",
+    "text/plain",
+    "application/rss+xml",
+    "application/atom+xml",
+)
+
+
+class FetchError(RuntimeError):
+    pass
+
+
+class PolicyBlockedError(FetchError):
+    pass
+
+
+class ContentRejectedError(FetchError):
+    pass
+
+
+class RateLimitedError(FetchError):
+    def __init__(self, retry_after_seconds: int | None, message: str = "HTTP 429") -> None:
+        self.retry_after_seconds = retry_after_seconds
+        suffix = f"; Retry-After={retry_after_seconds}s" if retry_after_seconds else ""
+        super().__init__(f"{message}{suffix}".strip())
+
+
+class FetchTimeoutError(FetchError):
+    pass
+
+
+def _connected_peer_ip(
+    response: httpx.Response,
+) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
+    """Return the actual remote peer when httpcore exposes its network stream."""
+    stream = response.extensions.get("network_stream")
+    if stream is None or not hasattr(stream, "get_extra_info"):
+        return None
+    try:
+        server_address = stream.get_extra_info("server_addr")
+        raw_address = server_address[0] if server_address else None
+        return ipaddress.ip_address(raw_address) if raw_address else None
+    except (IndexError, TypeError, ValueError, OSError):
+        return None
+
+
+def _validate_connected_peer(
+    response: httpx.Response,
+    *,
+    before: ValidatedUrl,
+    after: ValidatedUrl,
+    require_peer: bool,
+) -> None:
+    peer_ip = _connected_peer_ip(response)
+    if peer_ip is not None:
+        if not peer_ip.is_global:
+            raise FetchError(f"Niet-publiek verbonden IP-adres geblokkeerd: {peer_ip}")
+        return
+    if require_peer:
+        raise FetchError(
+            "Verbonden IP-adres kon niet veilig worden vastgesteld; aanvraag geblokkeerd."
+        )
+    # Mock/custom transports do not always expose the peer socket. Keep the conservative
+    # DNS-overlap fallback only for explicitly injected test/custom clients.
+    if not set(before.addresses).intersection(after.addresses):
+        raise FetchError("DNS-rebindcontrole faalde bij het benaderen van bron.")
+
+
+def parse_retry_after(value: str | None) -> int | None:
+    if not value:
+        return None
+    value = value.strip()
+    if not value:
+        return None
+    try:
+        return max(0, int(value))
+    except ValueError:
+        pass
+    try:
+        retry_datetime = parsedate_to_datetime(value)
+    except (TypeError, ValueError):
+        return None
+    if retry_datetime is None:
+        return None
+    if retry_datetime.tzinfo is None:
+        retry_datetime = retry_datetime.replace(tzinfo=utc)
+    retry_aware = retry_datetime.astimezone(utc)
+    now = datetime.now(utc)
+    delta = (retry_aware - now).total_seconds()
+    return max(0, int(delta))
+
+
+@dataclass(slots=True)
+class FetchedDocument:
+    requested_url: str
+    final_url: str
+    status_code: int
+    headers: dict[str, str]
+    content: bytes
+
+    @property
+    def text(self) -> str:
+        encoding = "utf-8"
+        content_type = self.headers.get("content-type", "")
+        if "charset=" in content_type:
+            encoding = content_type.split("charset=", 1)[1].split(";", 1)[0].strip()
+        return self.content.decode(encoding, errors="replace")
+
+    @property
+    def sha256(self) -> str:
+        return hashlib.sha256(self.content).hexdigest()
+
+
+def fetch_url(
+    url: str,
+    *,
+    source: Source | None = None,
+    conditional_headers: Mapping[str, str] | None = None,
+    client: httpx.Client | None = None,
+) -> FetchedDocument:
+    decision = assess_url(url, source=source)
+    if not decision.allowed:
+        raise PolicyBlockedError(decision.reason)
+
+    headers = {
+        "User-Agent": settings.FETCHER_USER_AGENT,
+        "Accept": (
+            "text/html,application/xhtml+xml,application/xml,application/json,"
+            "text/plain;q=0.8,*/*;q=0.1"
+        ),
+    }
+    if conditional_headers:
+        headers.update(conditional_headers)
+
+    own_client = client is None
+    http_client = client or httpx.Client(
+        timeout=httpx.Timeout(settings.FETCHER_TIMEOUT_SECONDS),
+        follow_redirects=False,
+        headers=headers,
+        verify=trusted_tls_context(),
+    )
+    current_url = url
+    try:
+        for _ in range(settings.FETCHER_MAX_REDIRECTS + 1):
+            validation = validate_public_url(
+                current_url,
+                allow_nonstandard_ports=settings.FETCHER_ALLOW_NONSTANDARD_PORTS,
+            )
+            response = http_client.get(current_url, headers=headers)
+            post_validation = validate_public_url(
+                str(response.url) if response.url else current_url,
+                allow_nonstandard_ports=settings.FETCHER_ALLOW_NONSTANDARD_PORTS,
+            )
+            _validate_connected_peer(
+                response,
+                before=validation,
+                after=post_validation,
+                require_peer=own_client,
+            )
+            if response.status_code in {301, 302, 303, 307, 308}:
+                location = response.headers.get("location")
+                if not location:
+                    raise FetchError("Redirect zonder Location-header.")
+                current_url = urljoin(current_url, location)
+                next_decision = assess_url(current_url, source=source)
+                if not next_decision.allowed:
+                    raise PolicyBlockedError(next_decision.reason)
+                continue
+            if response.status_code == 304:
+                return FetchedDocument(url, current_url, 304, dict(response.headers), b"")
+            if response.status_code == 429:
+                retry_after = parse_retry_after(response.headers.get("retry-after"))
+                raise RateLimitedError(retry_after)
+            if response.status_code >= 400:
+                raise FetchError(f"HTTP {response.status_code}")
+            content_type = response.headers.get("content-type", "").lower()
+            if content_type and not any(
+                allowed in content_type for allowed in ALLOWED_CONTENT_TYPES
+            ):
+                raise ContentRejectedError(f"Content-Type niet toegestaan: {content_type}")
+            content = response.content
+            if len(content) > settings.FETCHER_MAX_BYTES:
+                raise ContentRejectedError("Document overschrijdt de ingestelde groottebeperking.")
+            return FetchedDocument(
+                url, current_url, response.status_code, dict(response.headers), content
+            )
+        raise FetchError("Te veel redirects.")
+    except httpx.TimeoutException as exc:
+        raise FetchTimeoutError("Timeout tijdens HTTP-opvraag") from exc
+    except httpx.RequestError as exc:
+        raise FetchError(f"Netwerkfout tijdens HTTP-opvraag: {exc}") from exc
+    finally:
+        if own_client:
+            http_client.close()

apps/sources/services/health.py

@@ -0,0 +1,365 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+from django.db.models import Prefetch
+from django.utils import timezone
+
+from apps.sources.models import Source, SourcePolicyReview, SourceRun
+
+from .policy import create_policy_review
+
+
+@dataclass(frozen=True)
+class SourceHealth:
+    source_id: int
+    source_name: str
+    source_type: str
+    source_status: str
+    monitored_runs: int
+    success_ratio: float
+    avg_latency_ms: float | None
+    error_counts: dict[str, int]
+    http_status_counts: dict[str, int]
+    extracted_count: int
+    updated_count: int
+    duplicate_count: int
+    last_parser: str | None
+    last_parser_warnings: int
+    last_health_action: str | None
+    last_health_reason: str | None
+
+
+SOURCE_HEALTH_RUN_WINDOW = 30
+SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE = 6
+SOURCE_HEALTH_TEMPORARY_ERROR_RATIO = 0.7
+SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK = 3
+SOURCE_HEALTH_PARSER_MIN_WARNINGS = 2
+SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX = 0
+SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS = 12
+
+TEMPORARY_ERROR_CATEGORIES = {
+    "timeout",
+    "rate_limited",
+    "FetchTimeoutError",
+    "FetchError",
+    "unexpected",
+    "NetworkError",
+}
+POLICY_ERROR_CATEGORIES = {"policy"}
+
+
+def _normalize_metrics(raw: object) -> dict[str, object]:
+    if isinstance(raw, dict):
+        return raw
+    return {}
+
+
+def _warnings_from_run(run: SourceRun) -> list[str]:
+    metrics = _normalize_metrics(run.metrics)
+    warnings = metrics.get("warnings", [])
+    if not isinstance(warnings, list):
+        return []
+    return [str(item) for item in warnings if isinstance(item, str)]
+
+
+def _parser_from_run(run: SourceRun) -> str | None:
+    metrics = _normalize_metrics(run.metrics)
+    parser = metrics.get("parser")
+    if isinstance(parser, str) and parser:
+        return parser
+    return None
+
+
+def _int(value: object) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+def _float(value: object) -> float:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def _to_iso(dt: datetime | None) -> str | None:
+    if dt is None:
+        return None
+    return dt.isoformat()
+
+
+def _from_iso(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        dt = datetime.fromisoformat(value)
+    except ValueError:
+        return None
+    if timezone.is_naive(dt):
+        return timezone.make_aware(dt)
+    return dt
+
+
+def _health_metadata(source: Source) -> dict[str, object]:
+    metadata = source.metadata
+    if not isinstance(metadata, dict):
+        return {}
+    health = metadata.get("source_health")
+    return health if isinstance(health, dict) else {}
+
+
+def _set_health_metadata(source: Source, health_data: dict[str, object]) -> None:
+    metadata = source.metadata
+    if not isinstance(metadata, dict):
+        metadata = {}
+    metadata["source_health"] = health_data
+    source.metadata = metadata
+    source.save(update_fields=["metadata", "updated_at"])
+
+
+def _run_counts(runs: Iterable[SourceRun]) -> tuple[dict[str, int], dict[str, int]]:
+    error_counts: dict[str, int] = {}
+    http_status_counts: dict[str, int] = {}
+    for run in runs:
+        if run.status != SourceRun.Status.SUCCESS:
+            category = run.error_category or "unknown"
+            error_counts[category] = error_counts.get(category, 0) + 1
+        if run.http_status:
+            status = str(run.http_status)
+            http_status_counts[status] = http_status_counts.get(status, 0) + 1
+    return error_counts, http_status_counts
+
+
+def _last_parser_output(runs: list[SourceRun]) -> tuple[str | None, int]:
+    for run in runs:
+        if run.status != SourceRun.Status.SUCCESS:
+            continue
+        parser = _parser_from_run(run)
+        if parser:
+            warnings = _warnings_from_run(run)
+            return parser, len(warnings)
+    return None, 0
+
+
+def _latency_ms(runs: list[SourceRun]) -> float | None:
+    latencies = []
+    for run in runs:
+        if run.finished_at is None or run.started_at is None:
+            continue
+        latencies.append(max(0.0, (run.finished_at - run.started_at).total_seconds() * 1000))
+    if not latencies:
+        return None
+    return sum(latencies) / len(latencies)
+
+
+def _is_parser_drift_run(run: SourceRun) -> bool:
+    if run.status != SourceRun.Status.SUCCESS:
+        return False
+    if _int(run.extracted_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
+        return False
+    if _int(run.created_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
+        return False
+    if _int(run.updated_count) > SOURCE_HEALTH_PARSER_DRIFT_EXTRACTED_MAX:
+        return False
+    if _parser_from_run(run) in {None, "not-modified"}:
+        return False
+    warnings = _warnings_from_run(run)
+    return len(warnings) >= SOURCE_HEALTH_PARSER_MIN_WARNINGS
+
+
+def _determine_health_action(runs: list[SourceRun]) -> tuple[str | None, str | None]:
+    recent_runs = runs[:SOURCE_HEALTH_RUN_WINDOW]
+    considered_failures = [
+        run
+        for run in recent_runs
+        if run.status in {SourceRun.Status.FAILED, SourceRun.Status.SKIPPED}
+    ]
+
+    if any(
+        run.error_category in POLICY_ERROR_CATEGORIES
+        for run in considered_failures[:3]
+        if run.error_category
+    ):
+        return "quarantine", "Herhaald beleid-/securityprobleem in bronruns."
+
+    if len(considered_failures) >= SOURCE_HEALTH_MIN_RUNS_FOR_TEMPORARY_QUARANTINE:
+        temporary_count = sum(
+            1
+            for run in considered_failures
+            if (run.error_category or "") in TEMPORARY_ERROR_CATEGORIES
+        )
+        ratio = _float(temporary_count) / _float(len(considered_failures))
+        if ratio >= SOURCE_HEALTH_TEMPORARY_ERROR_RATIO:
+            return "quarantine", "Herhaald tijdelijk foutgedrag tijdens bronruns."
+
+    streak = 0
+    for run in recent_runs:
+        if _is_parser_drift_run(run):
+            streak += 1
+            if streak >= SOURCE_HEALTH_PARSER_DRIFT_SUCCESS_STREAK:
+                return (
+                    "quarantine",
+                    "Parserdrift vermoed: opeenvolgende succesvolle runs met minimale output.",
+                )
+            continue
+        streak = 0
+
+    return None, None
+
+
+def collect_source_health(
+    *, runs_to_consider: int = SOURCE_HEALTH_RUN_WINDOW
+) -> list[SourceHealth]:
+    sources = Source.objects.prefetch_related(
+        Prefetch(
+            "runs",
+            queryset=SourceRun.objects.order_by("-started_at")[:runs_to_consider],
+            to_attr="health_runs",
+        )
+    ).order_by("name")
+    rows: list[SourceHealth] = []
+
+    for source in sources:
+        runs = source.health_runs
+        monitored_runs = len(runs)
+        success_runs = [run for run in runs if run.status == SourceRun.Status.SUCCESS]
+
+        success_ratio = _float(len(success_runs) / monitored_runs) if monitored_runs else 0.0
+        avg_latency_ms = _latency_ms(runs)
+        error_counts, http_status_counts = _run_counts(runs)
+
+        extracted_count = sum(_int(run.extracted_count) for run in runs)
+        updated_count = sum(_int(run.updated_count) for run in runs)
+        duplicate_count = sum(_int(run.duplicate_count) for run in runs)
+
+        last_parser, last_warnings = _last_parser_output(runs)
+        action, reason = _determine_health_action(runs)
+
+        rows.append(
+            SourceHealth(
+                source_id=source.pk,
+                source_name=source.name,
+                source_type=source.source_type,
+                source_status=source.status,
+                monitored_runs=monitored_runs,
+                success_ratio=success_ratio,
+                avg_latency_ms=avg_latency_ms,
+                error_counts=error_counts,
+                http_status_counts=http_status_counts,
+                extracted_count=extracted_count,
+                updated_count=updated_count,
+                duplicate_count=duplicate_count,
+                last_parser=last_parser,
+                last_parser_warnings=last_warnings,
+                last_health_action=action,
+                last_health_reason=reason,
+            )
+        )
+
+    return rows
+
+
+def evaluate_source_health(*, now: datetime | None = None) -> dict[str, int]:
+    now = now or timezone.now()
+    rows = [
+        row
+        for row in collect_source_health()
+        if row.source_status in {Source.Status.ACTIVE, Source.Status.TRIAL}
+    ]
+    counts = {"evaluated": len(rows), "quarantined": 0}
+
+    for row in rows:
+        if row.last_health_action != "quarantine":
+            continue
+        counts["evaluated"] += 1
+
+        source = Source.objects.get(pk=row.source_id)
+        source.status = Source.Status.QUARANTINED
+        source.policy = Source.Policy.DENY
+        source.policy_reason = row.last_health_reason or "Bronhealth detecteert instabiele bron"
+        _set_health_metadata(
+            source,
+            {
+                "state": "quarantined",
+                "quarantine_reason": source.policy_reason,
+                "quarantined_at": _to_iso(now),
+                "recovery_due_at": _to_iso(
+                    now + timedelta(hours=SOURCE_HEALTH_RECOVERY_COOLDOWN_HOURS)
+                ),
+                "canary_started": False,
+            },
+        )
+        source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
+
+        latest_review = source.policy_reviews.order_by("-created_at").first()
+        if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.DENY:
+            create_policy_review(
+                source,
+                actor=None,
+                decision=SourcePolicyReview.Decision.DENY,
+                reason=source.policy_reason,
+                scope=SourcePolicyReview.Scope.SOURCE,
+                notes="Automatische bronhealth",
+            )
+
+        counts["quarantined"] += 1
+
+    return counts
+
+
+def canary_recovery_sources(*, now: datetime | None = None) -> list[Source]:
+    now = now or timezone.now()
+    sources: list[Source] = []
+    for source in Source.objects.filter(status=Source.Status.QUARANTINED):
+        health = _health_metadata(source)
+        if health.get("state") != "quarantined":
+            continue
+
+        if bool(health.get("canary_started", False)):
+            continue
+
+        recovery_due = _from_iso(
+            health.get("recovery_due_at") if isinstance(health, dict) else None
+        )
+        if recovery_due and recovery_due > now:
+            continue
+
+        sources.append(source)
+
+    return sources
+
+
+def start_health_canary(source: Source, *, now: datetime | None = None) -> None:
+    now = now or timezone.now()
+    health = _health_metadata(source)
+
+    latest_review = source.policy_reviews.order_by("-created_at").first()
+    if latest_review is None or latest_review.decision != SourcePolicyReview.Decision.TRIAL:
+        create_policy_review(
+            source,
+            actor=None,
+            decision=SourcePolicyReview.Decision.TRIAL,
+            reason="Automatische bronrecovery via canary",
+            scope=SourcePolicyReview.Scope.SOURCE,
+            notes="Canaryherstel",
+        )
+
+    source.status = Source.Status.TRIAL
+    source.policy = Source.Policy.REVIEW
+    source.policy_reason = "Bronherstel via geautomatiseerde canary"
+    source.next_run_at = now
+    _set_health_metadata(
+        source,
+        {
+            **health,
+            "state": "canary_in_progress",
+            "canary_started": True,
+            "canary_started_at": _to_iso(now),
+        },
+    )
+    source.save(update_fields=["status", "policy", "policy_reason", "next_run_at", "updated_at"])

apps/sources/services/imap_import.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import imaplib
+from collections.abc import Callable
+from contextlib import suppress
+from typing import Any
+
+from django.conf import settings
+
+from apps.core.services.microsoft365 import acquire_exchange_access_token, xoauth2_bytes
+from apps.sources.models import EmailMessageRecord, MailboxConnection
+from apps.sources.services.email_import import ingest_email, message_identity
+from apps.sources.services.mailbox_connections import decrypt_mailbox_password
+from apps.sources.services.tls import trusted_tls_context
+from apps.sources.services.url_security import validate_public_url
+
+ImapClientFactory = Callable[..., Any]
+HostValidator = Callable[[str], object]
+
+
+def poll_imap_mailbox(
+    connection: MailboxConnection,
+    *,
+    client_factory: ImapClientFactory | None = None,
+    host_validator: HostValidator = validate_public_url,
+) -> dict[str, int | str]:
+    """Import one bounded mailbox batch without letting one bad message abort the batch."""
+    if not connection.enabled:
+        return {"status": "disabled", "imported": 0}
+
+    if client_factory is None:
+        client_factory = imaplib.IMAP4_SSL
+    host_validator(f"https://{connection.imap_host}")
+    password = (
+        ""
+        if connection.provider == MailboxConnection.Provider.OUTLOOK
+        else decrypt_mailbox_password(connection)
+    )
+    client_options = {"timeout": settings.IMAP_CONNECT_TIMEOUT_SECONDS}
+    if client_factory is imaplib.IMAP4_SSL:
+        client_options["ssl_context"] = trusted_tls_context()
+    client = client_factory(connection.imap_host, connection.port, **client_options)
+    counters = {
+        "imported": 0,
+        "duplicates": 0,
+        "skipped_oversized": 0,
+        "failed": 0,
+    }
+    try:
+        if connection.provider == MailboxConnection.Provider.OUTLOOK:
+            access_token = acquire_exchange_access_token()
+            client.authenticate(
+                "XOAUTH2", lambda _challenge: xoauth2_bytes(connection.username, access_token)
+            )
+        else:
+            client.login(connection.username, password)
+        status, _ = client.select(connection.mailbox, readonly=True)
+        if status != "OK":
+            raise RuntimeError("IMAP-mailbox kon niet worden geopend")
+        status, data = client.uid("search", None, "ALL")
+        if status != "OK":
+            raise RuntimeError("IMAP-zoekopdracht mislukt")
+        uids = (data[0] or b"").split()[-settings.IMAP_MAX_MESSAGES_PER_POLL :]
+        for uid in uids:
+            try:
+                status, payload = client.uid("fetch", uid, "(RFC822)")
+                if status != "OK" or not payload:
+                    counters["failed"] += 1
+                    continue
+                raw = next((part[1] for part in payload if isinstance(part, tuple)), None)
+                if not isinstance(raw, bytes):
+                    counters["failed"] += 1
+                    continue
+                if len(raw) > settings.IMAP_MAX_MESSAGE_BYTES:
+                    counters["skipped_oversized"] += 1
+                    continue
+                identity = message_identity(raw)
+                if EmailMessageRecord.objects.filter(
+                    mailbox_connection=connection, message_id=identity
+                ).exists():
+                    counters["duplicates"] += 1
+                    continue
+                ingest_email(
+                    raw,
+                    mailbox=connection.mailbox,
+                    mailbox_connection=connection,
+                    platform=connection.platform,
+                )
+                counters["imported"] += 1
+            except Exception:
+                counters["failed"] += 1
+        return {"status": "ok", **counters}
+    finally:
+        with suppress(Exception):
+            client.logout()

apps/sources/services/mailbox_connections.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from uuid import uuid4
+
+from cryptography.fernet import Fernet, InvalidToken, MultiFernet
+from django.conf import settings
+from django.db import transaction
+from django.utils import timezone
+
+from apps.sources.models import MailboxConnection
+
+
+class MailboxCredentialError(RuntimeError):
+    """Safe credential configuration error without secret material."""
+
+
+def _cipher() -> MultiFernet:
+    keys = settings.MAILBOX_CREDENTIAL_KEYS
+    if not keys:
+        raise MailboxCredentialError(
+            "Credentialopslag is niet geconfigureerd. Stel MAILBOX_CREDENTIAL_KEYS in."
+        )
+    try:
+        return MultiFernet([Fernet(key.encode("ascii")) for key in keys])
+    except (ValueError, TypeError) as exc:
+        raise MailboxCredentialError(
+            "MAILBOX_CREDENTIAL_KEYS bevat een ongeldige sleutel."
+        ) from exc
+
+
+def encrypt_mailbox_password(password: str) -> str:
+    if not password:
+        raise MailboxCredentialError("Een app-wachtwoord is verplicht.")
+    return _cipher().encrypt(password.encode("utf-8")).decode("ascii")
+
+
+def decrypt_mailbox_password(connection: MailboxConnection) -> str:
+    try:
+        return _cipher().decrypt(connection.encrypted_password.encode("ascii")).decode("utf-8")
+    except (InvalidToken, ValueError, UnicodeError) as exc:
+        raise MailboxCredentialError(
+            "Het opgeslagen app-wachtwoord kan niet worden ontsleuteld. Sla het opnieuw op."
+        ) from exc
+
+
+@transaction.atomic
+def rotate_mailbox_credentials() -> int:
+    """Re-encrypt every mailbox password with the first configured key."""
+    rotated = 0
+    for connection in MailboxConnection.objects.select_for_update().exclude(
+        provider=MailboxConnection.Provider.OUTLOOK
+    ):
+        password = decrypt_mailbox_password(connection)
+        connection.encrypted_password = encrypt_mailbox_password(password)
+        connection.save(update_fields=["encrypted_password", "updated_at"])
+        rotated += 1
+    return rotated
+
+
+@transaction.atomic
+def save_mailbox_connection(
+    *,
+    user,
+    cleaned_data: dict[str, object],
+    instance: MailboxConnection | None = None,
+) -> MailboxConnection:
+    connection = instance or MailboxConnection(user=user)
+    if connection.pk and connection.user_id != user.pk:
+        raise PermissionError("Mailboxkoppeling behoort niet tot deze gebruiker.")
+
+    password = str(cleaned_data.pop("password", "") or "")
+    for field in (
+        "platform",
+        "provider",
+        "custom_host",
+        "port",
+        "username",
+        "mailbox",
+        "enabled",
+        "poll_interval_minutes",
+    ):
+        setattr(connection, field, cleaned_data[field])
+    uses_oauth = connection.provider == MailboxConnection.Provider.OUTLOOK
+    if uses_oauth:
+        connection.encrypted_password = ""
+    elif password:
+        connection.encrypted_password = encrypt_mailbox_password(password)
+    elif not connection.encrypted_password:
+        raise MailboxCredentialError("Een app-wachtwoord is verplicht.")
+    connection.next_poll_at = timezone.now()
+    connection.last_error_category = ""
+    connection.last_error_message = ""
+    connection.full_clean()
+    connection.save()
+    return connection
+
+
+@transaction.atomic
+def claim_mailbox_connection(
+    *, connection_id: int, worker_token: str | None = None, force: bool = False
+) -> tuple[MailboxConnection, str] | None:
+    now = timezone.now()
+    connection = MailboxConnection.objects.select_for_update().get(pk=connection_id)
+    if not connection.enabled:
+        return None
+    if connection.lease_expires_at and connection.lease_expires_at > now:
+        return None
+    if not force and connection.next_poll_at > now:
+        return None
+    token = worker_token or uuid4().hex
+    connection.lease_token = token
+    connection.lease_expires_at = now + timedelta(minutes=5)
+    connection.save(update_fields=["lease_token", "lease_expires_at", "updated_at"])
+    return connection, token
+
+
+@transaction.atomic
+def finish_mailbox_poll(
+    *,
+    connection_id: int,
+    worker_token: str,
+    success: bool,
+    error_category: str = "",
+    error_message: str = "",
+) -> None:
+    connection = MailboxConnection.objects.select_for_update().get(pk=connection_id)
+    if connection.lease_token != worker_token:
+        return
+    now = timezone.now()
+    connection.last_polled_at = now
+    connection.next_poll_at = now + timedelta(minutes=connection.poll_interval_minutes)
+    connection.lease_token = ""
+    connection.lease_expires_at = None
+    if success:
+        connection.last_success_at = now
+        connection.last_error_category = ""
+        connection.last_error_message = ""
+    else:
+        connection.last_error_category = error_category[:80]
+        connection.last_error_message = error_message[:500]
+    connection.save(
+        update_fields=[
+            "last_polled_at",
+            "next_poll_at",
+            "lease_token",
+            "lease_expires_at",
+            "last_success_at",
+            "last_error_category",
+            "last_error_message",
+            "updated_at",
+        ]
+    )

apps/sources/services/manual_import.py

@@ -0,0 +1,348 @@
+from __future__ import annotations
+
+import hashlib
+import html
+from dataclasses import dataclass
+from datetime import timedelta
+from urllib.parse import urlsplit
+from uuid import uuid4
+
+import bleach
+from django.conf import settings
+from django.db import transaction
+from django.db.models import QuerySet
+from django.utils import timezone
+
+from apps.jobs.models import JobSourceAlias
+from apps.jobs.services.pipeline import process_raw_document
+from apps.sources.models import RawDocument, Source, SourcePolicyReview, SourceRun
+from apps.sources.services.canonicalize import canonicalize_url
+from apps.sources.services.fetcher import (
+    FetchedDocument,
+    FetchError,
+    FetchTimeoutError,
+    PolicyBlockedError,
+    RateLimitedError,
+    fetch_url,
+)
+from apps.sources.services.policy import assess_url, create_policy_review
+
+
+class ManualImportError(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True)
+class ManualImportSummary:
+    source_id: int
+    source_name: str
+    source_url: str
+    mode: str
+    extracted_count: int
+    created_count: int
+    duplicate_count: int
+    warnings: list[str]
+    jobs: list[dict[str, str]]
+
+    def to_session_payload(self) -> dict[str, object]:
+        return {
+            "source_id": self.source_id,
+            "source_name": self.source_name,
+            "source_url": self.source_url,
+            "mode": self.mode,
+            "extracted_count": self.extracted_count,
+            "created_count": self.created_count,
+            "duplicate_count": self.duplicate_count,
+            "warnings": self.warnings,
+            "jobs": self.jobs,
+        }
+
+
+def _sanitize_pasted_text(value: str) -> str:
+    text = bleach.clean(value or "", tags=[], attributes={}, strip=True).strip()
+    if not text:
+        raise ManualImportError("Het geplakte tekstveld bevat geen bruikbare inhoud.")
+    if len(text.encode("utf-8")) > settings.MANUAL_IMPORT_PASTE_MAX_BYTES:
+        raise ManualImportError(
+            "Het tekstveld is te groot voor veilige import. Verwijder overtollige tekst."
+        )
+    return text
+
+
+def _kind_for(content_type: str) -> str:
+    content_type = (content_type or "").lower()
+    if "html" in content_type:
+        return RawDocument.Kind.HTML
+    if "xml" in content_type or "rss" in content_type or "atom" in content_type:
+        return RawDocument.Kind.XML
+    if "json" in content_type:
+        return RawDocument.Kind.JSON
+    return RawDocument.Kind.TEXT
+
+
+def _mode_source_name(domain: str, *, mode: str) -> str:
+    if mode == "paste":
+        return f"Handmatige tekstimport {domain}"
+    return f"Handmatige URL-import {domain}"
+
+
+def _ensure_manual_review(source: Source, actor) -> None:
+    review = source.latest_policy_review
+    if review and not review.is_expired and review.decision == SourcePolicyReview.Decision.ALLOW:
+        return
+    create_policy_review(
+        source,
+        actor=actor if actor and getattr(actor, "pk", None) else None,
+        decision=SourcePolicyReview.Decision.ALLOW,
+        reason="Handmatige import uitgevoerd.",
+        scope=SourcePolicyReview.Scope.SOURCE,
+    )
+
+
+def _ensure_manual_source(*, domain: str, source_url: str, actor, mode: str) -> Source:
+    defaults = {
+        "name": _mode_source_name(domain, mode=mode),
+        "base_url": source_url,
+        "status": Source.Status.CANDIDATE,
+        "policy": Source.Policy.ALLOW,
+    }
+    source, created = Source.objects.get_or_create(
+        domain=domain,
+        source_type=Source.Type.MANUAL,
+        defaults=defaults,
+    )
+    if not created:
+        source.name = _mode_source_name(domain, mode=mode)
+        source.base_url = source_url
+        source.status = Source.Status.CANDIDATE
+        source.policy = Source.Policy.ALLOW
+        source.save(update_fields=["name", "base_url", "status", "policy", "updated_at"])
+    _ensure_manual_review(source, actor=actor)
+    return source
+
+
+def _create_raw_document(
+    source: Source,
+    *,
+    source_run: SourceRun,
+    requested_url: str,
+    final_url: str,
+    content_type: str,
+    content: bytes,
+) -> RawDocument:
+    return RawDocument.objects.create(
+        source=source,
+        source_run=source_run,
+        url=requested_url,
+        final_url=final_url,
+        kind=_kind_for(content_type),
+        content_type=content_type[:200],
+        http_status=None,
+        response_headers={},
+        content_hash=hashlib.sha256(content).hexdigest(),
+        body_text=content.decode("utf-8", errors="replace"),
+        byte_length=len(content),
+        retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
+    )
+
+
+def _build_jobs_from_document(document: RawDocument) -> list[dict[str, str]]:
+    alias_qs: QuerySet[JobSourceAlias] = JobSourceAlias.objects.select_related("job").filter(
+        raw_document=document
+    )
+    jobs: list[dict[str, str]] = []
+    for alias in alias_qs:
+        title = alias.source_title or (alias.job.original_title if alias.job else "Vacature")
+        employer = alias.source_employer or "Onbekende werkgever"
+        jobs.append(
+            {
+                "id": str(alias.job_id),
+                "title": title,
+                "employer": employer,
+            }
+        )
+    return jobs
+
+
+def _run_pipeline(document: RawDocument, *, source_run: SourceRun) -> ManualImportSummary:
+    metrics = process_raw_document(document)
+    source = document.source
+    source_name = source.name if source else ""
+    warnings: list[str] = list(metrics.get("warnings", []))
+    len(warnings)
+    source_run.finish(
+        SourceRun.Status.SUCCESS,
+        http_status=document.source_run.http_status if document.source_run else None,
+        extracted_count=int(metrics["extracted"]),
+        created_count=int(metrics["created"]),
+        updated_count=int(metrics["updated"]),
+        duplicate_count=int(metrics["duplicates"]),
+        metrics={"parser": metrics["parser"], "warnings": warnings},
+    )
+    jobs = _build_jobs_from_document(document)
+    if metrics["created"] == 0 and metrics["updated"] == 0 and not warnings:
+        warnings.append("De bron leverde geen herkenbare vacaturedata op.")
+    if not warnings:
+        # keep stable, machine-readable payload shape
+        warnings = []
+    return ManualImportSummary(
+        source_id=source.pk,
+        source_name=source_name,
+        source_url=document.final_url,
+        mode="url",
+        extracted_count=int(metrics["extracted"]),
+        created_count=int(metrics["created"]),
+        duplicate_count=int(metrics["duplicates"]),
+        warnings=warnings,
+        jobs=jobs,
+    )
+
+
+def _build_synthetic_paste_payload(raw_text: str) -> tuple[str, bytes, str]:
+    lines = [html.escape(line.strip()) for line in raw_text.splitlines() if line.strip()]
+    title = lines[0] if lines else "Handmatige vacature"
+    body = "".join(f"<p>{line}</p>" for line in lines)
+    source_domain = f"manual-{uuid4().hex[:16]}"
+    synthetic_url = f"https://{source_domain}.vacature.local/"
+    html_content = (
+        f"<html><body><main><h1>{title}</h1>{body}</main>"
+        "<footer>Handmatige import; geen externe fetch</footer></body></html>"
+    )
+    return synthetic_url, html_content.encode("utf-8"), source_domain
+
+
+def import_manual_source(
+    *, actor, source_url: str | None = None, pasted_text: str | None = None
+) -> ManualImportSummary:
+    source_url = (source_url or "").strip()
+    pasted_text = (pasted_text or "").strip()
+    if not source_url and not pasted_text:
+        raise ManualImportError("Vul een URL of tekst in.")
+
+    with transaction.atomic():
+        if source_url:
+            normalized = canonicalize_url(source_url)
+            parsed = urlsplit(normalized)
+            domain = (parsed.hostname or "").lower()
+            if not domain:
+                raise ManualImportError("De bron-URL bevat geen geldig domein.")
+            decision = assess_url(normalized)
+            if not decision.allowed:
+                raise ManualImportError(f"Import geblokkeerd: {decision.reason}")
+            source = _ensure_manual_source(
+                domain=domain,
+                source_url=normalized,
+                actor=actor,
+                mode="url",
+            )
+            source_run = SourceRun.objects.create(source=source)
+            try:
+                fetched: FetchedDocument = fetch_url(normalized, source=source)
+            except PolicyBlockedError as exc:
+                source.status = Source.Status.QUARANTINED
+                source.policy = Source.Policy.DENY
+                source.policy_reason = str(exc)[:1000]
+                source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
+                source_run.finish(
+                    SourceRun.Status.SKIPPED,
+                    error_category="policy",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Import geblokkeerd: {exc}") from exc
+            except RateLimitedError as exc:
+                source_run.finish(
+                    SourceRun.Status.FAILED,
+                    error_category="rate_limited",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Rate limiting tijdens import: {exc}") from exc
+            except FetchTimeoutError as exc:
+                source_run.finish(
+                    SourceRun.Status.FAILED,
+                    error_category="timeout",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Time-out tijdens import: {exc}") from exc
+            except FetchError as exc:
+                source_run.finish(
+                    SourceRun.Status.FAILED,
+                    error_category="fetch",
+                    error_message=str(exc)[:1000],
+                )
+                raise ManualImportError(f"Fetch mislukt: {exc}") from exc
+
+            if fetched.status_code == 304:
+                source_run.finish(
+                    SourceRun.Status.SUCCESS,
+                    http_status=304,
+                    extracted_count=0,
+                    created_count=0,
+                    updated_count=0,
+                    duplicate_count=0,
+                    metrics={"parser": "not-modified", "warnings": []},
+                )
+                return ManualImportSummary(
+                    source_id=source.pk,
+                    source_name=source.name,
+                    source_url=source.base_url,
+                    mode="url",
+                    extracted_count=0,
+                    created_count=0,
+                    duplicate_count=0,
+                    warnings=["Geen inhoudsverandering (304)."],
+                    jobs=[],
+                )
+
+            document = _create_raw_document(
+                source,
+                source_run=source_run,
+                requested_url=fetched.requested_url,
+                final_url=fetched.final_url,
+                content_type=fetched.headers.get("content-type", ""),
+                content=fetched.content,
+            )
+            source.etag = fetched.headers.get("etag", source.etag)
+            source.last_modified = fetched.headers.get("last-modified", source.last_modified)
+            source.save(update_fields=["etag", "last_modified", "updated_at"])
+            summary = _run_pipeline(document, source_run=source_run)
+            return ManualImportSummary(
+                source_id=summary.source_id,
+                source_name=summary.source_name,
+                source_url=summary.source_url,
+                mode="url",
+                extracted_count=summary.extracted_count,
+                created_count=summary.created_count,
+                duplicate_count=summary.duplicate_count,
+                warnings=summary.warnings,
+                jobs=summary.jobs,
+            )
+
+        text = _sanitize_pasted_text(pasted_text)
+        synthetic_url, payload, synthetic_domain = _build_synthetic_paste_payload(text)
+        source = _ensure_manual_source(
+            domain=synthetic_domain,
+            source_url=synthetic_url,
+            actor=actor,
+            mode="paste",
+        )
+        source_run = SourceRun.objects.create(source=source)
+        document = _create_raw_document(
+            source,
+            source_run=source_run,
+            requested_url=synthetic_url,
+            final_url=synthetic_url,
+            content_type="text/html; charset=utf-8",
+            content=payload,
+        )
+        summary = _run_pipeline(document, source_run=source_run)
+        return ManualImportSummary(
+            source_id=summary.source_id,
+            source_name=source.name,
+            source_url=synthetic_url,
+            mode="paste",
+            extracted_count=summary.extracted_count,
+            created_count=summary.created_count,
+            duplicate_count=summary.duplicate_count,
+            warnings=summary.warnings,
+            jobs=summary.jobs,
+        )

apps/sources/services/policy.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from urllib.parse import urlsplit
+
+from django.conf import settings
+from django.utils import timezone
+
+from apps.sources.models import Source, SourcePolicyReview
+
+from .canonicalize import domain_matches
+from .robots import assess_robots
+
+DEFAULT_DENYLIST = {
+    "linkedin.com",
+    "indeed.com",
+    "indeed.be",
+    "stepstone.be",
+    "jobat.be",
+    "vdab.be",
+    "ictjob.be",
+    "careerjet.be",
+    "randstad.be",
+    "roberthalf.com",
+}
+
+
+@dataclass(frozen=True)
+class PolicyDecision:
+    allowed: bool
+    status: str
+    reason: str
+
+
+def _review_expiry(now: datetime | None = None) -> datetime:
+    return (now or timezone.now()) + timedelta(days=getattr(settings, "SOURCE_REVIEW_TTL_DAYS", 90))
+
+
+def is_denied_domain(hostname: str, denylist: set[str] | None = None) -> bool:
+    denylist = denylist or DEFAULT_DENYLIST
+    return any(domain_matches(hostname, domain) for domain in denylist)
+
+
+def create_policy_review(
+    source: Source,
+    *,
+    actor,
+    decision: SourcePolicyReview.Decision,
+    reason: str,
+    scope: SourcePolicyReview.Scope = SourcePolicyReview.Scope.SOURCE,
+    notes: str = "",
+    evidence_link: str = "",
+    expires_at: datetime | None = None,
+    metadata: dict | None = None,
+) -> SourcePolicyReview:
+    return SourcePolicyReview.objects.create(
+        source=source,
+        actor=actor if actor and getattr(actor, "pk", None) else None,
+        decision=decision,
+        scope=scope,
+        reason=reason,
+        notes=notes,
+        evidence_link=evidence_link,
+        expires_at=expires_at or _review_expiry(),
+        metadata=metadata or {},
+    )
+
+
+def _active_review(source: Source) -> SourcePolicyReview | None:
+    review = source.latest_policy_review
+    if review and not review.is_expired:
+        return review
+    return None
+
+
+def _check_review_gate(source: Source) -> PolicyDecision | None:
+    review = _active_review(source)
+    if source.policy == Source.Policy.REVIEW and source.status in {
+        Source.Status.CANDIDATE,
+        Source.Status.TRIAL,
+    }:
+        if not review:
+            return PolicyDecision(False, Source.Policy.REVIEW, "Review vereist")
+        if review.decision == SourcePolicyReview.Decision.DENY:
+            return PolicyDecision(
+                False, Source.Policy.DENY, review.reason or "Review blokkeert bron"
+            )
+        if review.decision == SourcePolicyReview.Decision.PAUSE:
+            return PolicyDecision(
+                False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze"
+            )
+        return None
+
+    if source.policy == Source.Policy.ALLOW:
+        if not review:
+            return PolicyDecision(
+                False, Source.Policy.REVIEW, "Review ontbreekt voor actief beleid"
+            )
+        if review.decision == SourcePolicyReview.Decision.DENY:
+            return PolicyDecision(
+                False, Source.Policy.DENY, review.reason or "Review blokkeert bron"
+            )
+        if review.decision == SourcePolicyReview.Decision.PAUSE:
+            return PolicyDecision(
+                False, Source.Policy.REVIEW, review.reason or "Review vraagt pauze"
+            )
+    return None
+
+
+def assess_url(url: str, *, source: Source | None = None) -> PolicyDecision:
+    hostname = (urlsplit(url).hostname or "").lower()
+    if not hostname:
+        return PolicyDecision(False, Source.Policy.DENY, "URL zonder hostname")
+    if is_denied_domain(hostname):
+        return PolicyDecision(False, Source.Policy.DENY, "Platformdomein staat op de denylist")
+    if source:
+        if source.status in {Source.Status.DISABLED, Source.Status.PAUSED}:
+            return PolicyDecision(False, Source.Policy.REVIEW, f"Bronstatus: {source.status}")
+        if source.policy == Source.Policy.DENY:
+            return PolicyDecision(
+                False, Source.Policy.DENY, source.policy_reason or "Bron geblokkeerd"
+            )
+        review_decision = _check_review_gate(source)
+        if review_decision is not None:
+            return review_decision
+
+    robots = assess_robots(url, source=source)
+    if not robots.allowed:
+        return PolicyDecision(False, Source.Policy.REVIEW, robots.reason)
+
+    return PolicyDecision(True, Source.Policy.ALLOW, "Toegestane publieke bron")

apps/sources/services/robots.py

@@ -0,0 +1,293 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from urllib.parse import urljoin, urlsplit, urlunsplit
+
+import httpx
+from django.conf import settings
+from django.utils import timezone
+
+from apps.sources.models import Source, SourceRobotsCache
+
+from .tls import trusted_tls_context
+from .url_security import UnsafeUrlError, validate_public_url
+
+ALLOW = "allow"
+DISALLOW = "disallow"
+
+
+@dataclass(frozen=True)
+class RobotsDecision:
+    allowed: bool
+    reason: str
+
+
+def _origin_for(url: str) -> str:
+    parts = urlsplit(url)
+    if not parts.scheme:
+        raise ValueError("Ongeldige URL voor robotscontrole.")
+    host = (parts.hostname or "").lower().rstrip(".")
+    if not host:
+        raise ValueError("Host ontbreekt voor robotscontrole.")
+    port = parts.port
+    if (
+        (parts.scheme == "http" and port == 80)
+        or (parts.scheme == "https" and port == 443)
+        or not port
+    ):
+        netloc = host
+    else:
+        netloc = f"{host}:{port}"
+    return f"{parts.scheme}://{netloc}"
+
+
+def _path_for(url: str) -> str:
+    path = urlsplit(url).path or "/"
+    return path if path.startswith("/") else f"/{path}"
+
+
+def _robots_url(origin: str) -> str:
+    parts = urlsplit(origin)
+    return urlunsplit((parts.scheme, parts.netloc, "/robots.txt", "", ""))
+
+
+def _rules_from_text(text: str) -> dict[str, dict[str, list[str]]]:
+    bucket: dict[str, dict[str, list[str]]] = {}
+    active_agents: set[str] = set()
+
+    for raw_line in text.splitlines():
+        line = raw_line.split("#", 1)[0].strip()
+        if ":" not in line:
+            continue
+        key, value = (part.strip() for part in line.split(":", 1))
+        if not key:
+            continue
+        key_lower = key.lower()
+        if key_lower == "user-agent":
+            token = value.lower()
+            active_agents = {token} if token else set()
+            continue
+        if key_lower not in {ALLOW, DISALLOW}:
+            continue
+        if not active_agents:
+            continue
+        for agent in active_agents:
+            section = bucket.setdefault(agent, {ALLOW: [], DISALLOW: []})
+            section[key_lower].append(value.strip() or "/")
+
+    for entry in bucket.values():
+        entry[ALLOW] = list(dict.fromkeys(entry[ALLOW]))
+        entry[DISALLOW] = list(dict.fromkeys(entry[DISALLOW]))
+    return bucket
+
+
+def _pick_rules(rules: dict[str, dict[str, list[str]]], user_agent: str) -> dict[str, list[str]]:
+    normalized = user_agent.lower()
+    selected = {ALLOW: [], DISALLOW: []}
+
+    for agent, values in rules.items():
+        if agent == "*" or (agent and agent in normalized):
+            selected[ALLOW].extend(values[ALLOW])
+            selected[DISALLOW].extend(values[DISALLOW])
+
+    selected[ALLOW] = list(dict.fromkeys(selected[ALLOW]))
+    selected[DISALLOW] = list(dict.fromkeys(selected[DISALLOW]))
+    return selected
+
+
+def _longest_prefix(path: str, rules: Iterable[str]) -> int:
+    return max(
+        (len(rule.rstrip("/")) for rule in rules if rule and path.startswith(rule)), default=0
+    )
+
+
+def _evaluate_path(
+    path: str, user_agent: str, rules: dict[str, dict[str, list[str]]]
+) -> RobotsDecision:
+    selected = _pick_rules(rules, user_agent=user_agent)
+    allow_len = _longest_prefix(path, selected[ALLOW])
+    disallow_len = _longest_prefix(path, selected[DISALLOW])
+    if disallow_len > allow_len:
+        return RobotsDecision(False, "Toegang geblokkeerd door robotsregels")
+    return RobotsDecision(True, "Robotsregels staan toegang toe")
+
+
+def _cache_ttl_seconds() -> float:
+    return float(getattr(settings, "ROBOTS_CACHE_TTL_SECONDS", 3600))
+
+
+def _max_bytes() -> int:
+    return int(getattr(settings, "ROBOTS_MAX_BYTES", 131072))
+
+
+def _fetch_robots(origin: str, *, client: httpx.Client | None = None) -> tuple[str, int, str, str]:
+    robots_url = _robots_url(origin)
+    own_client = client is None
+    http_client = client or httpx.Client(
+        timeout=httpx.Timeout(getattr(settings, "ROBOTS_FETCH_TIMEOUT_SECONDS", 5)),
+        follow_redirects=False,
+        verify=trusted_tls_context(),
+    )
+    try:
+        current_url = robots_url
+        for _ in range(getattr(settings, "FETCHER_MAX_REDIRECTS", 5) + 1):
+            validate_public_url(
+                current_url,
+                allow_nonstandard_ports=getattr(settings, "FETCHER_ALLOW_NONSTANDARD_PORTS", False),
+            )
+            response = http_client.get(
+                current_url,
+                headers={"User-Agent": getattr(settings, "FETCHER_USER_AGENT", "VacatureRadar")},
+            )
+            if response.status_code not in {301, 302, 303, 307, 308}:
+                break
+            location = response.headers.get("location")
+            if not location:
+                raise httpx.HTTPStatusError(
+                    "Robotsredirect zonder Location-header",
+                    request=response.request,
+                    response=response,
+                )
+            current_url = urljoin(current_url, location)
+        else:
+            raise httpx.TooManyRedirects(
+                "Te veel redirects bij robotscontrole", request=response.request
+            )
+    finally:
+        if own_client:
+            http_client.close()
+    return (
+        response.text,
+        response.status_code,
+        response.headers.get("etag", ""),
+        response.headers.get("last-modified", ""),
+    )
+
+
+def _load_cached(origin: str, now: datetime) -> SourceRobotsCache | None:
+    try:
+        cache = SourceRobotsCache.objects.get(origin=origin)
+    except SourceRobotsCache.DoesNotExist:
+        return None
+    if cache.expires_at <= now:
+        return None
+    return cache
+
+
+def _load_stale_cache(origin: str) -> SourceRobotsCache | None:
+    try:
+        return SourceRobotsCache.objects.get(origin=origin)
+    except SourceRobotsCache.DoesNotExist:
+        return None
+
+
+def _persist_cache(
+    origin: str,
+    *,
+    status_code: int,
+    content: str,
+    etag: str = "",
+    last_modified: str = "",
+    now: datetime,
+) -> SourceRobotsCache:
+    max_bytes = _max_bytes()
+    byte_length = len(content.encode("utf-8"))
+    if byte_length > max_bytes:
+        return SourceRobotsCache.objects.update_or_create(
+            origin=origin,
+            defaults={
+                "expires_at": now + timedelta(seconds=_cache_ttl_seconds()),
+                "allow_rules": {},
+                "disallow_rules": {},
+                "error": f"robots.txt te groot ({byte_length} bytes)",
+                "etag": etag,
+                "last_modified": last_modified,
+                "byte_length": byte_length,
+            },
+        )[0]
+
+    rules = {} if status_code in {404, 410} or status_code >= 400 else _rules_from_text(content)
+
+    return SourceRobotsCache.objects.update_or_create(
+        origin=origin,
+        defaults={
+            "expires_at": now + timedelta(seconds=_cache_ttl_seconds()),
+            "allow_rules": {agent: value[ALLOW] for agent, value in rules.items()},
+            "disallow_rules": {agent: value[DISALLOW] for agent, value in rules.items()},
+            "error": "",
+            "etag": etag,
+            "last_modified": last_modified,
+            "byte_length": byte_length,
+        },
+    )[0]
+
+
+def _build_ruleset(cache: SourceRobotsCache) -> dict[str, dict[str, list[str]]]:
+    rules = {"allow": {}, "disallow": {}}
+    for agent, entries in cache.allow_rules.items():
+        rules["allow"][agent] = list(entries)
+    for agent, entries in cache.disallow_rules.items():
+        rules["disallow"][agent] = list(entries)
+
+    normalized_rules: dict[str, dict[str, list[str]]] = {}
+    all_agents = set(rules["allow"].keys()) | set(rules["disallow"].keys())
+    for agent in all_agents:
+        normalized_rules[agent] = {
+            ALLOW: rules["allow"].get(agent, []),
+            DISALLOW: rules["disallow"].get(agent, []),
+        }
+    return normalized_rules
+
+
+def assess_robots(
+    url: str,
+    *,
+    source: Source | None = None,
+    user_agent: str | None = None,
+    client: httpx.Client | None = None,
+    now: datetime | None = None,
+) -> RobotsDecision:
+    if source is None or not source.honor_robots:
+        return RobotsDecision(True, "Robotscontrole niet vereist")
+
+    now = now or timezone.now()
+    try:
+        origin = _origin_for(url)
+        path = _path_for(url)
+    except ValueError as exc:
+        return RobotsDecision(False, str(exc))
+
+    cache = _load_cached(origin, now=now)
+    stale = _load_stale_cache(origin)
+    if cache is None:
+        try:
+            content, status_code, etag, last_modified = _fetch_robots(origin, client=client)
+            cache = _persist_cache(
+                origin,
+                status_code=status_code,
+                content=content,
+                etag=etag,
+                last_modified=last_modified,
+                now=now,
+            )
+        except UnsafeUrlError as exc:
+            return RobotsDecision(False, f"Robotscontrole mislukt: {exc}")
+        except httpx.HTTPError as exc:
+            if stale is not None:
+                return _evaluate_path(
+                    path,
+                    user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""),
+                    rules=_build_ruleset(stale),
+                )
+            return RobotsDecision(True, f"Robotscontrole tijdelijk niet beschikbaar: {exc}")
+
+    if cache.error:
+        return RobotsDecision(True, cache.error)
+
+    rules = _build_ruleset(cache)
+    decision = _evaluate_path(
+        path, user_agent=user_agent or getattr(settings, "FETCHER_USER_AGENT", ""), rules=rules
+    )
+    return decision

apps/sources/services/scheduling.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import hashlib
+from datetime import timedelta
+from urllib.parse import urlparse
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models import Max
+from django.utils import timezone
+
+from apps.sources.models import Source, SourceLease, SourceOriginState
+
+
+def origin_key(source: Source) -> str:
+    if source.domain:
+        return source.domain.lower()
+    hostname = (urlparse(source.base_url).hostname or "").lower()
+    return hostname.rstrip(".")
+
+
+def max_origin_minimum_interval_seconds(source: Source) -> int:
+    domain = origin_key(source)
+    aggregate = Source.objects.filter(domain=domain).aggregate(
+        maximum_interval=Max("minimum_interval_seconds")
+    )
+    max_interval = aggregate["maximum_interval"]
+    if max_interval is None:
+        return int(source.minimum_interval_seconds)
+    return int(max_interval)
+
+
+def _lease_ttl_seconds() -> int:
+    ttl = settings.SOURCE_LEASE_TTL_SECONDS
+    if ttl <= 0:
+        return 180
+    return int(ttl)
+
+
+def _max_origin_interval_seconds(source: Source) -> int:
+    explicit = settings.SOURCE_ORIGIN_MIN_INTERVAL_SECONDS
+    if explicit > 0:
+        return int(explicit)
+    return max_origin_minimum_interval_seconds(source)
+
+
+def calculate_jitter_seconds(source: Source, *, max_seconds: int) -> int:
+    max_seconds = int(max_seconds)
+    if max_seconds <= 0:
+        return 0
+    jitter_seed = f"{source.pk}:{source.domain}:{source.minimum_interval_seconds}"
+    digest = hashlib.sha256(jitter_seed.encode("utf-8")).digest()
+    jitter_span = max_seconds + 1
+    return int(int.from_bytes(digest[:8], "big") % jitter_span)
+
+
+def calculate_failure_backoff_seconds(
+    source: Source,
+    *,
+    failure_count: int,
+    retry_after_seconds: int | None = None,
+    timeout: bool = False,
+) -> int:
+    failure_count = max(1, failure_count)
+    base_seconds = settings.SOURCE_FAILURE_BACKOFF_BASE_SECONDS
+    if timeout:
+        base_seconds = max(base_seconds, settings.SOURCE_FAILURE_TIMEOUT_BASE_SECONDS)
+    factor = 2 ** min(failure_count - 1, 10)
+    backoff_seconds = base_seconds * factor
+    if retry_after_seconds:
+        backoff_seconds = max(backoff_seconds, retry_after_seconds)
+    max_seconds = settings.SOURCE_FAILURE_BACKOFF_MAX_SECONDS
+    return min(max_seconds, backoff_seconds) + calculate_jitter_seconds(
+        source, max_seconds=settings.SOURCE_FAILURE_JITTER_SECONDS
+    )
+
+
+def calculate_success_jitter_seconds(source: Source) -> int:
+    return calculate_jitter_seconds(source, max_seconds=settings.SOURCE_SUCCESS_JITTER_SECONDS)
+
+
+def acquire_source_lease(*, source_id: int, worker_token: str, now=None) -> SourceLease | None:
+    now = now or timezone.now()
+    with transaction.atomic():
+        source = Source.objects.select_for_update().get(pk=source_id)
+        if not source.domain:
+            return None
+
+        domain = origin_key(source)
+        origin_state = SourceOriginState.objects.select_for_update().get_or_create(
+            domain=domain, defaults={"next_allowed_at": now - timedelta(seconds=1)}
+        )[0]
+
+        if origin_state.next_allowed_at and origin_state.next_allowed_at > now:
+            existing_lease = SourceLease.objects.select_for_update().filter(source=source).first()
+            if not existing_lease or existing_lease.token != worker_token:
+                return None
+
+        lease = SourceLease.objects.select_for_update().filter(source=source).first()
+        if lease is not None and not lease.is_expired and lease.token != worker_token:
+            return None
+
+        active_leases = SourceLease.objects.select_for_update().filter(
+            source__domain=domain, expires_at__gt=now
+        )
+        active_count = active_leases.exclude(source=source).count()
+        max_concurrency = max(1, int(source.max_concurrency))
+        if active_count >= max_concurrency:
+            return None
+
+        if lease is None:
+            lease = SourceLease(source=source)
+        lease.token = worker_token
+        lease.worker_id = worker_token
+        lease.expires_at = now + timedelta(seconds=_lease_ttl_seconds())
+        if lease.pk is None:
+            lease.save()
+        else:
+            lease.save(update_fields=["token", "worker_id", "expires_at", "updated_at"])
+        return lease
+
+
+def release_source_lease(*, source_id: int, worker_token: str, now=None) -> bool:
+    now = now or timezone.now()
+    with transaction.atomic():
+        source = Source.objects.select_for_update().get(pk=source_id)
+        if not source.domain:
+            return False
+
+        lease = (
+            SourceLease.objects.select_for_update()
+            .filter(source=source, token=worker_token)
+            .first()
+        )
+        if not lease:
+            return False
+
+        lease.expires_at = now
+        lease.save(update_fields=["expires_at", "updated_at"])
+
+        interval_seconds = _max_origin_interval_seconds(source)
+        domain = origin_key(source)
+        origin_state = SourceOriginState.objects.select_for_update().get_or_create(
+            domain=domain, defaults={"next_allowed_at": now}
+        )[0]
+        origin_state.next_allowed_at = now + timedelta(seconds=max(0, interval_seconds))
+        origin_state.save(update_fields=["next_allowed_at", "updated_at"])
+        return True

apps/sources/services/tls.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+import ssl
+from functools import lru_cache
+from pathlib import Path
+
+import certifi
+
+
+@lru_cache(maxsize=1)
+def trusted_tls_context() -> ssl.SSLContext:
+    """Build a strict TLS context from the pinned application CA bundle."""
+    bundle = Path(certifi.where()).resolve(strict=True)
+    if not bundle.is_file():
+        raise RuntimeError("De vertrouwde CA-bundel ontbreekt.")
+    return ssl.create_default_context(cafile=str(bundle))

apps/sources/services/url_security.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import ipaddress
+import socket
+from collections.abc import Callable
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+Resolver = Callable[..., list[tuple]]
+
+
+class UnsafeUrlError(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class ValidatedUrl:
+    url: str
+    hostname: str
+    port: int
+    addresses: tuple[str, ...]
+
+
+def _extract_addresses(results: list[tuple]) -> tuple[str, ...]:
+    addresses: list[str] = []
+    for result in results:
+        sockaddr = result[4]
+        if sockaddr:
+            addresses.append(str(sockaddr[0]))
+    return tuple(dict.fromkeys(addresses))
+
+
+def validate_public_url(
+    url: str,
+    *,
+    resolver: Resolver = socket.getaddrinfo,
+    allow_nonstandard_ports: bool = False,
+) -> ValidatedUrl:
+    parts = urlsplit(url)
+    if parts.scheme.lower() not in {"http", "https"}:
+        raise UnsafeUrlError("Alleen http en https zijn toegestaan.")
+    if parts.username or parts.password:
+        raise UnsafeUrlError("URLs met ingebedde credentials zijn niet toegestaan.")
+    hostname = (parts.hostname or "").lower().rstrip(".")
+    if not hostname:
+        raise UnsafeUrlError("URL bevat geen hostname.")
+    if hostname == "localhost" or hostname.endswith(".localhost"):
+        raise UnsafeUrlError("Lokale hostnames zijn niet toegestaan.")
+    port = parts.port or (443 if parts.scheme.lower() == "https" else 80)
+    if not allow_nonstandard_ports and port not in {80, 443}:
+        raise UnsafeUrlError("Niet-standaard poorten zijn niet toegestaan.")
+    try:
+        direct_ip = ipaddress.ip_address(hostname.strip("[]"))
+        addresses = (str(direct_ip),)
+    except ValueError:
+        try:
+            results = resolver(hostname, port, type=socket.SOCK_STREAM)
+        except OSError as exc:
+            raise UnsafeUrlError("Hostname kan niet veilig worden opgelost.") from exc
+        addresses = _extract_addresses(results)
+    if not addresses:
+        raise UnsafeUrlError("Hostname leverde geen IP-adressen op.")
+    for raw in addresses:
+        try:
+            ip = ipaddress.ip_address(raw)
+        except ValueError as exc:
+            raise UnsafeUrlError("Ongeldig IP-adres na DNS-resolutie.") from exc
+        if not ip.is_global:
+            raise UnsafeUrlError(f"Niet-publiek IP-adres geblokkeerd: {ip}")
+    return ValidatedUrl(url=url, hostname=hostname, port=port, addresses=addresses)

apps/sources/tasks.py

@@ -0,0 +1,279 @@
+from __future__ import annotations
+
+from datetime import timedelta
+from uuid import uuid4
+
+from celery import shared_task
+from django.conf import settings
+from django.db.models import Q
+from django.utils import timezone
+
+from apps.jobs.services.pipeline import process_raw_document
+from apps.sources.models import MailboxConnection, RawDocument, Source, SourceRun
+from apps.sources.services.fetcher import (
+    FetchError,
+    FetchTimeoutError,
+    PolicyBlockedError,
+    RateLimitedError,
+    fetch_url,
+)
+from apps.sources.services.health import (
+    canary_recovery_sources,
+    evaluate_source_health,
+    start_health_canary,
+)
+from apps.sources.services.imap_import import poll_imap_mailbox
+from apps.sources.services.mailbox_connections import (
+    MailboxCredentialError,
+    claim_mailbox_connection,
+    finish_mailbox_poll,
+)
+from apps.sources.services.policy import assess_url
+from apps.sources.services.scheduling import (
+    acquire_source_lease,
+    calculate_failure_backoff_seconds,
+    calculate_success_jitter_seconds,
+    release_source_lease,
+)
+
+
+def _kind_for(content_type: str) -> str:
+    content_type = (content_type or "").lower()
+    if "html" in content_type:
+        return RawDocument.Kind.HTML
+    if "xml" in content_type or "rss" in content_type or "atom" in content_type:
+        return RawDocument.Kind.XML
+    if "json" in content_type:
+        return RawDocument.Kind.JSON
+    return RawDocument.Kind.TEXT
+
+
+@shared_task(
+    bind=True,
+    name="apps.sources.tasks.fetch_source",
+)
+def fetch_source(self, source_id: int, force: bool = False) -> dict[str, object]:
+    source = Source.objects.get(pk=source_id)
+    run = SourceRun.objects.create(source=source)
+    now = timezone.now()
+    lease_token = str(self.request.id or uuid4())
+    lease = None
+    decision = assess_url(source.base_url, source=source)
+    if source.status not in {Source.Status.ACTIVE, Source.Status.TRIAL, Source.Status.CANDIDATE}:
+        run.finish(SourceRun.Status.SKIPPED, error_category="status")
+        return {"status": "skipped", "reason": "status"}
+    if not decision.allowed:
+        run.finish(
+            SourceRun.Status.SKIPPED,
+            error_category="policy",
+            error_message=decision.reason,
+        )
+        return {"status": "skipped", "reason": "policy"}
+    if not force and source.next_run_at and source.next_run_at > now:
+        run.finish(SourceRun.Status.SKIPPED, error_category="not_due")
+        return {"status": "skipped", "reason": "not_due"}
+
+    lease = acquire_source_lease(source_id=source.pk, worker_token=lease_token, now=now)
+    if lease is None:
+        run.finish(
+            SourceRun.Status.SKIPPED,
+            error_category="lease",
+            error_message="Bron staat al in verwerking.",
+        )
+        return {"status": "skipped", "reason": "lease"}
+    headers: dict[str, str] = {}
+    if source.etag:
+        headers["If-None-Match"] = source.etag
+    if source.last_modified:
+        headers["If-Modified-Since"] = source.last_modified
+    try:
+        fetched = fetch_url(source.base_url, source=source, conditional_headers=headers)
+        if fetched.status_code == 304:
+            source.schedule_after_success(jitter_seconds=calculate_success_jitter_seconds(source))
+            run.finish(SourceRun.Status.SUCCESS, http_status=304)
+            return {"status": "not_modified"}
+        content_type = fetched.headers.get("content-type", "")
+        document = RawDocument.objects.create(
+            source=source,
+            source_run=run,
+            url=fetched.requested_url,
+            final_url=fetched.final_url,
+            kind=_kind_for(content_type),
+            content_type=content_type[:200],
+            http_status=fetched.status_code,
+            response_headers={
+                key: value
+                for key, value in fetched.headers.items()
+                if key.lower() in {"etag", "last-modified", "content-type", "cache-control"}
+            },
+            content_hash=fetched.sha256,
+            body_text=fetched.text,
+            byte_length=len(fetched.content),
+            retain_until=timezone.now() + timedelta(days=settings.RAW_DOCUMENT_RETENTION_DAYS),
+        )
+        metrics = process_raw_document(document)
+        source.etag = fetched.headers.get("etag", source.etag)
+        source.last_modified = fetched.headers.get("last-modified", source.last_modified)
+        source.save(update_fields=["etag", "last_modified", "updated_at"])
+        source.schedule_after_success(jitter_seconds=calculate_success_jitter_seconds(source))
+        run.finish(
+            SourceRun.Status.SUCCESS,
+            http_status=fetched.status_code,
+            extracted_count=int(metrics["extracted"]),
+            created_count=int(metrics["created"]),
+            updated_count=int(metrics["updated"]),
+            duplicate_count=int(metrics["duplicates"]),
+            metrics={"parser": metrics["parser"], "warnings": metrics["warnings"]},
+        )
+        return metrics
+    except PolicyBlockedError as exc:
+        source.status = Source.Status.QUARANTINED
+        source.policy = Source.Policy.DENY
+        source.policy_reason = str(exc)
+        source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
+        run.finish(
+            SourceRun.Status.SKIPPED,
+            error_category="policy",
+            error_message=str(exc)[:1000],
+        )
+        return {"status": "blocked", "reason": str(exc)}
+    except RateLimitedError as exc:
+        backoff = calculate_failure_backoff_seconds(
+            source,
+            failure_count=source.failure_count + 1,
+            retry_after_seconds=exc.retry_after_seconds,
+        )
+        source.schedule_after_failure(now=timezone.now(), backoff_seconds=backoff)
+        run.finish(
+            SourceRun.Status.FAILED,
+            error_category="rate_limited",
+            error_message=str(exc)[:1000],
+        )
+        return {"status": "rate_limited", "backoff": backoff}
+    except FetchTimeoutError as exc:
+        backoff = calculate_failure_backoff_seconds(
+            source, failure_count=source.failure_count + 1, timeout=True
+        )
+        source.schedule_after_failure(now=timezone.now(), backoff_seconds=backoff)
+        run.finish(
+            SourceRun.Status.FAILED,
+            error_category="timeout",
+            error_message=str(exc)[:1000],
+        )
+        return {"status": "timeout", "backoff": backoff}
+    except FetchError as exc:
+        backoff = calculate_failure_backoff_seconds(source, failure_count=source.failure_count + 1)
+        source.schedule_after_failure(now=timezone.now(), backoff_seconds=backoff)
+        run.finish(
+            SourceRun.Status.FAILED,
+            error_category=exc.__class__.__name__,
+            error_message=str(exc)[:1000],
+        )
+        return {"status": "failed", "error_category": exc.__class__.__name__, "backoff": backoff}
+    except Exception as exc:
+        source.schedule_after_failure()
+        run.finish(
+            SourceRun.Status.FAILED,
+            error_category="unexpected",
+            error_message=str(exc)[:1000],
+        )
+        return {"status": "failed", "error_category": "unexpected"}
+    finally:
+        if lease is not None:
+            release_source_lease(source_id=source.pk, worker_token=lease_token)
+
+
+@shared_task(name="apps.sources.tasks.schedule_due_sources")
+def schedule_due_sources() -> dict[str, int]:
+    now = timezone.now()
+    due = Source.objects.filter(
+        status__in=[Source.Status.ACTIVE, Source.Status.TRIAL],
+    ).exclude(source_type=Source.Type.EMAIL)
+    due = due.filter(Q(next_run_at__isnull=True) | Q(next_run_at__lte=now))
+    ids = list(due.values_list("id", flat=True).distinct()[:200])
+    for source_id in ids:
+        fetch_source.delay(source_id)
+    return {"scheduled": len(ids)}
+
+
+@shared_task(name="apps.sources.tasks.schedule_mailbox_polls")
+def schedule_mailbox_polls() -> dict[str, int]:
+    now = timezone.now()
+    due_ids = list(
+        MailboxConnection.objects.filter(enabled=True, next_poll_at__lte=now)
+        .exclude(user__username=settings.DEMO_USERNAME)
+        .filter(Q(lease_expires_at__isnull=True) | Q(lease_expires_at__lte=now))
+        .values_list("id", flat=True)[:100]
+    )
+    for connection_id in due_ids:
+        poll_mailbox.delay(connection_id)
+    return {"scheduled": len(due_ids)}
+
+
+@shared_task(bind=True, name="apps.sources.tasks.poll_mailbox")
+def poll_mailbox(self, connection_id: int, force: bool = False) -> dict[str, int | str]:
+    worker_token = str(self.request.id or uuid4())
+    claimed = claim_mailbox_connection(
+        connection_id=connection_id, worker_token=worker_token, force=force
+    )
+    if claimed is None:
+        return {"status": "skipped", "imported": 0}
+    connection, token = claimed
+    try:
+        result = poll_imap_mailbox(connection)
+        finish_mailbox_poll(
+            connection_id=connection.pk,
+            worker_token=token,
+            success=result["status"] == "ok",
+        )
+        return result
+    except MailboxCredentialError as exc:
+        finish_mailbox_poll(
+            connection_id=connection.pk,
+            worker_token=token,
+            success=False,
+            error_category="credentials",
+            error_message=str(exc),
+        )
+        return {"status": "failed", "error_category": "credentials", "imported": 0}
+    except Exception as exc:
+        error_category = "imap"
+        if exc.__class__.__name__.lower().startswith("unsafe"):
+            error_category = "host_policy"
+        finish_mailbox_poll(
+            connection_id=connection.pk,
+            worker_token=token,
+            success=False,
+            error_category=error_category,
+            error_message="IMAP-verbinding of authenticatie mislukt.",
+        )
+        return {"status": "failed", "error_category": error_category, "imported": 0}
+
+
+@shared_task(name="apps.sources.tasks.poll_imap_inbox")
+def poll_imap_inbox() -> dict[str, int]:
+    """Compatibility alias for old deployments; it now schedules due per-platform mailboxes."""
+    return schedule_mailbox_polls()
+
+
+@shared_task(name="apps.sources.tasks.cleanup_raw_documents")
+def cleanup_raw_documents() -> dict[str, int]:
+    deleted, _ = RawDocument.objects.filter(
+        retain_until__lt=timezone.now(), quarantined=False
+    ).delete()
+    return {"deleted": deleted}
+
+
+@shared_task(name="apps.sources.tasks.enforce_source_health")
+def enforce_source_health() -> dict[str, int]:
+    return evaluate_source_health()
+
+
+@shared_task(name="apps.sources.tasks.recover_source_health")
+def recover_source_health() -> dict[str, int]:
+    started = 0
+    for source in canary_recovery_sources():
+        start_health_canary(source)
+        fetch_source.delay(source.pk, force=True)
+        started += 1
+    return {"canary_started": started}

apps/sources/urls.py

@@ -0,0 +1,23 @@
+from django.urls import path
+
+from .views import (
+    SourceListView,
+    bulk_candidate_action,
+    edit_mailbox,
+    manual_import_view,
+    retry_source,
+    save_mailbox,
+    sync_platform_alerts,
+    toggle_mailbox,
+)
+
+urlpatterns = [
+    path("", SourceListView.as_view(), name="list"),
+    path("manual-import/", manual_import_view, name="manual_import"),
+    path("mailboxes/add/", save_mailbox, name="mailbox_add"),
+    path("mailboxes/<int:pk>/edit/", edit_mailbox, name="mailbox_edit"),
+    path("mailboxes/<int:pk>/sync/", sync_platform_alerts, name="sync_platform_alerts"),
+    path("mailboxes/<int:pk>/toggle/", toggle_mailbox, name="mailbox_toggle"),
+    path("bulk/", bulk_candidate_action, name="bulk"),
+    path("<int:pk>/retry/", retry_source, name="retry"),
+]

apps/sources/views.py

@@ -0,0 +1,357 @@
+from __future__ import annotations
+
+from django.conf import settings
+from django.contrib import messages
+from django.contrib.auth.decorators import login_required
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.db.models import Count, Prefetch, Q
+from django.http import HttpRequest
+from django.shortcuts import get_object_or_404, redirect, render
+from django.urls import reverse
+from django.utils import timezone
+from django.views.decorators.cache import never_cache
+from django.views.decorators.http import require_POST
+from django.views.generic import ListView
+
+from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
+from apps.profiles.models import SearchProfile
+
+from .forms import MailboxConnectionForm, ManualImportForm
+from .models import MailboxConnection, Source, SourcePolicyReview, SourceRun
+from .platforms import PLATFORM_ALERTS
+from .services.mailbox_connections import MailboxCredentialError, save_mailbox_connection
+from .services.manual_import import ManualImportError, ManualImportSummary, import_manual_source
+from .services.policy import create_policy_review
+from .tasks import fetch_source, poll_mailbox
+
+
+def _source_list_queryset():
+    return (
+        Source.objects.prefetch_related(
+            "policy_reviews",
+            Prefetch(
+                "runs",
+                queryset=SourceRun.objects.order_by("-started_at")[:1],
+                to_attr="latest_runs",
+            ),
+        )
+        .exclude(domain="jobs.example.org", status=Source.Status.DISABLED)
+        .filter(
+            Q(metadata__hidden_from_source_list__isnull=True)
+            | Q(metadata__hidden_from_source_list=False)
+        )
+        .filter(Q(metadata__watchlist__isnull=True) | Q(metadata__watchlist=False))
+        .annotate(job_count=Count("job_aliases", distinct=True))
+        .order_by("name")
+    )
+
+
+def _watchlist_queryset():
+    return (
+        Source.objects.filter(metadata__watchlist=True)
+        .filter(
+            Q(metadata__hidden_from_source_list__isnull=True)
+            | Q(metadata__hidden_from_source_list=False)
+        )
+        .order_by("name")
+    )
+
+
+def _manual_import_context(
+    request: HttpRequest,
+    *,
+    form: ManualImportForm,
+    manual_result=None,
+    mailbox_form: MailboxConnectionForm | None = None,
+    editing_mailbox: MailboxConnection | None = None,
+):
+    base_queryset = _source_list_queryset()
+    watchlist_queryset = _watchlist_queryset()
+    mailbox_connections = list(
+        request.user.mailbox_connections.filter(platform__in=PLATFORM_ALERTS)
+    )
+    active_search_profile = SearchProfile.objects.filter(user=request.user, is_active=True).first()
+    return {
+        "sources": base_queryset,
+        "active_source_count": Source.objects.filter(status=Source.Status.ACTIVE).count(),
+        "source_total_count": base_queryset.count(),
+        "review_pending_count": Source.objects.filter(policy=Source.Policy.REVIEW).count(),
+        "failed_source_count": Source.objects.filter(
+            Q(status=Source.Status.QUARANTINED) | Q(failure_count__gt=0)
+        ).count(),
+        "watchlist_sources": watchlist_queryset,
+        "watchlist_count": watchlist_queryset.count(),
+        "manual_import_form": form,
+        "manual_import_bookmarklet_endpoint": request.build_absolute_uri(
+            reverse("sources:manual_import")
+        ),
+        "manual_import_paste_max_bytes": settings.MANUAL_IMPORT_PASTE_MAX_BYTES,
+        "manual_import_result": manual_result,
+        "platform_alert_ingress_ready": any(
+            connection.enabled for connection in mailbox_connections
+        ),
+        "mailbox_connections": mailbox_connections,
+        "platform_alerts": PLATFORM_ALERTS.values(),
+        "mailbox_form": mailbox_form or MailboxConnectionForm(),
+        "editing_mailbox": editing_mailbox,
+        "mailbox_credential_key_ready": bool(settings.MAILBOX_CREDENTIAL_KEYS),
+        "active_search_profile": active_search_profile,
+    }
+
+
+class SourceListView(LoginRequiredMixin, ListView):
+    model = Source
+    template_name = "sources/list.html"
+    context_object_name = "sources"
+
+    def get_queryset(self):
+        return _source_list_queryset()
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context.update(
+            _manual_import_context(
+                self.request,
+                form=ManualImportForm(
+                    initial={"source_url": self.request.GET.get("source_url", "")}
+                ),
+            )
+        )
+        return context
+
+
+def _bulk_summary(action: str) -> str:
+    mapping = {
+        "allow": "toestaan",
+        "approve_to_trial": "proefrun",
+        "pause": "pauzeren",
+        "dismiss": "afwijzen",
+    }
+    return mapping.get(action, "onbekend")
+
+
+def _action_to_review(
+    action: str,
+) -> tuple[SourcePolicyReview.Decision, str, str, str]:
+    if action == "allow":
+        return (
+            SourcePolicyReview.Decision.ALLOW,
+            Source.Status.ACTIVE,
+            Source.Policy.ALLOW,
+            "Bron actief gemaakt door expliciete goedkeuring",
+        )
+    if action == "approve_to_trial":
+        return (
+            SourcePolicyReview.Decision.TRIAL,
+            Source.Status.TRIAL,
+            Source.Policy.REVIEW,
+            "Bron naar proefrun voor technische controle",
+        )
+    if action == "pause":
+        return (
+            SourcePolicyReview.Decision.PAUSE,
+            Source.Status.PAUSED,
+            Source.Policy.REVIEW,
+            "Bron tijdelijk gepauzeerd",
+        )
+    return (
+        SourcePolicyReview.Decision.DENY,
+        Source.Status.DISABLED,
+        Source.Policy.DENY,
+        "Bron afgewezen",
+    )
+
+
+@login_required
+@require_POST
+def bulk_candidate_action(request):
+    action = request.POST.get("action")
+    source_ids = [int(raw_id) for raw_id in request.POST.getlist("source_ids") if raw_id.isdigit()]
+    sources = _source_list_queryset().filter(id__in=source_ids).exclude(policy=Source.Policy.DENY)
+    if not sources.exists():
+        messages.info(request, "Geen bruikbare bronregels geselecteerd.")
+        return redirect("sources:list")
+
+    if action not in {"allow", "approve_to_trial", "pause", "dismiss"}:
+        messages.error(request, "Onbekende actie.")
+        return redirect("sources:list")
+
+    decision, status, policy, reason = _action_to_review(action)
+    for source in sources:
+        create_policy_review(
+            source,
+            actor=request.user,
+            decision=decision,
+            scope=SourcePolicyReview.Scope.SOURCE,
+            reason=reason,
+            notes="Bulkactie",
+        )
+        source.status = status
+        source.policy = policy
+        source.policy_reason = reason
+        source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
+
+    messages.success(request, f"{sources.count()} bron(nen) naar {_bulk_summary(action)}.")
+    return redirect("sources:list")
+
+
+@login_required
+def retry_source(request, pk: int):
+    if request.method == "POST":
+        source = get_object_or_404(Source, pk=pk)
+        if source.policy == Source.Policy.DENY:
+            messages.error(request, "Deze bron is door het bronbeleid geblokkeerd.")
+        else:
+            fetch_source.delay(source.pk, force=True)
+            messages.success(request, "Broncontrole ingepland.")
+    return redirect("sources:list")
+
+
+@login_required
+@require_POST
+def save_mailbox(request, pk: int | None = None):
+    instance = None
+    if pk is not None:
+        instance = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
+    form = MailboxConnectionForm(request.POST, instance=instance)
+    if form.is_valid():
+        try:
+            connection = save_mailbox_connection(
+                user=request.user,
+                cleaned_data=dict(form.cleaned_data),
+                instance=instance,
+            )
+        except MailboxCredentialError as exc:
+            form.add_error("password", str(exc))
+        else:
+            poll_mailbox.delay(connection.pk, force=True)
+            messages.success(
+                request,
+                f"{connection.get_platform_display()}-mailbox opgeslagen; "
+                "verbindingstest ingepland.",
+            )
+            return redirect("sources:list")
+    messages.error(request, "Controleer de mailboxinstellingen.")
+    return render(
+        request,
+        "sources/list.html",
+        _manual_import_context(
+            request,
+            form=ManualImportForm(),
+            mailbox_form=form,
+            editing_mailbox=instance,
+        ),
+    )
+
+
+@login_required
+def edit_mailbox(request, pk: int):
+    connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
+    if request.method == "POST":
+        return save_mailbox(request, pk=pk)
+    return render(
+        request,
+        "sources/list.html",
+        _manual_import_context(
+            request,
+            form=ManualImportForm(),
+            mailbox_form=MailboxConnectionForm(instance=connection),
+            editing_mailbox=connection,
+        ),
+    )
+
+
+@login_required
+@require_POST
+def sync_platform_alerts(request, pk: int):
+    connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
+    if not connection.enabled:
+        messages.error(request, "Activeer deze mailbox voordat je synchroniseert.")
+    else:
+        poll_mailbox.delay(connection.pk, force=True)
+        messages.success(request, f"{connection.get_platform_display()}-mailimport ingepland.")
+    return redirect("sources:list")
+
+
+@login_required
+@require_POST
+def toggle_mailbox(request, pk: int):
+    connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
+    connection.enabled = not connection.enabled
+    connection.next_poll_at = timezone.now()
+    connection.save(update_fields=["enabled", "next_poll_at", "updated_at"])
+    state = "geactiveerd" if connection.enabled else "gepauzeerd"
+    messages.success(request, f"{connection.get_platform_display()}-mailbox {state}.")
+    return redirect("sources:list")
+
+
+@login_required
+@never_cache
+def manual_import_view(request):
+    rate_limit = is_rate_limited(
+        request,
+        namespace="manual_import",
+        max_attempts=settings.MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS,
+        window_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_WINDOW_SECONDS,
+        block_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_BLOCK_SECONDS,
+    )
+    if rate_limit.is_blocked:
+        messages.error(
+            request,
+            "Te veel handmatige importverzoeken. Wacht 5 minuten en probeer daarna opnieuw.",
+        )
+        return render(
+            request,
+            "sources/list.html",
+            _manual_import_context(request, form=ManualImportForm(), manual_result=None),
+        )
+
+    if request.method == "POST":
+        form = ManualImportForm(request.POST)
+        result_payload = None
+        if not form.is_valid():
+            messages.error(request, "Controleer de handmatige importvelden.")
+            return render(
+                request,
+                "sources/list.html",
+                _manual_import_context(request, form=form, manual_result=result_payload),
+            )
+
+        cleaned = form.cleaned_data
+        try:
+            summary: ManualImportSummary = import_manual_source(
+                actor=request.user,
+                source_url=cleaned.get("source_url"),
+                pasted_text=cleaned.get("pasted_text"),
+            )
+            result_payload = summary.to_session_payload()
+            messages.success(
+                request,
+                "Handmatige import uitgevoerd. Bekijk het resultaat hieronder.",
+            )
+            clear_rate_limit(request, namespace="manual_import")
+            form = ManualImportForm()
+        except ManualImportError as exc:
+            messages.error(request, str(exc))
+            register_rate_limit_failure(
+                request,
+                namespace="manual_import",
+                max_attempts=settings.MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS,
+                window_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_WINDOW_SECONDS,
+                block_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_BLOCK_SECONDS,
+            )
+
+        return render(
+            request,
+            "sources/list.html",
+            _manual_import_context(request, form=form, manual_result=result_payload),
+        )
+
+    return render(
+        request,
+        "sources/list.html",
+        _manual_import_context(
+            request,
+            form=ManualImportForm(initial={"source_url": request.GET.get("source_url", "")}),
+        ),
+    )