Publish curated VacatureRadar source snapshot

apps/sources/adapters/__init__.py

@@ -0,0 +1,30 @@
+from .ats import (
+    GreenhouseAdapter,
+    LeverAdapter,
+    RecruiteeAdapter,
+    SmartRecruitersAdapter,
+    WorkableAdapter,
+)
+from .base import ExtractedJob, ExtractionResult
+from .generic_html import GenericHtmlAdapter
+from .jsonld import JsonLdJobPostingAdapter
+from .kempen import KempenEmployerAdapter
+from .mol_region import MolRegionEmployerAdapter
+from .regional import LocalEmployerListingAdapter
+from .rss import RssAdapter
+
+__all__ = [
+    "ExtractedJob",
+    "ExtractionResult",
+    "GenericHtmlAdapter",
+    "GreenhouseAdapter",
+    "JsonLdJobPostingAdapter",
+    "KempenEmployerAdapter",
+    "LeverAdapter",
+    "LocalEmployerListingAdapter",
+    "MolRegionEmployerAdapter",
+    "RecruiteeAdapter",
+    "RssAdapter",
+    "SmartRecruitersAdapter",
+    "WorkableAdapter",
+]

apps/sources/adapters/ats.py

@@ -0,0 +1,591 @@
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+CLOSED_STATUSES = {
+    "closed",
+    "inactive",
+    "withdrawn",
+    "removed",
+    "filled",
+    "expired",
+    "archived",
+}
+
+
+def _to_text(value):
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value.strip()
+    if isinstance(value, bool):
+        return str(value).lower()
+    if isinstance(value, int | float):
+        return str(value)
+    return ""
+
+
+def _find_nested(data, path: str):
+    if data is None:
+        return None
+    node = data
+    for part in path.split("."):
+        if not isinstance(node, dict) or part not in node:
+            return None
+        node = node[part]
+    return node
+
+
+def _first_text(data, *candidates):
+    for candidate in candidates:
+        if "." in candidate:
+            value = _find_nested(data, candidate)
+        else:
+            value = data.get(candidate) if isinstance(data, dict) else None
+        if value is None:
+            continue
+        if isinstance(value, list | tuple):
+            for item in value:
+                text = _to_text(item)
+                if text:
+                    return text
+            continue
+        text = _to_text(value)
+        if text:
+            return text
+        if isinstance(value, dict):
+            nested = value.values()
+            for nested_value in nested:
+                nested_text = _to_text(nested_value)
+                if nested_text:
+                    return nested_text
+    return ""
+
+
+def _collect_texts(value) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value.strip()] if value.strip() else []
+    if isinstance(value, list):
+        out: list[str] = []
+        for item in value:
+            for inner in _collect_texts(item):
+                if inner:
+                    out.append(inner)
+        return out
+    if isinstance(value, dict):
+        out: list[str] = []
+        for key in ("name", "city", "location", "address", "raw", "region", "country"):
+            if key in value:
+                for inner in _collect_texts(value[key]):
+                    if inner:
+                        out.append(inner)
+        return out
+    if isinstance(value, bool | int | float):
+        return [_to_text(value)]
+    return []
+
+
+def _to_list(value) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [item.strip() for item in (_to_text(v) for v in value) if item.strip()]
+    text = _to_text(value)
+    if not text:
+        return []
+    return [part.strip() for part in text.replace(";", ",").split(",") if part.strip()]
+
+
+def _as_text(value) -> str:
+    text = _to_text(value)
+    if not text:
+        return ""
+    return BeautifulSoup(text, "lxml").get_text(" ", strip=True)
+
+
+def _extract_payload(content: str):
+    try:
+        return json.loads(content.lstrip("\ufeff"))
+    except json.JSONDecodeError:
+        pass
+    soup = BeautifulSoup(content, "lxml")
+    for script in soup.find_all("script", attrs={"type": "application/json"}):
+        script_text = script.string or script.get_text("", strip=True)
+        if not script_text:
+            continue
+        try:
+            return json.loads(script_text.lstrip("\ufeff"))
+        except json.JSONDecodeError:
+            continue
+    return None
+
+
+class _AtsAdapter(ABC):
+    parser_key = "ats-provider"
+    parser_version = "1.0.0"
+    source_hosts: tuple[str, ...] = ()
+    support_markers: tuple[str, ...] = ()
+    listing_paths: tuple[tuple[str, ...], ...] = ()
+    detail_paths: tuple[tuple[str, ...], ...] = ()
+    closed_statuses = CLOSED_STATUSES
+
+    @abstractmethod
+    def _extract_records(self, payload) -> list[dict[str, object]]: ...
+
+    def _supports_url(self, url: str) -> bool:
+        host = (urlsplit(url).hostname or "").lower()
+        return any(host.endswith(suffix) for suffix in self.source_hosts)
+
+    def _supports_payload(self, payload, content: str) -> bool:
+        if not payload:
+            return False
+        payload_text = str(payload).lower()
+        return any(
+            marker in payload_text or marker in content.lower() for marker in self.support_markers
+        )
+
+    def _job_is_closed(self, record: dict[str, object]) -> bool:
+        status = _first_text(record, "status", "state", "job_status", "data.status").lower()
+        if status in self.closed_statuses:
+            return True
+        active = record.get("active")
+        if isinstance(active, bool):
+            return not active
+        return False
+
+    def _coerce_url(self, raw_url: str, base_url: str) -> str:
+        if raw_url:
+            return urljoin(base_url, raw_url)
+        return base_url
+
+    def _extract_location(self, record: dict[str, object]) -> tuple[str, str, str, str]:
+        location_raw = _first_text(
+            record,
+            "location.city",
+            "location.name",
+            "location.address",
+            "location.raw",
+            "categories.location",
+            "city",
+            "cityName",
+            "place",
+            "office",
+            "address",
+            "data.location",
+            "officeLocation",
+            "locationName",
+            "location",
+        )
+        if not location_raw:
+            location_raw = _first_text(record, "data.city", "data.location")
+            if not location_raw:
+                location_values = []
+                for key in ("city", "address", "location", "region", "office"):
+                    location_values.extend(_collect_texts(record.get(key, "")))
+                location_raw = ", ".join(location_values)
+
+        location = location_raw
+        [part.strip() for part in _to_text(location).split(",") if part.strip()]
+        region = _first_text(
+            record,
+            "region",
+            "data.region",
+            "location.region",
+            "address.region",
+        )
+        postal_code = _first_text(
+            record,
+            "postal_code",
+            "postalCode",
+            "location.postalCode",
+            "zipCode",
+        )
+        country = _first_text(
+            record,
+            "country",
+            "countryName",
+            "location.country",
+            "address.country",
+            "data.country",
+        )
+        return location, region, postal_code, country
+
+    def _extract_workplace(self, record: dict[str, object]) -> str:
+        workplace_raw = _first_text(
+            record,
+            "workplaceType",
+            "workplace",
+            "remoteType",
+            "remote_type",
+            "job_type",
+        ).lower()
+        if not workplace_raw:
+            if record.get("hybrid") is True:
+                return "hybrid"
+            if record.get("remote") is True:
+                return "remote"
+            if record.get("on_site") is True:
+                return "on_site"
+        if "remote" in workplace_raw:
+            return "remote" if "hybrid" not in workplace_raw else "hybrid"
+        if "hybrid" in workplace_raw:
+            return "hybrid"
+        if "onsite" in workplace_raw or "on site" in workplace_raw or "on-site" in workplace_raw:
+            return "on_site"
+        return ""
+
+    def _to_job(self, record: dict[str, object], base_url: str) -> ExtractedJob:
+        external_id = _first_text(
+            record,
+            "id",
+            "jobId",
+            "requisitionId",
+            "postingId",
+            "referenceId",
+            "positionId",
+        )
+        title = _first_text(
+            record,
+            "title",
+            "name",
+            "position",
+            "positionName",
+            "jobTitle",
+            "text",
+            "title.value",
+        )
+        raw_url = _first_text(
+            record,
+            "url",
+            "jobUrl",
+            "job_url",
+            "hostedUrl",
+            "applyUrl",
+            "link",
+            "absoluteUrl",
+            "careers_url",
+            "data.url",
+        )
+        employer_name = _first_text(
+            record,
+            "company_name",
+            "company.name",
+            "company",
+            "employer",
+            "organization",
+            "organizationName",
+            "department",
+            "hiringOrganization",
+        )
+        location_text, region, postal_code, country = self._extract_location(record)
+        description_raw = _first_text(
+            record,
+            "description",
+            "jobDescription",
+            "content",
+            "descriptionHtml",
+            "descriptionText",
+        )
+        description_text = _as_text(description_raw)
+        date_posted = _first_text(
+            record,
+            "createdAt",
+            "created",
+            "datePosted",
+            "publishedAt",
+            "published_at",
+            "created_at",
+            "jobCreated",
+            "releasedDate",
+        )
+        valid_through = _first_text(
+            record,
+            "closeDate",
+            "expiresAt",
+            "validThrough",
+            "valid_until",
+            "expirationDate",
+            "expiryDate",
+        )
+        employment_types = _to_list(
+            _first_text(
+                record,
+                "employmentType",
+                "employment_types",
+                "jobType",
+                "type",
+                "data.employmentType",
+                "employment_type_code",
+                "categories.commitment",
+                "typeOfEmployment.label",
+            )
+        )
+        workplace_type = self._extract_workplace(record)
+
+        evidence = [
+            FieldEvidence("external_id", "ats-id", 0.95, external_id[:240]),
+            FieldEvidence("url", "ats-url", 0.9, raw_url[:240]),
+            FieldEvidence("location_text", "ats-location", 0.88, location_text[:240]),
+            FieldEvidence("date_posted", "ats-date", 0.8, date_posted[:240]),
+            FieldEvidence("valid_through", "ats-date", 0.8, valid_through[:240]),
+            FieldEvidence(
+                "employment_types",
+                "ats-employment",
+                0.9,
+                ", ".join(employment_types)[:240],
+            ),
+        ]
+
+        return ExtractedJob(
+            url=self._coerce_url(raw_url, base_url),
+            title=title,
+            employer_name=employer_name,
+            external_id=external_id,
+            location_text=location_text,
+            region=region,
+            postal_code=postal_code,
+            country=country,
+            description_html=description_raw,
+            description_text=description_text,
+            date_posted=date_posted,
+            valid_through=valid_through,
+            employment_types=employment_types,
+            workplace_type=workplace_type,
+            raw=record.get("__raw__", record),
+            evidence=evidence,
+        )
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        if not self._supports_url(url):
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Onherkenbare ATS-host"],
+            )
+
+        payload = _extract_payload(content)
+        if not payload:
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Geen parseerbare ATS-response"],
+            )
+
+        if not self._supports_payload(payload, content):
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Geen herkenbare ATS-markup voor deze adapter"],
+            )
+
+        jobs: list[ExtractedJob] = []
+        for record in self._extract_records(payload):
+            if not isinstance(record, dict):
+                continue
+            if self._job_is_closed(record):
+                continue
+            job = self._to_job(record, base_url=url)
+            if job.title:
+                jobs.append(job)
+
+        warnings: list[str] = []
+        if not jobs:
+            warnings.append("Geen actieve ATS-vacatures gevonden")
+        return ExtractionResult(
+            jobs, self.parser_key, self.parser_version, 0.9 if jobs else 0.0, warnings
+        )
+
+
+class GreenhouseAdapter(_AtsAdapter):
+    parser_key = "ats-greenhouse"
+    source_hosts = ("greenhouse.io", "boards.greenhouse.io")
+    support_markers = ("greenhouse", "job board", "jobboard")
+    listing_paths = (("jobs",), ("data", "jobs"), ("data", "results"))
+    detail_paths = (("job",), ("data", "job"), ("result", "job"), ("result", "position"))
+    closed_statuses = CLOSED_STATUSES | {"published", "draft", "deleted"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+            if isinstance(value, dict) and "results" in value:
+                possible = value.get("results")
+                if isinstance(possible, list):
+                    return [item for item in possible if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []
+
+
+class LeverAdapter(_AtsAdapter):
+    parser_key = "ats-lever"
+    source_hosts = (
+        "jobs.lever.co",
+        "jobs.eu.lever.co",
+        "api.lever.co",
+        "api.eu.lever.co",
+    )
+    support_markers = ("lever", "requisition", "posting")
+    listing_paths = (("data",), ("jobs",), ("results",))
+    detail_paths = (("data",), ("job",), ("position",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"archived", "deleted"}
+
+    def _extract_records(self, payload):
+        if isinstance(payload, list):
+            return [item for item in payload if isinstance(item, dict)]
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                if path == ("position",):
+                    raw_payload = {
+                        "position": value,
+                        "work_type": _to_text(value.get("workplaceType")),
+                    }
+                    return [{**value, "__raw__": raw_payload}]
+                return [value]
+        return []
+
+
+class RecruiteeAdapter(_AtsAdapter):
+    parser_key = "ats-recruitee"
+    source_hosts = ("recruitee.com",)
+    support_markers = ("recruitee", "career", "vacancy")
+    listing_paths = (("jobs",), ("offers",), ("data", "jobs"), ("vacancies",))
+    detail_paths = (("job",), ("data", "job"), ("vacancy",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"hidden", "paused"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []
+
+
+class SmartRecruitersAdapter(_AtsAdapter):
+    parser_key = "ats-smartrecruiters"
+    parser_version = "1.1.0"
+    source_hosts = ("smartrecruiters.com",)
+    support_markers = ("smartrecruiters", "smart recruiter")
+    listing_paths = (("jobs",), ("data", "jobs"), ("results",), ("content",))
+    detail_paths = (("job",), ("data", "job"), ("posting",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"unpublished", "expired"}
+
+    def _to_job(self, record: dict[str, object], base_url: str) -> ExtractedJob:
+        prepared = dict(record)
+        if not _first_text(prepared, "url", "jobUrl", "hostedUrl", "applyUrl", "link"):
+            company_identifier = _first_text(prepared, "company.identifier")
+            posting_id = _first_text(prepared, "id", "uuid")
+            if company_identifier and posting_id:
+                prepared["url"] = (
+                    f"https://jobs.smartrecruiters.com/{company_identifier}/{posting_id}"
+                )
+        if _find_nested(prepared, "location.remote") is True:
+            prepared["remote"] = True
+        return super()._to_job(prepared, base_url)
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []
+
+
+class WorkableAdapter(_AtsAdapter):
+    parser_key = "ats-workable"
+    source_hosts = ("apply.workable.com",)
+    support_markers = ("workable", "workable job")
+    listing_paths = (("jobs",), ("data", "jobs"), ("results",))
+    detail_paths = (("job",), ("data", "job"), ("position",), ("result",))
+    closed_statuses = CLOSED_STATUSES | {"draft", "inactive"}
+
+    def _extract_records(self, payload):
+        for path in self.listing_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, list):
+                return [item for item in value if isinstance(item, dict)]
+        for path in self.detail_paths:
+            value = payload
+            for key in path:
+                if not isinstance(value, dict) or key not in value:
+                    value = None
+                    break
+                value = value[key]
+            if isinstance(value, dict):
+                return [value]
+        return []

apps/sources/adapters/base.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+
+@dataclass(slots=True)
+class FieldEvidence:
+    field_name: str
+    method: str
+    confidence: float
+    evidence: str = ""
+
+
+@dataclass(slots=True)
+class ExtractedJob:
+    url: str
+    title: str
+    employer_name: str = ""
+    external_id: str = ""
+    location_text: str = ""
+    region: str = ""
+    postal_code: str = ""
+    country: str = ""
+    description_html: str = ""
+    description_text: str = ""
+    language: str = ""
+    date_posted: str = ""
+    valid_through: str = ""
+    employment_types: list[str] = field(default_factory=list)
+    workplace_type: str = ""
+    compensation: dict[str, Any] = field(default_factory=dict)
+    skills_required: list[str] = field(default_factory=list)
+    skills_preferred: list[str] = field(default_factory=list)
+    raw: dict[str, Any] = field(default_factory=dict)
+    evidence: list[FieldEvidence] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class ExtractionResult:
+    jobs: list[ExtractedJob]
+    parser_key: str
+    parser_version: str
+    confidence: float
+    warnings: list[str] = field(default_factory=list)
+
+
+class SourceAdapter(Protocol):
+    parser_key: str
+    parser_version: str
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult: ...

apps/sources/adapters/email_alert.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import html
+import re
+from email import policy
+from email.message import Message
+from email.parser import BytesParser
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+
+from apps.sources.platforms import PLATFORM_ALERTS
+from apps.sources.services.canonicalize import canonicalize_url
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I)
+SKIP_TEXT = re.compile(
+    r"unsubscribe|afmelden|uitschrijven|privacy|view in browser|bekijk online|"
+    r"account|aanmelden|inloggen|login|voorkeuren|preferences|voorwaarden|terms|"
+    r"contact|help|hulp|over ons|about us",
+    re.I,
+)
+SKIP_PATH = re.compile(
+    r"/(?:unsubscribe|uitschrijven|afmelden|privacy|account|login|signin|preferences|"
+    r"settings|terms|legal|help|contact)(?:/|$)",
+    re.I,
+)
+
+
+def _matches_domain(hostname: str, domains: tuple[str, ...]) -> bool:
+    return any(hostname == domain or hostname.endswith(f".{domain}") for domain in domains)
+
+
+class EmailAlertAdapter:
+    parser_key = "email-alert"
+    parser_version = "1.1.0"
+
+    @staticmethod
+    def _provider_for_candidates(candidates: list[tuple[str, str]]) -> str:
+        hostnames = {
+            (urlsplit(href).hostname or "").lower()
+            for _, href in candidates
+            if href.lower().startswith(("http://", "https://"))
+        }
+        for provider, alert in PLATFORM_ALERTS.items():
+            if any(_matches_domain(hostname, alert.domains) for hostname in hostnames):
+                return provider
+        return "other"
+
+    @staticmethod
+    def _is_expected_platform_link(*, label: str, href: str, provider: str) -> bool:
+        alert = PLATFORM_ALERTS.get(provider)
+        if alert is None or len(label.strip()) < 4:
+            return False
+        parsed = urlsplit(href)
+        hostname = (parsed.hostname or "").lower()
+        return (
+            parsed.scheme.lower() == "https"
+            and _matches_domain(hostname, alert.domains)
+            and not SKIP_TEXT.search(label)
+            and not SKIP_PATH.search(parsed.path)
+        )
+
+    @staticmethod
+    def _decode_parts(message: Message) -> tuple[str, str]:
+        plain_parts: list[str] = []
+        html_parts: list[str] = []
+        for part in message.walk() if message.is_multipart() else [message]:
+            disposition = str(part.get("Content-Disposition") or "")
+            if "attachment" in disposition.lower():
+                continue
+            content_type = part.get_content_type()
+            try:
+                payload = part.get_content()
+            except Exception:
+                raw = part.get_payload(decode=True) or b""
+                payload = raw.decode(part.get_content_charset() or "utf-8", errors="replace")
+            if content_type == "text/plain":
+                plain_parts.append(str(payload))
+            elif content_type == "text/html":
+                html_parts.append(str(payload))
+        return "\n".join(plain_parts), "\n".join(html_parts)
+
+    def extract_message(
+        self, raw_message: bytes, *, expected_provider: str | None = None
+    ) -> ExtractionResult:
+        message = BytesParser(policy=policy.default).parsebytes(raw_message)
+        plain, html_body = self._decode_parts(message)
+        candidates: list[tuple[str, str]] = []
+        if html_body:
+            soup = BeautifulSoup(html_body, "lxml")
+            for anchor in soup.find_all("a", href=True):
+                label = " ".join(anchor.get_text(" ", strip=True).split())
+                href = html.unescape(str(anchor["href"]).strip())
+                if not href.lower().startswith(("http://", "https://")):
+                    continue
+                if SKIP_TEXT.search(label) or SKIP_TEXT.search(href):
+                    continue
+                candidates.append((label, href))
+        for href in URL_RE.findall(plain):
+            clean_href = href.rstrip(".,);]")
+            if SKIP_TEXT.search(clean_href):
+                continue
+            candidates.append(("", clean_href))
+
+        alert_provider = expected_provider or self._provider_for_candidates(candidates)
+        if expected_provider:
+            candidates = [
+                (label, href)
+                for label, href in candidates
+                if self._is_expected_platform_link(
+                    label=label,
+                    href=href,
+                    provider=expected_provider,
+                )
+            ]
+
+        jobs: list[ExtractedJob] = []
+        seen: set[str] = set()
+        subject = str(message.get("subject") or "Vacature uit e-mail").strip()
+        for label, href in candidates:
+            canonical = canonicalize_url(urljoin("https://invalid.local/", href))
+            if not canonical or canonical in seen:
+                continue
+            seen.add(canonical)
+            hostname = urlsplit(canonical).hostname or ""
+            title = label if len(label) >= 4 else subject
+            if len(title) > 300:
+                title = title[:300]
+            jobs.append(
+                ExtractedJob(
+                    url=canonical,
+                    title=title,
+                    employer_name="",
+                    description_text=plain[:5000]
+                    or BeautifulSoup(html_body, "lxml").get_text("\n", strip=True)[:5000],
+                    raw={
+                        "email_subject": subject,
+                        "email_sender": str(message.get("from") or ""),
+                        "target_domain": hostname,
+                        "alert_provider": alert_provider,
+                    },
+                    evidence=[FieldEvidence("url", "email-anchor", 0.75, label[:240])],
+                )
+            )
+        return ExtractionResult(
+            jobs,
+            self.parser_key,
+            self.parser_version,
+            0.65 if jobs else 0.0,
+            [] if jobs else ["Geen vacaturelinks in e-mail gevonden"],
+        )
+
+    def extract(self, content: str, *, url: str = "") -> ExtractionResult:
+        return self.extract_message(content.encode("utf-8", errors="replace"))

apps/sources/adapters/generic_html.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+LABEL_PATTERNS = {
+    "location": re.compile(r"^(locatie|location|lieu|plaats|standplaats)\s*:?$", re.I),
+    "date_posted": re.compile(r"^(publicatiedatum|geplaatst|date posted|published)\s*:?$", re.I),
+    "employment_type": re.compile(r"^(dienstverband|contract|employment type)\s*:?$", re.I),
+    "employer": re.compile(r"^(werkgever|employer|company|organisatie|société)\s*:?$", re.I),
+}
+
+
+class GenericHtmlAdapter:
+    parser_key = "generic-html"
+    parser_version = "1.1.0"
+
+    @staticmethod
+    def _meta(soup: BeautifulSoup, *names: str) -> str:
+        for name in names:
+            tag = soup.find("meta", attrs={"property": name}) or soup.find(
+                "meta", attrs={"name": name}
+            )
+            if tag and tag.get("content"):
+                return str(tag["content"]).strip()
+        return ""
+
+    @staticmethod
+    def _label_value(soup: BeautifulSoup, pattern: re.Pattern[str]) -> str:
+        label = soup.find(string=lambda value: bool(value and pattern.match(value.strip())))
+        if not label:
+            return ""
+        parent = label.parent
+        if not parent:
+            return ""
+        sibling = parent.find_next_sibling()
+        if sibling:
+            return sibling.get_text(" ", strip=True)
+        text = parent.get_text(" ", strip=True)
+        return pattern.sub("", text).strip(" :-")
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        soup = BeautifulSoup(content, "lxml")
+        for element in soup(["script", "style", "noscript", "template"]):
+            element.decompose()
+        h1 = soup.find("h1")
+        title = (h1.get_text(" ", strip=True) if h1 else "") or self._meta(
+            soup, "og:title", "twitter:title"
+        )
+        if not title and soup.title:
+            title = soup.title.get_text(" ", strip=True)
+        title = re.sub(r"\s+[|\\u2013\\u2014-]\s+.*$", "", title).strip()
+        if not title:
+            return ExtractionResult([], self.parser_key, self.parser_version, 0.0, ["Geen titel"])
+
+        employer = self._meta(soup, "og:site_name", "application-name") or self._label_value(
+            soup, LABEL_PATTERNS["employer"]
+        )
+        location = self._label_value(soup, LABEL_PATTERNS["location"])
+        date_posted = self._label_value(soup, LABEL_PATTERNS["date_posted"])
+        employment_type = self._label_value(soup, LABEL_PATTERNS["employment_type"])
+        main = soup.find("main") or soup.find("article") or soup.body
+        description_html = str(main) if main else ""
+        description_text = (
+            main.get_text("\n", strip=True) if main else soup.get_text("\n", strip=True)
+        )
+        canonical = soup.find("link", rel=lambda value: value and "canonical" in value)
+        job_url = (
+            urljoin(url, canonical.get("href")) if canonical and canonical.get("href") else url
+        )
+        evidence = [
+            FieldEvidence("title", "html-heading", 0.78, title[:240]),
+            FieldEvidence("employer_name", "html-meta-or-label", 0.65, employer[:240]),
+            FieldEvidence("location_text", "html-label", 0.62, location[:240]),
+            FieldEvidence("description", "html-main", 0.70, description_text[:300]),
+        ]
+        if date_posted:
+            evidence.append(FieldEvidence("date_posted", "html-label", 0.62, date_posted[:40]))
+        if employment_type:
+            evidence.append(
+                FieldEvidence("employment_types", "html-label", 0.62, employment_type[:120])
+            )
+        job = ExtractedJob(
+            url=job_url,
+            title=title,
+            employer_name=employer,
+            location_text=location,
+            description_html=description_html,
+            description_text=description_text,
+            date_posted=date_posted,
+            employment_types=[employment_type] if employment_type else [],
+            raw={"generic_html": True},
+            evidence=evidence,
+        )
+        return ExtractionResult([job], self.parser_key, self.parser_version, 0.70)

apps/sources/adapters/jsonld.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from typing import Any
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class JsonLdJobPostingAdapter:
+    parser_key = "jsonld-jobposting"
+    parser_version = "1.0.0"
+
+    @staticmethod
+    def _is_jobposting(value: Any) -> bool:
+        types = value if isinstance(value, list) else [value]
+        return any(str(item).lower() == "jobposting" for item in types)
+
+    def _walk(self, value: Any) -> Iterable[dict[str, Any]]:
+        if isinstance(value, dict):
+            if self._is_jobposting(value.get("@type")):
+                yield value
+            graph = value.get("@graph")
+            if graph is not None:
+                yield from self._walk(graph)
+            for child in value.values():
+                if isinstance(child, dict | list):
+                    yield from self._walk(child)
+        elif isinstance(value, list):
+            for child in value:
+                yield from self._walk(child)
+
+    @staticmethod
+    def _name(value: Any) -> str:
+        if isinstance(value, str):
+            return value.strip()
+        if isinstance(value, dict):
+            return str(value.get("name") or value.get("legalName") or "").strip()
+        return ""
+
+    @staticmethod
+    def _location(value: Any) -> tuple[str, str, str, str]:
+        locations = value if isinstance(value, list) else [value]
+        parts: list[str] = []
+        region = postal = country = ""
+        for location in locations:
+            if not isinstance(location, dict):
+                continue
+            address = location.get("address", location)
+            if isinstance(address, str):
+                parts.append(address)
+                continue
+            if not isinstance(address, dict):
+                continue
+            locality = str(address.get("addressLocality") or "").strip()
+            region = region or str(address.get("addressRegion") or "").strip()
+            postal = postal or str(address.get("postalCode") or "").strip()
+            country_value = address.get("addressCountry")
+            country = (
+                country
+                or JsonLdJobPostingAdapter._name(country_value)
+                or str(country_value or "").strip()
+            )
+            label = ", ".join(part for part in [locality, region, postal, country] if part)
+            if label:
+                parts.append(label)
+        return " | ".join(dict.fromkeys(parts)), region, postal, country
+
+    @staticmethod
+    def _employment_types(value: Any) -> list[str]:
+        if isinstance(value, list):
+            return [str(item).strip() for item in value if str(item).strip()]
+        if value:
+            return [str(value).strip()]
+        return []
+
+    @staticmethod
+    def _identifier(value: Any) -> str:
+        if isinstance(value, dict):
+            return str(value.get("value") or value.get("name") or "").strip()
+        return str(value or "").strip()
+
+    @staticmethod
+    def _salary(value: Any) -> dict[str, Any]:
+        if not isinstance(value, dict):
+            return {}
+        result: dict[str, Any] = {}
+        currency = value.get("currency")
+        if currency:
+            result["currency"] = currency
+        raw_value = value.get("value")
+        if isinstance(raw_value, dict):
+            for key in ("minValue", "maxValue", "value", "unitText"):
+                if key in raw_value:
+                    result[key] = raw_value[key]
+        elif raw_value is not None:
+            result["value"] = raw_value
+        return result
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        soup = BeautifulSoup(content, "lxml")
+        records: list[dict[str, Any]] = []
+        warnings: list[str] = []
+        for script in soup.find_all(
+            "script", attrs={"type": lambda value: value and "ld+json" in value}
+        ):
+            raw = script.string or script.get_text("", strip=True)
+            if not raw:
+                continue
+            try:
+                payload = json.loads(raw)
+            except json.JSONDecodeError:
+                warnings.append("Ongeldige JSON-LD overgeslagen")
+                continue
+            records.extend(self._walk(payload))
+
+        jobs: list[ExtractedJob] = []
+        for record in records:
+            title = str(record.get("title") or record.get("name") or "").strip()
+            if not title:
+                warnings.append("JobPosting zonder titel overgeslagen")
+                continue
+            description_html = str(record.get("description") or "").strip()
+            description_text = BeautifulSoup(description_html, "lxml").get_text("\n", strip=True)
+            employer = self._name(record.get("hiringOrganization"))
+            location, region, postal, country = self._location(record.get("jobLocation"))
+            workplace_type = str(record.get("jobLocationType") or "").strip()
+            if not location and record.get("applicantLocationRequirements"):
+                location, region, postal, country = self._location(
+                    record.get("applicantLocationRequirements")
+                )
+            job_url = str(record.get("url") or url).strip()
+            job_url = urljoin(url, job_url)
+            evidence = [
+                FieldEvidence("title", "jsonld", 0.98, title[:240]),
+                FieldEvidence("employer_name", "jsonld", 0.95, employer[:240]),
+                FieldEvidence("location_text", "jsonld", 0.92, location[:240]),
+                FieldEvidence("description", "jsonld", 0.95, description_text[:300]),
+            ]
+            jobs.append(
+                ExtractedJob(
+                    url=job_url,
+                    title=title,
+                    employer_name=employer,
+                    external_id=self._identifier(record.get("identifier")),
+                    location_text=location,
+                    region=region,
+                    postal_code=postal,
+                    country=country,
+                    description_html=description_html,
+                    description_text=description_text,
+                    language=str(record.get("inLanguage") or "").strip(),
+                    date_posted=str(record.get("datePosted") or "").strip(),
+                    valid_through=str(record.get("validThrough") or "").strip(),
+                    employment_types=self._employment_types(record.get("employmentType")),
+                    workplace_type=workplace_type,
+                    compensation=self._salary(record.get("baseSalary")),
+                    raw=record,
+                    evidence=evidence,
+                )
+            )
+        confidence = 0.95 if jobs else 0.0
+        return ExtractionResult(jobs, self.parser_key, self.parser_version, confidence, warnings)

apps/sources/adapters/kempen.py

@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import parse_qs, urljoin, urlsplit
+
+from bs4 import BeautifulSoup, Tag
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class KempenEmployerAdapter:
+    """Extract public employer lists around Mol from exact reviewed routes."""
+
+    parser_key = "regional-kempen-employers"
+    parser_version = "1.0.0"
+    supported_routes = {
+        "ziekenhuisgeel.careersite.be": {"/nl/vacatures"},
+        "geel.hro.be": {"/"},
+        "jobs.turnhout.be": {"/"},
+        "jobs.renotec.be": {"/nl/alle-jobs"},
+        "ravago.softgarden.io": {"/en/vacancies"},
+        "jobs.sanofi.com": {"/en/belgium"},
+        "www.daf.com": {"/nl-nl/werken-bij-daf/vacatures"},
+    }
+
+    def _route(self, url: str) -> tuple[str, str] | None:
+        parts = urlsplit(url)
+        host = (parts.hostname or "").lower()
+        path = parts.path.rstrip("/") or "/"
+        if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
+            return None
+        return host, path
+
+    def _supports_url(self, url: str) -> bool:
+        return self._route(url) is not None
+
+    @staticmethod
+    def _same_host_url(base_url: str, href: str) -> str:
+        job_url = urljoin(base_url, href)
+        parts = urlsplit(job_url)
+        if parts.scheme != "https":
+            return ""
+        if (parts.hostname or "").lower() != (urlsplit(base_url).hostname or "").lower():
+            return ""
+        return job_url
+
+    @staticmethod
+    def _job(
+        *,
+        url: str,
+        title: str,
+        employer: str,
+        location: str,
+        postal_code: str,
+        description: str,
+        external_id: str,
+        evidence_source: str,
+    ) -> ExtractedJob:
+        return ExtractedJob(
+            url=url,
+            title=title,
+            employer_name=employer,
+            external_id=external_id,
+            location_text=location,
+            postal_code=postal_code,
+            region="Antwerpen",
+            country="BE",
+            description_text=description,
+            raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
+            evidence=[
+                FieldEvidence("title", evidence_source, 0.94, title[:240]),
+                FieldEvidence("employer_name", "reviewed-source", 0.97, employer),
+                FieldEvidence("location_text", "kempen-location-marker", 0.9, location),
+            ],
+        )
+
+    def _ziekenhuis_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select(".vacature-tegel"):
+            link = card.select_one("a.vacature-tegel__link[href]")
+            title_node = card.select_one(".vacature-tegel__titel")
+            if not isinstance(link, Tag) or not isinstance(title_node, Tag):
+                continue
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            match = re.fullmatch(r"/nl/vacature/(\d+)/[^/]+", urlsplit(job_url).path)
+            title = title_node.get_text(" ", strip=True)
+            if not title or not match:
+                continue
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Ziekenhuis Geel",
+                    location="Geel, Antwerpen",
+                    postal_code="2440",
+                    description=card.get_text(" ", strip=True),
+                    external_id=match.group(1),
+                    evidence_source="ziekenhuis-geel-card",
+                )
+            )
+        return jobs
+
+    def _stad_geel(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select(".vacatureKader[data-href]"):
+            href = str(card.get("data-href") or "")
+            match = re.fullmatch(r"vacature\.php\?id=(\d+)", href)
+            title_node = card.find("h4")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            if not title or not match:
+                continue
+            jobs.append(
+                self._job(
+                    url=self._same_host_url(url, href),
+                    title=title,
+                    employer="Lokaal bestuur Geel",
+                    location="Geel, Antwerpen",
+                    postal_code="2440",
+                    description=card.get_text(" ", strip=True),
+                    external_id=match.group(1),
+                    evidence_source="stad-geel-hro-card",
+                )
+            )
+        return jobs
+
+    def _stad_turnhout(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.find_all("a", href=True):
+            href = str(link.get("href") or "")
+            match = re.fullmatch(r"vacature\.php\?id=(\d+)", href)
+            title_node = link.select_one(".block-update__body__title__inner")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            if not title or not match:
+                continue
+            jobs.append(
+                self._job(
+                    url=self._same_host_url(url, href),
+                    title=title,
+                    employer="Stad Turnhout",
+                    location="Turnhout, Antwerpen",
+                    postal_code="2300",
+                    description=link.get_text(" ", strip=True),
+                    external_id=match.group(1),
+                    evidence_source="stad-turnhout-hro-card",
+                )
+            )
+        return jobs
+
+    def _renotec(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select(".s-tile.s-card"):
+            card_text = card.get_text(" ", strip=True)
+            if "Geel" not in card_text:
+                continue
+            title_node = card.find("h3")
+            link = card.find("a", href=True)
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
+            parts = urlsplit(job_url)
+            query = parse_qs(parts.query)
+            external_id = (query.get("id") or [""])[0]
+            if parts.path != "/nl/detail/" or not external_id.isdigit() or not title:
+                continue
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Group Renotec",
+                    location="Geel, Antwerpen (mogelijk meerdere werfregio's)",
+                    postal_code="2440",
+                    description=card_text,
+                    external_id=external_id,
+                    evidence_source="renotec-geel-card",
+                )
+            )
+        return jobs
+
+    def _ravago(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select(".matchElement"):
+            locations = {
+                node.get_text(" ", strip=True) for node in card.select(".location-view-item")
+            }
+            local_places = locations.intersection({"Arendonk", "Olen"})
+            if not local_places:
+                continue
+            link = card.find("a", href=True)
+            title = link.get_text(" ", strip=True) if link else ""
+            job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
+            match = re.fullmatch(r"/job/(\d+)/[^/]+/?", urlsplit(job_url).path)
+            if not title or not match:
+                continue
+            place = "Arendonk" if "Arendonk" in local_places else "Olen"
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Ravago",
+                    location=f"{place}, Antwerpen",
+                    postal_code="2370" if place == "Arendonk" else "2250",
+                    description=card.get_text(" ", strip=True),
+                    external_id=match.group(1),
+                    evidence_source="ravago-softgarden-card",
+                )
+            )
+        return jobs
+
+    def _sanofi(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.select(".job-list a[data-job-id][href]"):
+            location_node = link.select_one(".job-location")
+            title_node = link.select_one(".job-title")
+            location = location_node.get_text(" ", strip=True) if location_node else ""
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            external_id = str(link.get("data-job-id") or "")
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            if location != "Geel, Belgium" or not title or not external_id.isdigit():
+                continue
+            if not re.fullmatch(r"/en/job/geel/[^/]+/\d+/\d+", urlsplit(job_url).path):
+                continue
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Sanofi",
+                    location="Geel, Antwerpen",
+                    postal_code="2440",
+                    description=link.get_text(" ", strip=True),
+                    external_id=external_id,
+                    evidence_source="sanofi-belgium-geel-card",
+                )
+            )
+        return jobs
+
+    def _daf(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select("li.itemlist__item"):
+            location_node = card.select_one(".js-vac-metalocation")
+            title_link = card.select_one("a.js-vac-title[href]")
+            location = location_node.get_text(" ", strip=True) if location_node else ""
+            title = title_link.get_text(" ", strip=True) if title_link else ""
+            job_url = (
+                self._same_host_url(url, str(title_link.get("href") or "")) if title_link else ""
+            )
+            path = urlsplit(job_url).path
+            prefix = "/nl-nl/werken-bij-daf/vacatures/"
+            if location != "Westerlo" or not title or not path.startswith(prefix):
+                continue
+            slug = path.removeprefix(prefix).strip("/")
+            if not slug or "/" in slug:
+                continue
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="DAF Trucks",
+                    location="Westerlo, Antwerpen",
+                    postal_code="2260",
+                    description=card.get_text(" ", strip=True),
+                    external_id=slug,
+                    evidence_source="daf-westerlo-card",
+                )
+            )
+        return jobs
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        route = self._route(url)
+        if route is None:
+            return ExtractionResult(
+                [], self.parser_key, self.parser_version, 0.0, ["Onherkende Kempen-bron"]
+            )
+        host, _ = route
+        extractors = {
+            "ziekenhuisgeel.careersite.be": self._ziekenhuis_geel,
+            "geel.hro.be": self._stad_geel,
+            "jobs.turnhout.be": self._stad_turnhout,
+            "jobs.renotec.be": self._renotec,
+            "ravago.softgarden.io": self._ravago,
+            "jobs.sanofi.com": self._sanofi,
+            "www.daf.com": self._daf,
+        }
+        jobs = extractors[host](BeautifulSoup(content, "lxml"), url)
+        unique_jobs = list({job.url: job for job in jobs if job.url}.values())
+        return ExtractionResult(
+            unique_jobs,
+            self.parser_key,
+            self.parser_version,
+            0.92 if unique_jobs else 0.0,
+            [] if unique_jobs else ["Geen actuele regionale vacatures gevonden"],
+        )

apps/sources/adapters/mol_region.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import parse_qs, urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class MolRegionEmployerAdapter:
+    """Extract reviewed employer listings around postcode 2400 without detail fetches."""
+
+    parser_key = "regional-mol-employers"
+    parser_version = "1.0.0"
+    thomas_more_company_guid = "eab9ca13-ee10-4504-8a87-a785d0b037ef"
+    supported_routes = {
+        "www.sckcen.be": {"/nl/carriere/vacatures"},
+        "cipalschaubroeck.teamtailor.com": {"/jobs"},
+        "www.vanroey.be": {"/en/job-overview"},
+        "netropolix.recruitee.com": {"/"},
+        "jobpage.cvwarehouse.com": {"/"},
+    }
+
+    def _route(self, url: str) -> tuple[str, str] | None:
+        parts = urlsplit(url)
+        host = (parts.hostname or "").lower()
+        path = parts.path.rstrip("/") or "/"
+        if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
+            return None
+        if host == "jobpage.cvwarehouse.com":
+            query = parse_qs(parts.query)
+            if query.get("companyGuid") != [self.thomas_more_company_guid] or "job" in query:
+                return None
+        return host, path
+
+    def _supports_url(self, url: str) -> bool:
+        return self._route(url) is not None
+
+    @staticmethod
+    def _same_host_url(base_url: str, href: str) -> str:
+        job_url = urljoin(base_url, href)
+        if urlsplit(job_url).scheme != "https":
+            return ""
+        if (urlsplit(job_url).hostname or "").lower() != (
+            urlsplit(base_url).hostname or ""
+        ).lower():
+            return ""
+        return job_url
+
+    @staticmethod
+    def _job(
+        *,
+        url: str,
+        title: str,
+        employer: str,
+        location: str,
+        postal_code: str,
+        description: str,
+        external_id: str,
+        evidence_source: str,
+    ) -> ExtractedJob:
+        return ExtractedJob(
+            url=url,
+            title=title,
+            employer_name=employer,
+            external_id=external_id,
+            location_text=location,
+            postal_code=postal_code,
+            region="Antwerpen",
+            country="BE",
+            description_text=description,
+            raw={"regional_listing": evidence_source, "region_center": "2400 Mol"},
+            evidence=[
+                FieldEvidence("title", evidence_source, 0.94, title[:240]),
+                FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
+                FieldEvidence("location_text", "mol-region-review", 0.82, location),
+            ],
+        )
+
+    def _sck_cen(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.select("article a[href]"):
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            path = urlsplit(job_url).path if job_url else ""
+            if not path.startswith("/nl/carriere/vacatures/"):
+                continue
+            title = link.get_text(" ", strip=True)
+            if not title:
+                continue
+            card = link.find_parent("article")
+            description = card.get_text(" ", strip=True) if card else title
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="SCK CEN",
+                    location="Mol, Antwerpen",
+                    postal_code="2400",
+                    description=description,
+                    external_id=path.rstrip("/").rsplit("/", 1)[-1],
+                    evidence_source="sck-cen-vacancy-card",
+                )
+            )
+        return jobs
+
+    def _cipal(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select("li"):
+            card_text = card.get_text(" ", strip=True)
+            if "Westerlo" not in card_text and "Geel" not in card_text:
+                continue
+            link = card.find("a", href=True)
+            if not link:
+                continue
+            title = link.get_text(" ", strip=True)
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            path = urlsplit(job_url).path if job_url else ""
+            match = re.fullmatch(r"/jobs/(\d+)-[^/]+", path.rstrip("/"))
+            if not title or not match:
+                continue
+            location = "Geel, Antwerpen" if "Geel" in card_text else "Westerlo, Antwerpen"
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Cipal Schaubroeck",
+                    location=location,
+                    postal_code="2440" if "Geel" in card_text else "2260",
+                    description=card_text,
+                    external_id=match.group(1),
+                    evidence_source="cipal-teamtailor-card",
+                )
+            )
+        return jobs
+
+    def _vanroey(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for heading in soup.select("h3.elementor-heading-title"):
+            link = heading.find("a", href=True)
+            if not link:
+                continue
+            title = link.get_text(" ", strip=True)
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            path = urlsplit(job_url).path if job_url else ""
+            if not title or not re.fullmatch(r"/en/job/[^/]+/", path):
+                continue
+            slug = path.rstrip("/").rsplit("/", 1)[-1]
+            if "oost-vlaanderen" in slug:
+                continue
+            card = heading.find_parent("section") or heading.parent
+            description = card.get_text(" ", strip=True) if card else title
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="VanRoey",
+                    location="Turnhout/Geel (hybride; controleer vacaturedetail)",
+                    postal_code="",
+                    description=description,
+                    external_id=slug,
+                    evidence_source="vanroey-job-card",
+                )
+            )
+        return jobs
+
+    def _netropolix(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.find_all("a", href=True):
+            title = link.get_text(" ", strip=True)
+            if "Geel" not in title:
+                continue
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            path = urlsplit(job_url).path if job_url else ""
+            if not re.fullmatch(r"/o/[^/]+", path.rstrip("/")):
+                continue
+            card = link.find_parent("div")
+            description = card.get_text(" ", strip=True) if card else title
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="NTX (Netropolix)",
+                    location="Geel, Antwerpen",
+                    postal_code="2440",
+                    description=description,
+                    external_id=path.rstrip("/").rsplit("/", 1)[-1],
+                    evidence_source="netropolix-recruitee-card",
+                )
+            )
+        return jobs
+
+    def _thomas_more(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.select("a.jobLink[data-item='readmore'][data-jobid][href]"):
+            title = link.get_text(" ", strip=True)
+            title_casefold = title.casefold()
+            if not any(place in title_casefold for place in ("geel", "turnhout", "vorselaar")):
+                continue
+            external_id = str(link.get("data-jobid") or "")
+            if not external_id.isdigit():
+                continue
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            query = parse_qs(urlsplit(job_url).query) if job_url else {}
+            if query.get("companyGuid") != [self.thomas_more_company_guid] or query.get("job") != [
+                external_id
+            ]:
+                continue
+            if "turnhout" in title_casefold:
+                location, postal_code = "Turnhout, Antwerpen", "2300"
+            elif "vorselaar" in title_casefold:
+                location, postal_code = "Vorselaar, Antwerpen", "2290"
+            else:
+                location, postal_code = "Geel, Antwerpen", "2440"
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Thomas More",
+                    location=location,
+                    postal_code=postal_code,
+                    description=f"Publieke regionale vacature bij Thomas More: {title}.",
+                    external_id=external_id,
+                    evidence_source="thomas-more-cvwarehouse-card",
+                )
+            )
+        return jobs
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        route = self._route(url)
+        if route is None:
+            return ExtractionResult(
+                [], self.parser_key, self.parser_version, 0.0, ["Onherkende Mol-regiobron"]
+            )
+        host, _ = route
+        soup = BeautifulSoup(content, "lxml")
+        extractors = {
+            "www.sckcen.be": self._sck_cen,
+            "cipalschaubroeck.teamtailor.com": self._cipal,
+            "www.vanroey.be": self._vanroey,
+            "netropolix.recruitee.com": self._netropolix,
+            "jobpage.cvwarehouse.com": self._thomas_more,
+        }
+        jobs = extractors[host](soup, url)
+        unique_jobs = list({job.url: job for job in jobs}.values())
+        warnings = [] if unique_jobs else ["Geen actuele vacatures rond Mol gevonden"]
+        return ExtractionResult(
+            unique_jobs,
+            self.parser_key,
+            self.parser_version,
+            0.9 if unique_jobs else 0.0,
+            warnings,
+        )

apps/sources/adapters/regional.py

@@ -0,0 +1,312 @@
+from __future__ import annotations
+
+from urllib.parse import urljoin, urlsplit
+
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class CordaCampusAdapter:
+    """Extract the public job cards intentionally published by Corda Campus."""
+
+    parser_key = "regional-corda-campus"
+    parser_version = "1.0.0"
+    source_hosts = ("cordacampus.com",)
+
+    def _supports_url(self, url: str) -> bool:
+        parts = urlsplit(url)
+        host = (parts.hostname or "").lower()
+        return host.endswith(self.source_hosts) and parts.path.rstrip("/") == "/jobs"
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        if not self._supports_url(url):
+            return ExtractionResult(
+                [], self.parser_key, self.parser_version, 0.0, ["Onherkenbare regiobron"]
+            )
+
+        source_host = (urlsplit(url).hostname or "").lower()
+        soup = BeautifulSoup(content, "lxml")
+        jobs: list[ExtractedJob] = []
+        seen_urls: set[str] = set()
+
+        for card in soup.select('a.event-item[href*="/job/"]'):
+            job_url = urljoin(url, str(card.get("href") or ""))
+            job_host = (urlsplit(job_url).hostname or "").lower()
+            if job_host != source_host or job_url in seen_urls:
+                continue
+            title_node = card.select_one(".job-title")
+            employer_node = card.select_one(".company-title")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            employer = employer_node.get_text(" ", strip=True) if employer_node else ""
+            if not title:
+                continue
+
+            date_node = card.select_one(".bottom-info")
+            date_posted = date_node.get_text(" ", strip=True) if date_node else ""
+            external_id = urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1]
+            description = f"Vacature van {employer or 'een Corda-werkgever'} via Corda Campus."
+            jobs.append(
+                ExtractedJob(
+                    url=job_url,
+                    title=title,
+                    employer_name=employer,
+                    external_id=external_id,
+                    location_text="Hasselt, Limburg",
+                    region="Limburg",
+                    country="BE",
+                    description_text=description,
+                    date_posted=date_posted,
+                    raw={"regional_listing": "corda-campus"},
+                    evidence=[
+                        FieldEvidence("title", "corda-job-card", 0.95, title[:240]),
+                        FieldEvidence("employer_name", "corda-job-card", 0.92, employer[:240]),
+                        FieldEvidence(
+                            "location_text",
+                            "regional-source-scope",
+                            0.72,
+                            "Corda Campus, Hasselt",
+                        ),
+                    ],
+                )
+            )
+            seen_urls.add(job_url)
+
+        warnings = [] if jobs else ["Geen actuele Corda-vacatures gevonden"]
+        return ExtractionResult(
+            jobs,
+            self.parser_key,
+            self.parser_version,
+            0.9 if jobs else 0.0,
+            warnings,
+        )
+
+
+class LocalEmployerListingAdapter:
+    """Extract reviewed public listing pages of employers in the Hasselt-Genk area."""
+
+    parser_key = "regional-local-employers"
+    parser_version = "1.0.0"
+    supported_routes = {
+        "acagroup.be": {"/en/jobs"},
+        "www.xploregroup.be": {"/en/jobs"},
+        "www.uhasselt.be": {"/vacatures"},
+        "ses.pxl.be": {"/"},
+        "ziekenhuis-oost-limburg.cvw.io": {"/"},
+    }
+
+    def _supports_url(self, url: str) -> bool:
+        return self._route(url) is not None
+
+    def _route(self, url: str) -> tuple[str, str] | None:
+        parts = urlsplit(url)
+        host = (parts.hostname or "").lower()
+        path = parts.path.rstrip("/") or "/"
+        if parts.scheme != "https" or path not in self.supported_routes.get(host, set()):
+            return None
+        return host, path
+
+    @staticmethod
+    def _job(
+        *,
+        url: str,
+        title: str,
+        employer: str,
+        location: str,
+        description: str,
+        external_id: str,
+        evidence_source: str,
+    ) -> ExtractedJob:
+        return ExtractedJob(
+            url=url,
+            title=title,
+            employer_name=employer,
+            external_id=external_id,
+            location_text=location,
+            region="Limburg",
+            country="BE",
+            description_text=description,
+            raw={"regional_listing": evidence_source},
+            evidence=[
+                FieldEvidence("title", evidence_source, 0.94, title[:240]),
+                FieldEvidence("employer_name", "reviewed-source", 0.96, employer),
+                FieldEvidence("location_text", "regional-source-scope", 0.78, location),
+            ],
+        )
+
+    @staticmethod
+    def _same_host_url(base_url: str, href: str) -> str:
+        job_url = urljoin(base_url, href)
+        if (urlsplit(job_url).hostname or "").lower() != (
+            urlsplit(base_url).hostname or ""
+        ).lower():
+            return ""
+        return job_url
+
+    def _aca(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.find_all("a", href=True):
+            href = str(link.get("href") or "")
+            if not href.startswith("/en/jobs/"):
+                continue
+            title_node = link.find("h3")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            job_url = self._same_host_url(url, href)
+            if not title or not job_url:
+                continue
+            description_node = link.find("p")
+            description = (
+                description_node.get_text(" ", strip=True)
+                if description_node
+                else f"Vacature bij ACA Group: {title}."
+            )
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="ACA Group",
+                    location="Hasselt (hybride; kantoorselectie per vacature)",
+                    description=description,
+                    external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1],
+                    evidence_source="aca-job-card",
+                )
+            )
+        return jobs
+
+    def _xplore(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for heading in soup.find_all("h3"):
+            link = heading.find("a", href=True)
+            if not link or not str(link.get("href") or "").startswith("/en/jobs/"):
+                continue
+            card = heading.parent
+            location_node = next(
+                (
+                    node
+                    for node in card.find_all("p")
+                    if "Hasselt" in node.get_text(" ", strip=True)
+                ),
+                None,
+            )
+            if not location_node:
+                continue
+            title = link.get_text(" ", strip=True)
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            if not title or not job_url:
+                continue
+            location = location_node.get_text(" ", strip=True)
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Xplore Group",
+                    location=location,
+                    description=(
+                        f"Vacature bij Xplore Group met Hasselt als mogelijke werklocatie: {title}."
+                    ),
+                    external_id=urlsplit(job_url).path.rstrip("/").rsplit("/", 1)[-1],
+                    evidence_source="xplore-job-card",
+                )
+            )
+        return jobs
+
+    def _uhasselt(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select("section.vacancy-item"):
+            title_node = card.find("h3")
+            link = card.find("a", href=True)
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            job_url = self._same_host_url(url, str(link.get("href") or "")) if link else ""
+            if not title or not job_url or "/vacatures/detail/" not in urlsplit(job_url).path:
+                continue
+            external_id = urlsplit(job_url).path.split("/detail/", 1)[-1].split("-", 1)[0]
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Universiteit Hasselt",
+                    location="Hasselt/Diepenbeek, Limburg",
+                    description=card.get_text(" ", strip=True),
+                    external_id=external_id,
+                    evidence_source="uhasselt-vacancy-card",
+                )
+            )
+        return jobs
+
+    def _pxl(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for card in soup.select(".vacature-card[id^='Vacature_']"):
+            external_id = str(card.get("id") or "").removeprefix("Vacature_")
+            title_node = card.select_one(".vacature-card-titel")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            if not title or not external_id.isdigit():
+                continue
+            cells = [
+                node.get_text(" ", strip=True) for node in card.select("td.vacature-card-td-text")
+            ]
+            campus = next((value for value in cells if value.startswith("Campus ")), "Hasselt")
+            job_url = f"{url.rstrip('/')}?vacature_id={external_id}"
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Hogeschool PXL",
+                    location=f"{campus}, Limburg",
+                    description=" · ".join(cells),
+                    external_id=external_id,
+                    evidence_source="pxl-vacancy-card",
+                )
+            )
+        return jobs
+
+    def _zol(self, soup: BeautifulSoup, url: str) -> list[ExtractedJob]:
+        jobs = []
+        for link in soup.select("a[data-item='readmore'][data-jobid][href]"):
+            external_id = str(link.get("data-jobid") or "")
+            title_node = link.select_one(".job-title")
+            title = title_node.get_text(" ", strip=True) if title_node else ""
+            job_url = self._same_host_url(url, str(link.get("href") or ""))
+            if not title or not external_id.isdigit() or not job_url:
+                continue
+            jobs.append(
+                self._job(
+                    url=job_url,
+                    title=title,
+                    employer="Ziekenhuis Oost-Limburg",
+                    location="Genk/Lanaken/Maaseik, Limburg",
+                    description=f"Publieke vacature van Ziekenhuis Oost-Limburg: {title}.",
+                    external_id=external_id,
+                    evidence_source="zol-cvwarehouse-card",
+                )
+            )
+        return jobs
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        route = self._route(url)
+        if route is None:
+            return ExtractionResult(
+                [],
+                self.parser_key,
+                self.parser_version,
+                0.0,
+                ["Onherkenbare lokale werkgeversbron"],
+            )
+        host, _ = route
+        soup = BeautifulSoup(content, "lxml")
+        extractors = {
+            "acagroup.be": self._aca,
+            "www.xploregroup.be": self._xplore,
+            "www.uhasselt.be": self._uhasselt,
+            "ses.pxl.be": self._pxl,
+            "ziekenhuis-oost-limburg.cvw.io": self._zol,
+        }
+        jobs = extractors[host](soup, url)
+        unique_jobs = list({job.url: job for job in jobs}.values())
+        warnings = [] if unique_jobs else ["Geen actuele lokale vacatures gevonden"]
+        return ExtractionResult(
+            unique_jobs,
+            self.parser_key,
+            self.parser_version,
+            0.9 if unique_jobs else 0.0,
+            warnings,
+        )

apps/sources/adapters/registry.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from apps.sources.models import RawDocument
+
+from .ats import (
+    GreenhouseAdapter,
+    LeverAdapter,
+    RecruiteeAdapter,
+    SmartRecruitersAdapter,
+    WorkableAdapter,
+)
+from .base import ExtractionResult
+from .generic_html import GenericHtmlAdapter
+from .jsonld import JsonLdJobPostingAdapter
+from .kempen import KempenEmployerAdapter
+from .mol_region import MolRegionEmployerAdapter
+from .regional import CordaCampusAdapter, LocalEmployerListingAdapter
+from .rss import RssAdapter
+
+
+class AdapterRegistry:
+    def __init__(self) -> None:
+        self.greenhouse = GreenhouseAdapter()
+        self.lever = LeverAdapter()
+        self.recruitee = RecruiteeAdapter()
+        self.smartrecruiters = SmartRecruitersAdapter()
+        self.workable = WorkableAdapter()
+        self.corda_campus = CordaCampusAdapter()
+        self.local_employers = LocalEmployerListingAdapter()
+        self.mol_region_employers = MolRegionEmployerAdapter()
+        self.kempen_employers = KempenEmployerAdapter()
+        self.jsonld = JsonLdJobPostingAdapter()
+        self.generic = GenericHtmlAdapter()
+        self.rss = RssAdapter()
+        self.providers = [
+            self.corda_campus,
+            self.local_employers,
+            self.mol_region_employers,
+            self.kempen_employers,
+            self.greenhouse,
+            self.lever,
+            self.recruitee,
+            self.smartrecruiters,
+            self.workable,
+        ]
+
+    def extract(self, document: RawDocument) -> ExtractionResult:
+        content = document.body_text
+        url = document.final_url or document.url
+        content_type = (document.content_type or "").lower()
+        if document.kind == RawDocument.Kind.XML or "rss" in content_type or "atom" in content_type:
+            return self.rss.extract(content, url=url)
+
+        for provider in self.providers:
+            if provider._supports_url(url):
+                result = provider.extract(content, url=url)
+                if any(
+                    msg in result.warnings
+                    for msg in (
+                        "Geen parseerbare ATS-response",
+                        "Geen herkenbare ATS-markup voor deze adapter",
+                    )
+                ):
+                    continue
+                return result
+
+        jsonld_result = self.jsonld.extract(content, url=url)
+        if jsonld_result.jobs:
+            return jsonld_result
+        generic_result = self.generic.extract(content, url=url)
+        generic_result.warnings = jsonld_result.warnings + generic_result.warnings
+        return generic_result
+
+
+registry = AdapterRegistry()

apps/sources/adapters/rss.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from datetime import datetime
+from time import mktime
+
+import feedparser
+from bs4 import BeautifulSoup
+
+from .base import ExtractedJob, ExtractionResult, FieldEvidence
+
+
+class RssAdapter:
+    parser_key = "rss-atom"
+    parser_version = "1.0.0"
+
+    def extract(self, content: str, *, url: str) -> ExtractionResult:
+        feed = feedparser.parse(content)
+        jobs: list[ExtractedJob] = []
+        for entry in feed.entries:
+            title = str(entry.get("title") or "").strip()
+            link = str(entry.get("link") or "").strip()
+            if not title or not link:
+                continue
+            summary = str(entry.get("summary") or entry.get("description") or "")
+            text = BeautifulSoup(summary, "lxml").get_text("\n", strip=True)
+            published = ""
+            if entry.get("published_parsed"):
+                published = datetime.fromtimestamp(mktime(entry.published_parsed)).isoformat()
+            jobs.append(
+                ExtractedJob(
+                    url=link,
+                    title=title,
+                    external_id=str(entry.get("id") or ""),
+                    description_html=summary,
+                    description_text=text,
+                    date_posted=published,
+                    raw=dict(entry),
+                    evidence=[
+                        FieldEvidence("title", "rss", 0.88, title[:240]),
+                        FieldEvidence("description", "rss", 0.80, text[:300]),
+                    ],
+                )
+            )
+        warnings: list[str] = []
+        if getattr(feed, "bozo", False):
+            warnings.append(str(getattr(feed, "bozo_exception", "Ongeldige feed")))
+        return ExtractionResult(
+            jobs,
+            self.parser_key,
+            self.parser_version,
+            0.82 if jobs else 0.0,
+            warnings,
+        )