Publish curated VacatureRadar source snapshot

tests/security/test_policy.py

@@ -0,0 +1,109 @@
+from datetime import timedelta
+
+import pytest
+from django.utils import timezone
+
+from apps.sources.models import Source, SourcePolicyReview
+from apps.sources.services.policy import assess_url, create_policy_review, is_denied_domain
+from apps.sources.services.robots import RobotsDecision
+
+
+def test_platform_domains_are_denied_including_subdomains():
+    for domain in (
+        "www.linkedin.com",
+        "be.indeed.com",
+        "www.ictjob.be",
+        "www.jobat.be",
+        "www.stepstone.be",
+        "www.careerjet.be",
+        "jobs.randstad.be",
+        "www.roberthalf.com",
+    ):
+        assert is_denied_domain(domain)
+    assert not is_denied_domain("jobs.example.org")
+    assert not assess_url("https://www.linkedin.com/jobs/view/123").allowed
+    assert not assess_url("https://www.ictjob.be/nl/it-vacatures-zoeken/example").allowed
+
+
+@pytest.mark.security
+def test_source_policy_without_review_is_blocked(db):
+    source = Source.objects.create(
+        name="Example jobs",
+        source_type=Source.Type.EMPLOYER,
+        base_url="https://jobs.example.org/vacatures/",
+        domain="jobs.example.org",
+        status=Source.Status.ACTIVE,
+        policy=Source.Policy.ALLOW,
+    )
+    decision = assess_url(source.base_url, source=source)
+    assert not decision.allowed
+    assert decision.status == Source.Policy.REVIEW
+
+
+@pytest.mark.security
+def test_source_review_expiry_blocks_fetch_decision(db):
+    source = Source.objects.create(
+        name="Example jobs",
+        source_type=Source.Type.EMPLOYER,
+        base_url="https://jobs.example.org/vacatures/",
+        domain="jobs.example.org",
+        status=Source.Status.ACTIVE,
+        policy=Source.Policy.ALLOW,
+    )
+    create_policy_review(
+        source,
+        actor=None,
+        decision=SourcePolicyReview.Decision.ALLOW,
+        reason="Aanvankelijk goedgekeurd",
+        scope=SourcePolicyReview.Scope.SOURCE,
+        expires_at=timezone.now() - timedelta(days=1),
+    )
+    decision = assess_url(source.base_url, source=source)
+    assert not decision.allowed
+
+
+@pytest.mark.security
+def test_policy_conflict_denies_even_when_allow_policy_set(db):
+    source = Source.objects.create(
+        name="Example jobs",
+        source_type=Source.Type.EMPLOYER,
+        base_url="https://jobs.example.org/vacatures/",
+        domain="jobs.example.org",
+        status=Source.Status.ACTIVE,
+        policy=Source.Policy.ALLOW,
+    )
+    create_policy_review(
+        source,
+        actor=None,
+        decision=SourcePolicyReview.Decision.DENY,
+        reason="Juridische review: geblokkeerd",
+        scope=SourcePolicyReview.Scope.SOURCE,
+    )
+    decision = assess_url(source.base_url, source=source)
+    assert not decision.allowed
+    assert decision.status == Source.Policy.DENY
+
+
+@pytest.mark.security
+def test_trial_review_can_allow_fetch(db, monkeypatch):
+    monkeypatch.setattr(
+        "apps.sources.services.policy.assess_robots",
+        lambda *args, **kwargs: RobotsDecision(True, "Testrobots staan toegang toe"),
+    )
+    source = Source.objects.create(
+        name="Example jobs",
+        source_type=Source.Type.EMPLOYER,
+        base_url="https://jobs.example.org/vacatures/",
+        domain="jobs.example.org",
+        status=Source.Status.TRIAL,
+        policy=Source.Policy.REVIEW,
+    )
+    create_policy_review(
+        source,
+        actor=None,
+        decision=SourcePolicyReview.Decision.TRIAL,
+        reason="Technische proefrun gestart",
+        scope=SourcePolicyReview.Scope.SOURCE,
+    )
+    decision = assess_url(source.base_url, source=source)
+    assert decision.allowed

tests/security/test_robots.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+import httpx
+import pytest
+from django.utils import timezone
+
+from apps.sources.models import Source, SourceRobotsCache
+from apps.sources.services.robots import assess_robots
+from apps.sources.services.url_security import ValidatedUrl
+
+
+def _allow_test_dns(monkeypatch):
+    monkeypatch.setattr(
+        "apps.sources.services.robots.validate_public_url",
+        lambda url, **kwargs: ValidatedUrl(
+            url=url,
+            hostname="jobs.example.org",
+            port=443,
+            addresses=("93.184.216.34",),
+        ),
+    )
+
+
+@pytest.mark.security
+def test_robots_uses_user_agent_matching_and_allows_default(db, monkeypatch):
+    _allow_test_dns(monkeypatch)
+    source = Source.objects.create(
+        name="Example jobs",
+        source_type=Source.Type.EMPLOYER,
+        base_url="https://jobs.example.org/vacatures/",
+        domain="jobs.example.org",
+        status=Source.Status.ACTIVE,
+        policy=Source.Policy.ALLOW,
+    )
+    calls = {"count": 0}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls["count"] += 1
+        assert request.url.path == "/robots.txt"
+        body = (
+            "User-agent: *\n"
+            "Disallow: /admin\n"
+            "User-agent: vacatureradar/0.1 (+local-personal-use)\n"
+            "Allow: /admin\n"
+        )
+        return httpx.Response(200, text=body)
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    decision = assess_robots(
+        source.base_url + "admin",
+        source=source,
+        user_agent="VacatureRadar/0.1 (+local-personal-use)",
+        client=client,
+        now=timezone.now(),
+    )
+    client.close()
+    assert decision.allowed
+    assert calls["count"] == 1
+
+
+@pytest.mark.security
+def test_robots_cache_refreshes_after_ttl(db, settings, monkeypatch):
+    _allow_test_dns(monkeypatch)
+    source = Source.objects.create(
+        name="Example jobs",
+        source_type=Source.Type.EMPLOYER,
+        base_url="https://jobs.example.org/vacatures/",
+        domain="jobs.example.org",
+        status=Source.Status.ACTIVE,
+        policy=Source.Policy.ALLOW,
+    )
+    settings.ROBOTS_CACHE_TTL_SECONDS = 0
+
+    calls = {"count": 0}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls["count"] += 1
+        return httpx.Response(200, text="User-agent: *\nAllow: /")
+
+    client = httpx.Client(transport=httpx.MockTransport(handler))
+    assess_robots(
+        source.base_url, source=source, user_agent="test-agent", client=client, now=timezone.now()
+    )
+    assert calls["count"] == 1
+    assess_robots(
+        source.base_url,
+        source=source,
+        user_agent="test-agent",
+        client=client,
+        now=timezone.now(),
+    )
+    assert calls["count"] == 2
+    assert SourceRobotsCache.objects.filter(origin__contains="jobs.example.org").exists()
+    client.close()
+
+
+@pytest.mark.security
+def test_robots_follows_and_revalidates_redirects(db, monkeypatch):
+    validated_urls = []
+
+    def validate(url, **kwargs):
+        validated_urls.append(url)
+        if "127.0.0.1" in url:
+            from apps.sources.services.url_security import UnsafeUrlError
+
+            raise UnsafeUrlError("Privéadres")
+        return ValidatedUrl(
+            url=url,
+            hostname="jobs.example.org",
+            port=443,
+            addresses=("93.184.216.34",),
+        )
+
+    monkeypatch.setattr("apps.sources.services.robots.validate_public_url", validate)
+    source = Source.objects.create(
+        name="Redirect jobs",
+        source_type=Source.Type.ATS,
+        base_url="https://ats.example.org/jobs",
+        domain="ats.example.org",
+        status=Source.Status.ACTIVE,
+        policy=Source.Policy.ALLOW,
+    )
+
+    def safe_handler(request: httpx.Request) -> httpx.Response:
+        if request.url.host == "ats.example.org":
+            return httpx.Response(
+                302,
+                headers={"location": "https://jobs.example.org/robots.txt"},
+                request=request,
+            )
+        return httpx.Response(200, text="User-agent: *\nAllow: /", request=request)
+
+    client = httpx.Client(transport=httpx.MockTransport(safe_handler))
+    assert assess_robots(source.base_url, source=source, client=client).allowed is True
+    client.close()
+    assert validated_urls == [
+        "https://ats.example.org/robots.txt",
+        "https://jobs.example.org/robots.txt",
+    ]
+
+    SourceRobotsCache.objects.all().delete()
+
+    def unsafe_handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            302, headers={"location": "http://127.0.0.1/robots.txt"}, request=request
+        )
+
+    client = httpx.Client(transport=httpx.MockTransport(unsafe_handler))
+    blocked = assess_robots(source.base_url, source=source, client=client)
+    client.close()
+    assert blocked.allowed is False
+    assert "Privéadres" in blocked.reason

tests/security/test_sanitize.py

@@ -0,0 +1,18 @@
+import pytest
+
+from apps.jobs.services.sanitize import sanitize_job_html
+
+
+@pytest.mark.security
+def test_sanitize_removes_script_handlers_forms_and_iframes():
+    dirty = (
+        '<p onclick="alert(1)">Hallo</p><script>alert(1)</script>'
+        '<iframe src="x"></iframe><form><input></form>'
+        '<a href="javascript:alert(1)">x</a>'
+    )
+    clean = sanitize_job_html(dirty)
+    assert "script" not in clean.lower()
+    assert "onclick" not in clean.lower()
+    assert "iframe" not in clean.lower()
+    assert "form" not in clean.lower()
+    assert "javascript:" not in clean.lower()

tests/security/test_url_security.py

@@ -0,0 +1,34 @@
+import socket
+
+import pytest
+
+from apps.sources.services.url_security import UnsafeUrlError, validate_public_url
+
+
+def fake_resolver(address: str):
+    def resolver(host, port, type=socket.SOCK_STREAM):
+        return [(socket.AF_INET, type, 6, "", (address, port))]
+
+    return resolver
+
+
+@pytest.mark.security
+def test_blocks_private_ipv4_after_dns_resolution():
+    with pytest.raises(UnsafeUrlError, match="Niet-publiek"):
+        validate_public_url("https://jobs.example.org/test", resolver=fake_resolver("10.0.0.5"))
+
+
+@pytest.mark.security
+def test_blocks_loopback_and_embedded_credentials():
+    with pytest.raises(UnsafeUrlError):
+        validate_public_url("http://127.0.0.1/admin")
+    with pytest.raises(UnsafeUrlError):
+        validate_public_url("https://user:pass@example.org/")
+
+
+@pytest.mark.security
+def test_accepts_global_address():
+    result = validate_public_url(
+        "https://jobs.example.org/test", resolver=fake_resolver("93.184.216.34")
+    )
+    assert result.hostname == "jobs.example.org"