Publish curated VacatureRadar source snapshot
This commit is contained in:
Jens committed
commit
e29774cb7f
338 files changed
+32699
No files matched your search
@@ -0,0 +1,357 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.contrib.auth.mixins import LoginRequiredMixin
|
||||
from django.db.models import Count, Prefetch, Q
|
||||
from django.http import HttpRequest
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
from django.views.decorators.cache import never_cache
|
||||
from django.views.decorators.http import require_POST
|
||||
from django.views.generic import ListView
|
||||
|
||||
from apps.core.rate_limit import clear_rate_limit, is_rate_limited, register_rate_limit_failure
|
||||
from apps.profiles.models import SearchProfile
|
||||
|
||||
from .forms import MailboxConnectionForm, ManualImportForm
|
||||
from .models import MailboxConnection, Source, SourcePolicyReview, SourceRun
|
||||
from .platforms import PLATFORM_ALERTS
|
||||
from .services.mailbox_connections import MailboxCredentialError, save_mailbox_connection
|
||||
from .services.manual_import import ManualImportError, ManualImportSummary, import_manual_source
|
||||
from .services.policy import create_policy_review
|
||||
from .tasks import fetch_source, poll_mailbox
|
||||
|
||||
|
||||
def _source_list_queryset():
|
||||
return (
|
||||
Source.objects.prefetch_related(
|
||||
"policy_reviews",
|
||||
Prefetch(
|
||||
"runs",
|
||||
queryset=SourceRun.objects.order_by("-started_at")[:1],
|
||||
to_attr="latest_runs",
|
||||
),
|
||||
)
|
||||
.exclude(domain="jobs.example.org", status=Source.Status.DISABLED)
|
||||
.filter(
|
||||
Q(metadata__hidden_from_source_list__isnull=True)
|
||||
| Q(metadata__hidden_from_source_list=False)
|
||||
)
|
||||
.filter(Q(metadata__watchlist__isnull=True) | Q(metadata__watchlist=False))
|
||||
.annotate(job_count=Count("job_aliases", distinct=True))
|
||||
.order_by("name")
|
||||
)
|
||||
|
||||
|
||||
def _watchlist_queryset():
|
||||
return (
|
||||
Source.objects.filter(metadata__watchlist=True)
|
||||
.filter(
|
||||
Q(metadata__hidden_from_source_list__isnull=True)
|
||||
| Q(metadata__hidden_from_source_list=False)
|
||||
)
|
||||
.order_by("name")
|
||||
)
|
||||
|
||||
|
||||
def _manual_import_context(
|
||||
request: HttpRequest,
|
||||
*,
|
||||
form: ManualImportForm,
|
||||
manual_result=None,
|
||||
mailbox_form: MailboxConnectionForm | None = None,
|
||||
editing_mailbox: MailboxConnection | None = None,
|
||||
):
|
||||
base_queryset = _source_list_queryset()
|
||||
watchlist_queryset = _watchlist_queryset()
|
||||
mailbox_connections = list(
|
||||
request.user.mailbox_connections.filter(platform__in=PLATFORM_ALERTS)
|
||||
)
|
||||
active_search_profile = SearchProfile.objects.filter(user=request.user, is_active=True).first()
|
||||
return {
|
||||
"sources": base_queryset,
|
||||
"active_source_count": Source.objects.filter(status=Source.Status.ACTIVE).count(),
|
||||
"source_total_count": base_queryset.count(),
|
||||
"review_pending_count": Source.objects.filter(policy=Source.Policy.REVIEW).count(),
|
||||
"failed_source_count": Source.objects.filter(
|
||||
Q(status=Source.Status.QUARANTINED) | Q(failure_count__gt=0)
|
||||
).count(),
|
||||
"watchlist_sources": watchlist_queryset,
|
||||
"watchlist_count": watchlist_queryset.count(),
|
||||
"manual_import_form": form,
|
||||
"manual_import_bookmarklet_endpoint": request.build_absolute_uri(
|
||||
reverse("sources:manual_import")
|
||||
),
|
||||
"manual_import_paste_max_bytes": settings.MANUAL_IMPORT_PASTE_MAX_BYTES,
|
||||
"manual_import_result": manual_result,
|
||||
"platform_alert_ingress_ready": any(
|
||||
connection.enabled for connection in mailbox_connections
|
||||
),
|
||||
"mailbox_connections": mailbox_connections,
|
||||
"platform_alerts": PLATFORM_ALERTS.values(),
|
||||
"mailbox_form": mailbox_form or MailboxConnectionForm(),
|
||||
"editing_mailbox": editing_mailbox,
|
||||
"mailbox_credential_key_ready": bool(settings.MAILBOX_CREDENTIAL_KEYS),
|
||||
"active_search_profile": active_search_profile,
|
||||
}
|
||||
|
||||
|
||||
class SourceListView(LoginRequiredMixin, ListView):
|
||||
model = Source
|
||||
template_name = "sources/list.html"
|
||||
context_object_name = "sources"
|
||||
|
||||
def get_queryset(self):
|
||||
return _source_list_queryset()
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
context = super().get_context_data(**kwargs)
|
||||
context.update(
|
||||
_manual_import_context(
|
||||
self.request,
|
||||
form=ManualImportForm(
|
||||
initial={"source_url": self.request.GET.get("source_url", "")}
|
||||
),
|
||||
)
|
||||
)
|
||||
return context
|
||||
|
||||
|
||||
def _bulk_summary(action: str) -> str:
|
||||
mapping = {
|
||||
"allow": "toestaan",
|
||||
"approve_to_trial": "proefrun",
|
||||
"pause": "pauzeren",
|
||||
"dismiss": "afwijzen",
|
||||
}
|
||||
return mapping.get(action, "onbekend")
|
||||
|
||||
|
||||
def _action_to_review(
|
||||
action: str,
|
||||
) -> tuple[SourcePolicyReview.Decision, str, str, str]:
|
||||
if action == "allow":
|
||||
return (
|
||||
SourcePolicyReview.Decision.ALLOW,
|
||||
Source.Status.ACTIVE,
|
||||
Source.Policy.ALLOW,
|
||||
"Bron actief gemaakt door expliciete goedkeuring",
|
||||
)
|
||||
if action == "approve_to_trial":
|
||||
return (
|
||||
SourcePolicyReview.Decision.TRIAL,
|
||||
Source.Status.TRIAL,
|
||||
Source.Policy.REVIEW,
|
||||
"Bron naar proefrun voor technische controle",
|
||||
)
|
||||
if action == "pause":
|
||||
return (
|
||||
SourcePolicyReview.Decision.PAUSE,
|
||||
Source.Status.PAUSED,
|
||||
Source.Policy.REVIEW,
|
||||
"Bron tijdelijk gepauzeerd",
|
||||
)
|
||||
return (
|
||||
SourcePolicyReview.Decision.DENY,
|
||||
Source.Status.DISABLED,
|
||||
Source.Policy.DENY,
|
||||
"Bron afgewezen",
|
||||
)
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def bulk_candidate_action(request):
|
||||
action = request.POST.get("action")
|
||||
source_ids = [int(raw_id) for raw_id in request.POST.getlist("source_ids") if raw_id.isdigit()]
|
||||
sources = _source_list_queryset().filter(id__in=source_ids).exclude(policy=Source.Policy.DENY)
|
||||
if not sources.exists():
|
||||
messages.info(request, "Geen bruikbare bronregels geselecteerd.")
|
||||
return redirect("sources:list")
|
||||
|
||||
if action not in {"allow", "approve_to_trial", "pause", "dismiss"}:
|
||||
messages.error(request, "Onbekende actie.")
|
||||
return redirect("sources:list")
|
||||
|
||||
decision, status, policy, reason = _action_to_review(action)
|
||||
for source in sources:
|
||||
create_policy_review(
|
||||
source,
|
||||
actor=request.user,
|
||||
decision=decision,
|
||||
scope=SourcePolicyReview.Scope.SOURCE,
|
||||
reason=reason,
|
||||
notes="Bulkactie",
|
||||
)
|
||||
source.status = status
|
||||
source.policy = policy
|
||||
source.policy_reason = reason
|
||||
source.save(update_fields=["status", "policy", "policy_reason", "updated_at"])
|
||||
|
||||
messages.success(request, f"{sources.count()} bron(nen) naar {_bulk_summary(action)}.")
|
||||
return redirect("sources:list")
|
||||
|
||||
|
||||
@login_required
|
||||
def retry_source(request, pk: int):
|
||||
if request.method == "POST":
|
||||
source = get_object_or_404(Source, pk=pk)
|
||||
if source.policy == Source.Policy.DENY:
|
||||
messages.error(request, "Deze bron is door het bronbeleid geblokkeerd.")
|
||||
else:
|
||||
fetch_source.delay(source.pk, force=True)
|
||||
messages.success(request, "Broncontrole ingepland.")
|
||||
return redirect("sources:list")
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def save_mailbox(request, pk: int | None = None):
|
||||
instance = None
|
||||
if pk is not None:
|
||||
instance = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
|
||||
form = MailboxConnectionForm(request.POST, instance=instance)
|
||||
if form.is_valid():
|
||||
try:
|
||||
connection = save_mailbox_connection(
|
||||
user=request.user,
|
||||
cleaned_data=dict(form.cleaned_data),
|
||||
instance=instance,
|
||||
)
|
||||
except MailboxCredentialError as exc:
|
||||
form.add_error("password", str(exc))
|
||||
else:
|
||||
poll_mailbox.delay(connection.pk, force=True)
|
||||
messages.success(
|
||||
request,
|
||||
f"{connection.get_platform_display()}-mailbox opgeslagen; "
|
||||
"verbindingstest ingepland.",
|
||||
)
|
||||
return redirect("sources:list")
|
||||
messages.error(request, "Controleer de mailboxinstellingen.")
|
||||
return render(
|
||||
request,
|
||||
"sources/list.html",
|
||||
_manual_import_context(
|
||||
request,
|
||||
form=ManualImportForm(),
|
||||
mailbox_form=form,
|
||||
editing_mailbox=instance,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@login_required
|
||||
def edit_mailbox(request, pk: int):
|
||||
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
|
||||
if request.method == "POST":
|
||||
return save_mailbox(request, pk=pk)
|
||||
return render(
|
||||
request,
|
||||
"sources/list.html",
|
||||
_manual_import_context(
|
||||
request,
|
||||
form=ManualImportForm(),
|
||||
mailbox_form=MailboxConnectionForm(instance=connection),
|
||||
editing_mailbox=connection,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def sync_platform_alerts(request, pk: int):
|
||||
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
|
||||
if not connection.enabled:
|
||||
messages.error(request, "Activeer deze mailbox voordat je synchroniseert.")
|
||||
else:
|
||||
poll_mailbox.delay(connection.pk, force=True)
|
||||
messages.success(request, f"{connection.get_platform_display()}-mailimport ingepland.")
|
||||
return redirect("sources:list")
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def toggle_mailbox(request, pk: int):
|
||||
connection = get_object_or_404(MailboxConnection, pk=pk, user=request.user)
|
||||
connection.enabled = not connection.enabled
|
||||
connection.next_poll_at = timezone.now()
|
||||
connection.save(update_fields=["enabled", "next_poll_at", "updated_at"])
|
||||
state = "geactiveerd" if connection.enabled else "gepauzeerd"
|
||||
messages.success(request, f"{connection.get_platform_display()}-mailbox {state}.")
|
||||
return redirect("sources:list")
|
||||
|
||||
|
||||
@login_required
|
||||
@never_cache
|
||||
def manual_import_view(request):
|
||||
rate_limit = is_rate_limited(
|
||||
request,
|
||||
namespace="manual_import",
|
||||
max_attempts=settings.MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS,
|
||||
window_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_WINDOW_SECONDS,
|
||||
block_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_BLOCK_SECONDS,
|
||||
)
|
||||
if rate_limit.is_blocked:
|
||||
messages.error(
|
||||
request,
|
||||
"Te veel handmatige importverzoeken. Wacht 5 minuten en probeer daarna opnieuw.",
|
||||
)
|
||||
return render(
|
||||
request,
|
||||
"sources/list.html",
|
||||
_manual_import_context(request, form=ManualImportForm(), manual_result=None),
|
||||
)
|
||||
|
||||
if request.method == "POST":
|
||||
form = ManualImportForm(request.POST)
|
||||
result_payload = None
|
||||
if not form.is_valid():
|
||||
messages.error(request, "Controleer de handmatige importvelden.")
|
||||
return render(
|
||||
request,
|
||||
"sources/list.html",
|
||||
_manual_import_context(request, form=form, manual_result=result_payload),
|
||||
)
|
||||
|
||||
cleaned = form.cleaned_data
|
||||
try:
|
||||
summary: ManualImportSummary = import_manual_source(
|
||||
actor=request.user,
|
||||
source_url=cleaned.get("source_url"),
|
||||
pasted_text=cleaned.get("pasted_text"),
|
||||
)
|
||||
result_payload = summary.to_session_payload()
|
||||
messages.success(
|
||||
request,
|
||||
"Handmatige import uitgevoerd. Bekijk het resultaat hieronder.",
|
||||
)
|
||||
clear_rate_limit(request, namespace="manual_import")
|
||||
form = ManualImportForm()
|
||||
except ManualImportError as exc:
|
||||
messages.error(request, str(exc))
|
||||
register_rate_limit_failure(
|
||||
request,
|
||||
namespace="manual_import",
|
||||
max_attempts=settings.MANUAL_IMPORT_RATE_LIMIT_MAX_ATTEMPTS,
|
||||
window_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_WINDOW_SECONDS,
|
||||
block_seconds=settings.MANUAL_IMPORT_RATE_LIMIT_BLOCK_SECONDS,
|
||||
)
|
||||
|
||||
return render(
|
||||
request,
|
||||
"sources/list.html",
|
||||
_manual_import_context(request, form=form, manual_result=result_payload),
|
||||
)
|
||||
|
||||
return render(
|
||||
request,
|
||||
"sources/list.html",
|
||||
_manual_import_context(
|
||||
request,
|
||||
form=ManualImportForm(initial={"source_url": request.GET.get("source_url", "")}),
|
||||
),
|
||||
)
|
||||
Reference in new issue
Block a user