Publish curated VacatureRadar source snapshot
This commit is contained in:
Jens committed
commit
e29774cb7f
338 files changed
+32699
No files matched your search
@@ -0,0 +1,488 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import field as dataclass_field
|
||||
from decimal import Decimal
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from django.db import IntegrityError, transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.jobs.models import (
|
||||
Employer,
|
||||
FieldProvenance,
|
||||
JobPosting,
|
||||
JobSourceAlias,
|
||||
JobVersion,
|
||||
ScoreRun,
|
||||
)
|
||||
from apps.profiles.models import SearchProfile
|
||||
from apps.sources.adapters.base import FieldEvidence
|
||||
from apps.sources.adapters.registry import registry
|
||||
from apps.sources.models import RawDocument, Source
|
||||
from apps.sources.services.policy import is_denied_domain
|
||||
|
||||
from .dedupe import DedupeDecision, find_existing_job
|
||||
from .features import extract_deterministic_features
|
||||
from .normalization import CanonicalJobDraft, normalize_extracted_job, normalize_token
|
||||
from .scoring import score_and_save
|
||||
|
||||
RECRUITER_TERMS = {
|
||||
"recruitment",
|
||||
"recruiter",
|
||||
"staffing",
|
||||
"interim",
|
||||
"consultancy",
|
||||
"consulting",
|
||||
"talent",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PersistenceContext:
|
||||
"""Run-scoped reference cache; never shared between workers or imports."""
|
||||
|
||||
employers: dict[tuple[str, str], Employer | None] = dataclass_field(default_factory=dict)
|
||||
active_profiles: list[SearchProfile] | None = None
|
||||
latest_scores: dict[tuple[object, int], ScoreRun] = dataclass_field(default_factory=dict)
|
||||
|
||||
|
||||
def _confidence(value: float) -> Decimal:
|
||||
return Decimal(str(max(0.0, min(1.0, value))))
|
||||
|
||||
|
||||
def resolve_employer(
|
||||
draft: CanonicalJobDraft, *, context: PersistenceContext | None = None
|
||||
) -> Employer | None:
|
||||
name = draft.employer_name.strip()
|
||||
domain = draft.employer_domain.strip()
|
||||
if not name and not domain:
|
||||
return None
|
||||
display_name = name or domain
|
||||
normalized = normalize_token(display_name)
|
||||
cache_key = (normalized, domain)
|
||||
if context is not None and cache_key in context.employers:
|
||||
return context.employers[cache_key]
|
||||
recruiter = any(term in normalized for term in RECRUITER_TERMS)
|
||||
employer, _ = Employer.objects.get_or_create(
|
||||
normalized_name=normalized,
|
||||
domain=domain,
|
||||
defaults={
|
||||
"name": display_name,
|
||||
"is_direct_employer": not recruiter,
|
||||
"is_recruiter": recruiter,
|
||||
"confidence": _confidence(0.75 if name else 0.45),
|
||||
},
|
||||
)
|
||||
changed: list[str] = []
|
||||
if name and employer.name != name and len(name) > len(employer.name):
|
||||
employer.name = name
|
||||
changed.append("name")
|
||||
if recruiter and not employer.is_recruiter:
|
||||
employer.is_recruiter = True
|
||||
employer.is_direct_employer = False
|
||||
changed.extend(["is_recruiter", "is_direct_employer"])
|
||||
if changed:
|
||||
employer.save(update_fields=[*changed, "updated_at"])
|
||||
if context is not None:
|
||||
context.employers[cache_key] = employer
|
||||
return employer
|
||||
|
||||
|
||||
def job_snapshot(job: JobPosting) -> dict[str, object]:
|
||||
return {
|
||||
"id": str(job.id),
|
||||
"title": job.original_title,
|
||||
"normalized_title": job.normalized_title,
|
||||
"employer": job.employer_name,
|
||||
"canonical_url": job.canonical_url,
|
||||
"location": job.raw_location,
|
||||
"region": job.region,
|
||||
"municipality": job.municipality,
|
||||
"workplace_type": job.workplace_type,
|
||||
"employment_types": job.employment_types,
|
||||
"description_text": job.description_text,
|
||||
"skills_required": job.skills_required,
|
||||
"skills_preferred": job.skills_preferred,
|
||||
"date_posted": job.date_posted.isoformat() if job.date_posted else None,
|
||||
"valid_through": job.valid_through.isoformat() if job.valid_through else None,
|
||||
"status": job.status,
|
||||
"content_hash": job.content_hash,
|
||||
}
|
||||
|
||||
|
||||
def _source_is_direct(source: Source | None, draft: CanonicalJobDraft) -> bool:
|
||||
host = (urlsplit(draft.canonical_url).hostname or "").lower()
|
||||
if is_denied_domain(host):
|
||||
return False
|
||||
if not source:
|
||||
return False
|
||||
metadata = source.metadata if isinstance(source.metadata, dict) else {}
|
||||
return source.source_type == Source.Type.EMPLOYER or metadata.get("direct_employer") is True
|
||||
|
||||
|
||||
def _alias_payload(
|
||||
raw_payload: object,
|
||||
decision: DedupeDecision,
|
||||
*,
|
||||
fallback_canonical_url: str | None,
|
||||
) -> dict[str, object]:
|
||||
if not isinstance(raw_payload, dict):
|
||||
raw_payload = {}
|
||||
payload: dict[str, object] = dict(raw_payload)
|
||||
if decision.reason == "review_direct_conflict" or decision.resolved_direct:
|
||||
payload["employer_resolution"] = {
|
||||
"reason": decision.reason,
|
||||
"confidence": float(decision.similarity),
|
||||
"canonical_url": decision.canonical_url or fallback_canonical_url,
|
||||
"resolved_direct": decision.resolved_direct,
|
||||
"evidence": decision.evidence,
|
||||
}
|
||||
return payload
|
||||
|
||||
|
||||
def _apply_draft(
|
||||
job: JobPosting,
|
||||
draft: CanonicalJobDraft,
|
||||
employer: Employer | None,
|
||||
*,
|
||||
direct: bool,
|
||||
resolved_canonical_url: str | None = None,
|
||||
) -> list[str]:
|
||||
canonical_url = resolved_canonical_url if direct else draft.canonical_url
|
||||
fields = {
|
||||
"employer": employer,
|
||||
"original_title": draft.title,
|
||||
"normalized_title": draft.normalized_title,
|
||||
"job_family": draft.job_family,
|
||||
"language": draft.language,
|
||||
"description_html_sanitized": draft.description_html,
|
||||
"description_text": draft.description_text,
|
||||
"raw_location": draft.location_text,
|
||||
"country": draft.country,
|
||||
"region": draft.region,
|
||||
"municipality": draft.municipality,
|
||||
"postal_code": draft.postal_code,
|
||||
"workplace_type": draft.workplace_type,
|
||||
"employment_types": draft.employment_types,
|
||||
"compensation": draft.compensation,
|
||||
"skills_required": draft.skills_required,
|
||||
"skills_preferred": draft.skills_preferred,
|
||||
"date_posted": draft.date_posted,
|
||||
"valid_through": draft.valid_through,
|
||||
"content_hash": draft.content_hash,
|
||||
"analysis_features": extract_deterministic_features(draft.title, draft.description_text),
|
||||
"status": JobPosting.Status.ACTIVE,
|
||||
"last_seen": timezone.now(),
|
||||
"direct_employer": direct or (employer.is_direct_employer if employer else False),
|
||||
"recruiter": employer.is_recruiter if employer else not direct,
|
||||
}
|
||||
changed: list[str] = []
|
||||
for field, value in fields.items():
|
||||
if (value not in (None, "", [], {}) or field in {"status", "last_seen"}) and (
|
||||
getattr(job, field) != value
|
||||
):
|
||||
setattr(job, field, value)
|
||||
changed.append(field)
|
||||
if direct and canonical_url and job.canonical_url != canonical_url:
|
||||
job.canonical_url = canonical_url
|
||||
changed.append("canonical_url")
|
||||
substantive_changes = [field for field in changed if field != "last_seen"]
|
||||
if substantive_changes and "last_changed" not in changed:
|
||||
job.last_changed = timezone.now()
|
||||
changed.append("last_changed")
|
||||
return changed
|
||||
|
||||
|
||||
def _sync_evidence(
|
||||
*,
|
||||
job: JobPosting,
|
||||
alias: JobSourceAlias,
|
||||
evidence_items: list[FieldEvidence],
|
||||
parser_version: str,
|
||||
) -> None:
|
||||
existing = {
|
||||
(item.field_name, item.extraction_method): item
|
||||
for item in FieldProvenance.objects.filter(job=job, source_alias=alias)
|
||||
}
|
||||
creates: list[FieldProvenance] = []
|
||||
updates: list[FieldProvenance] = []
|
||||
for evidence in evidence_items:
|
||||
key = (evidence.field_name, evidence.method)
|
||||
values = {
|
||||
"confidence": _confidence(evidence.confidence),
|
||||
"evidence_excerpt": evidence.evidence[:1000],
|
||||
"parser_version": parser_version,
|
||||
}
|
||||
current = existing.get(key)
|
||||
if current is None:
|
||||
creates.append(
|
||||
FieldProvenance(
|
||||
job=job,
|
||||
source_alias=alias,
|
||||
field_name=evidence.field_name,
|
||||
extraction_method=evidence.method,
|
||||
**values,
|
||||
)
|
||||
)
|
||||
continue
|
||||
changed = False
|
||||
for field_name, value in values.items():
|
||||
if getattr(current, field_name) != value:
|
||||
setattr(current, field_name, value)
|
||||
changed = True
|
||||
if changed:
|
||||
current.updated_at = timezone.now()
|
||||
updates.append(current)
|
||||
if creates:
|
||||
FieldProvenance.objects.bulk_create(creates, batch_size=250)
|
||||
if updates:
|
||||
FieldProvenance.objects.bulk_update(
|
||||
updates,
|
||||
["confidence", "evidence_excerpt", "parser_version", "updated_at"],
|
||||
batch_size=250,
|
||||
)
|
||||
|
||||
|
||||
def _copy_score(score: ScoreRun) -> ScoreRun:
|
||||
return ScoreRun.objects.create(
|
||||
job=score.job,
|
||||
profile=score.profile,
|
||||
profile_version=score.profile_version,
|
||||
score=score.score,
|
||||
confidence=score.confidence,
|
||||
recommendation=score.recommendation,
|
||||
components=score.components,
|
||||
positives=score.positives,
|
||||
concerns=score.concerns,
|
||||
hard_exclusions=score.hard_exclusions,
|
||||
evidence=score.evidence,
|
||||
model_version=score.model_version,
|
||||
prompt_version=score.prompt_version,
|
||||
)
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def persist_draft(
|
||||
draft: CanonicalJobDraft,
|
||||
*,
|
||||
document: RawDocument,
|
||||
parser_key: str,
|
||||
parser_version: str,
|
||||
extraction_confidence: float,
|
||||
context: PersistenceContext | None = None,
|
||||
) -> tuple[JobPosting, DedupeDecision, bool]:
|
||||
source = document.source
|
||||
employer = resolve_employer(draft, context=context)
|
||||
decision = find_existing_job(draft, source=source)
|
||||
direct = _source_is_direct(source, draft)
|
||||
if decision.resolved_direct:
|
||||
direct = True
|
||||
created = False
|
||||
substantive_change = False
|
||||
|
||||
if decision.job is None:
|
||||
try:
|
||||
job = JobPosting.objects.create(
|
||||
employer=employer,
|
||||
original_title=draft.title,
|
||||
normalized_title=draft.normalized_title,
|
||||
job_family=draft.job_family,
|
||||
language=draft.language,
|
||||
canonical_url=draft.canonical_url,
|
||||
canonical_key=draft.canonical_key,
|
||||
content_hash=draft.content_hash,
|
||||
description_html_sanitized=draft.description_html,
|
||||
description_text=draft.description_text,
|
||||
raw_location=draft.location_text,
|
||||
country=draft.country,
|
||||
region=draft.region,
|
||||
municipality=draft.municipality,
|
||||
postal_code=draft.postal_code,
|
||||
workplace_type=draft.workplace_type,
|
||||
employment_types=draft.employment_types,
|
||||
compensation=draft.compensation,
|
||||
skills_required=draft.skills_required,
|
||||
skills_preferred=draft.skills_preferred,
|
||||
date_posted=draft.date_posted,
|
||||
valid_through=draft.valid_through,
|
||||
direct_employer=direct or (employer.is_direct_employer if employer else False),
|
||||
recruiter=employer.is_recruiter if employer else not direct,
|
||||
extraction_confidence=_confidence(extraction_confidence),
|
||||
analysis_features=extract_deterministic_features(
|
||||
draft.title, draft.description_text
|
||||
),
|
||||
status=JobPosting.Status.ACTIVE,
|
||||
)
|
||||
created = True
|
||||
except IntegrityError:
|
||||
job = JobPosting.objects.get(canonical_key=draft.canonical_key)
|
||||
decision = DedupeDecision(job, "canonical_key_race", 1.0)
|
||||
else:
|
||||
job = decision.job
|
||||
if job.content_hash != draft.content_hash:
|
||||
JobVersion.objects.get_or_create(
|
||||
job=job,
|
||||
content_hash=job.content_hash,
|
||||
defaults={"snapshot": job_snapshot(job), "changed_fields": []},
|
||||
)
|
||||
changed = _apply_draft(
|
||||
job,
|
||||
draft,
|
||||
employer,
|
||||
direct=direct,
|
||||
resolved_canonical_url=decision.canonical_url,
|
||||
)
|
||||
if extraction_confidence > float(job.extraction_confidence):
|
||||
job.extraction_confidence = _confidence(extraction_confidence)
|
||||
changed.append("extraction_confidence")
|
||||
substantive_change = any(field not in {"last_seen", "last_changed"} for field in changed)
|
||||
if changed:
|
||||
job.save(update_fields=list(dict.fromkeys([*changed, "updated_at"])))
|
||||
|
||||
alias = (
|
||||
JobSourceAlias.objects.filter(
|
||||
job=job,
|
||||
source=source,
|
||||
canonical_url=draft.canonical_url,
|
||||
external_id=draft.external_id,
|
||||
)
|
||||
.order_by("-last_seen")
|
||||
.first()
|
||||
)
|
||||
if alias is None:
|
||||
alias = JobSourceAlias.objects.create(
|
||||
job=job,
|
||||
source=source,
|
||||
raw_document=document,
|
||||
url=draft.source_url,
|
||||
canonical_url=draft.canonical_url,
|
||||
external_id=draft.external_id,
|
||||
source_title=draft.title,
|
||||
source_employer=draft.employer_name,
|
||||
extraction_method=parser_key,
|
||||
extraction_confidence=_confidence(extraction_confidence),
|
||||
is_canonical=direct,
|
||||
payload=_alias_payload(
|
||||
draft.raw,
|
||||
decision,
|
||||
fallback_canonical_url=draft.canonical_url,
|
||||
),
|
||||
)
|
||||
else:
|
||||
alias.last_seen = timezone.now()
|
||||
alias.raw_document = document
|
||||
alias.payload = _alias_payload(
|
||||
alias.payload, decision, fallback_canonical_url=draft.canonical_url
|
||||
)
|
||||
if direct:
|
||||
alias.is_canonical = True
|
||||
alias.save(
|
||||
update_fields=["last_seen", "raw_document", "payload", "is_canonical", "updated_at"]
|
||||
)
|
||||
|
||||
_sync_evidence(
|
||||
job=job,
|
||||
alias=alias,
|
||||
evidence_items=draft.evidence,
|
||||
parser_version=parser_version,
|
||||
)
|
||||
|
||||
if created or substantive_change:
|
||||
JobVersion.objects.get_or_create(
|
||||
job=job,
|
||||
content_hash=job.content_hash,
|
||||
defaults={"snapshot": job_snapshot(job), "changed_fields": []},
|
||||
)
|
||||
if context is not None:
|
||||
if context.active_profiles is None:
|
||||
context.active_profiles = list(SearchProfile.objects.filter(is_active=True))
|
||||
profiles = context.active_profiles
|
||||
else:
|
||||
profiles = SearchProfile.objects.filter(is_active=True)
|
||||
for profile in profiles:
|
||||
score = score_and_save(job, profile)
|
||||
if context is not None:
|
||||
context.latest_scores[(job.pk, profile.pk)] = score
|
||||
else:
|
||||
if context is not None:
|
||||
if context.active_profiles is None:
|
||||
context.active_profiles = list(SearchProfile.objects.filter(is_active=True))
|
||||
profiles = context.active_profiles
|
||||
else:
|
||||
profiles = SearchProfile.objects.filter(is_active=True)
|
||||
for profile in profiles:
|
||||
cache_key = (job.pk, profile.pk)
|
||||
previous = context.latest_scores.get(cache_key) if context is not None else None
|
||||
if previous is None:
|
||||
previous = (
|
||||
ScoreRun.objects.filter(job=job, profile=profile)
|
||||
.order_by("-created_at")
|
||||
.first()
|
||||
)
|
||||
if previous is None or previous.profile_version != profile.version:
|
||||
score = score_and_save(job, profile)
|
||||
else:
|
||||
score = _copy_score(previous)
|
||||
if context is not None:
|
||||
context.latest_scores[cache_key] = score
|
||||
return job, decision, created
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def process_raw_document(
|
||||
document: RawDocument, *, context: PersistenceContext | None = None
|
||||
) -> dict[str, int | str | list[str]]:
|
||||
result = registry.extract(document)
|
||||
document.parser_key = result.parser_key
|
||||
document.parser_version = result.parser_version
|
||||
document.extraction_confidence = _confidence(result.confidence)
|
||||
document.save(
|
||||
update_fields=["parser_key", "parser_version", "extraction_confidence", "updated_at"]
|
||||
)
|
||||
created = updated = duplicates = 0
|
||||
for extracted in result.jobs:
|
||||
source_metadata = (
|
||||
document.source.metadata
|
||||
if document.source and isinstance(document.source.metadata, dict)
|
||||
else {}
|
||||
)
|
||||
configured_employer = str(source_metadata.get("employer_name") or "").strip()
|
||||
if configured_employer:
|
||||
extracted.employer_name = configured_employer
|
||||
extracted.evidence.append(
|
||||
FieldEvidence(
|
||||
"employer_name",
|
||||
"reviewed-source-config",
|
||||
0.99,
|
||||
configured_employer,
|
||||
)
|
||||
)
|
||||
draft = normalize_extracted_job(extracted)
|
||||
_, decision, was_created = persist_draft(
|
||||
draft,
|
||||
document=document,
|
||||
parser_key=result.parser_key,
|
||||
parser_version=result.parser_version,
|
||||
extraction_confidence=result.confidence,
|
||||
context=context,
|
||||
)
|
||||
if was_created:
|
||||
created += 1
|
||||
elif (
|
||||
decision.reason.startswith("exact")
|
||||
or decision.reason.startswith("fuzzy")
|
||||
or decision.reason == "resolved_direct_match"
|
||||
):
|
||||
duplicates += 1
|
||||
updated += 1
|
||||
else:
|
||||
updated += 1
|
||||
return {
|
||||
"extracted": len(result.jobs),
|
||||
"created": created,
|
||||
"updated": updated,
|
||||
"duplicates": duplicates,
|
||||
"parser": result.parser_key,
|
||||
"warnings": result.warnings,
|
||||
}
|
||||
Reference in new issue
Block a user