Publish curated VacatureRadar source snapshot
This commit is contained in:
Jens committed
commit
e29774cb7f
338 files changed
+32699
No files matched your search
@@ -0,0 +1,136 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-20 23:57
|
||||
|
||||
import django.db.models.deletion
|
||||
import django.utils.timezone
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='RawDocument',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('url', models.URLField(blank=True, max_length=2000)),
|
||||
('final_url', models.URLField(blank=True, max_length=2000)),
|
||||
('kind', models.CharField(choices=[('html', 'HTML'), ('xml', 'XML'), ('email', 'E-mail'), ('json', 'JSON'), ('text', 'Tekst')], max_length=16)),
|
||||
('content_type', models.CharField(blank=True, max_length=200)),
|
||||
('http_status', models.PositiveSmallIntegerField(blank=True, null=True)),
|
||||
('response_headers', models.JSONField(blank=True, default=dict)),
|
||||
('content_hash', models.CharField(db_index=True, max_length=64)),
|
||||
('body_text', models.TextField(blank=True)),
|
||||
('byte_length', models.PositiveIntegerField(default=0)),
|
||||
('parser_key', models.CharField(blank=True, max_length=120)),
|
||||
('parser_version', models.CharField(blank=True, max_length=80)),
|
||||
('extraction_confidence', models.DecimalField(decimal_places=3, default=0, max_digits=4)),
|
||||
('retain_until', models.DateTimeField(blank=True, null=True)),
|
||||
('quarantined', models.BooleanField(default=False)),
|
||||
('quarantine_reason', models.CharField(blank=True, max_length=500)),
|
||||
('metadata', models.JSONField(blank=True, default=dict)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-created_at'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='EmailMessageRecord',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('message_id', models.CharField(max_length=998, unique=True)),
|
||||
('mailbox', models.CharField(default='INBOX', max_length=255)),
|
||||
('sender', models.CharField(blank=True, max_length=500)),
|
||||
('subject', models.CharField(blank=True, max_length=998)),
|
||||
('received_at', models.DateTimeField(blank=True, null=True)),
|
||||
('links', models.JSONField(blank=True, default=list)),
|
||||
('processed', models.BooleanField(default=False)),
|
||||
('error_message', models.CharField(blank=True, max_length=1000)),
|
||||
('raw_document', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='email_record', to='sources.rawdocument')),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-received_at', '-created_at'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Source',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('name', models.CharField(max_length=200)),
|
||||
('source_type', models.CharField(choices=[('employer', 'Werkgeverspagina'), ('ats', 'Publieke ATS-pagina'), ('rss', 'RSS/Atom'), ('sitemap', 'Sitemap'), ('email', 'Vacaturemail'), ('manual', 'Handmatige import')], max_length=24)),
|
||||
('base_url', models.URLField(blank=True, max_length=1000)),
|
||||
('domain', models.CharField(db_index=True, max_length=255)),
|
||||
('status', models.CharField(choices=[('candidate', 'Kandidaat'), ('trial', 'Proefrun'), ('active', 'Actief'), ('quarantined', 'Quarantaine'), ('paused', 'Gepauzeerd'), ('disabled', 'Uitgeschakeld')], default='candidate', max_length=24)),
|
||||
('policy', models.CharField(choices=[('allow', 'Toestaan'), ('review', 'Te beoordelen'), ('deny', 'Blokkeren')], default='review', max_length=16)),
|
||||
('policy_reason', models.CharField(blank=True, max_length=500)),
|
||||
('parser_key', models.CharField(default='auto', max_length=120)),
|
||||
('strict_mode', models.BooleanField(default=True)),
|
||||
('allow_public_endpoint', models.BooleanField(default=False)),
|
||||
('honor_robots', models.BooleanField(default=True)),
|
||||
('crawl_interval_minutes', models.PositiveIntegerField(default=720)),
|
||||
('minimum_interval_seconds', models.PositiveIntegerField(default=30)),
|
||||
('max_concurrency', models.PositiveSmallIntegerField(default=1)),
|
||||
('next_run_at', models.DateTimeField(blank=True, db_index=True, null=True)),
|
||||
('last_success_at', models.DateTimeField(blank=True, null=True)),
|
||||
('last_failure_at', models.DateTimeField(blank=True, null=True)),
|
||||
('failure_count', models.PositiveIntegerField(default=0)),
|
||||
('etag', models.CharField(blank=True, max_length=500)),
|
||||
('last_modified', models.CharField(blank=True, max_length=500)),
|
||||
('robots_checked_at', models.DateTimeField(blank=True, null=True)),
|
||||
('terms_checked_at', models.DateTimeField(blank=True, null=True)),
|
||||
('metadata', models.JSONField(blank=True, default=dict)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['name'],
|
||||
'constraints': [models.UniqueConstraint(fields=('domain', 'source_type'), name='unique_domain_source_type')],
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='rawdocument',
|
||||
name='source',
|
||||
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='sources.source'),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='SourceRun',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('status', models.CharField(choices=[('running', 'Bezig'), ('success', 'Geslaagd'), ('partial', 'Gedeeltelijk'), ('failed', 'Mislukt'), ('skipped', 'Overgeslagen')], default='running', max_length=16)),
|
||||
('started_at', models.DateTimeField(default=django.utils.timezone.now)),
|
||||
('finished_at', models.DateTimeField(blank=True, null=True)),
|
||||
('http_status', models.PositiveSmallIntegerField(blank=True, null=True)),
|
||||
('discovered_count', models.PositiveIntegerField(default=0)),
|
||||
('extracted_count', models.PositiveIntegerField(default=0)),
|
||||
('created_count', models.PositiveIntegerField(default=0)),
|
||||
('updated_count', models.PositiveIntegerField(default=0)),
|
||||
('duplicate_count', models.PositiveIntegerField(default=0)),
|
||||
('error_category', models.CharField(blank=True, max_length=80)),
|
||||
('error_message', models.CharField(blank=True, max_length=1000)),
|
||||
('metrics', models.JSONField(blank=True, default=dict)),
|
||||
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='runs', to='sources.source')),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-started_at'],
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='rawdocument',
|
||||
name='source_run',
|
||||
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='sources.sourcerun'),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name='rawdocument',
|
||||
index=models.Index(fields=['source', 'content_hash'], name='sources_raw_source__4a7b2a_idx'),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,51 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-21 for VR-102
|
||||
|
||||
import django.conf
|
||||
import django.db.models.deletion
|
||||
import django.utils.timezone
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
("sources", "0001_initial"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="SourcePolicyReview",
|
||||
fields=[
|
||||
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
|
||||
("created_at", models.DateTimeField(auto_now_add=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("scope", models.CharField(choices=[("source", "Bronspecifiek"), ("domain", "Domeinbrede controle")], default="source", max_length=20)),
|
||||
("decision", models.CharField(choices=[("allow", "Toestaan"), ("trial", "Proefrun"), ("pause", "Pauzeren"), ("deny", "Blokkeren"),], max_length=16)),
|
||||
("reason", models.CharField(max_length=500)),
|
||||
("evidence_link", models.URLField(blank=True, max_length=500)),
|
||||
("notes", models.TextField(blank=True)),
|
||||
("expires_at", models.DateTimeField(blank=True, null=True)),
|
||||
("metadata", models.JSONField(blank=True, default=dict)),
|
||||
("actor", models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="policy_reviews", to=django.conf.settings.AUTH_USER_MODEL)),
|
||||
("source", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name="policy_reviews", to="sources.source")),
|
||||
],
|
||||
options={"ordering": ["-created_at"]},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="SourceRobotsCache",
|
||||
fields=[
|
||||
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
|
||||
("created_at", models.DateTimeField(auto_now_add=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("origin", models.CharField(max_length=255, unique=True)),
|
||||
("expires_at", models.DateTimeField()),
|
||||
("etag", models.CharField(blank=True, max_length=255)),
|
||||
("last_modified", models.CharField(blank=True, max_length=255)),
|
||||
("allow_rules", models.JSONField(blank=True, default=dict)),
|
||||
("disallow_rules", models.JSONField(blank=True, default=dict)),
|
||||
("error", models.CharField(blank=True, max_length=500)),
|
||||
("byte_length", models.PositiveIntegerField(default=0)),
|
||||
],
|
||||
options={"ordering": ["origin"]},
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,45 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-21 for VR-103
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
("sources", "0002_policy_review_and_robots_cache"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="SourceLease",
|
||||
fields=[
|
||||
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
|
||||
("created_at", models.DateTimeField(auto_now_add=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("token", models.CharField(default="", max_length=64)),
|
||||
("worker_id", models.CharField(blank=True, max_length=128)),
|
||||
("expires_at", models.DateTimeField(db_index=True)),
|
||||
(
|
||||
"source",
|
||||
models.OneToOneField(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="lease",
|
||||
to="sources.source",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={"ordering": ["source"]},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="SourceOriginState",
|
||||
fields=[
|
||||
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
|
||||
("created_at", models.DateTimeField(auto_now_add=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("domain", models.CharField(max_length=255, unique=True)),
|
||||
("next_allowed_at", models.DateTimeField()),
|
||||
],
|
||||
options={"ordering": ["domain"]},
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,21 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-21 22:59
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sources', '0003_source_lease_and_origin_state'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveConstraint(
|
||||
model_name='source',
|
||||
name='unique_domain_source_type',
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='source',
|
||||
constraint=models.UniqueConstraint(condition=models.Q(('base_url', ''), _negated=True), fields=('source_type', 'base_url'), name='unique_source_type_base_url_nonempty'),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,67 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-22 00:12
|
||||
|
||||
import django.db.models.deletion
|
||||
import django.utils.timezone
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sources', '0004_remove_source_unique_domain_source_type_and_more'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='emailmessagerecord',
|
||||
name='message_id',
|
||||
field=models.CharField(max_length=998),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='MailboxConnection',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('platform', models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed')], max_length=24)),
|
||||
('provider', models.CharField(choices=[('gmail', 'Gmail'), ('outlook', 'Outlook / Microsoft 365'), ('custom', 'Andere IMAP-provider')], max_length=24)),
|
||||
('custom_host', models.CharField(blank=True, max_length=255)),
|
||||
('port', models.PositiveIntegerField(default=993)),
|
||||
('username', models.CharField(max_length=320)),
|
||||
('encrypted_password', models.TextField()),
|
||||
('mailbox', models.CharField(default='INBOX', max_length=255)),
|
||||
('enabled', models.BooleanField(default=True)),
|
||||
('poll_interval_minutes', models.PositiveIntegerField(choices=[(5, 'Elke 5 minuten'), (15, 'Elke 15 minuten'), (30, 'Elke 30 minuten'), (60, 'Elk uur'), (180, 'Elke 3 uur'), (360, 'Elke 6 uur')], default=30)),
|
||||
('next_poll_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
|
||||
('last_polled_at', models.DateTimeField(blank=True, null=True)),
|
||||
('last_success_at', models.DateTimeField(blank=True, null=True)),
|
||||
('last_error_category', models.CharField(blank=True, max_length=80)),
|
||||
('last_error_message', models.CharField(blank=True, max_length=500)),
|
||||
('lease_token', models.CharField(blank=True, max_length=64)),
|
||||
('lease_expires_at', models.DateTimeField(blank=True, db_index=True, null=True)),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='mailbox_connections', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['platform'],
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='emailmessagerecord',
|
||||
name='mailbox_connection',
|
||||
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='sources.mailboxconnection'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='emailmessagerecord',
|
||||
constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', False)), fields=('mailbox_connection', 'message_id'), name='unique_message_per_mailbox_connection'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='emailmessagerecord',
|
||||
constraint=models.UniqueConstraint(condition=models.Q(('mailbox_connection__isnull', True)), fields=('message_id',), name='unique_legacy_email_message_id'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='mailboxconnection',
|
||||
constraint=models.UniqueConstraint(fields=('user', 'platform'), name='unique_mailbox_platform_per_user'),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-22 00:48
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sources', '0005_alter_emailmessagerecord_message_id_and_more'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='mailboxconnection',
|
||||
name='platform',
|
||||
field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('eurobrussels', 'EuroBrussels'), ('brusselsjobs', 'BrusselsJobs')], max_length=24),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,26 @@
|
||||
# Generated by Django 5.2.16 on 2026-07-22 01:25
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
def disable_retired_brussels_mailboxes(apps, schema_editor):
|
||||
mailbox_connection = apps.get_model("sources", "MailboxConnection")
|
||||
mailbox_connection.objects.filter(
|
||||
platform__in=("eurobrussels", "brusselsjobs")
|
||||
).update(enabled=False)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('sources', '0006_alter_mailboxconnection_platform'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(disable_retired_brussels_mailboxes, migrations.RunPython.noop),
|
||||
migrations.AlterField(
|
||||
model_name='mailboxconnection',
|
||||
name='platform',
|
||||
field=models.CharField(choices=[('vdab', 'VDAB'), ('indeed', 'Indeed'), ('linkedin', 'LinkedIn'), ('ictjob', 'ictjob.be'), ('jobat', 'Jobat'), ('stepstone', 'StepStone'), ('careerjet', 'Careerjet'), ('randstad', 'Randstad'), ('roberthalf', 'Robert Half')], max_length=24),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,13 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [("sources", "0007_alter_mailboxconnection_platform")]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="mailboxconnection",
|
||||
name="encrypted_password",
|
||||
field=models.TextField(blank=True),
|
||||
)
|
||||
]
|
||||
Whitespace-only changes.
Reference in new issue
Block a user