Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

This commit is contained in:
Public Source Publisher committed 2026-10-01 01:34:21 +02:00
commit 7ca726caf2
659 files changed
+98171

No files matched your search

+185
View File
@@ -0,0 +1,185 @@
from __future__ import annotations
import asyncio
import sys
from dataclasses import replace
from datetime import UTC, datetime, timedelta
from uuid import uuid4
import pytest
import pytest_asyncio
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ragcore.application.jobs import (
Job,
JobContext,
JobWorker,
LeaseLostError,
RetryableJobError,
TerminalJobError,
)
from ragcore.config import Settings
from ragcore.infrastructure.database import create_database_engine, create_session_factory
from ragcore.infrastructure.database.jobs import PostgresJobRepository
@pytest.fixture(scope="session")
def event_loop_policy() -> asyncio.AbstractEventLoopPolicy:
"""psycopg's async mode refuses Windows' default proactor loop; the other
PostgreSQL-backed modules already declare this, so this one does too."""
if sys.platform == "win32":
return asyncio.WindowsSelectorEventLoopPolicy()
return asyncio.get_event_loop_policy()
async def _empty_the_queue(factory: async_sessionmaker[AsyncSession]) -> None:
"""Leave ``jobs.jobs`` as this test found it --- before *and* after.
The queue is one shared table, and the ingestion and system gates later
in a full run drive a real ``JobWorker``, which claims the oldest
*available* job in it. A leftover fixture job is therefore claimed instead
of theirs, and they fail for a reason that has nothing to do with what
they test.
"""
async with factory() as session:
await session.execute(text("TRUNCATE jobs.events, jobs.attempts, jobs.jobs CASCADE"))
await session.commit()
@pytest_asyncio.fixture
async def session_factory() -> async_sessionmaker[AsyncSession]:
engine = create_database_engine(Settings(env="test"))
factory = create_session_factory(engine)
await _empty_the_queue(factory)
try:
yield factory
finally:
await _empty_the_queue(factory)
await engine.dispose()
WORKSPACE_ID = uuid4()
def _job(
now: datetime, *, key: str = "job-1", operation: str = "parse", max_attempts: int = 3
) -> Job:
return Job(
id=uuid4(),
# Every job the platform enqueues is owned by exactly one workspace;
# the repository refuses to write one that is not.
workspace_id=WORKSPACE_ID,
job_key=key,
operation=operation,
target_key="blob:abc",
profile_version="parser-v1",
payload={"source": "fixture"},
available_at=now,
max_attempts=max_attempts,
trace_id="trace-job",
)
@pytest.mark.integration
async def test_concurrent_workers_only_one_claims_a_ready_job(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
now = datetime.now(UTC)
first = PostgresJobRepository(session_factory)
second = PostgresJobRepository(session_factory)
await first.enqueue(_job(now))
claims = await asyncio.gather(
first.claim_next("worker-a", now=now, lease_duration=timedelta(minutes=1)),
second.claim_next("worker-b", now=now, lease_duration=timedelta(minutes=1)),
)
assert sum(claim is not None for claim in claims) == 1
@pytest.mark.integration
async def test_expired_lease_is_reclaimed_and_old_owner_cannot_complete(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
now = datetime.now(UTC)
repository = PostgresJobRepository(session_factory)
await repository.enqueue(_job(now))
first = await repository.claim_next(
"worker-a", now=now, lease_duration=timedelta(seconds=1)
)
assert first is not None
reclaimed_at = now + timedelta(seconds=2)
second = await repository.claim_next(
"worker-b", now=reclaimed_at, lease_duration=timedelta(minutes=1)
)
assert second is not None
assert second.attempt_number == 2
with pytest.raises(LeaseLostError):
await repository.complete(first, now=reclaimed_at)
@pytest.mark.integration
async def test_worker_distinguishes_retryable_and_terminal_errors(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
now = datetime.now(UTC)
repository = PostgresJobRepository(session_factory)
retry_job = _job(now, key="retry", max_attempts=2)
terminal_job = replace(
_job(now, key="terminal", operation="terminal", max_attempts=2),
available_at=now + timedelta(seconds=1),
)
await repository.enqueue(retry_job)
await repository.enqueue(terminal_job)
received: list[JobContext] = []
async def retry(context: JobContext) -> None:
received.append(context)
raise RetryableJobError("temporary", retry_after_seconds=30)
async def terminal(_context: JobContext) -> None:
raise TerminalJobError("invalid input")
worker = JobWorker(
repository,
{"parse": retry},
worker_id="worker-retry",
clock=lambda: now,
)
assert await worker.run_once()
worker_terminal = JobWorker(
repository,
{"terminal": terminal},
worker_id="worker-terminal",
clock=lambda: now + timedelta(seconds=2),
)
assert await worker_terminal.run_once()
assert received[0].idempotency_key == "retry"
assert isinstance(received[0].cancellation, asyncio.Event)
async with session_factory() as session:
rows = (
await session.execute(
text("SELECT job_key, status, last_error_code FROM jobs.jobs ORDER BY job_key")
)
).all()
assert rows == [
("retry", "retry_wait", "retryable_error"),
("terminal", "terminal", "terminal_error"),
]
async with session_factory() as session:
event_rows = (
await session.execute(
text(
"SELECT event_type, error_code FROM jobs.events "
"WHERE event_type IN ('retry_scheduled', 'terminal') "
"ORDER BY event_type"
)
)
).all()
assert ("retry_scheduled", "retryable_error") in event_rows
assert ("terminal", "terminal_error") in event_rows
@@ -0,0 +1,236 @@
"""The jobs listing filters and pages in SQL, against real PostgreSQL.
The bug these exist for was invisible to every fake: the screen asked for the
most recently updated hundred jobs of a workspace and then dropped the
non-matching ones in Python. With fewer than a hundred jobs in the fixture
that is indistinguishable from a real query -- and with more than a hundred it
silently answers "no jobs match" for a job that plainly exists.
So the fixtures here are deliberately larger than one page, and the failed job
under test is deliberately *older* than everything on it.
"""
from __future__ import annotations
import asyncio
import sys
from datetime import UTC, datetime, timedelta
from uuid import UUID, uuid4
import pytest
import pytest_asyncio
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ragcore.application.jobs import (
Job,
JobListFilter,
JobStatus,
RetryableJobError,
)
from ragcore.config import Settings
from ragcore.infrastructure.database import create_database_engine, create_session_factory
from ragcore.infrastructure.database.jobs import PostgresJobRepository
pytestmark = [pytest.mark.integration, pytest.mark.security]
WORKSPACE_A = uuid4()
WORKSPACE_B = uuid4()
PAGE_SIZE = 50
# What the screen used to fetch before filtering in Python. The fixture for
# the filtering test has to exceed it, or the old implementation would happen
# to pass: with fewer jobs than this, "fetch a bounded page, then filter" and
# "filter, then fetch a bounded page" return the same rows.
LEGACY_FETCH_LIMIT = 100
BASE = datetime(2026, 8, 1, 12, 0, tzinfo=UTC)
@pytest.fixture(scope="session")
def event_loop_policy() -> asyncio.AbstractEventLoopPolicy:
"""psycopg's async mode cannot run on Windows' default proactor loop."""
if sys.platform == "win32":
return asyncio.WindowsSelectorEventLoopPolicy()
return asyncio.get_event_loop_policy()
async def _empty_the_queue(factory: async_sessionmaker[AsyncSession]) -> None:
"""Leave ``jobs.jobs`` as this test found it --- before *and* after.
The queue is one shared table, and the ingestion and system gates later
in a full run drive a real ``JobWorker``, which claims the oldest
*available* job in it. A leftover fixture job is therefore claimed instead
of theirs, and they fail for a reason that has nothing to do with what
they test.
"""
async with factory() as session:
await session.execute(text("TRUNCATE jobs.events, jobs.attempts, jobs.jobs CASCADE"))
await session.commit()
@pytest_asyncio.fixture
async def session_factory() -> async_sessionmaker[AsyncSession]:
engine = create_database_engine(Settings(env="test"))
factory = create_session_factory(engine)
await _empty_the_queue(factory)
try:
yield factory
finally:
await _empty_the_queue(factory)
await engine.dispose()
def _job(
workspace_id: UUID,
key: str,
*,
operation: str = "ingest_upload",
available_at: datetime,
) -> Job:
return Job(
id=uuid4(),
workspace_id=workspace_id,
job_key=key,
operation=operation,
target_key="blob:abc",
profile_version="upload-v1",
payload={"original_name": f"{key}.md"},
available_at=available_at,
max_attempts=1,
trace_id=f"trace-{key}",
)
async def _fail_terminally(
repository: PostgresJobRepository, *, at: datetime
) -> None:
"""Drive the next claimable job to TERMINAL through the real transition.
``now`` is a parameter of both calls, so the failure can be stamped in the
past -- which is the whole point: a job that failed long ago is exactly the
one a truncated listing loses.
"""
claim = await repository.claim_next("worker-1", now=at, lease_duration=timedelta(minutes=5))
assert claim is not None
await repository.fail(claim, RetryableJobError("nope"), now=at)
async def _plant_old_failure(
repository: PostgresJobRepository,
*,
workspace_id: UUID,
key: str,
operation: str = "ingest_upload",
) -> UUID:
job = await repository.enqueue(
_job(workspace_id, key, operation=operation, available_at=BASE)
)
await _fail_terminally(repository, at=BASE + timedelta(seconds=1))
return job.id
async def _pad_with_newer_jobs(
repository: PostgresJobRepository, workspace_id: UUID, count: int
) -> None:
for index in range(count):
await repository.enqueue(
_job(
workspace_id,
f"pad-{workspace_id.hex[:6]}-{index}",
available_at=BASE + timedelta(hours=1, minutes=index),
)
)
async def test_a_failure_older_than_one_page_is_still_found_by_the_status_filter(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
failed_id = await _plant_old_failure(repository, workspace_id=WORKSPACE_A, key="old-failure")
padding = LEGACY_FETCH_LIMIT + 20
await _pad_with_newer_jobs(repository, WORKSPACE_A, padding)
# It is genuinely out of reach of the old implementation: not on the first
# page, and not even among the rows that page was cut from.
unfiltered = await repository.list_jobs(
workspace_id=WORKSPACE_A, filters=JobListFilter(limit=PAGE_SIZE)
)
assert unfiltered.total == padding + 1
assert failed_id not in {job.id for job in unfiltered.items}
failed = await repository.list_jobs(
workspace_id=WORKSPACE_A,
filters=JobListFilter(status=JobStatus.TERMINAL, limit=PAGE_SIZE),
)
assert [job.id for job in failed.items] == [failed_id]
assert failed.total == 1
async def test_the_status_filter_never_widens_the_workspace_scope(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
mine = await _plant_old_failure(repository, workspace_id=WORKSPACE_A, key="mine-failed")
theirs = await _plant_old_failure(repository, workspace_id=WORKSPACE_B, key="theirs-failed")
failed_for_a = await repository.list_jobs(
workspace_id=WORKSPACE_A, filters=JobListFilter(status=JobStatus.TERMINAL)
)
assert [job.id for job in failed_for_a.items] == [mine]
assert failed_for_a.total == 1
assert theirs not in {job.id for job in failed_for_a.items}
# The platform-global scope is the only one that sees both, and it has to
# be asked for explicitly.
everything = await repository.list_jobs(
workspace_id=None, filters=JobListFilter(status=JobStatus.TERMINAL)
)
assert {job.id for job in everything.items} == {mine, theirs}
async def test_status_and_operation_narrow_the_same_query(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
failed_upload = await _plant_old_failure(
repository, workspace_id=WORKSPACE_A, key="failed-upload"
)
failed_sync = await _plant_old_failure(
repository, workspace_id=WORKSPACE_A, key="failed-sync", operation="sync_source"
)
await _pad_with_newer_jobs(repository, WORKSPACE_A, PAGE_SIZE + 10)
page = await repository.list_jobs(
workspace_id=WORKSPACE_A,
filters=JobListFilter(status=JobStatus.TERMINAL, operation="sync_source"),
)
assert [job.id for job in page.items] == [failed_sync]
assert page.total == 1
assert failed_upload not in {job.id for job in page.items}
async def test_paging_covers_every_matching_job_exactly_once(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
total = PAGE_SIZE + 10
await _pad_with_newer_jobs(repository, WORKSPACE_A, total)
first = await repository.list_jobs(
workspace_id=WORKSPACE_A, filters=JobListFilter(status=JobStatus.READY, limit=PAGE_SIZE)
)
second = await repository.list_jobs(
workspace_id=WORKSPACE_A,
filters=JobListFilter(status=JobStatus.READY, limit=PAGE_SIZE, offset=PAGE_SIZE),
)
assert first.total == second.total == total
assert len(first.items) == PAGE_SIZE
assert len(second.items) == total - PAGE_SIZE
seen = [job.id for job in first.items] + [job.id for job in second.items]
assert len(set(seen)) == total
@@ -0,0 +1,232 @@
"""Job ownership against real PostgreSQL.
The web-level tests prove the routes ask for the right scope. These prove the
storage layer answers it correctly -- including the two things a fake can
never really check:
* that ``workspace_id = :id`` genuinely excludes legacy ``NULL`` rows (SQL
three-valued logic, not Python's ``==``);
* that a foreign retry is refused by the ``UPDATE``'s own predicate, so the
row is provably unchanged afterwards rather than merely un-returned.
"""
from __future__ import annotations
import asyncio
import sys
from datetime import UTC, datetime
from uuid import UUID, uuid4
import pytest
import pytest_asyncio
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ragcore.application.jobs import Job, JobListFilter, JobStatus
from ragcore.config import Settings
from ragcore.infrastructure.database import create_database_engine, create_session_factory
from ragcore.infrastructure.database.jobs import PostgresJobRepository
pytestmark = [pytest.mark.integration, pytest.mark.security]
WORKSPACE_A = uuid4()
WORKSPACE_B = uuid4()
@pytest.fixture(scope="session")
def event_loop_policy() -> asyncio.AbstractEventLoopPolicy:
"""psycopg's async mode cannot run on Windows' default proactor loop."""
if sys.platform == "win32":
return asyncio.WindowsSelectorEventLoopPolicy()
return asyncio.get_event_loop_policy()
async def _empty_the_queue(factory: async_sessionmaker[AsyncSession]) -> None:
"""Leave ``jobs.jobs`` as this test found it --- before *and* after.
The queue is one shared table, and the ingestion and system gates later
in a full run drive a real ``JobWorker``, which claims the oldest
*available* job in it. A leftover fixture job is therefore claimed instead
of theirs, and they fail for a reason that has nothing to do with what
they test.
"""
async with factory() as session:
await session.execute(text("TRUNCATE jobs.events, jobs.attempts, jobs.jobs CASCADE"))
await session.commit()
@pytest_asyncio.fixture
async def session_factory() -> async_sessionmaker[AsyncSession]:
engine = create_database_engine(Settings(env="test"))
factory = create_session_factory(engine)
await _empty_the_queue(factory)
try:
yield factory
finally:
await _empty_the_queue(factory)
await engine.dispose()
def _job(workspace_id: UUID, key: str) -> Job:
return Job(
id=uuid4(),
workspace_id=workspace_id,
job_key=key,
operation="ingest_upload",
target_key="blob:abc",
profile_version="upload-v1",
payload={"workspace_id": str(workspace_id)},
available_at=datetime.now(UTC),
trace_id=f"trace-{key}",
)
async def _plant_legacy_job(factory: async_sessionmaker[AsyncSession]) -> UUID:
"""A row exactly as migration 0018 leaves an unbackfillable job: a payload
that still names a workspace, and no ``workspace_id`` column value."""
job_id = uuid4()
now = datetime.now(UTC)
async with factory() as session, session.begin():
await session.execute(
text(
"""
INSERT INTO jobs.jobs (id, job_key, operation, target_key, profile_version,
payload, status, available_at, max_attempts,
attempt_count, trace_id, last_error_code,
last_error_details, created_at, updated_at)
VALUES (:id, 'legacy', 'ingest_upload', 'blob:legacy', 'upload-v1',
CAST(:payload AS json), 'terminal', :now, 3, 3, 'trace-legacy',
'retryable_error', CAST('{}' AS json), :now, :now)
"""
),
{
"id": job_id,
"payload": f'{{"workspace_id": "{WORKSPACE_A}"}}',
"now": now,
},
)
return job_id
async def test_a_new_job_cannot_be_written_without_an_owner(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
with pytest.raises(ValueError, match="must name the workspace"):
await repository.enqueue(
Job(
id=uuid4(),
job_key="ownerless",
operation="ingest_upload",
target_key="blob:abc",
profile_version="upload-v1",
available_at=datetime.now(UTC),
)
)
async def test_listing_and_counting_are_confined_to_one_workspace(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
await repository.enqueue(_job(WORKSPACE_A, "a1"))
await repository.enqueue(_job(WORKSPACE_A, "a2"))
await repository.enqueue(_job(WORKSPACE_B, "b1"))
legacy_id = await _plant_legacy_job(session_factory)
page_a = await repository.list_jobs(workspace_id=WORKSPACE_A, filters=JobListFilter())
assert {job.job_key for job in page_a.items} == {"a1", "a2"}
assert page_a.total == 2
assert legacy_id not in {job.id for job in page_a.items}
counts_a = await repository.status_counts(workspace_id=WORKSPACE_A)
assert counts_a == {JobStatus.READY.value: 2}
# The platform-global scope is a different, explicit query.
page_all = await repository.list_jobs(workspace_id=None, filters=JobListFilter())
assert {job.job_key for job in page_all.items} == {"a1", "a2", "b1", "legacy"}
assert page_all.total == 4
async def test_a_foreign_or_legacy_job_reads_as_missing(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
foreign = await repository.enqueue(_job(WORKSPACE_B, "b1"))
legacy_id = await _plant_legacy_job(session_factory)
assert await repository.get_owned(foreign.id, workspace_id=WORKSPACE_A) is None
assert await repository.get_owned(legacy_id, workspace_id=WORKSPACE_A) is None
# Both exist; only the explicit global scope may see them.
assert await repository.get_owned(foreign.id, workspace_id=None) is not None
assert await repository.get_owned(legacy_id, workspace_id=None) is not None
async def test_a_foreign_retry_leaves_the_job_byte_for_byte_unchanged(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
foreign = await repository.enqueue(_job(WORKSPACE_B, "b1"))
now = datetime.now(UTC)
async with session_factory() as session, session.begin():
await session.execute(
text(
"UPDATE jobs.jobs SET status = 'terminal', attempt_count = 3, "
"last_error_code = 'retryable_error' WHERE id = :id"
),
{"id": foreign.id},
)
async def snapshot() -> tuple[object, ...]:
async with session_factory() as session:
row = (
await session.execute(
text(
"SELECT status, attempt_count, available_at, updated_at, leased_by "
"FROM jobs.jobs WHERE id = :id"
),
{"id": foreign.id},
)
).one()
return tuple(row)
before = await snapshot()
assert await repository.requeue(foreign.id, workspace_id=WORKSPACE_A, now=now) is None
assert await snapshot() == before
# And the owner can still recover it, so the refusal is scoping and not a
# blanket "requeue is broken".
requeued = await repository.requeue(foreign.id, workspace_id=WORKSPACE_B, now=now)
assert requeued is not None and requeued.status is JobStatus.READY
async def test_a_legacy_job_is_immutable_for_workspace_scoped_callers(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
legacy_id = await _plant_legacy_job(session_factory)
now = datetime.now(UTC)
assert await repository.requeue(legacy_id, workspace_id=WORKSPACE_A, now=now) is None
still = await repository.get(legacy_id)
assert still is not None and still.status is JobStatus.TERMINAL
assert still.workspace_id is None
# Only the explicit global scope may act on it.
assert await repository.requeue(legacy_id, workspace_id=None, now=now) is not None
async def test_enqueued_jobs_round_trip_their_owner(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
stored = await repository.enqueue(_job(WORKSPACE_A, "a1"))
assert stored.workspace_id == WORKSPACE_A
reread = await repository.get_owned(stored.id, workspace_id=WORKSPACE_A)
assert reread is not None and reread.workspace_id == WORKSPACE_A
@@ -0,0 +1,89 @@
"""Real Postgres coverage for ``JobRepository.get`` -- the read side ``GET
/v1/jobs/{job_id}`` needs so ``POST /v1/uploads``'s ``status_url`` resolves
instead of 404ing."""
from __future__ import annotations
import asyncio
import sys
from datetime import UTC, datetime
from uuid import uuid4
import pytest
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ragcore.application.jobs.models import Job
from ragcore.config import Settings
from ragcore.infrastructure.database import create_database_engine, create_session_factory
from ragcore.infrastructure.database.jobs import PostgresJobRepository
pytestmark = pytest.mark.integration
@pytest.fixture(scope="session")
def event_loop_policy() -> asyncio.AbstractEventLoopPolicy:
if sys.platform == "win32":
return asyncio.WindowsSelectorEventLoopPolicy()
return asyncio.get_event_loop_policy()
async def _empty_the_queue(factory: async_sessionmaker[AsyncSession]) -> None:
"""Leave ``jobs.jobs`` as this test found it --- before *and* after.
The queue is one shared table, and the ingestion and system gates later in
a full run drive a real ``JobWorker``, which claims the oldest *available*
job in it. A single leftover READY job is enough to make one of those
gates process the wrong work and fail for a reason that has nothing to do
with what it tests.
"""
async with factory() as session:
await session.execute(text("TRUNCATE jobs.events, jobs.attempts, jobs.jobs CASCADE"))
await session.commit()
@pytest.fixture
async def session_factory() -> async_sessionmaker[AsyncSession]:
engine = create_database_engine(Settings(env="test"))
factory = create_session_factory(engine)
await _empty_the_queue(factory)
try:
yield factory
finally:
await _empty_the_queue(factory)
await engine.dispose()
async def test_get_returns_the_enqueued_job(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
workspace_id = uuid4()
job = Job(
id=uuid4(),
workspace_id=workspace_id,
job_key=f"get-job-{uuid4()}",
operation="ingest_upload",
target_key="blob:abc",
profile_version="upload-v1",
payload={"workspace_id": str(workspace_id)},
available_at=datetime.now(UTC),
trace_id="trace-get-job",
)
await repository.enqueue(job)
found = await repository.get(job.id)
assert found is not None
assert found.id == job.id
assert found.operation == "ingest_upload"
assert found.payload == job.payload
assert found.workspace_id == workspace_id
async def test_get_returns_none_for_an_unknown_id(
session_factory: async_sessionmaker[AsyncSession],
) -> None:
repository = PostgresJobRepository(session_factory)
assert await repository.get(uuid4()) is None