Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

tests/unit/application/sources/test_source_scheduler.py

@@ -0,0 +1,178 @@
+"""Sources that carry an interval get synced without anyone asking.
+
+Until now a sync ran only when an operator pressed the button. For a platform
+whose promise is that an index keeps step with a folder or a repository, that
+made staleness the default and freshness a chore someone had to remember.
+
+The scheduler does not sync anything. It enqueues the same ``sync_source`` job
+the Control Plane button enqueues, so every ownership, convergence and
+authorization rule downstream is the one already in place.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import UTC, datetime, timedelta
+from uuid import UUID, uuid4
+
+import pytest
+
+from ragcore.application.sources.models import SourceRecord, SourceStatus
+from ragcore.application.sources.scheduler import SourceSyncScheduler
+from ragcore.domain.sources import ConnectorKind, SourceSyncState
+
+NOW = datetime(2026, 8, 23, 12, 0, tzinfo=UTC)
+WORKSPACE = uuid4()
+SPACE = uuid4()
+
+
+def _source(
+    *,
+    interval: int | None = 3600,
+    last_sync_at: datetime | None = None,
+    status: SourceStatus = SourceStatus.ACTIVE,
+    sync_state: SourceSyncState = SourceSyncState.COMPLETED,
+    kind: ConnectorKind = ConnectorKind.FOLDER,
+) -> SourceRecord:
+    return SourceRecord(
+        id=uuid4(),
+        workspace_id=WORKSPACE,
+        space_id=SPACE,
+        name="Runbooks",
+        kind=kind,
+        status=status,
+        sync_state=sync_state,
+        last_sync_at=last_sync_at,
+        sync_interval_seconds=interval,
+    )
+
+
+class _Registry:
+    def __init__(self, sources: list[SourceRecord]) -> None:
+        self._sources = sources
+        self.asked_at: list[datetime] = []
+
+    async def list_due_for_sync(self, *, now: datetime, limit: int) -> tuple[SourceRecord, ...]:
+        self.asked_at.append(now)
+        return tuple(self._sources[:limit])
+
+
+class _Jobs:
+    def __init__(self) -> None:
+        self.enqueued: list[tuple[UUID, str]] = []
+        self.any_enqueued = asyncio.Event()
+
+    async def enqueue(self, **kwargs: object) -> object:
+        self.enqueued.append((kwargs["workspace_id"], str(kwargs["idempotency_key"])))  # type: ignore[arg-type]
+        self.any_enqueued.set()
+        return object()
+
+
+@pytest.mark.asyncio
+async def test_a_due_source_is_enqueued_as_an_ordinary_sync_job() -> None:
+    due = _source(interval=3600, last_sync_at=NOW - timedelta(hours=2))
+    jobs = _Jobs()
+    scheduler = SourceSyncScheduler(_Registry([due]), jobs)  # type: ignore[arg-type]
+
+    enqueued = await scheduler.tick(now=NOW)
+
+    assert enqueued == 1
+    assert [workspace for workspace, _ in jobs.enqueued] == [WORKSPACE]
+
+
+@pytest.mark.asyncio
+async def test_two_schedulers_ticking_together_produce_one_job() -> None:
+    """Several worker replicas each run a scheduler. They must not each start
+    the same sync -- and they do not need a lock to avoid it: the key is
+    derived from when the sync came due, so both compute the same one and the
+    queue's own uniqueness collapses them."""
+
+    due = _source(interval=3600, last_sync_at=NOW - timedelta(hours=2))
+    first, second = _Jobs(), _Jobs()
+
+    await SourceSyncScheduler(_Registry([due]), first).tick(now=NOW)  # type: ignore[arg-type]
+    await SourceSyncScheduler(_Registry([due]), second).tick(  # type: ignore[arg-type]
+        now=NOW + timedelta(seconds=17)
+    )
+
+    assert first.enqueued and second.enqueued
+    assert first.enqueued[0][1] == second.enqueued[0][1]
+
+
+@pytest.mark.asyncio
+async def test_a_source_that_has_never_synced_is_due_immediately() -> None:
+    jobs = _Jobs()
+    scheduler = SourceSyncScheduler(_Registry([_source(last_sync_at=None)]), jobs)  # type: ignore[arg-type]
+
+    assert await scheduler.tick(now=NOW) == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "source",
+    [
+        pytest.param(_source(interval=None), id="no interval configured"),
+        pytest.param(
+            _source(last_sync_at=NOW - timedelta(minutes=5)), id="synced within the interval"
+        ),
+        pytest.param(_source(status=SourceStatus.PAUSED), id="paused"),
+        pytest.param(_source(status=SourceStatus.DELETED), id="deleted"),
+        pytest.param(
+            _source(sync_state=SourceSyncState.SCANNING), id="a run is already in flight"
+        ),
+        pytest.param(
+            _source(sync_state=SourceSyncState.AWAITING_INGEST),
+            id="waiting on its own child ingests",
+        ),
+        pytest.param(_source(kind=ConnectorKind.UPLOAD), id="fed by its callers, not synced"),
+    ],
+)
+async def test_sources_that_must_not_be_scheduled(source: SourceRecord) -> None:
+    """The registry query is the first filter, but it is not the only one: the
+    scheduler refuses these itself, so a widened query cannot quietly start
+    syncing something that should be left alone."""
+
+    jobs = _Jobs()
+    scheduler = SourceSyncScheduler(_Registry([source]), jobs)  # type: ignore[arg-type]
+
+    assert await scheduler.tick(now=NOW) == 0
+    assert jobs.enqueued == []
+
+
+class _FailingRegistry(_Registry):
+    def __init__(self, sources: list[SourceRecord]) -> None:
+        super().__init__(sources)
+        self.calls = 0
+
+    async def list_due_for_sync(self, *, now: datetime, limit: int) -> tuple[SourceRecord, ...]:
+        self.calls += 1
+        if self.calls == 1:
+            raise RuntimeError("the database blinked")
+        return await super().list_due_for_sync(now=now, limit=limit)
+
+
+@pytest.mark.asyncio
+async def test_a_failing_tick_does_not_end_the_loop() -> None:
+    """The scheduler shares a process with the claim loops. A transient
+    database error while looking for due sources must cost one tick, not the
+    worker -- the next tick finds the same sources still due."""
+
+    registry = _FailingRegistry([_source(last_sync_at=None)])
+    jobs = _Jobs()
+    stop = asyncio.Event()
+    scheduler = SourceSyncScheduler(registry, jobs)  # type: ignore[arg-type]
+
+    async def until_enqueued() -> None:
+        await jobs.any_enqueued.wait()
+        stop.set()
+
+    await asyncio.wait_for(
+        asyncio.gather(
+            scheduler.run(stop, interval_seconds=0.01),
+            until_enqueued(),
+        ),
+        timeout=5.0,
+    )
+
+    assert registry.calls >= 2
+    assert len(jobs.enqueued) >= 1

tests/unit/application/sources/test_source_sync.py

@@ -0,0 +1,666 @@
+"""Source synchronisation semantics: discover -> compare -> ingest new/changed
+-> preserve unchanged -> retire disappeared -> persist the outcome."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import AsyncIterable, AsyncIterator
+from dataclasses import replace
+from datetime import UTC, datetime
+from uuid import UUID, uuid4
+
+import pytest
+
+from ragcore.application.documents import DocumentRecord
+from ragcore.application.jobs import Job, JobService
+from ragcore.application.sources import (
+    SourceItemRecord,
+    SourceItemState,
+    SourceItemSummary,
+    SourceRecord,
+    SourceStatus,
+    SourceSyncOutcome,
+    SourceSyncRunRecord,
+    SourceSyncService,
+)
+from ragcore.application.storage import BlobReference
+from ragcore.domain.sources import (
+    Acquisition,
+    ConnectorKind,
+    ScanResult,
+    SourceCursor,
+    SourceFingerprint,
+    SourceItem,
+    SourceSyncState,
+)
+
+WORKSPACE_ID = uuid4()
+SPACE_ID = uuid4()
+
+
+class FakeConnector:
+    """Serves a mutable set of path -> content; pages when asked to."""
+
+    def __init__(self, source_id: UUID, files: dict[str, bytes], *, page_size: int = 100) -> None:
+        self.source_id = source_id
+        self.files = files
+        self.page_size = page_size
+        self.acquired: list[str] = []
+        self.fail_scan = False
+
+    def _items(self) -> list[SourceItem]:
+        return [
+            SourceItem(
+                source_id=self.source_id,
+                external_id=path,
+                locator=path,
+                name=path.rsplit("/", 1)[-1],
+                media_type="text/plain",
+                metadata={"sha256": hashlib.sha256(content).hexdigest(), "commit": "abc1234def"},
+            )
+            for path, content in sorted(self.files.items())
+        ]
+
+    async def scan(self, cursor: SourceCursor | None = None) -> ScanResult:
+        if self.fail_scan:
+            raise RuntimeError("mount gone")
+        items = self._items()
+        start = 0
+        if cursor is not None and cursor.value:
+            start = next(i for i, item in enumerate(items) if item.external_id == cursor.value) + 1
+        page = items[start : start + self.page_size]
+        has_more = start + self.page_size < len(items)
+        return ScanResult(
+            tuple(page),
+            SourceCursor(page[-1].external_id) if has_more and page else None,
+            complete=not has_more and cursor is None,
+        )
+
+    async def acquire(self, item: SourceItem) -> Acquisition:
+        self.acquired.append(item.locator)
+        content = self.files[item.locator]
+
+        async def stream() -> AsyncIterator[bytes]:
+            yield content
+
+        return Acquisition(
+            stream=stream(),
+            source_item=item,
+            fingerprint=SourceFingerprint(hashlib.sha256(content).hexdigest()),
+            media_type="text/plain",
+            original_name=item.name,
+        )
+
+    def fingerprint(self, item: SourceItem) -> SourceFingerprint:
+        return SourceFingerprint(str(item.metadata["sha256"]))
+
+
+_CONVERGEABLE = (SourceSyncState.AWAITING_INGEST, SourceSyncState.COMPLETED_WITH_FAILURES)
+
+
+class FakeRegistry:
+    """Mirrors PostgresSourceRegistry's convergence semantics, which is the
+    point of these tests: the counters and the settled state must come out of
+    remembered items, not out of what discovery happened to see."""
+
+    def __init__(self) -> None:
+        self.items: dict[tuple[UUID, str], SourceItemRecord] = {}
+        self.runs: dict[UUID, SourceSyncRunRecord] = {}
+        self.states: list[tuple[SourceSyncState, bool]] = []
+        self.sync_state = SourceSyncState.IDLE
+
+    async def record_sync_run(self, run: SourceSyncRunRecord) -> SourceSyncRunRecord:
+        self.runs[run.id] = run
+        return run
+
+    async def set_sync_state(
+        self, source_id, state, *, now, error=None, revision=None, successful=False
+    ):
+        self.states.append((state, successful))
+        self.sync_state = state
+        return None
+
+    async def list_items(self, source_id: UUID) -> tuple[SourceItemRecord, ...]:
+        return tuple(record for (sid, _), record in self.items.items() if sid == source_id)
+
+    async def upsert_items(self, items: tuple[SourceItemRecord, ...]) -> None:
+        for item in items:
+            self.items[(item.source_id, item.external_id)] = item
+
+    async def mark_items_removed(self, source_id, external_ids, *, now) -> None:
+        for external_id in external_ids:
+            current = self.items[(source_id, external_id)]
+            self.items[(source_id, external_id)] = replace(
+                current,
+                removed=True,
+                last_seen_at=now,
+                # Retirement *is* the applied state: the desired outcome (no
+                # searchable content) has been reached.
+                applied_fingerprint=current.fingerprint,
+                applied_at=now,
+                pending_job_id=None,
+                last_failure_code=None,
+                last_failure_retryable=None,
+            )
+
+    async def mark_item_ingested(
+        self, source_id, external_id, *, fingerprint, document_id, now
+    ) -> bool:
+        current = self.items.get((source_id, external_id))
+        if current is None or current.fingerprint != fingerprint:
+            return False
+        self.items[(source_id, external_id)] = replace(
+            current,
+            applied_fingerprint=fingerprint,
+            applied_at=now,
+            pending_job_id=None,
+            attempts=0,
+            last_failure_code=None,
+            last_failure_retryable=None,
+            last_failure_job_id=None,
+            document_id=document_id if document_id is not None else current.document_id,
+        )
+        return True
+
+    async def mark_item_failed(
+        self, source_id, external_id, *, fingerprint, failure_code, retryable, now, job_id=None
+    ) -> bool:
+        current = self.items.get((source_id, external_id))
+        if current is None or current.fingerprint != fingerprint:
+            return False
+        self.items[(source_id, external_id)] = replace(
+            current,
+            pending_job_id=None,
+            last_seen_at=now,
+            last_failure_code=failure_code[:64],
+            last_failure_retryable=retryable,
+            last_failure_job_id=job_id,
+        )
+        return True
+
+    async def summarize_items(self, source_id: UUID) -> SourceItemSummary:
+        records = [record for (sid, _), record in self.items.items() if sid == source_id]
+        live = [record for record in records if not record.removed]
+        return SourceItemSummary(
+            total=len(records),
+            removed=sum(1 for record in records if record.removed),
+            applied=sum(1 for record in live if record.applied),
+            failed=sum(1 for record in live if record.state is SourceItemState.FAILED),
+            pending=sum(1 for record in live if record.state is SourceItemState.PENDING),
+        )
+
+    async def converge_sync_state(self, source_id, state, *, now, successful=False):
+        if self.sync_state not in _CONVERGEABLE:
+            return None
+        self.states.append((state, successful))
+        self.sync_state = state
+        return object()
+
+
+class FakeDocuments:
+    def __init__(self) -> None:
+        self.documents: dict[tuple[str, str], DocumentRecord] = {}
+        self.retired: list[UUID] = []
+
+    def seed(self, source_id: UUID, external_id: str) -> DocumentRecord:
+        document = DocumentRecord(
+            id=uuid4(),
+            workspace_id=WORKSPACE_ID,
+            space_id=SPACE_ID,
+            source_key=str(source_id),
+            external_id=external_id,
+            title=external_id,
+            source_id=source_id,
+        )
+        self.documents[(str(source_id), external_id)] = document
+        return document
+
+    async def get_document_by_source(self, workspace_id, source_key, external_id):
+        return self.documents.get((source_key, external_id))
+
+    async def mark_source_deleted(self, document_id: UUID) -> None:
+        self.retired.append(document_id)
+
+
+class FakeBlobs:
+    def __init__(self) -> None:
+        self.stored: list[tuple[str, str]] = []
+
+    async def put(
+        self,
+        chunks: AsyncIterable[bytes],
+        *,
+        media_type=None,
+        original_name=None,
+        expected_sha256=None,
+    ):
+        body = b"".join([chunk async for chunk in chunks])
+        digest = hashlib.sha256(body).hexdigest()
+        assert expected_sha256 == digest
+        self.stored.append((original_name or "", digest))
+        return BlobReference(
+            digest=digest,
+            size=len(body),
+            media_type=media_type,
+            original_name=original_name,
+            storage_key=f"blobs/{digest}",
+        )
+
+
+class RecordingJobs:
+    def __init__(self) -> None:
+        self.jobs: list[Job] = []
+
+    async def enqueue(self, job: Job) -> Job:
+        # Mirrors the real repository's idempotency on job_key.
+        existing = next((j for j in self.jobs if j.job_key == job.job_key), None)
+        if existing is not None:
+            return existing
+        self.jobs.append(job)
+        return job
+
+
+def _source(source_id: UUID, kind: ConnectorKind = ConnectorKind.FOLDER) -> SourceRecord:
+    return SourceRecord(
+        id=source_id,
+        workspace_id=WORKSPACE_ID,
+        space_id=SPACE_ID,
+        name="Runbooks",
+        kind=kind,
+        status=SourceStatus.ACTIVE,
+        config={"root": "/mnt/runbooks", "trust_level": "trusted_internal"},
+    )
+
+
+def _service(
+    registry: FakeRegistry, documents: FakeDocuments, blobs: FakeBlobs, jobs: RecordingJobs
+) -> SourceSyncService:
+    return SourceSyncService(
+        registry=registry,  # type: ignore[arg-type]
+        documents=documents,  # type: ignore[arg-type]
+        blob_storage=blobs,  # type: ignore[arg-type]
+        jobs=JobService(jobs),  # type: ignore[arg-type]
+    )
+
+
+@pytest.mark.asyncio
+async def test_first_sync_ingests_every_item_as_an_ingest_upload_job() -> None:
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha", "dir/b.md": b"bravo"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+
+    report = await _service(registry, documents, blobs, jobs).run(
+        _source(source_id), connector, job_id=uuid4()
+    )
+
+    # Enqueued is not ingested: the run cannot honestly say "completed" while
+    # the child jobs that make the content searchable are still in flight.
+    assert report.run.outcome is SourceSyncOutcome.AWAITING_INGEST
+    assert (report.run.discovered, report.run.new, report.run.changed, report.run.unchanged) == (
+        2,
+        2,
+        0,
+        0,
+    )
+    assert report.run.removed == 0 and report.run.failed == 0
+    assert report.run.pending == 2
+    assert report.run.revision == "abc1234def"
+    assert len(jobs.jobs) == 2
+    for job in jobs.jobs:
+        assert job.operation == "ingest_upload"
+        assert job.payload["source_id"] == str(source_id)
+        assert job.payload["space_id"] == str(SPACE_ID)
+        assert job.payload["workspace_id"] == str(WORKSPACE_ID)
+        assert job.payload["trust_level"] == "trusted_internal"
+        assert job.payload["revision"] == "abc1234def"
+        assert job.payload["external_id"] in {"a.md", "dir/b.md"}
+    assert registry.states[-1] == (SourceSyncState.AWAITING_INGEST, False)
+    assert {record.external_id for record in registry.items.values()} == {"a.md", "dir/b.md"}
+    assert all(record.state is SourceItemState.PENDING for record in registry.items.values())
+
+
+@pytest.mark.asyncio
+async def test_resync_skips_unchanged_ingests_changed_and_retires_removed() -> None:
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha", "b.md": b"bravo", "c.md": b"charlie"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    await service.run(_source(source_id), connector)
+    first_jobs = len(jobs.jobs)
+    retired_doc = documents.seed(source_id, "c.md")
+
+    connector.files["b.md"] = b"bravo v2"
+    del connector.files["c.md"]
+    connector.acquired.clear()
+    report = await service.run(_source(source_id), connector)
+
+    assert (
+        report.run.discovered,
+        report.run.new,
+        report.run.changed,
+        report.run.unchanged,
+        report.run.removed,
+    ) == (2, 0, 1, 1, 1)
+    assert connector.acquired == ["b.md"], "unchanged content is never re-acquired"
+    assert len(jobs.jobs) == first_jobs + 1
+    assert documents.retired == [retired_doc.id]
+    assert report.retired_document_ids == (retired_doc.id,)
+    assert registry.items[(source_id, "c.md")].removed is True
+    assert registry.items[(source_id, "a.md")].removed is False
+
+
+@pytest.mark.asyncio
+async def test_multi_page_discovery_is_still_complete_and_reconciles_removals() -> None:
+    source_id = uuid4()
+    files = {f"doc{i:02d}.md": f"body {i}".encode() for i in range(7)}
+    connector = FakeConnector(source_id, files, page_size=3)
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    first = await service.run(_source(source_id), connector)
+    assert first.run.discovered == 7
+
+    removed = documents.seed(source_id, "doc03.md")
+    del connector.files["doc03.md"]
+    second = await service.run(_source(source_id), connector)
+    assert second.run.removed == 1
+    assert documents.retired == [removed.id]
+
+
+@pytest.mark.asyncio
+async def test_discovery_failure_records_a_failed_run_and_retires_nothing() -> None:
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    await service.run(_source(source_id), connector)
+    documents.seed(source_id, "a.md")
+
+    connector.fail_scan = True
+    with pytest.raises(Exception, match="discovery failed"):
+        await service.run(_source(source_id), connector)
+
+    failed_runs = [run for run in registry.runs.values() if run.outcome is SourceSyncOutcome.FAILED]
+    assert len(failed_runs) == 1
+    assert failed_runs[0].error_code == "discovery_failed"
+    assert documents.retired == [], "an incomplete discovery never retires documents"
+    assert registry.states[-1] == (SourceSyncState.FAILED, False)
+
+
+@pytest.mark.asyncio
+async def test_upload_sources_are_not_syncable() -> None:
+    source_id = uuid4()
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    with pytest.raises(Exception, match="only connector-backed"):
+        await service.run(_source(source_id, ConnectorKind.UPLOAD), FakeConnector(source_id, {}))
+
+
+@pytest.mark.asyncio
+async def test_re_enqueueing_the_same_content_collapses_onto_one_job() -> None:
+    """Same source item + same fingerprint -> same idempotency key: a sync
+    that re-sees content whose earlier job is still queued does not duplicate it."""
+
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    await service.run(_source(source_id), connector)
+    # Forget the registry memory (as if the first run's bookkeeping were lost)
+    # and sync again: the job key still dedups.
+    registry.items.clear()
+    await service.run(_source(source_id), connector)
+    assert len(jobs.jobs) == 1
+    assert jobs.jobs[0].available_at <= datetime.now(UTC)
+
+
+async def _apply_all(
+    registry: FakeRegistry, source_id: UUID, *, only: set[str] | None = None
+) -> None:
+    """Stand in for the child ingest jobs reporting success."""
+
+    now = datetime.now(UTC)
+    for (sid, external_id), record in list(registry.items.items()):
+        if sid == source_id and not record.removed and (only is None or external_id in only):
+            await registry.mark_item_ingested(
+                source_id,
+                external_id,
+                fingerprint=record.fingerprint,
+                document_id=uuid4(),
+                now=now,
+            )
+
+
+@pytest.mark.asyncio
+async def test_a_source_only_completes_once_every_child_ingestion_applied() -> None:
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha", "b.md": b"bravo"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+
+    first = await service.run(_source(source_id), connector)
+    assert first.run.outcome is SourceSyncOutcome.AWAITING_INGEST
+
+    await _apply_all(registry, source_id)
+    second = await service.run(_source(source_id), connector)
+
+    assert second.run.outcome is SourceSyncOutcome.COMPLETED
+    assert (second.run.unchanged, second.run.pending, second.run.failed) == (2, 0, 0)
+    assert registry.states[-1] == (SourceSyncState.COMPLETED, True)
+    assert len(jobs.jobs) == 2, "nothing is re-enqueued once the content is applied"
+
+
+@pytest.mark.asyncio
+async def test_an_unchanged_item_keeps_the_fingerprint_its_ingestion_applied() -> None:
+    """The regression that makes convergence meaningless: a re-sync that
+    rebuilds the remembered record loses ``applied_fingerprint`` and every
+    already-searchable item silently reverts to pending."""
+
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    await service.run(_source(source_id), connector)
+    await _apply_all(registry, source_id)
+    applied_at = registry.items[(source_id, "a.md")].applied_at
+
+    await service.run(_source(source_id), connector)
+
+    record = registry.items[(source_id, "a.md")]
+    assert record.applied is True
+    assert record.state is SourceItemState.APPLIED
+    assert record.applied_at == applied_at, "a no-op re-sync must not restamp convergence"
+
+
+@pytest.mark.asyncio
+async def test_a_failed_child_ingestion_settles_the_source_with_failures() -> None:
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha", "b.md": b"bravo"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    await service.run(_source(source_id), connector)
+
+    # "a.md" became searchable; "b.md"'s ingestion failed for good.
+    await _apply_all(registry, source_id, only={"a.md"})
+    await registry.mark_item_failed(
+        source_id,
+        "b.md",
+        fingerprint=registry.items[(source_id, "b.md")].fingerprint,
+        failure_code="security_quarantined",
+        retryable=False,
+        now=datetime.now(UTC),
+    )
+
+    report = await service.run(_source(source_id), connector)
+
+    assert report.run.outcome is SourceSyncOutcome.COMPLETED_WITH_FAILURES
+    assert (report.run.failed, report.run.pending) == (1, 0)
+    assert report.run.error_code == "partial_failures"
+    assert registry.states[-1] == (SourceSyncState.COMPLETED_WITH_FAILURES, False)
+    assert registry.items[(source_id, "b.md")].state is SourceItemState.FAILED
+    assert len(jobs.jobs) == 2, (
+        "a terminally failed fingerprint is not re-enqueued: the child job for "
+        "that exact content already exists and will never report again"
+    )
+
+
+@pytest.mark.asyncio
+async def test_recovery_from_a_terminal_failure_runs_through_the_child_job_not_a_resync() -> None:
+    """The full recovery loop for content that has not changed.
+
+    The chosen semantics are that a terminally failed item whose *discovered*
+    fingerprint is unchanged is never re-enqueued by a later sync -- the child
+    job for that exact content already exists, so re-enqueueing would either
+    collapse onto it or spin. That is a deliberate choice and it only holds up
+    if the failure is recoverable some other way, so this test walks the whole
+    path: fail -> the failure names its job -> sync again changes nothing ->
+    that job is retried -> the item converges.
+
+    Without the last two steps the "no re-enqueue" rule would be a permanent
+    dead end dressed up as a policy.
+    """
+
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+
+    first = await service.run(_source(source_id), connector)
+    ingest_job_id = first.enqueued_job_ids[0]
+    fingerprint = registry.items[(source_id, "a.md")].fingerprint
+
+    # The child ingestion fails terminally, naming the job that failed.
+    await registry.mark_item_failed(
+        source_id,
+        "a.md",
+        fingerprint=fingerprint,
+        failure_code="embedding_failed",
+        retryable=True,
+        now=datetime.now(UTC),
+    )
+    failed = registry.items[(source_id, "a.md")]
+    assert failed.state is SourceItemState.FAILED
+    assert failed.applied is False
+
+    # Syncing again is honest about it: nothing new is enqueued, the item is
+    # still failed, and the source does not claim to have completed cleanly.
+    again = await service.run(_source(source_id), connector)
+    assert len(jobs.jobs) == 1, "the same terminal fingerprint is never re-enqueued"
+    assert again.run.outcome is SourceSyncOutcome.COMPLETED_WITH_FAILURES
+    assert again.run.failed == 1
+    assert registry.items[(source_id, "a.md")].state is SourceItemState.FAILED
+
+    # Recovery: an authorized operator retries that child ingest job, and this
+    # time it reports success for the same fingerprint.
+    await registry.mark_item_ingested(
+        source_id,
+        "a.md",
+        fingerprint=fingerprint,
+        document_id=uuid4(),
+        now=datetime.now(UTC),
+    )
+
+    recovered = registry.items[(source_id, "a.md")]
+    assert recovered.applied_fingerprint == fingerprint
+    assert recovered.applied is True
+    assert recovered.state is SourceItemState.APPLIED
+    assert recovered.last_failure_code is None
+    summary = await registry.summarize_items(source_id)
+    assert (summary.failed, summary.pending) == (0, 0)
+    assert summary.converged is True
+    assert ingest_job_id is not None
+
+
+@pytest.mark.asyncio
+async def test_a_terminal_failure_remembers_the_job_an_operator_must_retry() -> None:
+    """"Retry the failed ingest job" needs an actual job to point at."""
+
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    report = await service.run(_source(source_id), connector)
+    job_id = report.enqueued_job_ids[0]
+
+    await registry.mark_item_failed(
+        source_id,
+        "a.md",
+        fingerprint=registry.items[(source_id, "a.md")].fingerprint,
+        failure_code="embedding_failed",
+        retryable=True,
+        now=datetime.now(UTC),
+        job_id=job_id,
+    )
+
+    assert registry.items[(source_id, "a.md")].last_failure_job_id == job_id
+
+
+@pytest.mark.asyncio
+async def test_changed_content_clears_a_previous_failure_and_keeps_what_was_applied() -> None:
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    await service.run(_source(source_id), connector)
+    await _apply_all(registry, source_id)
+    applied = registry.items[(source_id, "a.md")].applied_fingerprint
+
+    connector.files["a.md"] = b"alpha v2"
+    report = await service.run(_source(source_id), connector)
+
+    record = registry.items[(source_id, "a.md")]
+    assert report.run.outcome is SourceSyncOutcome.AWAITING_INGEST
+    assert record.state is SourceItemState.PENDING
+    assert record.applied is False
+    assert record.applied_fingerprint == applied, (
+        "the old content stays searchable until the new content is really applied"
+    )
+    assert record.pending_job_id == report.enqueued_job_ids[0]
+
+
+@pytest.mark.asyncio
+async def test_a_retired_item_counts_as_converged_not_as_outstanding_work() -> None:
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha", "gone.md": b"gone"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    await service.run(_source(source_id), connector)
+    await _apply_all(registry, source_id)
+
+    del connector.files["gone.md"]
+    report = await service.run(_source(source_id), connector)
+
+    assert report.run.outcome is SourceSyncOutcome.COMPLETED
+    assert report.run.removed == 1
+    summary = await registry.summarize_items(source_id)
+    assert (summary.removed, summary.applied, summary.pending, summary.failed) == (1, 1, 0, 0)
+    assert summary.converged is True
+
+
+@pytest.mark.asyncio
+async def test_a_retired_item_that_reappears_unchanged_is_pending_again() -> None:
+    """Retirement records "no searchable content" as the *applied* state. When
+    the file comes back -- even byte-identical -- that applied state is stale:
+    the document is still retired until the re-enqueued ingest reactivates it,
+    so carrying the retirement's fingerprint forward would report the source
+    converged while the reactivation is still in flight."""
+
+    source_id = uuid4()
+    connector = FakeConnector(source_id, {"a.md": b"alpha", "back.md": b"returns"})
+    registry, documents, blobs, jobs = FakeRegistry(), FakeDocuments(), FakeBlobs(), RecordingJobs()
+    service = _service(registry, documents, blobs, jobs)
+    await service.run(_source(source_id), connector)
+    await _apply_all(registry, source_id)
+
+    returning = dict(connector.files)
+    del connector.files["back.md"]
+    await service.run(_source(source_id), connector)
+    assert registry.items[(source_id, "back.md")].state is SourceItemState.REMOVED
+
+    connector.files = returning
+    report = await service.run(_source(source_id), connector)
+
+    record = registry.items[(source_id, "back.md")]
+    assert record.removed is False
+    assert record.state is SourceItemState.PENDING
+    assert record.applied is False
+    assert report.run.outcome is SourceSyncOutcome.AWAITING_INGEST