from __future__ import annotations import logging import os from datetime import UTC, datetime from threading import Event, Lock, Thread, current_thread from time import monotonic from .acquisition import AcquisitionError, GitAcquirer from .gitea import GiteaClient from .models import ( Repository, RepositorySyncResult, ScanJob, ScanJobStatus, ScanSource, ScanTrigger, SourceOrigin, ) from .scanner import scan_repository from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds LOGGER = logging.getLogger(__name__) def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int: try: value = max(1, int(os.getenv(name, str(default)))) except ValueError: value = default return min(value, maximum) if maximum else value def _positive_float(name: str, default: float) -> float: try: return max(0.2, float(os.getenv(name, str(default)))) except ValueError: return default class ScanJobRunner: """Single-process durable worker and bounded Gitea scheduler.""" def __init__( self, store: ScanStore, *, gitea: GiteaClient | None = None, acquirer: GitAcquirer | None = None, poll_seconds: float = 0.5, sync_interval_seconds: float | None = None, max_queue: int | None = None, ) -> None: self.store = store self.gitea = gitea or GiteaClient() self.acquirer = acquirer or GitAcquirer() self.poll_seconds = max(0.2, poll_seconds) self.sync_interval_seconds = sync_interval_seconds or _positive_float( "REPOPULSE_GITEA_SYNC_INTERVAL_SECONDS", 300.0 ) self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000) self._stop = Event() self._thread: Thread | None = None self._last_sync = float("-inf") self._sync_lock = Lock() @property def started(self) -> bool: return self._thread is not None and self._thread.is_alive() def start(self) -> None: if self.started: return self._stop.clear() self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True) self._thread.start() def stop(self) -> None: self._stop.set() thread = self._thread if thread and thread is not current_thread(): thread.join(timeout=5) if thread is None or not thread.is_alive(): self._thread = None def _run(self) -> None: while not self._stop.is_set(): try: job = self._run_iteration() except MaintenanceModeError: job = None except Exception: self._last_sync = monotonic() LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure") job = None if job is None: self._stop.wait(self.poll_seconds) def _run_iteration(self) -> ScanJob | None: """Perform one scheduler pass unless durable maintenance has paused work.""" if self.store.maintenance_status()["enabled"]: return None if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()): return None if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds: self.sync_now() if self.store.maintenance_status()["enabled"]: return None return self.process_one() def sync_now(self) -> RepositorySyncResult: with self._sync_lock: if self.store.maintenance_status()["enabled"]: raise MaintenanceModeError("RepoPulse is in maintenance mode") if not self.gitea.configured: result = RepositorySyncResult( provider="gitea", errors=["not_configured: Gitea connector is not configured"], synced_at=datetime.now(UTC), ) self.store.record_sync_result(result) self._last_sync = monotonic() return result try: result = self.gitea.sync(self.store) except Exception: self._last_sync = monotonic() raise self._last_sync = monotonic() if not result.errors: self._schedule_changed_repositories() return result def _schedule_changed_repositories(self) -> None: if self.store.maintenance_status()["enabled"]: return for summary in self.store.list_repositories(limit=1000): if summary.provider != "gitea" or summary.archived or not summary.latest_revision: continue if not summary.policy.auto_scan: continue if summary.last_scan_revision and summary.last_scan_revision == summary.latest_revision: continue try: self.store.enqueue_job( summary.id, summary.latest_revision, ScanTrigger.SCHEDULED, max_queue=self.max_queue, ) except (MaintenanceModeError, QueueFullError, KeyError): continue def set_maintenance_mode(self, enabled: bool) -> dict[str, object]: """Serialize maintenance changes with catalog reconciliation and scan scheduling.""" with self._sync_lock: return self.store.set_maintenance_mode(enabled) def enqueue( self, repository: Repository, *, force: bool = False, trigger: ScanTrigger = ScanTrigger.MANUAL, ) -> ScanJob: requested_revision = repository.latest_revision return self.store.enqueue_job( repository.id, requested_revision, trigger, force=force, max_queue=self.max_queue, ) def process_one(self) -> ScanJob | None: job = self.store.claim_next_job() if job is None: return None try: self._process(job) except Exception: LOGGER.exception("RepoPulse scan job failed outside the isolated job handler") try: self.store.update_job( job.id, status=ScanJobStatus.FAILED, phase="failed", progress=100, error_code="internal_error", error_detail="Unexpected repository analysis failure", message="Repository analysis failed", ) except Exception: LOGGER.exception("RepoPulse could not persist an unexpected job failure") return self.store.get_job(job.id) class _LeaseHeartbeat: def __init__(self, store: ScanStore, job_id: str) -> None: self.store = store self.job_id = job_id self.stop_event = Event() self.thread = Thread(target=self._run, name=f"repopulse-lease-{job_id[:8]}", daemon=True) def start(self) -> None: self.thread.start() def stop(self) -> None: self.stop_event.set() self.thread.join(timeout=2) def _run(self) -> None: interval = max(1.0, job_lease_seconds() / 3) while not self.stop_event.wait(interval): try: self.store.renew_job_lease(self.job_id) except Exception: LOGGER.warning("RepoPulse could not renew a scan-job lease", exc_info=True) def _process(self, job: ScanJob) -> None: repository = self.store.get_repository(job.repository_id) if repository is None: self.store.update_job( job.id, status=ScanJobStatus.FAILED, phase="failed", progress=100, error_code="repository_not_found", error_detail="Repository catalog entry no longer exists", ) return source_path = None acquired = None lease_heartbeat = self._LeaseHeartbeat(self.store, job.id) lease_heartbeat.start() try: if repository.provider != "gitea": source_path = self.store.get_repository_source_path(repository.id) if not source_path: raise AcquisitionError("source_unavailable", "Local repository source is unavailable") actual_revision = job.requested_revision branch = repository.default_branch else: acquired = self.acquirer.acquire(repository, job.id, job.requested_revision) source_path = str(acquired.path) actual_revision = acquired.actual_revision branch = acquired.branch self.store.update_job( job.id, status=ScanJobStatus.SCANNING, phase="scanning", progress=30, actual_revision=actual_revision, message="Running deterministic repository analysis", ) source = ScanSource( origin=SourceOrigin.GITEA if repository.provider == "gitea" else SourceOrigin.LOCAL, provider=repository.provider, repository_id=repository.id, repository_full_name=repository.full_name, branch=branch, requested_revision=job.requested_revision, actual_revision=actual_revision, trigger=job.trigger, ) report = scan_repository(source_path, source=source, display_name=repository.name) self.store.update_job( job.id, status=ScanJobStatus.ANALYZING, phase="analyzing", progress=80, message="Persisting report and comparison baseline", ) self.store.update_job( job.id, status=ScanJobStatus.PERSISTING, phase="persisting", progress=90, message="Saving report and revision history", ) saved = self.store.save(report) self.store.upsert_repository(repository.model_copy(update={"policy": saved.policy})) self.store.update_job( job.id, status=ScanJobStatus.COMPLETED, phase="completed", progress=100, actual_revision=saved.source.actual_revision, scan_id=saved.id, message="Scan completed", ) except AcquisitionError as exc: self.store.update_job( job.id, status=ScanJobStatus.FAILED, phase="failed", progress=100, error_code=exc.code, error_detail=exc.detail, message="Repository source acquisition failed", ) except (ValueError, PermissionError, OSError) as exc: self.store.update_job( job.id, status=ScanJobStatus.FAILED, phase="failed", progress=100, error_code="scan_failed", error_detail=str(exc) if isinstance(exc, ValueError) else "Repository analysis failed", message="Repository analysis failed", ) except Exception: LOGGER.exception("RepoPulse scan job failed without exposing job details") self.store.update_job( job.id, status=ScanJobStatus.FAILED, phase="failed", progress=100, error_code="internal_error", error_detail="Unexpected repository analysis failure", message="Repository analysis failed", ) finally: lease_heartbeat.stop() if acquired is not None and repository is not None: try: self.acquirer.cleanup(acquired, repository) except AcquisitionError: LOGGER.warning("Unable to clean one RepoPulse worktree")