Initial curated RepoPulse source from 2524ccc3c1c27739996dccb2b7992989f88cd26d
This commit is contained in:
Public Source Publisher committed
commit
0adf8ef5d9
90 files changed
+14315
No files matched your search
@@ -0,0 +1,330 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from threading import Event, Lock, Thread, current_thread
|
||||
from time import monotonic
|
||||
|
||||
from .acquisition import AcquisitionError, GitAcquirer
|
||||
from .gitea import GiteaClient
|
||||
from .models import (
|
||||
Repository,
|
||||
RepositorySyncResult,
|
||||
ScanJob,
|
||||
ScanJobStatus,
|
||||
ScanSource,
|
||||
ScanTrigger,
|
||||
SourceOrigin,
|
||||
)
|
||||
from .scanner import scan_repository
|
||||
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
|
||||
try:
|
||||
value = max(1, int(os.getenv(name, str(default))))
|
||||
except ValueError:
|
||||
value = default
|
||||
return min(value, maximum) if maximum else value
|
||||
|
||||
|
||||
def _positive_float(name: str, default: float) -> float:
|
||||
try:
|
||||
return max(0.2, float(os.getenv(name, str(default))))
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
class ScanJobRunner:
|
||||
"""Single-process durable worker and bounded Gitea scheduler."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: ScanStore,
|
||||
*,
|
||||
gitea: GiteaClient | None = None,
|
||||
acquirer: GitAcquirer | None = None,
|
||||
poll_seconds: float = 0.5,
|
||||
sync_interval_seconds: float | None = None,
|
||||
max_queue: int | None = None,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.gitea = gitea or GiteaClient()
|
||||
self.acquirer = acquirer or GitAcquirer()
|
||||
self.poll_seconds = max(0.2, poll_seconds)
|
||||
self.sync_interval_seconds = sync_interval_seconds or _positive_float(
|
||||
"REPOPULSE_GITEA_SYNC_INTERVAL_SECONDS", 300.0
|
||||
)
|
||||
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
|
||||
self._stop = Event()
|
||||
self._thread: Thread | None = None
|
||||
self._last_sync = float("-inf")
|
||||
self._sync_lock = Lock()
|
||||
|
||||
@property
|
||||
def started(self) -> bool:
|
||||
return self._thread is not None and self._thread.is_alive()
|
||||
|
||||
def start(self) -> None:
|
||||
if self.started:
|
||||
return
|
||||
self._stop.clear()
|
||||
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
thread = self._thread
|
||||
if thread and thread is not current_thread():
|
||||
thread.join(timeout=5)
|
||||
if thread is None or not thread.is_alive():
|
||||
self._thread = None
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
job = self._run_iteration()
|
||||
except MaintenanceModeError:
|
||||
job = None
|
||||
except Exception:
|
||||
self._last_sync = monotonic()
|
||||
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
|
||||
job = None
|
||||
if job is None:
|
||||
self._stop.wait(self.poll_seconds)
|
||||
|
||||
def _run_iteration(self) -> ScanJob | None:
|
||||
"""Perform one scheduler pass unless durable maintenance has paused work."""
|
||||
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return None
|
||||
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
|
||||
return None
|
||||
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
|
||||
self.sync_now()
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return None
|
||||
return self.process_one()
|
||||
|
||||
def sync_now(self) -> RepositorySyncResult:
|
||||
with self._sync_lock:
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
raise MaintenanceModeError("RepoPulse is in maintenance mode")
|
||||
if not self.gitea.configured:
|
||||
result = RepositorySyncResult(
|
||||
provider="gitea",
|
||||
errors=["not_configured: Gitea connector is not configured"],
|
||||
synced_at=datetime.now(UTC),
|
||||
)
|
||||
self.store.record_sync_result(result)
|
||||
self._last_sync = monotonic()
|
||||
return result
|
||||
try:
|
||||
result = self.gitea.sync(self.store)
|
||||
except Exception:
|
||||
self._last_sync = monotonic()
|
||||
raise
|
||||
self._last_sync = monotonic()
|
||||
if not result.errors:
|
||||
self._schedule_changed_repositories()
|
||||
return result
|
||||
|
||||
def _schedule_changed_repositories(self) -> None:
|
||||
if self.store.maintenance_status()["enabled"]:
|
||||
return
|
||||
for summary in self.store.list_repositories(limit=1000):
|
||||
if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
|
||||
continue
|
||||
if not summary.policy.auto_scan:
|
||||
continue
|
||||
if summary.last_scan_revision and summary.last_scan_revision == summary.latest_revision:
|
||||
continue
|
||||
try:
|
||||
self.store.enqueue_job(
|
||||
summary.id,
|
||||
summary.latest_revision,
|
||||
ScanTrigger.SCHEDULED,
|
||||
max_queue=self.max_queue,
|
||||
)
|
||||
except (MaintenanceModeError, QueueFullError, KeyError):
|
||||
continue
|
||||
|
||||
def set_maintenance_mode(self, enabled: bool) -> dict[str, object]:
|
||||
"""Serialize maintenance changes with catalog reconciliation and scan scheduling."""
|
||||
|
||||
with self._sync_lock:
|
||||
return self.store.set_maintenance_mode(enabled)
|
||||
|
||||
def enqueue(
|
||||
self,
|
||||
repository: Repository,
|
||||
*,
|
||||
force: bool = False,
|
||||
trigger: ScanTrigger = ScanTrigger.MANUAL,
|
||||
) -> ScanJob:
|
||||
requested_revision = repository.latest_revision
|
||||
return self.store.enqueue_job(
|
||||
repository.id,
|
||||
requested_revision,
|
||||
trigger,
|
||||
force=force,
|
||||
max_queue=self.max_queue,
|
||||
)
|
||||
|
||||
def process_one(self) -> ScanJob | None:
|
||||
job = self.store.claim_next_job()
|
||||
if job is None:
|
||||
return None
|
||||
try:
|
||||
self._process(job)
|
||||
except Exception:
|
||||
LOGGER.exception("RepoPulse scan job failed outside the isolated job handler")
|
||||
try:
|
||||
self.store.update_job(
|
||||
job.id,
|
||||
status=ScanJobStatus.FAILED,
|
||||
phase="failed",
|
||||
progress=100,
|
||||
error_code="internal_error",
|
||||
error_detail="Unexpected repository analysis failure",
|
||||
message="Repository analysis failed",
|
||||
)
|
||||
except Exception:
|
||||
LOGGER.exception("RepoPulse could not persist an unexpected job failure")
|
||||
return self.store.get_job(job.id)
|
||||
|
||||
class _LeaseHeartbeat:
|
||||
def __init__(self, store: ScanStore, job_id: str) -> None:
|
||||
self.store = store
|
||||
self.job_id = job_id
|
||||
self.stop_event = Event()
|
||||
self.thread = Thread(target=self._run, name=f"repopulse-lease-{job_id[:8]}", daemon=True)
|
||||
|
||||
def start(self) -> None:
|
||||
self.thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self.stop_event.set()
|
||||
self.thread.join(timeout=2)
|
||||
|
||||
def _run(self) -> None:
|
||||
interval = max(1.0, job_lease_seconds() / 3)
|
||||
while not self.stop_event.wait(interval):
|
||||
try:
|
||||
self.store.renew_job_lease(self.job_id)
|
||||
except Exception:
|
||||
LOGGER.warning("RepoPulse could not renew a scan-job lease", exc_info=True)
|
||||
|
||||
def _process(self, job: ScanJob) -> None:
|
||||
repository = self.store.get_repository(job.repository_id)
|
||||
if repository is None:
|
||||
self.store.update_job(
|
||||
job.id,
|
||||
status=ScanJobStatus.FAILED,
|
||||
phase="failed",
|
||||
progress=100,
|
||||
error_code="repository_not_found",
|
||||
error_detail="Repository catalog entry no longer exists",
|
||||
)
|
||||
return
|
||||
source_path = None
|
||||
acquired = None
|
||||
lease_heartbeat = self._LeaseHeartbeat(self.store, job.id)
|
||||
lease_heartbeat.start()
|
||||
try:
|
||||
if repository.provider != "gitea":
|
||||
source_path = self.store.get_repository_source_path(repository.id)
|
||||
if not source_path:
|
||||
raise AcquisitionError("source_unavailable", "Local repository source is unavailable")
|
||||
actual_revision = job.requested_revision
|
||||
branch = repository.default_branch
|
||||
else:
|
||||
acquired = self.acquirer.acquire(repository, job.id, job.requested_revision)
|
||||
source_path = str(acquired.path)
|
||||
actual_revision = acquired.actual_revision
|
||||
branch = acquired.branch
|
||||
self.store.update_job(
|
||||
job.id,
|
||||
status=ScanJobStatus.SCANNING,
|
||||
phase="scanning",
|
||||
progress=30,
|
||||
actual_revision=actual_revision,
|
||||
message="Running deterministic repository analysis",
|
||||
)
|
||||
source = ScanSource(
|
||||
origin=SourceOrigin.GITEA if repository.provider == "gitea" else SourceOrigin.LOCAL,
|
||||
provider=repository.provider,
|
||||
repository_id=repository.id,
|
||||
repository_full_name=repository.full_name,
|
||||
branch=branch,
|
||||
requested_revision=job.requested_revision,
|
||||
actual_revision=actual_revision,
|
||||
trigger=job.trigger,
|
||||
)
|
||||
report = scan_repository(source_path, source=source, display_name=repository.name)
|
||||
self.store.update_job(
|
||||
job.id,
|
||||
status=ScanJobStatus.ANALYZING,
|
||||
phase="analyzing",
|
||||
progress=80,
|
||||
message="Persisting report and comparison baseline",
|
||||
)
|
||||
self.store.update_job(
|
||||
job.id,
|
||||
status=ScanJobStatus.PERSISTING,
|
||||
phase="persisting",
|
||||
progress=90,
|
||||
message="Saving report and revision history",
|
||||
)
|
||||
saved = self.store.save(report)
|
||||
self.store.upsert_repository(repository.model_copy(update={"policy": saved.policy}))
|
||||
self.store.update_job(
|
||||
job.id,
|
||||
status=ScanJobStatus.COMPLETED,
|
||||
phase="completed",
|
||||
progress=100,
|
||||
actual_revision=saved.source.actual_revision,
|
||||
scan_id=saved.id,
|
||||
message="Scan completed",
|
||||
)
|
||||
except AcquisitionError as exc:
|
||||
self.store.update_job(
|
||||
job.id,
|
||||
status=ScanJobStatus.FAILED,
|
||||
phase="failed",
|
||||
progress=100,
|
||||
error_code=exc.code,
|
||||
error_detail=exc.detail,
|
||||
message="Repository source acquisition failed",
|
||||
)
|
||||
except (ValueError, PermissionError, OSError) as exc:
|
||||
self.store.update_job(
|
||||
job.id,
|
||||
status=ScanJobStatus.FAILED,
|
||||
phase="failed",
|
||||
progress=100,
|
||||
error_code="scan_failed",
|
||||
error_detail=str(exc) if isinstance(exc, ValueError) else "Repository analysis failed",
|
||||
message="Repository analysis failed",
|
||||
)
|
||||
except Exception:
|
||||
LOGGER.exception("RepoPulse scan job failed without exposing job details")
|
||||
self.store.update_job(
|
||||
job.id,
|
||||
status=ScanJobStatus.FAILED,
|
||||
phase="failed",
|
||||
progress=100,
|
||||
error_code="internal_error",
|
||||
error_detail="Unexpected repository analysis failure",
|
||||
message="Repository analysis failed",
|
||||
)
|
||||
finally:
|
||||
lease_heartbeat.stop()
|
||||
if acquired is not None and repository is not None:
|
||||
try:
|
||||
self.acquirer.cleanup(acquired, repository)
|
||||
except AcquisitionError:
|
||||
LOGGER.warning("Unable to clean one RepoPulse worktree")
|
||||
Reference in new issue
Block a user