331 lines
12 KiB
Python
331 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from datetime import UTC, datetime
|
|
from threading import Event, Lock, Thread, current_thread
|
|
from time import monotonic
|
|
|
|
from .acquisition import AcquisitionError, GitAcquirer
|
|
from .gitea import GiteaClient
|
|
from .models import (
|
|
Repository,
|
|
RepositorySyncResult,
|
|
ScanJob,
|
|
ScanJobStatus,
|
|
ScanSource,
|
|
ScanTrigger,
|
|
SourceOrigin,
|
|
)
|
|
from .scanner import scan_repository
|
|
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
|
|
|
|
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
|
|
try:
|
|
value = max(1, int(os.getenv(name, str(default))))
|
|
except ValueError:
|
|
value = default
|
|
return min(value, maximum) if maximum else value
|
|
|
|
|
|
def _positive_float(name: str, default: float) -> float:
|
|
try:
|
|
return max(0.2, float(os.getenv(name, str(default))))
|
|
except ValueError:
|
|
return default
|
|
|
|
|
|
class ScanJobRunner:
|
|
"""Single-process durable worker and bounded Gitea scheduler."""
|
|
|
|
def __init__(
|
|
self,
|
|
store: ScanStore,
|
|
*,
|
|
gitea: GiteaClient | None = None,
|
|
acquirer: GitAcquirer | None = None,
|
|
poll_seconds: float = 0.5,
|
|
sync_interval_seconds: float | None = None,
|
|
max_queue: int | None = None,
|
|
) -> None:
|
|
self.store = store
|
|
self.gitea = gitea or GiteaClient()
|
|
self.acquirer = acquirer or GitAcquirer()
|
|
self.poll_seconds = max(0.2, poll_seconds)
|
|
self.sync_interval_seconds = sync_interval_seconds or _positive_float(
|
|
"REPOPULSE_GITEA_SYNC_INTERVAL_SECONDS", 300.0
|
|
)
|
|
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
|
|
self._stop = Event()
|
|
self._thread: Thread | None = None
|
|
self._last_sync = float("-inf")
|
|
self._sync_lock = Lock()
|
|
|
|
@property
|
|
def started(self) -> bool:
|
|
return self._thread is not None and self._thread.is_alive()
|
|
|
|
def start(self) -> None:
|
|
if self.started:
|
|
return
|
|
self._stop.clear()
|
|
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
|
|
self._thread.start()
|
|
|
|
def stop(self) -> None:
|
|
self._stop.set()
|
|
thread = self._thread
|
|
if thread and thread is not current_thread():
|
|
thread.join(timeout=5)
|
|
if thread is None or not thread.is_alive():
|
|
self._thread = None
|
|
|
|
def _run(self) -> None:
|
|
while not self._stop.is_set():
|
|
try:
|
|
job = self._run_iteration()
|
|
except MaintenanceModeError:
|
|
job = None
|
|
except Exception:
|
|
self._last_sync = monotonic()
|
|
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
|
|
job = None
|
|
if job is None:
|
|
self._stop.wait(self.poll_seconds)
|
|
|
|
def _run_iteration(self) -> ScanJob | None:
|
|
"""Perform one scheduler pass unless durable maintenance has paused work."""
|
|
|
|
if self.store.maintenance_status()["enabled"]:
|
|
return None
|
|
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
|
|
return None
|
|
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
|
|
self.sync_now()
|
|
if self.store.maintenance_status()["enabled"]:
|
|
return None
|
|
return self.process_one()
|
|
|
|
def sync_now(self) -> RepositorySyncResult:
|
|
with self._sync_lock:
|
|
if self.store.maintenance_status()["enabled"]:
|
|
raise MaintenanceModeError("RepoPulse is in maintenance mode")
|
|
if not self.gitea.configured:
|
|
result = RepositorySyncResult(
|
|
provider="gitea",
|
|
errors=["not_configured: Gitea connector is not configured"],
|
|
synced_at=datetime.now(UTC),
|
|
)
|
|
self.store.record_sync_result(result)
|
|
self._last_sync = monotonic()
|
|
return result
|
|
try:
|
|
result = self.gitea.sync(self.store)
|
|
except Exception:
|
|
self._last_sync = monotonic()
|
|
raise
|
|
self._last_sync = monotonic()
|
|
if not result.errors:
|
|
self._schedule_changed_repositories()
|
|
return result
|
|
|
|
def _schedule_changed_repositories(self) -> None:
|
|
if self.store.maintenance_status()["enabled"]:
|
|
return
|
|
for summary in self.store.list_repositories(limit=1000):
|
|
if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
|
|
continue
|
|
if not summary.policy.auto_scan:
|
|
continue
|
|
if summary.last_scan_revision and summary.last_scan_revision == summary.latest_revision:
|
|
continue
|
|
try:
|
|
self.store.enqueue_job(
|
|
summary.id,
|
|
summary.latest_revision,
|
|
ScanTrigger.SCHEDULED,
|
|
max_queue=self.max_queue,
|
|
)
|
|
except (MaintenanceModeError, QueueFullError, KeyError):
|
|
continue
|
|
|
|
def set_maintenance_mode(self, enabled: bool) -> dict[str, object]:
|
|
"""Serialize maintenance changes with catalog reconciliation and scan scheduling."""
|
|
|
|
with self._sync_lock:
|
|
return self.store.set_maintenance_mode(enabled)
|
|
|
|
def enqueue(
|
|
self,
|
|
repository: Repository,
|
|
*,
|
|
force: bool = False,
|
|
trigger: ScanTrigger = ScanTrigger.MANUAL,
|
|
) -> ScanJob:
|
|
requested_revision = repository.latest_revision
|
|
return self.store.enqueue_job(
|
|
repository.id,
|
|
requested_revision,
|
|
trigger,
|
|
force=force,
|
|
max_queue=self.max_queue,
|
|
)
|
|
|
|
def process_one(self) -> ScanJob | None:
|
|
job = self.store.claim_next_job()
|
|
if job is None:
|
|
return None
|
|
try:
|
|
self._process(job)
|
|
except Exception:
|
|
LOGGER.exception("RepoPulse scan job failed outside the isolated job handler")
|
|
try:
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.FAILED,
|
|
phase="failed",
|
|
progress=100,
|
|
error_code="internal_error",
|
|
error_detail="Unexpected repository analysis failure",
|
|
message="Repository analysis failed",
|
|
)
|
|
except Exception:
|
|
LOGGER.exception("RepoPulse could not persist an unexpected job failure")
|
|
return self.store.get_job(job.id)
|
|
|
|
class _LeaseHeartbeat:
|
|
def __init__(self, store: ScanStore, job_id: str) -> None:
|
|
self.store = store
|
|
self.job_id = job_id
|
|
self.stop_event = Event()
|
|
self.thread = Thread(target=self._run, name=f"repopulse-lease-{job_id[:8]}", daemon=True)
|
|
|
|
def start(self) -> None:
|
|
self.thread.start()
|
|
|
|
def stop(self) -> None:
|
|
self.stop_event.set()
|
|
self.thread.join(timeout=2)
|
|
|
|
def _run(self) -> None:
|
|
interval = max(1.0, job_lease_seconds() / 3)
|
|
while not self.stop_event.wait(interval):
|
|
try:
|
|
self.store.renew_job_lease(self.job_id)
|
|
except Exception:
|
|
LOGGER.warning("RepoPulse could not renew a scan-job lease", exc_info=True)
|
|
|
|
def _process(self, job: ScanJob) -> None:
|
|
repository = self.store.get_repository(job.repository_id)
|
|
if repository is None:
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.FAILED,
|
|
phase="failed",
|
|
progress=100,
|
|
error_code="repository_not_found",
|
|
error_detail="Repository catalog entry no longer exists",
|
|
)
|
|
return
|
|
source_path = None
|
|
acquired = None
|
|
lease_heartbeat = self._LeaseHeartbeat(self.store, job.id)
|
|
lease_heartbeat.start()
|
|
try:
|
|
if repository.provider != "gitea":
|
|
source_path = self.store.get_repository_source_path(repository.id)
|
|
if not source_path:
|
|
raise AcquisitionError("source_unavailable", "Local repository source is unavailable")
|
|
actual_revision = job.requested_revision
|
|
branch = repository.default_branch
|
|
else:
|
|
acquired = self.acquirer.acquire(repository, job.id, job.requested_revision)
|
|
source_path = str(acquired.path)
|
|
actual_revision = acquired.actual_revision
|
|
branch = acquired.branch
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.SCANNING,
|
|
phase="scanning",
|
|
progress=30,
|
|
actual_revision=actual_revision,
|
|
message="Running deterministic repository analysis",
|
|
)
|
|
source = ScanSource(
|
|
origin=SourceOrigin.GITEA if repository.provider == "gitea" else SourceOrigin.LOCAL,
|
|
provider=repository.provider,
|
|
repository_id=repository.id,
|
|
repository_full_name=repository.full_name,
|
|
branch=branch,
|
|
requested_revision=job.requested_revision,
|
|
actual_revision=actual_revision,
|
|
trigger=job.trigger,
|
|
)
|
|
report = scan_repository(source_path, source=source, display_name=repository.name)
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.ANALYZING,
|
|
phase="analyzing",
|
|
progress=80,
|
|
message="Persisting report and comparison baseline",
|
|
)
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.PERSISTING,
|
|
phase="persisting",
|
|
progress=90,
|
|
message="Saving report and revision history",
|
|
)
|
|
saved = self.store.save(report)
|
|
self.store.upsert_repository(repository.model_copy(update={"policy": saved.policy}))
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.COMPLETED,
|
|
phase="completed",
|
|
progress=100,
|
|
actual_revision=saved.source.actual_revision,
|
|
scan_id=saved.id,
|
|
message="Scan completed",
|
|
)
|
|
except AcquisitionError as exc:
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.FAILED,
|
|
phase="failed",
|
|
progress=100,
|
|
error_code=exc.code,
|
|
error_detail=exc.detail,
|
|
message="Repository source acquisition failed",
|
|
)
|
|
except (ValueError, PermissionError, OSError) as exc:
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.FAILED,
|
|
phase="failed",
|
|
progress=100,
|
|
error_code="scan_failed",
|
|
error_detail=str(exc) if isinstance(exc, ValueError) else "Repository analysis failed",
|
|
message="Repository analysis failed",
|
|
)
|
|
except Exception:
|
|
LOGGER.exception("RepoPulse scan job failed without exposing job details")
|
|
self.store.update_job(
|
|
job.id,
|
|
status=ScanJobStatus.FAILED,
|
|
phase="failed",
|
|
progress=100,
|
|
error_code="internal_error",
|
|
error_detail="Unexpected repository analysis failure",
|
|
message="Repository analysis failed",
|
|
)
|
|
finally:
|
|
lease_heartbeat.stop()
|
|
if acquired is not None and repository is not None:
|
|
try:
|
|
self.acquirer.cleanup(acquired, repository)
|
|
except AcquisitionError:
|
|
LOGGER.warning("Unable to clean one RepoPulse worktree")
|