Initial curated RAGcore source from c16667f0c9397e2bfc8dd0255b114fba23c8d05e

ops/alerts/queue_age_slo.yaml

@@ -0,0 +1,45 @@
+# Job queue-age SLO alert.
+#
+# Required by docs/09_OPERATIONS/OBSERVABILITY.md's "Alerts" list ("...
+# oude leases, queue-age SLO, repeated terminal jobs, ...").
+#
+# Fires when a job waits too long between being enqueued and being claimed
+# by a worker, signalling the worker pool is falling behind or stuck.
+#
+# Tool-agnostic on purpose (no Prometheus/Grafana/Alertmanager assumed) --
+# adapt `signal` to whichever metrics pipeline a given deployment already
+# runs.
+
+alert: JobQueueAgeSloBreached
+severity: warning
+description: >-
+  A job's age at the moment a worker claimed it exceeded the configured
+  SLO threshold, meaning jobs are waiting too long in the queue before
+  being picked up.
+
+signal:
+  type: metric
+  metric: ragcore_job_age_seconds
+  # Recorded by src/ragcore/platform/telemetry/metrics.py::
+  # TelemetryMetrics.record_job_claimed -- see
+  # ops/dashboards/operations_overview.yaml's "Job age at claim" panel.
+  aggregation: p95
+  comparison: greater_than
+  threshold_seconds: 300
+
+for: 5m
+
+runbook:
+  - >-
+    Check ragcore_job_queue_depth alongside this alert -- a simultaneously
+    rising queue depth points at worker capacity being outpaced by job
+    volume; a flat or falling depth with high age instead suggests a
+    specific stuck job or a worker that stopped claiming work.
+  - >-
+    Confirm workers are actually running and their lease-recovery loop is
+    healthy (see the paired "oude leases" / stale-lease alert also
+    required by OBSERVABILITY.md).
+  - >-
+    If job volume genuinely exceeds capacity, scale the worker pool; if a
+    specific job type is slow, investigate that job's own processing time
+    rather than the queue mechanism itself.