Files
projectbrain-Public/backend/test/runner-statistics.test.ts
T

411 lines
22 KiB
TypeScript

import { describe, expect, it, vi } from "vitest";
import type { FetchLike } from "@modelcontextprotocol/sdk/shared/transport.js";
import { loadEnv } from "../src/config/env.js";
import type { CompactRunnerJob, GiteaRuntimeSnapshot } from "../src/services/activityRuntime.js";
import { createAppOpsRunnerStatisticsLoader, createGiteaRunnerStatisticsFallback, createOriginBoundFetch, readAppOpsTool, type RunnerStatisticsSnapshot } from "../src/services/runnerStatistics.js";
const now = new Date("2026-10-01T09:00:00.000Z");
const env = (input: NodeJS.ProcessEnv = {}) => loadEnv({
NODE_ENV: "test",
DATABASE_URL: "postgresql://example.invalid/projectbrain",
PROJECTBRAIN_DATA_SOURCE: "database",
PROJECTBRAIN_OWNER_USERNAME: "owner",
PROJECTBRAIN_OWNER_PASSWORD: "correct-horse-battery-staple",
PROJECTBRAIN_AUTH_SECRET: "test-secret-with-at-least-thirty-two-characters",
...input,
});
const profile = (id: "checks" | "backend" | "frontend" | "security" | "browser" | "build", input: Partial<RunnerStatisticsSnapshot["profiles"][number]> = {}) => ({
id,
displayName: `ITWorx CI · ${id[0]!.toUpperCase()}${id.slice(1)}`,
labels: [`itworx-ci-${id}`],
runnerName: `${id}-1`,
status: "ready",
queueDepth: 2,
activeJobs: 1,
oldestQueuedSeconds: 60,
reason: null,
cpuLimit: 2,
memoryLimitBytes: 4_294_967_296,
cpuUsed: 1.25,
memoryUsedBytes: 1_073_741_824,
completedJobs: 4,
p50WaitSeconds: 30,
p95WaitSeconds: 120,
p50DurationSeconds: 90,
p95DurationSeconds: 300,
lastDurationSeconds: 100,
...input,
});
const stats = (generatedAt = now.toISOString()): RunnerStatisticsSnapshot => ({
schemaVersion: 1,
status: "healthy",
generatedAt,
scope: "ci-fleet",
budget: { cpuLimit: 8, memoryLimitBytes: 17_179_869_184, activeJobLimit: 2, heavyJobLimit: 1, reservedCpu: 4, reservedMemoryBytes: 8_589_934_592, activeJobs: 1, heavyJobs: 0 },
host: { cpuBusyPercent: 20, loadRatio: 0.5, availableMemoryBytes: 8_589_934_592, cacheFreeBytes: 17_179_869_184 },
profiles: ["checks", "backend", "frontend", "security", "browser", "build"].map((id) => profile(id as RunnerStatisticsSnapshot["profiles"][number]["id"])),
window: { startAt: "2026-10-01T08:00:00.000Z", endAt: now.toISOString(), completedJobs: 24, samples: 120, coverage: "complete" },
history: [{ at: "2026-10-01T08:59:00.000Z", activeJobs: 1, queuedJobs: 2, cpuUsed: 1.25, memoryUsedBytes: 1_073_741_824 }],
});
const runtimeJob = (input: Partial<CompactRunnerJob> = {}): CompactRunnerJob => ({
id: 1,
runId: 10,
runAttempt: 1,
name: "Backend tests",
status: "completed",
queueState: "completed",
conclusion: "success",
labels: ["itworx-ci-backend"],
completedSteps: 4,
totalSteps: 4,
url: "https://gitea.example.com/Jens/projectbrain/actions/runs/10/jobs/1",
createdAt: "2026-10-01T08:55:00.000Z",
startedAt: "2026-10-01T08:56:00.000Z",
completedAt: "2026-10-01T08:58:00.000Z",
...input,
});
const runtime = (jobs: CompactRunnerJob[]): GiteaRuntimeSnapshot => ({
availability: "live",
runners: [{ id: 1, name: "backend-1", status: "online", busy: false, disabled: false, labels: ["itworx-ci-backend"] }],
jobs,
counts: { running: 0, runnable: 0, dependencyBlocked: 0 },
runnableJobScan: { complete: true, observed: 0, total: 0 },
runnerInventoryComplete: true,
generatedAt: now.toISOString(),
stateVersion: "snapshot-v1",
});
describe("runner statistics bridge and Gitea fallback", () => {
it("accepts only HTTPS AppOps endpoints while keeping the optional integration unset by default", () => {
expect(env().PROJECTBRAIN_APPOPS_MCP_URL).toBeUndefined();
expect(env({ PROJECTBRAIN_APPOPS_MCP_URL: "" }).PROJECTBRAIN_APPOPS_MCP_URL).toBeUndefined();
expect(env({ PROJECTBRAIN_APPOPS_MCP_URL: "https://appops.example.com/mcp" }).PROJECTBRAIN_APPOPS_MCP_URL)
.toBe("https://appops.example.com/mcp");
expect(() => env({ PROJECTBRAIN_APPOPS_MCP_URL: "http://appops.example.com/mcp" })).toThrow();
});
it("rejects an HTTP MCP endpoint before the statistics transport can send its credential", async () => {
const httpsFetch = vi.fn().mockResolvedValue(new Response(null, { status: 500 }));
await expect(readAppOpsTool("https://appops.example.com/mcp", "test-readonly-token-for-statistics", { fetch: httpsFetch })).rejects.toThrow();
expect(httpsFetch).toHaveBeenCalled();
const httpFetch = vi.fn().mockResolvedValue(new Response(null, { status: 500 }));
await expect(readAppOpsTool("http://appops.example.com/mcp", "test-readonly-token-for-statistics", { fetch: httpFetch }))
.rejects.toThrow(/HTTPS/i);
expect(httpFetch).not.toHaveBeenCalled();
});
it("reports not configured without calling the optional MCP bridge", async () => {
const read = vi.fn();
const load = createAppOpsRunnerStatisticsLoader(env(), { now: () => now, read });
const result = await load();
expect(result.snapshot.status).toBe("not_configured");
expect(result.snapshot.host.availableMemoryBytes).toBeNull();
expect(read).not.toHaveBeenCalled();
const partiallyConfigured = createAppOpsRunnerStatisticsLoader(env({ PROJECTBRAIN_APPOPS_MCP_URL: "https://appops.example.com/mcp" }), { now: () => now, read });
expect((await partiallyConfigured()).snapshot.status).toBe("unavailable");
expect(read).not.toHaveBeenCalled();
});
it("keeps complete v1 data, but degrades partial profile and window coverage", async () => {
const read = vi.fn().mockResolvedValue(stats());
const load = createAppOpsRunnerStatisticsLoader(env({
PROJECTBRAIN_APPOPS_MCP_URL: "https://appops.example.com/mcp",
PROJECTBRAIN_APPOPS_MCP_TOKEN: "scoped-readonly-machine-token-123",
}), { now: () => now, read });
const complete = await load();
expect(complete.snapshot.status).toBe("healthy");
expect(complete.snapshot.profiles[0]?.displayName).toBe("ITWorx CI · Checks");
expect(complete.snapshot.profiles[0]?.lastDurationSeconds).toBe(100);
const partial = createAppOpsRunnerStatisticsLoader(env({
PROJECTBRAIN_APPOPS_MCP_URL: "https://appops.example.com/mcp",
PROJECTBRAIN_APPOPS_MCP_TOKEN: "scoped-readonly-machine-token-123",
}), { now: () => now, read: async () => ({ ...stats(), status: "healthy", profiles: stats().profiles.slice(0, 4), window: { ...stats().window, coverage: "unavailable" } }) });
expect((await partial()).snapshot.status).toBe("degraded");
});
it("reads structured MCP content and strips fields outside the public statistics schema", async () => {
const load = createAppOpsRunnerStatisticsLoader(env({
PROJECTBRAIN_APPOPS_MCP_URL: "https://appops.example.com/mcp",
PROJECTBRAIN_APPOPS_MCP_TOKEN: "scoped-readonly-machine-token-123",
}), { now: () => now, read: async () => ({ isError: false, structuredContent: { ...stats(), secretEcho: "must-not-survive" } }) });
const result = await load();
expect(result.snapshot.status).toBe("healthy");
expect(JSON.stringify(result.snapshot)).not.toContain("must-not-survive");
});
it("rejects stale, future, out-of-range, and internally inconsistent metric samples", async () => {
const configuredEnv = env({
PROJECTBRAIN_APPOPS_MCP_URL: "https://appops.example.com/mcp",
PROJECTBRAIN_APPOPS_MCP_TOKEN: "scoped-readonly-machine-token-123",
});
const loadFor = (value: unknown) => createAppOpsRunnerStatisticsLoader(configuredEnv, { now: () => now, read: async () => value });
expect((await loadFor(stats("2026-10-01T08:57:59.000Z"))()).snapshot.status).toBe("unavailable");
expect((await loadFor(stats("2026-10-01T09:02:00.000Z"))()).snapshot.status).toBe("unavailable");
expect((await loadFor({ ...stats(), host: { ...stats().host, availableMemoryBytes: 9e20 } })()).snapshot.status).toBe("unavailable");
expect((await loadFor({ ...stats(), profiles: [profile("backend", { p50DurationSeconds: 300, p95DurationSeconds: 120 })] })()).snapshot.status).toBe("unavailable");
expect((await loadFor({ ...stats(), history: [{ ...stats().history[0]!, at: "2026-10-01T09:00:01.000Z" }] })()).snapshot.status).toBe("unavailable");
const aged = await loadFor({ ...stats(), history: [{ ...stats().history[0]!, at: "2026-09-29T08:00:00.000Z" }] })();
expect(aged.snapshot.status).toBe("degraded");
expect(aged.snapshot.history).toEqual([]);
});
it("rejects future measurement-window bounds while accepting unknown bounds", async () => {
const configuredEnv = env({
PROJECTBRAIN_APPOPS_MCP_URL: "https://appops.example.com/mcp",
PROJECTBRAIN_APPOPS_MCP_TOKEN: "scoped-readonly-machine-token-123",
});
const loadFor = (value: unknown) => createAppOpsRunnerStatisticsLoader(configuredEnv, { now: () => now, read: async () => value });
const futureStart = await loadFor({
...stats(),
window: { ...stats().window, startAt: "2026-10-01T09:00:01.000Z", endAt: null },
})();
expect(futureStart.snapshot.status).toBe("unavailable");
const futureEnd = await loadFor({
...stats(),
window: { ...stats().window, endAt: "2026-10-01T09:00:01.000Z" },
})();
expect(futureEnd.snapshot.status).toBe("unavailable");
const unknownBounds = await loadFor({
...stats(),
window: { ...stats().window, startAt: null, endAt: null, coverage: "bounded" },
})();
expect(unknownBounds.snapshot.status).toBe("healthy");
});
it("coalesces readers, enforces the 30-second minimum interval, and never returns a tool error or secret", async () => {
let clock = now;
const secret = "scoped-readonly-machine-token-123";
const read = vi.fn().mockRejectedValue(new Error(`Authorization: Bearer ${secret}`, { cause: new Error(secret) }));
const load = createAppOpsRunnerStatisticsLoader(env({
PROJECTBRAIN_APPOPS_MCP_URL: "https://appops.example.com/mcp",
PROJECTBRAIN_APPOPS_MCP_TOKEN: secret,
}), { now: () => clock, read });
const [first, simultaneous] = await Promise.all([load(), load()]);
expect(first).toEqual(simultaneous);
expect(first.snapshot.status).toBe("unavailable");
expect(first.warning).toBe("AppOps runner statistics could not be loaded.");
expect(JSON.stringify(first)).not.toContain(secret);
expect(read).toHaveBeenCalledTimes(1);
clock = new Date(now.getTime() + 29_999);
await load();
expect(read).toHaveBeenCalledTimes(1);
clock = new Date(now.getTime() + 30_000);
await load();
expect(read).toHaveBeenCalledTimes(2);
});
it("prevents authorization from crossing origins or following a redirect", async () => {
const fetcher = vi.fn().mockResolvedValue(new Response("ok"));
const safeFetch = createOriginBoundFetch("https://appops.example.com/mcp", fetcher);
await expect(safeFetch("https://evil.example/mcp", { headers: { Authorization: "Bearer secret" } })).rejects.toThrow();
await expect(safeFetch("https://appops.example.com/other", { headers: { Authorization: "Bearer secret" } })).rejects.toThrow();
await safeFetch("https://appops.example.com/mcp", { redirect: "follow" });
expect(fetcher).toHaveBeenCalledTimes(1);
expect(fetcher.mock.calls[0]?.[1]?.redirect).toBe("error");
});
it.each(["slow POST", "stalled JSON body", "stalled SSE body"] as const)(
"bounds the MCP operation and cancels a %s before cleanup completes",
async (mode) => {
let toolCallStarted = false;
let observedSignal: AbortSignal | undefined;
let bodyCancelled = false;
const fetchImpl: FetchLike = async (input, init) => {
const method = init?.method ?? (input instanceof Request ? input.method : "GET");
if (method === "GET") return new Response(null, { status: 405 });
const requestBody = typeof init?.body === "string" ? init.body : "";
const message = JSON.parse(requestBody) as { id?: number; method?: string };
if (message.method === "initialize") {
return new Response(JSON.stringify({
jsonrpc: "2.0",
id: message.id,
result: {
protocolVersion: "2025-06-18",
capabilities: { tools: {} },
serverInfo: { name: "AppOps test", version: "1.0.0" },
},
}), { headers: { "content-type": "application/json" } });
}
if (message.method === "notifications/initialized") return new Response(null, { status: 202 });
if (message.method !== "tools/call") return new Response(null, { status: 400 });
toolCallStarted = true;
observedSignal = init?.signal ?? undefined;
if (mode === "slow POST") {
return await new Promise<Response>((_resolve, reject) => {
const signal = init?.signal;
if (signal?.aborted) reject(new Error("request aborted"));
else signal?.addEventListener("abort", () => reject(new Error("request aborted")), { once: true });
});
}
const body = new ReadableStream<Uint8Array>({
pull: () => new Promise<void>(() => {}),
cancel: () => { bodyCancelled = true; },
});
return new Response(body, {
headers: { "content-type": mode === "stalled SSE body" ? "text/event-stream" : "application/json" },
});
};
let rejection: unknown;
const outcome = await Promise.race([
readAppOpsTool("https://appops.example.com/mcp", "scoped-readonly-machine-token-123", { timeoutMs: 30, fetch: fetchImpl })
.then(() => "resolved", (error: unknown) => { rejection = error; return "rejected"; }),
new Promise<string>((resolve) => setTimeout(() => resolve("still pending"), 250)),
]);
expect(outcome).toBe("rejected");
expect(rejection).toBeInstanceOf(Error);
expect((rejection as Error).message).toBe("AppOps statistics request timed out.");
expect((rejection as Error).cause).toBeInstanceOf(Error);
expect(((rejection as Error).cause as Error).message).toBe("AppOps statistics request timed out.");
expect(toolCallStarted).toBe(true);
expect(observedSignal?.aborted).toBe(true);
if (mode !== "slow POST") expect(bodyCancelled).toBe(true);
},
1_000,
);
it("redacts a configured machine credential echoed in a profile reason", async () => {
const secret = "scoped-readonly-machine-token-123";
const payload = { ...stats(), profiles: [profile("backend", { runnerName: secret, reason: `Bearer ${secret}` })] };
const load = createAppOpsRunnerStatisticsLoader(env({
PROJECTBRAIN_APPOPS_MCP_URL: "https://appops.example.com/mcp",
PROJECTBRAIN_APPOPS_MCP_TOKEN: secret,
}), { now: () => now, read: async () => payload });
const result = await load();
expect(JSON.stringify(result)).not.toContain(secret);
expect(result.snapshot.profiles[0]?.reason).toBe("Bearer [redacted]");
expect(result.snapshot.profiles[0]?.runnerName).toBe("[redacted]");
});
it("deduplicates retried job attempts and excludes invalid timestamps from bounded percentiles", () => {
const fallback = createGiteaRunnerStatisticsFallback(runtime([
runtimeJob({ id: 1, runAttempt: 1, createdAt: "2026-10-01T08:50:00.000Z", startedAt: "2026-10-01T08:51:00.000Z", completedAt: "2026-10-01T08:53:00.000Z" }),
runtimeJob({ id: 1, runAttempt: 2, createdAt: "2026-10-01T08:55:00.000Z", startedAt: "2026-10-01T08:56:00.000Z", completedAt: "2026-10-01T08:58:00.000Z" }),
runtimeJob({ id: 2, runId: 11, createdAt: "2026-10-01T08:40:00.000Z", startedAt: "2026-10-01T08:42:00.000Z", completedAt: "2026-10-01T08:46:00.000Z" }),
runtimeJob({ id: 3, runId: 12, createdAt: "2026-10-01T08:40:00.000Z", startedAt: "not-a-date", completedAt: "2026-10-01T08:46:00.000Z" }),
]), { now: () => now });
const backend = fallback.profiles.find((item) => item.id === "backend");
expect(backend).toMatchObject({ completedJobs: 2, samples: 2, p50WaitSeconds: 60, p95WaitSeconds: 120, p50DurationSeconds: 120, p95DurationSeconds: 240, lastDurationSeconds: 120, window: { coverage: "bounded" } });
expect(backend?.window.startAt).toBe("2026-10-01T08:40:00.000Z");
expect(backend?.window.endAt).toBe("2026-10-01T08:58:00.000Z");
});
it("keeps profile queue depth unknown when a complete queue scan has an unattributed queued job", () => {
const fallback = createGiteaRunnerStatisticsFallback({
...runtime([
runtimeJob({ id: 10, status: "queued", queueState: "runnable", labels: ["itworx-ci-backend"], runnerId: undefined, runnerName: undefined }),
runtimeJob({ id: 11, runId: 11, status: "queued", queueState: "runnable", labels: ["ubuntu-latest"], runnerId: undefined, runnerName: undefined }),
]),
counts: { running: 0, runnable: 2, dependencyBlocked: 0 },
runnableJobScan: { complete: true, observed: 2, total: 2 },
}, { now: () => now });
expect(fallback.jobs).toMatchObject({
queuedObserved: 2,
queuedTotal: 2,
queuedCoverage: "complete",
unattributedObserved: { queued: 1, total: 1 },
});
expect(fallback.profiles.find((item) => item.id === "backend")).toMatchObject({ queueDepth: null, queueObserved: 1 });
expect(fallback.profiles.find((item) => item.id === "frontend")).toMatchObject({ queueDepth: null, queueObserved: 0 });
});
it("keeps exact profile queue depth when all queued jobs are attributed, regardless of other unattributed states", () => {
const allQueuedJobsAttributed = createGiteaRunnerStatisticsFallback({
...runtime([
runtimeJob({ id: 12, status: "queued", queueState: "runnable", labels: ["itworx-ci-backend"], runnerId: undefined, runnerName: undefined }),
]),
counts: { running: 0, runnable: 1, dependencyBlocked: 0 },
runnableJobScan: { complete: true, observed: 1, total: 1 },
}, { now: () => now });
expect(allQueuedJobsAttributed.profiles.find((item) => item.id === "backend")).toMatchObject({ queueDepth: 1, queueObserved: 1 });
expect(allQueuedJobsAttributed.profiles.find((item) => item.id === "frontend")).toMatchObject({ queueDepth: 0, queueObserved: 0 });
const onlyUnattributedNonqueuedJobs = createGiteaRunnerStatisticsFallback({
...runtime([
runtimeJob({ id: 13, status: "in_progress", queueState: "running", labels: ["ubuntu-latest"], runnerId: undefined, runnerName: undefined }),
runtimeJob({ id: 14, runId: 14, status: "pending", queueState: "dependency-blocked", labels: ["linux-validation"], runnerId: undefined, runnerName: undefined }),
runtimeJob({ id: 15, runId: 15, labels: ["ubuntu-latest"], runnerId: undefined, runnerName: undefined }),
]),
counts: { running: 1, runnable: 0, dependencyBlocked: 1 },
runnableJobScan: { complete: true, observed: 0, total: 0 },
}, { now: () => now });
expect(onlyUnattributedNonqueuedJobs.jobs).toMatchObject({
queuedTotal: 0,
queuedCoverage: "complete",
unattributedObserved: { running: 1, dependencyBlocked: 1, completed: 1, total: 3 },
});
expect(onlyUnattributedNonqueuedJobs.profiles.find((item) => item.id === "backend")).toMatchObject({ queueDepth: 0, queueObserved: 0 });
});
it("does not guess profiles from shared legacy labels and exposes unattributed jobs", () => {
const fallback = createGiteaRunnerStatisticsFallback({
...runtime([]),
jobs: [
runtimeJob({ id: 4, status: "queued", queueState: "runnable", labels: ["ubuntu-latest"], runnerId: undefined, runnerName: undefined }),
runtimeJob({ id: 5, runId: 11, status: "queued", queueState: "runnable", labels: ["linux-validation"], runnerId: undefined, runnerName: undefined }),
runtimeJob({ id: 6, runId: 12, status: "queued", queueState: "runnable", labels: ["ubuntu-latest", "itworx-ci-backend"], runnerId: undefined, runnerName: undefined }),
],
runnableJobScan: { complete: false, observed: 3, total: 50 },
}, { now: () => now });
expect(fallback.jobs.queuedCoverage).toBe("bounded");
expect(fallback.profiles.find((item) => item.id === "frontend")).toMatchObject({ queueDepth: null, queueObserved: 0 });
expect(fallback.profiles.find((item) => item.id === "backend")).toMatchObject({ queueDepth: null, queueObserved: 1 });
expect(fallback.profiles.find((item) => item.id === "security")?.queueObserved).toBe(0);
expect(fallback.jobs.unattributedObserved).toMatchObject({ queued: 2, total: 2 });
expect(fallback.jobs.unattributedCoverage).toBe("bounded");
});
it("maps a fixed runner name only for assigned running/completed jobs and rejects conflicting specialized labels", () => {
const fallback = createGiteaRunnerStatisticsFallback(runtime([
runtimeJob({ id: 7, status: "in_progress", queueState: "running", runnerName: "backend-fixed", labels: ["linux-validation"] }),
runtimeJob({ id: 8, runId: 13, runnerName: "backend-fixed", labels: ["itworx-ci-backend", "itworx-ci-frontend"] }),
]), {
now: () => now,
authoritativeProfiles: [profile("backend", { runnerName: "backend-fixed" })],
});
expect(fallback.profiles.find((item) => item.id === "backend")?.activeJobsObserved).toBe(1);
expect(fallback.profiles.find((item) => item.id === "frontend")?.activeJobsObserved).toBe(0);
expect(fallback.jobs.unattributedObserved).toMatchObject({ running: 0, completed: 1, total: 1 });
});
it("limits AppOps response streams to one mebibyte", async () => {
const safeFetch = createOriginBoundFetch("https://appops.example.com/mcp", vi.fn().mockResolvedValue(new Response("x".repeat(1_048_577))));
const response = await safeFetch("https://appops.example.com/mcp");
await expect(response.text()).rejects.toThrow();
});
it("returns independent snapshots from the 30-second cache", async () => {
const read = vi.fn().mockResolvedValue(stats());
const load = createAppOpsRunnerStatisticsLoader(env({
PROJECTBRAIN_APPOPS_MCP_URL: "https://appops.example.com/mcp",
PROJECTBRAIN_APPOPS_MCP_TOKEN: "scoped-readonly-machine-token-123",
}), { now: () => now, read });
const first = await load();
first.snapshot.profiles[0]!.displayName = "poisoned";
expect((await load()).snapshot.profiles[0]?.displayName).toBe("ITWorx CI · Checks");
expect(read).toHaveBeenCalledTimes(1);
});
});