projectbrain-Public/backend/test/autonomy.test.ts

import { describe, expect, it } from "vitest";
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
import { loadEnv } from "../src/config/env.js";
import { compareAutoModeProjectRows, createProjectBrainMcpServer } from "../src/mcp/autonomyServer.js";
import { autonomyRetryDelayMinutes, canResumeAutonomyLease, classifyAutonomyBlocker, enqueueAutonomyTaskInState, evaluateAutonomyRunBudget, isAutonomyLeaseOwner, normalizeAutonomyState, parseRoadmapMarkdown, releaseActiveAutonomyLease, reorderAutonomyQueueInState, resetAutonomyQueueOrderInState, retryAutonomyBlockerIfDue } from "../src/repositories/autonomyRepository.js";
import { autonomyHeartbeatSlotId, compactAutonomyState, resolveAutonomyHeartbeat, summarizeAutonomyProject } from "../src/services/autonomyRuntime.js";

const env = loadEnv({
  NODE_ENV: "test",
  DATABASE_URL: "postgresql://example.invalid/projectbrain",
  PROJECTBRAIN_DATA_SOURCE: "database",
  PROJECTBRAIN_OWNER_USERNAME: "owner",
  PROJECTBRAIN_OWNER_PASSWORD: "correct-horse-battery-staple",
  PROJECTBRAIN_AUTH_SECRET: "test-secret-with-at-least-thirty-two-characters",
  PROJECTBRAIN_MCP_ENABLED: "true",
  PROJECTBRAIN_PUBLIC_URL: "https://projectbrain.example.com",
  PROJECTBRAIN_GITEA_URL: "https://gitea.example.com",
  PROJECTBRAIN_GITEA_TOKEN: "a-token-with-at-least-twenty-characters",
});

describe("Auto Mode roadmap parser", () => {
  it("turns markdown phases and list items into ordered durable work", () => {
    const roadmap = parseRoadmapMarkdown(`# Product roadmap

## Fundament
Maak de deliverylaag hervatbaar.
- [x] Bestaande status importeren
- [ ] Leasecontract toevoegen

## Product
Voltooi de gebruikersflow.
1. Auto toggle bouwen
2. Voortgang tonen`);

    expect(roadmap.title).toBe("Product roadmap");
    expect(roadmap.phases).toHaveLength(2);
    expect(roadmap.phases[0]).toMatchObject({ title: "Fundament", objective: "Maak de deliverylaag hervatbaar.", status: "active" });
    expect(roadmap.phases[0].items.map((item) => [item.title, item.status])).toEqual([
      ["Bestaande status importeren", "completed"],
      ["Leasecontract toevoegen", "pending"],
    ]);
    expect(roadmap.phases[1].status).toBe("pending");
    expect(roadmap.phases[1].items.map((item) => item.title)).toEqual(["Auto toggle bouwen", "Voortgang tonen"]);
  });

  it("accepts a flat task list by creating one execution phase", () => {
    const roadmap = parseRoadmapMarkdown("- Analyseer repo\n- Bouw oplossing\n- Valideer resultaat", "MCP automation");

    expect(roadmap.title).toBe("MCP automation");
    expect(roadmap.phases).toHaveLength(1);
    expect(roadmap.phases[0].title).toBe("Execution");
    expect(roadmap.phases[0].items).toHaveLength(3);
  });

  it("allows an empty phase so the brainstorm loop can discover its work", () => {
    const roadmap = parseRoadmapMarkdown("# Roadmap\n\n## Quality convergence\nZoek autonoom naar resterende gaten.");

    expect(roadmap.phases[0].items).toEqual([]);
    expect(roadmap.phases[0].brainstormConverged).toBe(false);
    expect(roadmap.phases[0].objective).toBe("Zoek autonoom naar resterende gaten.");
  });
});

describe("Auto Mode state safety", () => {
  it("keeps an hourly ChatGPT Chat executor fresh between scheduled runs", () => {
    expect(env.PROJECTBRAIN_AUTONOMY_EXECUTOR_STALE_SECONDS).toBe(5_400);
  });

  it("stores a user assignment durably even while Auto Mode is off", () => {
    const current = normalizeAutonomyState({}, "owner");
    const queued = enqueueAutonomyTaskInState(current, { instruction: "Verbeter de mobiele navigatie\nmet regressietests." }, "owner");

    expect(queued.state.status).toBe("off");
    expect(queued.task).toMatchObject({
      title: "Verbeter de mobiele navigatie",
      instruction: "Verbeter de mobiele navigatie\nmet regressietests.",
      source: "user_assignment",
      status: "pending",
    });
    expect(queued.state.roadmap?.phases[0].items).toEqual([queued.task]);
    expect(queued.state.events.at(-1)?.type).toBe("autonomy.task.enqueued");
  });

  it("prioritizes a new user assignment in the current claimable phase instead of a phase already passed", () => {
    const roadmap = parseRoadmapMarkdown("## Eerder\n- [ ] Eerste opdracht\n\n## Huidig\n- [ ] Lopend werk");
    roadmap.phases[0].id = "projectbrain-opdrachten-old";
    roadmap.phases[0].status = "completed";
    roadmap.phases[1].status = "active";
    roadmap.currentPhaseIndex = 1;
    const current = normalizeAutonomyState({ enabled: true, status: "running", roadmap }, "owner");

    const queued = enqueueAutonomyTaskInState(current, { instruction: "Nieuwe opdracht" }, "owner");

    expect(queued.state.status).toBe("running");
    expect(queued.state.roadmap?.phases).toHaveLength(2);
    expect(queued.state.roadmap?.phases[0].status).toBe("completed");
    expect(queued.state.roadmap?.phases[1].items[0]).toEqual(queued.task);
    expect(queued.state.roadmap?.phases[1].items[1].title).toBe("Lopend werk");
  });

  it("persists an exact owner-defined order for pending executable work only", () => {
    const roadmap = parseRoadmapMarkdown("## Fase\n- [ ] A\n- [ ] B\n- [ ] C");
    roadmap.phases[0].items[2].scope = "future";
    const current = normalizeAutonomyState({ enabled: true, roadmap }, "owner");
    const [a, b] = current.roadmap!.phases[0].items;

    const reordered = reorderAutonomyQueueInState(current, [b.id, a.id], "owner");

    expect(reordered.roadmap?.phases[0].items[0].manualOrder).toBe(1);
    expect(reordered.roadmap?.phases[0].items[1].manualOrder).toBe(0);
    expect(reordered.roadmap?.phases[0].items[2].manualOrder).toBeUndefined();
    expect(reordered.events.at(-1)?.type).toBe("autonomy.queue.reordered");
    expect(() => reorderAutonomyQueueInState(current, [a.id], "owner")).toThrow("AUTO_MODE_QUEUE_CHANGED");

    const automatic = resetAutonomyQueueOrderInState(reordered, "owner");
    expect(automatic.roadmap?.phases.flatMap((phase) => phase.items).every((item) => item.manualOrder === undefined)).toBe(true);
    expect(automatic.events.at(-1)?.type).toBe("autonomy.queue.order_reset");
  });

  it("derives the public status from durable facts instead of stale stored copy", () => {
    expect(normalizeAutonomyState({ enabled: true, status: "running" }).status).toBe("waiting_roadmap");
    expect(normalizeAutonomyState({}).profile).toBe("normal");
    expect(normalizeAutonomyState({ profile: "unsupported" }).profile).toBe("normal");
    const disabled = normalizeAutonomyState({ enabled: false, status: "running", activeLease: { token: "old", kind: "implement", phaseId: "phase" } });
    expect(disabled.status).toBe("off");
    expect(disabled.activeLease).toBeUndefined();
  });

  it("releases in-progress work safely when Auto Mode is disabled", () => {
    const roadmap = parseRoadmapMarkdown("## Fase\n- [ ] Werk");
    roadmap.phases[0].status = "executing";
    roadmap.phases[0].items[0].status = "in_progress";
    const released = releaseActiveAutonomyLease({
      version: 1,
      profile: "normal",
      enabled: true,
      status: "running",
      autoBrainstorm: true,
      maxBrainstormIterations: 5,
      maxConsecutiveFailures: 3,
      leaseMinutes: 30,
      maxWorkItemsPerRun: 8,
      maxRunMinutes: 45,
      runs: [],
      roadmap,
      activeLease: { token: "lease", kind: "implement", actor: "codex", roadmapId: roadmap.id, phaseId: roadmap.phases[0].id, itemId: roadmap.phases[0].items[0].id, createdAt: new Date().toISOString(), expiresAt: new Date(Date.now() + 60_000).toISOString() },
      completedLeases: [],
      events: [],
      updatedAt: new Date().toISOString(),
      updatedBy: "owner",
    }, "owner");

    expect(released.activeLease).toBeUndefined();
    expect(released.roadmap?.phases[0]).toMatchObject({ status: "active", items: [expect.objectContaining({ status: "pending" })] });
    expect(released.events.at(-1)?.type).toBe("autonomy.lease.released");
  });

  it("separates claimable work from a fresh or expired executor lease", () => {
    const roadmap = parseRoadmapMarkdown("## Fase\n- [ ] Werk");
    const base = normalizeAutonomyState({ enabled: true, roadmap });
    const fresh = summarizeAutonomyProject({
      projectId: "project-id", projectSlug: "project", projectName: "Project",
      autonomy: { ...base, activeLease: { token: "lease", kind: "implement", actor: "executor", roadmapId: roadmap.id, phaseId: roadmap.phases[0].id, itemId: roadmap.phases[0].items[0].id, createdAt: "2026-09-02T10:00:00.000Z", expiresAt: "2026-09-02T10:30:00.000Z" } },
    }, Date.parse("2026-09-02T10:15:00.000Z"));
    const expired = summarizeAutonomyProject({
      projectId: "project-id", projectSlug: "project", projectName: "Project",
      autonomy: { ...base, activeLease: { token: "lease", kind: "implement", actor: "executor", roadmapId: roadmap.id, phaseId: roadmap.phases[0].id, itemId: roadmap.phases[0].items[0].id, createdAt: "2026-09-02T10:00:00.000Z", expiresAt: "2026-09-02T10:30:00.000Z" } },
    }, Date.parse("2026-09-02T10:31:00.000Z"));

    expect(fresh).toMatchObject({ activeWorkKind: "implement", pendingWorkCount: 1, lifecycle: "running" });
    expect(expired.activeWorkKind).toBeUndefined();
    expect(expired.lifecycle).toBe("ready");
  });

  it("defaults and bounds the continuous executor run budget", () => {
    expect(normalizeAutonomyState({})).toMatchObject({ maxWorkItemsPerRun: 8, maxRunMinutes: 45 });
    expect(normalizeAutonomyState({ maxWorkItemsPerRun: 100, maxRunMinutes: 2 })).toMatchObject({ maxWorkItemsPerRun: 20, maxRunMinutes: 5 });
  });

  it("only lets the owning scheduled run resume a live lease", () => {
    const lease = {
      token: "lease",
      kind: "implement" as const,
      actor: "chatgpt-web",
      runId: "run-1",
      roadmapId: "roadmap",
      phaseId: "phase",
      createdAt: "2026-09-02T10:00:00.000Z",
      expiresAt: "2026-09-02T10:30:00.000Z",
    };

    expect(canResumeAutonomyLease(lease, "chatgpt-web", "run-1")).toBe(true);
    expect(canResumeAutonomyLease(lease, "chatgpt-web", "run-2")).toBe(false);
    expect(canResumeAutonomyLease(lease, "another-executor", "run-1")).toBe(false);
    expect(canResumeAutonomyLease(lease, "chatgpt-web")).toBe(false);
    expect(isAutonomyLeaseOwner(lease, "chatgpt-web")).toBe(true);
    expect(isAutonomyLeaseOwner(lease, "another-executor")).toBe(false);
  });

  it("enforces item and elapsed-time budgets from durable run state", () => {
    const base = normalizeAutonomyState({
      maxWorkItemsPerRun: 2,
      maxRunMinutes: 10,
      runs: [{ id: "item-run", actor: "chatgpt-web", startedAt: "2026-09-03T10:00:00.000Z", reportedWorkItems: 2 }],
    });
    expect(evaluateAutonomyRunBudget(base, "chatgpt-web", "item-run", Date.parse("2026-09-03T10:01:00.000Z"))).toMatchObject({ allowed: false, reason: "item_limit" });

    const timed = normalizeAutonomyState({
      maxWorkItemsPerRun: 8,
      maxRunMinutes: 10,
      runs: [{ id: "time-run", actor: "chatgpt-web", startedAt: "2026-09-03T10:00:00.000Z", reportedWorkItems: 1 }],
    });
    expect(evaluateAutonomyRunBudget(timed, "chatgpt-web", "time-run", Date.parse("2026-09-03T10:10:00.000Z"))).toMatchObject({ allowed: false, reason: "time_limit" });
    expect(evaluateAutonomyRunBudget(timed, "other-actor", "time-run", Date.parse("2026-09-03T10:01:00.000Z"))).toMatchObject({ allowed: false, reason: "owner_mismatch" });
  });

  it("classifies technical failures as retryable and reserves human blockers for real owner input", () => {
    expect(classifyAutonomyBlocker("Untrusted non-HTTPS Gitea server URL")).toBe("retryable");
    expect(classifyAutonomyBlocker("Runner unavailable during validation")).toBe("retryable");
    expect(classifyAutonomyBlocker("Missing credential required for production deployment")).toBe("human");
    expect(classifyAutonomyBlocker("Owner decision required between two product directions")).toBe("human");
    expect(classifyAutonomyBlocker("Temporary provider failure", "human")).toBe("retryable");
  });

  it("backs off repeated technical failures without requiring a human decision", () => {
    expect(autonomyRetryDelayMinutes(1)).toBe(15);
    expect(autonomyRetryDelayMinutes(2)).toBe(30);
    expect(autonomyRetryDelayMinutes(3)).toBe(60);
    expect(autonomyRetryDelayMinutes(20)).toBeLessThanOrEqual(1_440);
  });

  it("makes a due retryable blocker claimable without clearing failure counters", () => {
    const roadmap = parseRoadmapMarkdown("## Fase\n- [ ] Werk");
    roadmap.status = "blocked";
    roadmap.phases[0].status = "blocked";
    roadmap.phases[0].items[0].status = "blocked";
    roadmap.phases[0].items[0].attempts = 2;
    const blocked = normalizeAutonomyState({
      enabled: true,
      roadmap,
      blocker: { at: "2026-09-03T10:00:00.000Z", retryAt: "2026-09-03T11:00:00.000Z", kind: "retryable", phaseId: roadmap.phases[0].id, itemId: roadmap.phases[0].items[0].id, reason: "Temporary provider failure" },
    });

    const retried = retryAutonomyBlockerIfDue(blocked, Date.parse("2026-09-03T11:00:00.000Z"));

    expect(retried).toMatchObject({ status: "running", roadmap: { status: "active", phases: [{ status: "active", items: [{ status: "pending", attempts: 2 }] }] } });
    expect(retried.blocker).toBeUndefined();
    expect(retried.events.at(-1)?.type).toBe("autonomy.blocker.retry_ready");
  });

  it("fences heartbeat ownership between overlapping scheduled runs", () => {
    const current = { runId: "new-run", status: "working", startedAt: new Date("2026-09-03T10:00:00.000Z") };
    expect(resolveAutonomyHeartbeat(current, { instanceId: "ordinary-chat", runId: "old-run", status: "idle" })).toMatchObject({ accepted: false });
    expect(resolveAutonomyHeartbeat(current, { instanceId: "ordinary-chat", status: "degraded" })).toMatchObject({ accepted: false, reason: "run_owner_mismatch" });
    expect(resolveAutonomyHeartbeat(current, { instanceId: "ordinary-chat", runId: "new-run", status: "idle" })).toMatchObject({ accepted: true, startedAt: current.startedAt });
    expect(resolveAutonomyHeartbeat(current, { instanceId: "ordinary-chat", runId: "next-run", status: "working" }, new Date("2026-09-03T10:10:00.000Z"))).toMatchObject({ accepted: false, reason: "active_run" });
    expect(resolveAutonomyHeartbeat(current, { instanceId: "ordinary-chat", runId: "next-run", status: "working" }, new Date("2026-09-03T11:00:00.000Z"))).toMatchObject({ accepted: true, startedAt: new Date("2026-09-03T11:00:00.000Z") });
    expect(resolveAutonomyHeartbeat({ ...current, status: "idle" }, { instanceId: "ordinary-chat", runId: "next-run", status: "working" }, new Date("2026-09-03T10:10:00.000Z"))).toMatchObject({ accepted: true, startedAt: new Date("2026-09-03T10:10:00.000Z") });
    expect(resolveAutonomyHeartbeat({ ...current, activeProjectId: "project-a" }, { instanceId: "ordinary-chat", runId: "new-run", status: "working", activeProjectId: "project-b" })).toMatchObject({ accepted: false, reason: "active_project" });
  });

  it("allows takeover after a short project's configured run window plus grace", () => {
    const current = { runId: "old-run", status: "working", startedAt: new Date("2026-09-03T10:00:00.000Z") };
    const takeoverAfterMs = 25 * 60_000;
    expect(resolveAutonomyHeartbeat(current, { instanceId: "ordinary-chat", runId: "next-run", status: "working" }, new Date("2026-09-03T10:24:00.000Z"), takeoverAfterMs)).toMatchObject({ accepted: false, reason: "active_run" });
    expect(resolveAutonomyHeartbeat(current, { instanceId: "ordinary-chat", runId: "next-run", status: "working" }, new Date("2026-09-03T10:26:00.000Z"), takeoverAfterMs)).toMatchObject({ accepted: true });
  });

  it("isolates concurrent ordinary ChatGPT executors by project", () => {
    expect(autonomyHeartbeatSlotId("mediarr")).toBe("project:mediarr");
    expect(autonomyHeartbeatSlotId("rombuilder")).toBe("project:rombuilder");
    expect(autonomyHeartbeatSlotId("mediarr")).not.toBe(autonomyHeartbeatSlotId("rombuilder"));
    expect(autonomyHeartbeatSlotId()).toBe("primary");
  });

  it("projects large durable roadmaps into bounded MCP context", () => {
    const roadmap = parseRoadmapMarkdown(`## Current\n${Array.from({ length: 80 }, (_, index) => `- [ ] Item ${index + 1} ${"detail ".repeat(30)}`).join("\n")}\n## Later\n${Array.from({ length: 80 }, (_, index) => `- [ ] Later ${index + 1}`).join("\n")}`);
    const state = normalizeAutonomyState({
      enabled: true,
      roadmap,
      events: Array.from({ length: 30 }, (_, index) => ({ id: `event-${index}`, at: "2026-09-04T10:00:00.000Z", type: "test", actor: "chatgpt", summary: "evidence ".repeat(200) })),
    });
    for (const item of state.roadmap!.phases[0].items.slice(0, 40)) item.status = "completed";

    const compact = compactAutonomyState(state, { itemLimit: 10, eventLimit: 3 });
    const serialized = JSON.stringify(compact);

    expect(compact.roadmap?.currentPhase?.items).toHaveLength(10);
    expect(compact.roadmap?.currentPhase).toMatchObject({ totalItems: 80, itemWindowStart: 37, itemsReturned: 10, hasMoreItems: true });
    expect(compact.roadmap?.currentPhase?.items[3]).toMatchObject({ title: expect.stringContaining("Item 41"), status: "pending" });
    expect(compact.events).toHaveLength(3);
    expect(serialized).not.toContain("sourceMarkdown");
    expect(serialized.length).toBeLessThan(12_000);
    expect(serialized.length).toBeLessThan(JSON.stringify(state).length / 5);
  });
});

describe("Auto Mode MCP surface", () => {
  it("orders claimable projects fairly before considering backlog size", () => {
    const projects = [
      { name: "huge-recent", status: "running", pending_work_items: 201, last_run_at: "2026-09-05T09:00:00.000Z" },
      { name: "small-old", status: "running", pending_work_items: 4, last_run_at: "2026-09-04T09:00:00.000Z" },
      { name: "never-run", status: "running", pending_work_items: 2 },
      { name: "busy", status: "running", pending_work_items: 50, active_work_kind: "implement", last_run_at: "2026-09-01T09:00:00.000Z" },
      { name: "blocked", status: "blocked", pending_work_items: 500, last_run_at: "2026-08-01T09:00:00.000Z" },
    ].sort(compareAutoModeProjectRows);

    expect(projects.map((project) => project.name)).toEqual(["never-run", "small-old", "huge-recent", "busy", "blocked"]);
  });

  it("advertises the complete durable roadmap controller to MCP clients", async () => {
    const server = createProjectBrainMcpServer(env, "auto-contract-test");
    const client = new Client({ name: "auto-contract-test", version: "1.0.0" });
    const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();

    try {
      await server.connect(serverTransport);
      await client.connect(clientTransport);
      const advertisedTools = (await client.listTools()).tools;
      const tools = advertisedTools.map((tool) => tool.name);

      expect(tools).toEqual(expect.arrayContaining([
        "list_auto_mode_projects",
        "heartbeat_auto_executor",
        "get_auto_mode_state",
        "get_runner_claim_health",
        "configure_auto_mode",
        "import_auto_roadmap",
        "reorder_auto_queue",
        "reset_auto_queue_order",
        "resume_auto_mode",
        "claim_auto_work",
        "report_auto_work",
      ]));
      expect(advertisedTools.find((tool) => tool.name === "claim_auto_work")?.inputSchema).toMatchObject({
        required: expect.arrayContaining(["project_id", "run_id"]),
      });
      expect(advertisedTools.find((tool) => tool.name === "reorder_auto_queue")?.inputSchema).toMatchObject({
        required: expect.arrayContaining(["project_id", "item_ids"]),
      });
      expect(advertisedTools.find((tool) => tool.name === "configure_auto_mode")?.inputSchema).toMatchObject({
        properties: expect.objectContaining({ max_work_items_per_run: expect.any(Object), max_run_minutes: expect.any(Object), max_recovery_minutes: expect.any(Object), pause_after_current: expect.any(Object), scheduling_policy: expect.any(Object) }),
      });
      expect(advertisedTools.find((tool) => tool.name === "get_auto_mode_state")?.inputSchema).toMatchObject({
        properties: expect.objectContaining({ view: expect.any(Object), item_limit: expect.any(Object), event_limit: expect.any(Object) }),
      });
      expect(advertisedTools.find((tool) => tool.name === "report_auto_work")?.inputSchema).toMatchObject({
        properties: expect.objectContaining({ blocker_type: expect.any(Object), retry_after_minutes: expect.any(Object) }),
      });
    } finally {
      await client.close();
      await server.close();
    }
  });
});