Publish curated projectbrain source from 88fab8df1542019289e947f092c2b387e0825c60
This commit is contained in:
1 parent
48e72d07e0
commit
eca2891b07
45 files changed
+2697
-328
No files matched your search
@@ -1,4 +1,6 @@
|
||||
import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { Prisma } from "@prisma/client";
|
||||
import { prisma } from "../src/db/prisma.js";
|
||||
import { configureAutonomy, getAutonomyState, importAutonomyRoadmap, reportAutonomyWork } from "../src/repositories/autonomyRepository.js";
|
||||
import { claimAutonomyWork } from "../src/services/autonomyRuntimeControl.js";
|
||||
@@ -11,12 +13,204 @@ import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
||||
import { createProjectBrainMcpServer } from "../src/mcp/server.js";
|
||||
import { loadEnv } from "../src/config/env.js";
|
||||
import { GiteaApiError, GiteaClient } from "../src/integrations/gitea/client.js";
|
||||
import { claimReadyWork, getProjectWorkGraph, reclaimExpiredWork, recordWorkCompletion, syncProjectWorkGraph, updateWorkDependencies } from "../src/services/workGraph.js";
|
||||
|
||||
const databaseUrl = process.env.TEST_DATABASE_URL;
|
||||
if (databaseUrl) process.env.DATABASE_URL = databaseUrl;
|
||||
const databaseDescribe = databaseUrl ? describe : describe.skip;
|
||||
const slugs: string[] = [];
|
||||
|
||||
const createWorkGraphProject = async (markdown: string) => {
|
||||
const slug = `workgraph-${randomUUID()}`;
|
||||
slugs.push(slug);
|
||||
await createProject({ id: slug, name: "Work graph concurrency proof", description: "Isolated database regression" });
|
||||
await configureAutonomy(slug, { enabled: true, autoBrainstorm: false }, "database-test");
|
||||
await importAutonomyRoadmap(slug, { markdown }, "database-test");
|
||||
const project = await prisma.project.findUniqueOrThrow({ where: { slug } });
|
||||
const graph = await getProjectWorkGraph(slug);
|
||||
return { slug, project, graph };
|
||||
};
|
||||
|
||||
const nodeByTitle = (graph: Awaited<ReturnType<typeof getProjectWorkGraph>>, title: string) => {
|
||||
const node = graph.nodes.find((candidate) => candidate.title === title);
|
||||
if (!node) throw new Error(`Missing work graph fixture node '${title}'.`);
|
||||
return node;
|
||||
};
|
||||
|
||||
const setAutonomyDependencies = async (projectId: string, slug: string, title: string, dependsOnTitles: string[]) => {
|
||||
const state = await getAutonomyState(slug);
|
||||
const items = state.roadmap?.phases.flatMap((phase) => phase.items) ?? [];
|
||||
const target = items.find((item) => item.title === title);
|
||||
if (!target) throw new Error(`Missing autonomy fixture item '${title}'.`);
|
||||
target.dependsOn = dependsOnTitles.map((dependencyTitle) => {
|
||||
const dependency = items.find((item) => item.title === dependencyTitle);
|
||||
if (!dependency) throw new Error(`Missing autonomy fixture dependency '${dependencyTitle}'.`);
|
||||
return dependency.id;
|
||||
});
|
||||
const workspace = await prisma.workspaceState.findUniqueOrThrow({ where: { projectId }, select: { notesJson: true } });
|
||||
const notes = typeof workspace.notesJson === "object" && workspace.notesJson !== null && !Array.isArray(workspace.notesJson) ? workspace.notesJson : {};
|
||||
await prisma.workspaceState.update({ where: { projectId }, data: { notesJson: { ...notes, autonomy: state } as Prisma.InputJsonValue } });
|
||||
};
|
||||
|
||||
const setAutonomyPriority = async (projectId: string, slug: string, title: string, priority: number) => {
|
||||
const state = await getAutonomyState(slug);
|
||||
const target = state.roadmap?.phases.flatMap((phase) => phase.items).find((item) => item.title === title);
|
||||
if (!target) throw new Error(`Missing autonomy fixture item '${title}'.`);
|
||||
target.priority = priority;
|
||||
const workspace = await prisma.workspaceState.findUniqueOrThrow({ where: { projectId }, select: { notesJson: true } });
|
||||
const notes = typeof workspace.notesJson === "object" && workspace.notesJson !== null && !Array.isArray(workspace.notesJson) ? workspace.notesJson : {};
|
||||
await prisma.workspaceState.update({ where: { projectId }, data: { notesJson: { ...notes, autonomy: state } as Prisma.InputJsonValue } });
|
||||
};
|
||||
|
||||
const setAutonomyRoadmapOrder = async (projectId: string, slug: string, titles: string[]) => {
|
||||
const state = await getAutonomyState(slug);
|
||||
const itemOrder = new Map(titles.map((title, index) => [title, index]));
|
||||
const items = state.roadmap?.phases.flatMap((phase) => phase.items) ?? [];
|
||||
if (items.length !== titles.length || titles.some((title) => !items.some((item) => item.title === title))) {
|
||||
throw new Error("The requested roadmap order does not match the source fixture.");
|
||||
}
|
||||
for (const phase of state.roadmap?.phases ?? []) {
|
||||
phase.items.sort((left, right) => itemOrder.get(left.title)! - itemOrder.get(right.title)!);
|
||||
}
|
||||
const workspace = await prisma.workspaceState.findUniqueOrThrow({ where: { projectId }, select: { notesJson: true } });
|
||||
const notes = typeof workspace.notesJson === "object" && workspace.notesJson !== null && !Array.isArray(workspace.notesJson) ? workspace.notesJson : {};
|
||||
await prisma.workspaceState.update({ where: { projectId }, data: { notesJson: { ...notes, autonomy: state } as Prisma.InputJsonValue } });
|
||||
};
|
||||
|
||||
type DatabaseLockWaiter = { pid: number; query: string; blockingPids: number[]; hasProjectRowShareLock: boolean };
|
||||
const readDatabaseLockWaiters = () => prisma.$queryRaw<DatabaseLockWaiter[]>`
|
||||
SELECT activity.pid, activity.query, pg_blocking_pids(activity.pid) AS "blockingPids",
|
||||
EXISTS (
|
||||
SELECT 1 FROM pg_locks
|
||||
WHERE pg_locks.pid = activity.pid
|
||||
AND pg_locks.relation = '"Project"'::regclass
|
||||
AND pg_locks.mode = 'RowShareLock'
|
||||
AND pg_locks.granted
|
||||
) AS "hasProjectRowShareLock"
|
||||
FROM pg_stat_activity AS activity
|
||||
WHERE activity.datname = current_database()
|
||||
AND activity.pid <> pg_backend_pid()
|
||||
AND activity.state = 'active'
|
||||
AND activity.wait_event_type = 'Lock'
|
||||
`;
|
||||
|
||||
const isProjectCoordinationLockQuery = (query: string) => {
|
||||
const normalized = query.toLowerCase();
|
||||
return normalized.includes("for update") || normalized.includes("for no key update");
|
||||
};
|
||||
|
||||
const waitForDatabaseLockWaiter = async (predicate: (waiter: DatabaseLockWaiter) => boolean, timeoutMs = 2000) => {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const waiter = (await readDatabaseLockWaiters()).find(predicate);
|
||||
if (waiter) return waiter;
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const createAuditInsertBarrier = async (projectId: string, eventType: "work.completed" | "work.reclaimed") => {
|
||||
const suffix = randomUUID().replaceAll("-", "");
|
||||
const triggerName = `test_audit_${suffix}`;
|
||||
const functionName = `${triggerName}_fn`;
|
||||
const advisoryKey = [Number.parseInt(suffix.slice(0, 7), 16), Number.parseInt(suffix.slice(7, 14), 16)] as const;
|
||||
let blockerPid = 0;
|
||||
let releaseAdvisoryLock: () => void = () => undefined;
|
||||
let acquiredAdvisoryLock!: () => void;
|
||||
let functionCreated = false;
|
||||
let triggerCreated = false;
|
||||
let lockHolder: Promise<void> | undefined;
|
||||
const locked = new Promise<void>((resolve) => { acquiredAdvisoryLock = resolve; });
|
||||
const hold = new Promise<void>((resolve) => { releaseAdvisoryLock = resolve; });
|
||||
const cleanup = async () => {
|
||||
releaseAdvisoryLock();
|
||||
if (lockHolder) await lockHolder.catch(() => undefined);
|
||||
if (triggerCreated) await prisma.$executeRawUnsafe(`DROP TRIGGER IF EXISTS ${triggerName} ON "WorkGraphAuditEvent"`);
|
||||
if (functionCreated) await prisma.$executeRawUnsafe(`DROP FUNCTION IF EXISTS ${functionName}()`);
|
||||
};
|
||||
try {
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE FUNCTION ${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM pg_advisory_xact_lock(${advisoryKey[0]}, ${advisoryKey[1]});
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$
|
||||
`);
|
||||
functionCreated = true;
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE TRIGGER ${triggerName}
|
||||
BEFORE INSERT ON "WorkGraphAuditEvent"
|
||||
FOR EACH ROW WHEN (NEW."projectId" = '${projectId}' AND NEW."eventType" = '${eventType}')
|
||||
EXECUTE FUNCTION ${functionName}();
|
||||
`);
|
||||
triggerCreated = true;
|
||||
lockHolder = prisma.$transaction(async (transaction) => {
|
||||
const [backend] = await transaction.$queryRaw<Array<{ pid: number }>>`SELECT pg_backend_pid() AS pid`;
|
||||
blockerPid = backend!.pid;
|
||||
await transaction.$queryRaw`SELECT pg_advisory_xact_lock(${advisoryKey[0]}::integer, ${advisoryKey[1]}::integer) IS NULL`;
|
||||
acquiredAdvisoryLock();
|
||||
await hold;
|
||||
});
|
||||
await Promise.race([locked, lockHolder.then(() => { throw new Error("The advisory lock holder ended before acquiring its lock."); })]);
|
||||
return { blockerPid, release: releaseAdvisoryLock, cleanup };
|
||||
} catch (error) {
|
||||
await cleanup();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const createWorkGraphNodeUpdateBarrier = async (nodeId: string) => {
|
||||
const suffix = randomUUID().replaceAll("-", "");
|
||||
const triggerName = `test_node_${suffix}`;
|
||||
const functionName = `${triggerName}_fn`;
|
||||
const advisoryKey = [Number.parseInt(suffix.slice(0, 7), 16), Number.parseInt(suffix.slice(7, 14), 16)] as const;
|
||||
let blockerPid = 0;
|
||||
let releaseAdvisoryLock: () => void = () => undefined;
|
||||
let acquiredAdvisoryLock!: () => void;
|
||||
let functionCreated = false;
|
||||
let triggerCreated = false;
|
||||
let lockHolder: Promise<void> | undefined;
|
||||
const locked = new Promise<void>((resolve) => { acquiredAdvisoryLock = resolve; });
|
||||
const hold = new Promise<void>((resolve) => { releaseAdvisoryLock = resolve; });
|
||||
const cleanup = async () => {
|
||||
releaseAdvisoryLock();
|
||||
if (lockHolder) await lockHolder.catch(() => undefined);
|
||||
if (triggerCreated) await prisma.$executeRawUnsafe(`DROP TRIGGER IF EXISTS ${triggerName} ON "WorkGraphNode"`);
|
||||
if (functionCreated) await prisma.$executeRawUnsafe(`DROP FUNCTION IF EXISTS ${functionName}()`);
|
||||
};
|
||||
try {
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE FUNCTION ${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM pg_advisory_xact_lock(${advisoryKey[0]}, ${advisoryKey[1]});
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$
|
||||
`);
|
||||
functionCreated = true;
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE TRIGGER ${triggerName}
|
||||
BEFORE UPDATE ON "WorkGraphNode"
|
||||
FOR EACH ROW WHEN (NEW."id" = '${nodeId}')
|
||||
EXECUTE FUNCTION ${functionName}();
|
||||
`);
|
||||
triggerCreated = true;
|
||||
lockHolder = prisma.$transaction(async (transaction) => {
|
||||
const [backend] = await transaction.$queryRaw<Array<{ pid: number }>>`SELECT pg_backend_pid() AS pid`;
|
||||
blockerPid = backend!.pid;
|
||||
await transaction.$queryRaw`SELECT pg_advisory_xact_lock(${advisoryKey[0]}::integer, ${advisoryKey[1]}::integer) IS NULL`;
|
||||
acquiredAdvisoryLock();
|
||||
await hold;
|
||||
});
|
||||
await Promise.race([locked, lockHolder.then(() => { throw new Error("The advisory lock holder ended before acquiring its lock."); })]);
|
||||
return { blockerPid, release: releaseAdvisoryLock, cleanup };
|
||||
} catch (error) {
|
||||
await cleanup();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
databaseDescribe("autonomy database write isolation", () => {
|
||||
it("preserves progress and the active lease when the same roadmap is imported again", async () => {
|
||||
const slug = `autonomy-import-${Date.now()}`;
|
||||
@@ -107,7 +301,10 @@ databaseDescribe("autonomy database write isolation", () => {
|
||||
vi.spyOn(GiteaClient.prototype, "getBranch").mockResolvedValue({ name: "chatgpt/existing", commit: { id: head, message: "Repair" }, protected: false });
|
||||
vi.spyOn(GiteaClient.prototype, "getContents").mockResolvedValue({ type: "file", path: "src/feature.ts", name: "feature.ts", sha: "file-sha", size: 4, encoding: "base64", content: "dGVzdA==" });
|
||||
const write = vi.spyOn(GiteaClient.prototype, "updateFile").mockResolvedValue({});
|
||||
vi.spyOn(GiteaClient.prototype, "listActionRuns").mockResolvedValue({ total_count: 1, workflow_runs: [{ id: 2, run_number: 2, display_title: "CI", path: ".gitea/workflows/ci.yml", event: "pull_request", head_sha: head, head_branch: "chatgpt/existing", status: "completed", conclusion: "success", html_url: "https://gitea.example.com/runs/2" }] });
|
||||
vi.spyOn(GiteaClient.prototype, "listActionRuns").mockResolvedValue({ total_count: 2, workflow_runs: [
|
||||
{ id: 2, run_number: 2, display_title: "Managed validation / full", path: ".gitea/workflows/managed-validation.yml", event: "pull_request", head_sha: head, head_branch: "chatgpt/existing", status: "completed", conclusion: "success", html_url: "https://gitea.example.com/runs/2" },
|
||||
{ id: 3, run_number: 3, display_title: "CI", path: ".gitea/workflows/ci.yml", event: "pull_request", head_sha: head, head_branch: "chatgpt/existing", status: "completed", conclusion: "success", html_url: "https://gitea.example.com/runs/3" },
|
||||
] });
|
||||
vi.spyOn(GiteaClient.prototype, "getBranchProtection").mockResolvedValue({ enable_status_check: true, status_check_contexts: ["CI / test"] });
|
||||
vi.spyOn(GiteaClient.prototype, "getCombinedCommitStatus").mockResolvedValue({ state: "success", statuses: [{ context: "CI / test", status: "success" }] });
|
||||
const merge = vi.spyOn(GiteaClient.prototype, "mergePullRequest").mockImplementation(async () => { remoteMerged = true; });
|
||||
@@ -162,4 +359,379 @@ databaseDescribe("autonomy database write isolation", () => {
|
||||
expect(integrated.spec.validationEvidence).toBeUndefined();
|
||||
expect(compare).toHaveBeenLastCalledWith("AutonomyProof", slug, "b".repeat(40), "a".repeat(40));
|
||||
});
|
||||
|
||||
it("serializes concurrent dependency edits before checking for cycles", async () => {
|
||||
const { slug, project, graph } = await createWorkGraphProject("## Work\n- Task A\n- Task B");
|
||||
const taskA = nodeByTitle(graph, "Task A");
|
||||
const taskB = nodeByTitle(graph, "Task B");
|
||||
const suffix = randomUUID().replaceAll("-", "");
|
||||
const triggerName = `test_cycle_${suffix}`;
|
||||
const functionName = `${triggerName}_fn`;
|
||||
const advisoryKey = [Number.parseInt(suffix.slice(0, 7), 16), Number.parseInt(suffix.slice(7, 14), 16)] as const;
|
||||
let blockerPid = 0;
|
||||
let releaseAdvisoryLock: () => void = () => undefined;
|
||||
let acquiredAdvisoryLock!: () => void;
|
||||
let functionCreated = false;
|
||||
let triggerCreated = false;
|
||||
const locked = new Promise<void>((resolve) => { acquiredAdvisoryLock = resolve; });
|
||||
const hold = new Promise<void>((resolve) => { releaseAdvisoryLock = resolve; });
|
||||
let lockHolder: Promise<void> | undefined;
|
||||
let edits: Array<Promise<Awaited<ReturnType<typeof updateWorkDependencies>>>> | undefined;
|
||||
try {
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE FUNCTION ${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM pg_advisory_xact_lock(${advisoryKey[0]}, ${advisoryKey[1]});
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$
|
||||
`);
|
||||
functionCreated = true;
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE TRIGGER ${triggerName}
|
||||
BEFORE INSERT ON "WorkGraphDependency"
|
||||
FOR EACH ROW WHEN (NEW."projectId" = '${project.id}')
|
||||
EXECUTE FUNCTION ${functionName}();
|
||||
`);
|
||||
triggerCreated = true;
|
||||
lockHolder = prisma.$transaction(async (transaction) => {
|
||||
const [backend] = await transaction.$queryRaw<Array<{ pid: number }>>`SELECT pg_backend_pid() AS pid`;
|
||||
blockerPid = backend!.pid;
|
||||
await transaction.$queryRaw`SELECT pg_advisory_xact_lock(${advisoryKey[0]}::integer, ${advisoryKey[1]}::integer) IS NULL`;
|
||||
acquiredAdvisoryLock();
|
||||
await hold;
|
||||
});
|
||||
await Promise.race([locked, lockHolder.then(() => { throw new Error("The advisory lock holder ended before acquiring its lock."); })]);
|
||||
edits = [
|
||||
updateWorkDependencies(slug, taskA.id, [taskB.id], "database-test"),
|
||||
updateWorkDependencies(slug, taskB.id, [taskA.id], "database-test"),
|
||||
];
|
||||
const waitDeadline = Date.now() + 2000;
|
||||
let bothAtSerializedBoundary = false;
|
||||
while (!bothAtSerializedBoundary && Date.now() < waitDeadline) {
|
||||
const waiters = await readDatabaseLockWaiters();
|
||||
const blockedInserts = waiters.filter((row) => row.query.toLowerCase().includes("workgraphdependency") && row.blockingPids.includes(blockerPid));
|
||||
const projectLockWaiters = waiters.filter((row) => isProjectCoordinationLockQuery(row.query));
|
||||
const baselinePair = blockedInserts.length >= 2;
|
||||
const serializedPair = blockedInserts.length === 1 && projectLockWaiters.some((row) => row.blockingPids.includes(blockedInserts[0]!.pid));
|
||||
bothAtSerializedBoundary = baselinePair || serializedPair;
|
||||
if (!bothAtSerializedBoundary) await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
expect(bothAtSerializedBoundary).toBe(true);
|
||||
releaseAdvisoryLock();
|
||||
const results = await Promise.allSettled(edits);
|
||||
expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1);
|
||||
const rejected = results.filter((result): result is PromiseRejectedResult => result.status === "rejected");
|
||||
expect(rejected).toHaveLength(1);
|
||||
expect(rejected[0]?.reason).toMatchObject({ message: "WORK_GRAPH_DEPENDENCY_CYCLE" });
|
||||
const finalGraph = await getProjectWorkGraph(slug);
|
||||
const finalA = nodeByTitle(finalGraph, "Task A");
|
||||
const finalB = nodeByTitle(finalGraph, "Task B");
|
||||
expect(finalA.dependencies.some((edge) => edge.dependsOnNodeId === finalB.id)).not.toBe(finalB.dependencies.some((edge) => edge.dependsOnNodeId === finalA.id));
|
||||
} finally {
|
||||
releaseAdvisoryLock();
|
||||
if (lockHolder) await lockHolder.catch(() => undefined);
|
||||
if (edits) await Promise.allSettled(edits);
|
||||
if (triggerCreated) await prisma.$executeRawUnsafe(`DROP TRIGGER IF EXISTS ${triggerName} ON "WorkGraphDependency"`);
|
||||
if (functionCreated) await prisma.$executeRawUnsafe(`DROP FUNCTION IF EXISTS ${functionName}()`);
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("does not attach a new unresolved prerequisite to work with an active lease", async () => {
|
||||
const { slug, project } = await createWorkGraphProject("## Work\n- Runnable task\n- Prerequisite task");
|
||||
await setAutonomyPriority(project.id, slug, "Runnable task", 100);
|
||||
const currentGraph = await getProjectWorkGraph(slug);
|
||||
const task = nodeByTitle(currentGraph, "Runnable task");
|
||||
const prerequisite = nodeByTitle(currentGraph, "Prerequisite task");
|
||||
const beforeDependencies = await prisma.workGraphDependency.findMany({ where: { projectId: project.id, nodeId: task.id }, select: { dependsOnNodeId: true, dependencyType: true, createdBy: true } });
|
||||
expect(beforeDependencies).toHaveLength(0);
|
||||
const claim = await claimReadyWork(slug, "claimant");
|
||||
expect(claim?.node.id).toBe(task.id);
|
||||
expect(claim?.node.dependencies.map((edge) => edge.dependsOnNodeId)).toEqual(beforeDependencies.map((edge) => edge.dependsOnNodeId));
|
||||
await expect(updateWorkDependencies(slug, task.id, [prerequisite.id], "database-test")).rejects.toThrow("WORK_GRAPH_ACTIVE_LEASE");
|
||||
const persisted = await prisma.workGraphDependency.findMany({ where: { projectId: project.id, nodeId: task.id }, select: { dependsOnNodeId: true, dependencyType: true, createdBy: true } });
|
||||
expect(persisted).toEqual(beforeDependencies);
|
||||
});
|
||||
|
||||
it("serializes a dependency edit behind an in-flight claim", async () => {
|
||||
const { slug, project } = await createWorkGraphProject("## Work\n- Runnable task\n- Prerequisite task");
|
||||
await setAutonomyPriority(project.id, slug, "Runnable task", 100);
|
||||
const currentGraph = await getProjectWorkGraph(slug);
|
||||
const task = nodeByTitle(currentGraph, "Runnable task");
|
||||
const prerequisite = nodeByTitle(currentGraph, "Prerequisite task");
|
||||
const suffix = randomUUID().replaceAll("-", "");
|
||||
const triggerName = `test_claim_${suffix}`;
|
||||
const functionName = `${triggerName}_fn`;
|
||||
const advisoryKey = [Number.parseInt(suffix.slice(0, 7), 16), Number.parseInt(suffix.slice(7, 14), 16)] as const;
|
||||
|
||||
let releaseAdvisoryLock: () => void = () => undefined;
|
||||
let acquiredAdvisoryLock!: () => void;
|
||||
let blockerPid = 0;
|
||||
let functionCreated = false;
|
||||
let triggerCreated = false;
|
||||
const beforeDependencies = await prisma.workGraphDependency.findMany({ where: { projectId: project.id, nodeId: task.id }, select: { dependsOnNodeId: true, dependencyType: true, createdBy: true } });
|
||||
expect(beforeDependencies).toHaveLength(0);
|
||||
const locked = new Promise<void>((resolve) => { acquiredAdvisoryLock = resolve; });
|
||||
const hold = new Promise<void>((resolve) => { releaseAdvisoryLock = resolve; });
|
||||
let claim: ReturnType<typeof claimReadyWork> | undefined;
|
||||
let edit: Promise<{ status: "fulfilled" } | { status: "rejected"; error: unknown }> | undefined;
|
||||
let editSettled = false;
|
||||
let lockHolder: Promise<void> | undefined;
|
||||
try {
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE FUNCTION ${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM pg_advisory_xact_lock(${advisoryKey[0]}, ${advisoryKey[1]});
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$
|
||||
`);
|
||||
functionCreated = true;
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE TRIGGER ${triggerName}
|
||||
BEFORE UPDATE ON "WorkGraphNode"
|
||||
FOR EACH ROW WHEN (NEW."projectId" = '${project.id}' AND NEW."owner" IS NOT NULL AND OLD."owner" IS DISTINCT FROM NEW."owner")
|
||||
EXECUTE FUNCTION ${functionName}();
|
||||
`);
|
||||
triggerCreated = true;
|
||||
lockHolder = prisma.$transaction(async (transaction) => {
|
||||
const [backend] = await transaction.$queryRaw<Array<{ pid: number }>>`SELECT pg_backend_pid() AS pid`;
|
||||
blockerPid = backend!.pid;
|
||||
await transaction.$queryRaw`SELECT pg_advisory_xact_lock(${advisoryKey[0]}::integer, ${advisoryKey[1]}::integer) IS NULL`;
|
||||
acquiredAdvisoryLock();
|
||||
await hold;
|
||||
});
|
||||
await Promise.race([locked, lockHolder!.then(() => { throw new Error("The advisory lock holder ended before acquiring its lock."); })]);
|
||||
claim = claimReadyWork(slug, "claimant");
|
||||
const waitDeadline = Date.now() + 2000;
|
||||
let claimWaiter: DatabaseLockWaiter | undefined;
|
||||
while (!claimWaiter && Date.now() < waitDeadline) {
|
||||
claimWaiter = (await readDatabaseLockWaiters()).find((row) => row.query.toLowerCase().includes("update") && row.query.toLowerCase().includes("workgraphnode") && row.blockingPids.includes(blockerPid));
|
||||
if (!claimWaiter) await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
expect(claimWaiter).toBeDefined();
|
||||
|
||||
edit = updateWorkDependencies(slug, task.id, [prerequisite.id], "database-test").then(
|
||||
() => ({ status: "fulfilled" as const }),
|
||||
(error: unknown) => ({ status: "rejected" as const, error }),
|
||||
).finally(() => { editSettled = true; });
|
||||
const editWaitDeadline = Date.now() + 2000;
|
||||
let dependencyEditWaiter: DatabaseLockWaiter | undefined;
|
||||
while (!dependencyEditWaiter && !editSettled && Date.now() < editWaitDeadline) {
|
||||
dependencyEditWaiter = (await readDatabaseLockWaiters()).find((row) => isProjectCoordinationLockQuery(row.query) && row.blockingPids.includes(claimWaiter!.pid));
|
||||
if (!dependencyEditWaiter && !editSettled) await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
expect(editSettled).toBe(false);
|
||||
expect(dependencyEditWaiter).toBeDefined();
|
||||
} finally {
|
||||
releaseAdvisoryLock?.();
|
||||
if (lockHolder) await lockHolder.catch(() => undefined);
|
||||
if (claim) await claim.catch(() => undefined);
|
||||
if (edit) await edit.catch(() => undefined);
|
||||
if (triggerCreated) await prisma.$executeRawUnsafe(`DROP TRIGGER IF EXISTS ${triggerName} ON "WorkGraphNode"`);
|
||||
if (functionCreated) await prisma.$executeRawUnsafe(`DROP FUNCTION IF EXISTS ${functionName}()`);
|
||||
}
|
||||
|
||||
const claimResult = await claim!;
|
||||
expect(claimResult?.node.id).toBe(task.id);
|
||||
expect(claimResult?.node.dependencies.map((edge) => edge.dependsOnNodeId)).toEqual(beforeDependencies.map((edge) => edge.dependsOnNodeId));
|
||||
const editResult = await edit!;
|
||||
expect(editResult.status).toBe("rejected");
|
||||
if (editResult.status === "rejected") expect(editResult.error).toMatchObject({ message: "WORK_GRAPH_ACTIVE_LEASE" });
|
||||
const persisted = await prisma.workGraphDependency.findMany({ where: { projectId: project.id, nodeId: task.id }, select: { dependsOnNodeId: true, dependencyType: true, createdBy: true } });
|
||||
expect(persisted).toEqual(beforeDependencies);
|
||||
}, 15_000);
|
||||
|
||||
it("keeps completion audit foreign-key checks compatible with a waiting graph sync", async () => {
|
||||
const { slug, project, graph } = await createWorkGraphProject("## Work\n- Completing task\n- Other task");
|
||||
await setAutonomyPriority(project.id, slug, "Completing task", 100);
|
||||
const task = nodeByTitle(graph, "Completing task");
|
||||
const claim = await claimReadyWork(slug, "completion-worker");
|
||||
expect(claim?.node.id).toBe(task.id);
|
||||
const barrier = await createAuditInsertBarrier(project.id, "work.completed");
|
||||
let completion: ReturnType<typeof recordWorkCompletion> | undefined;
|
||||
let sync: ReturnType<typeof syncProjectWorkGraph> | undefined;
|
||||
try {
|
||||
completion = recordWorkCompletion(slug, task.id, "completion-worker", claim!.leaseToken, { summary: "Completion lock-order proof" });
|
||||
const completionWaiter = await waitForDatabaseLockWaiter((waiter) => waiter.query.toLowerCase().includes("workgraphauditevent") && waiter.blockingPids.includes(barrier.blockerPid));
|
||||
expect(completionWaiter).toBeDefined();
|
||||
|
||||
sync = syncProjectWorkGraph(slug);
|
||||
const syncWaiter = await waitForDatabaseLockWaiter((waiter) => waiter.query.toLowerCase().includes("workgraphnode") && waiter.blockingPids.includes(completionWaiter!.pid) && waiter.hasProjectRowShareLock);
|
||||
expect(syncWaiter).toBeDefined();
|
||||
|
||||
barrier.release();
|
||||
const results = await Promise.allSettled([completion, sync]);
|
||||
expect(results.map((result) => result.status)).toEqual(["fulfilled", "fulfilled"]);
|
||||
} finally {
|
||||
barrier.release();
|
||||
await Promise.allSettled([...(completion ? [completion] : []), ...(sync ? [sync] : [])]);
|
||||
await barrier.cleanup();
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("keeps expired-work audit foreign-key checks compatible with a waiting claim", async () => {
|
||||
const { slug, project, graph } = await createWorkGraphProject("## Work\n- Expired task\n- Other task");
|
||||
await setAutonomyPriority(project.id, slug, "Expired task", 100);
|
||||
const task = nodeByTitle(graph, "Expired task");
|
||||
const initialClaim = await claimReadyWork(slug, "expired-worker");
|
||||
expect(initialClaim?.node.id).toBe(task.id);
|
||||
await prisma.workGraphNode.update({ where: { id: task.id }, data: { leaseExpiresAt: new Date(Date.now() - 60_000) } });
|
||||
const barrier = await createAuditInsertBarrier(project.id, "work.reclaimed");
|
||||
let reclaim: ReturnType<typeof reclaimExpiredWork> | undefined;
|
||||
let claimAfterExpiry: ReturnType<typeof claimReadyWork> | undefined;
|
||||
try {
|
||||
reclaim = reclaimExpiredWork(slug);
|
||||
const reclaimWaiter = await waitForDatabaseLockWaiter((waiter) => waiter.query.toLowerCase().includes("workgraphauditevent") && waiter.blockingPids.includes(barrier.blockerPid));
|
||||
expect(reclaimWaiter).toBeDefined();
|
||||
|
||||
claimAfterExpiry = claimReadyWork(slug, "reclaim-race-claimant");
|
||||
const claimWaiter = await waitForDatabaseLockWaiter((waiter) => isProjectCoordinationLockQuery(waiter.query) && waiter.blockingPids.includes(reclaimWaiter!.pid));
|
||||
expect(claimWaiter).toBeDefined();
|
||||
|
||||
barrier.release();
|
||||
const [reclaimResult, claimResult] = await Promise.all([reclaim, claimAfterExpiry]);
|
||||
expect(reclaimResult.reclaimedCount).toBe(1);
|
||||
expect(claimResult?.node.id).toBe(task.id);
|
||||
} finally {
|
||||
barrier.release();
|
||||
await Promise.allSettled([...(reclaim ? [reclaim] : []), ...(claimAfterExpiry ? [claimAfterExpiry] : [])]);
|
||||
await barrier.cleanup();
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("serializes multi-node reclaim before it can invert locks with graph reconciliation", async () => {
|
||||
const { slug, project, graph } = await createWorkGraphProject("## Work\n- Task B\n- Task A");
|
||||
const taskA = nodeByTitle(graph, "Task A");
|
||||
const taskB = nodeByTitle(graph, "Task B");
|
||||
const now = Date.now();
|
||||
await prisma.workGraphNode.update({ where: { id: taskB.id }, data: { owner: "expired-worker", leaseToken: randomUUID(), leaseExpiresAt: new Date(now - 120_000), executionEligibility: "running", canonicalLifecycle: "running" } });
|
||||
await prisma.workGraphNode.update({ where: { id: taskA.id }, data: { owner: "expired-worker", leaseToken: randomUUID(), leaseExpiresAt: new Date(now - 60_000), executionEligibility: "running", canonicalLifecycle: "running" } });
|
||||
const reclaimWhere = { projectId: project.id, leaseExpiresAt: { lt: new Date() }, owner: { not: null }, executionEligibility: "running" as const };
|
||||
const reclaimOrder = await prisma.workGraphNode.findMany({ where: reclaimWhere, select: { id: true, title: true } });
|
||||
expect(reclaimOrder).toHaveLength(2);
|
||||
const graphOrder = reclaimOrder.map((node) => node.title).reverse();
|
||||
await setAutonomyRoadmapOrder(project.id, slug, graphOrder);
|
||||
|
||||
const graphFirstNode = graphOrder[0] === "Task A" ? taskA : taskB;
|
||||
const barrier = await createWorkGraphNodeUpdateBarrier(graphFirstNode.id);
|
||||
let sync: ReturnType<typeof syncProjectWorkGraph> | undefined;
|
||||
let reclaim: ReturnType<typeof reclaimExpiredWork> | undefined;
|
||||
try {
|
||||
sync = syncProjectWorkGraph(slug);
|
||||
const syncWaiter = await waitForDatabaseLockWaiter((waiter) => waiter.query.toLowerCase().includes("workgraphnode") && waiter.blockingPids.includes(barrier.blockerPid) && waiter.hasProjectRowShareLock);
|
||||
expect(syncWaiter).toBeDefined();
|
||||
|
||||
reclaim = reclaimExpiredWork(slug);
|
||||
const reclaimProjectWaiter = await waitForDatabaseLockWaiter((waiter) => isProjectCoordinationLockQuery(waiter.query) && waiter.blockingPids.includes(syncWaiter!.pid));
|
||||
const reclaimNodeWaiter = reclaimProjectWaiter ? undefined : await waitForDatabaseLockWaiter((waiter) => waiter.query.toLowerCase().includes("workgraphnode") && waiter.blockingPids.includes(syncWaiter!.pid));
|
||||
expect(reclaimProjectWaiter ?? reclaimNodeWaiter).toBeDefined();
|
||||
|
||||
barrier.release();
|
||||
if (reclaimNodeWaiter) {
|
||||
const graphReverseWaiter = await waitForDatabaseLockWaiter((waiter) => waiter.query.toLowerCase().includes("workgraphnode") && waiter.blockingPids.includes(reclaimNodeWaiter.pid) && waiter.hasProjectRowShareLock);
|
||||
expect(graphReverseWaiter).toBeDefined();
|
||||
}
|
||||
const results = await Promise.allSettled([sync, reclaim]);
|
||||
const failures = results.filter((result): result is PromiseRejectedResult => result.status === "rejected");
|
||||
expect(results.map((result) => result.status), failures.map((failure) => String(failure.reason)).join("\n")).toEqual(["fulfilled", "fulfilled"]);
|
||||
} finally {
|
||||
barrier.release();
|
||||
await Promise.allSettled([...(sync ? [sync] : []), ...(reclaim ? [reclaim] : [])]);
|
||||
await barrier.cleanup();
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("keeps a live lease's derived dependencies until expiry, then applies the latest source edges", async () => {
|
||||
const { slug, project } = await createWorkGraphProject("## Work\n- Runnable task\n- Prerequisite task");
|
||||
await setAutonomyPriority(project.id, slug, "Runnable task", 100);
|
||||
const graph = await getProjectWorkGraph(slug);
|
||||
const task = nodeByTitle(graph, "Runnable task");
|
||||
const prerequisite = nodeByTitle(graph, "Prerequisite task");
|
||||
const claim = await claimReadyWork(slug, "claimant");
|
||||
expect(claim?.node.id).toBe(task.id);
|
||||
expect(claim?.node.dependencies).toHaveLength(0);
|
||||
|
||||
await setAutonomyDependencies(project.id, slug, "Runnable task", ["Prerequisite task"]);
|
||||
const duringLease = await getProjectWorkGraph(slug);
|
||||
const leasedTask = nodeByTitle(duringLease, "Runnable task");
|
||||
expect(leasedTask.executionEligibility).toBe("running");
|
||||
expect(leasedTask.dependencies).toHaveLength(0);
|
||||
|
||||
await prisma.workGraphNode.update({ where: { id: task.id }, data: { leaseExpiresAt: new Date(Date.now() - 60_000) } });
|
||||
const afterExpiry = await getProjectWorkGraph(slug);
|
||||
const blockedTask = nodeByTitle(afterExpiry, "Runnable task");
|
||||
expect(blockedTask.executionEligibility).toBe("waiting");
|
||||
expect(blockedTask.dependencies.map((edge) => edge.dependsOnNodeId)).toEqual([prerequisite.id]);
|
||||
const nextClaim = await claimReadyWork(slug, "next-claimant");
|
||||
expect(nextClaim?.node.id).toBe(prerequisite.id);
|
||||
expect(nextClaim?.node.id).not.toBe(task.id);
|
||||
}, 15_000);
|
||||
|
||||
it("keeps the old complete derived edge set visible until reconciliation commits", async () => {
|
||||
const { slug, project } = await createWorkGraphProject("## Work\n- Task A\n- Task B\n- Task C");
|
||||
await setAutonomyDependencies(project.id, slug, "Task A", ["Task B"]);
|
||||
const initialGraph = await getProjectWorkGraph(slug);
|
||||
const taskA = nodeByTitle(initialGraph, "Task A");
|
||||
const taskB = nodeByTitle(initialGraph, "Task B");
|
||||
const taskC = nodeByTitle(initialGraph, "Task C");
|
||||
await setAutonomyDependencies(project.id, slug, "Task A", ["Task C"]);
|
||||
|
||||
let releaseProjectLock!: () => void;
|
||||
let acquiredProjectLock!: () => void;
|
||||
let blockerPid = 0;
|
||||
const projectLocked = new Promise<void>((resolve) => { acquiredProjectLock = resolve; });
|
||||
const hold = new Promise<void>((resolve) => { releaseProjectLock = resolve; });
|
||||
const lockHolder = prisma.$transaction(async (transaction) => {
|
||||
const [backend] = await transaction.$queryRaw<Array<{ pid: number }>>`SELECT pg_backend_pid() AS pid`;
|
||||
blockerPid = backend!.pid;
|
||||
await transaction.$queryRaw`SELECT "id" FROM "Project" WHERE "id" = ${project.id} FOR UPDATE`;
|
||||
acquiredProjectLock();
|
||||
await hold;
|
||||
});
|
||||
let sync: Promise<string> | undefined;
|
||||
try {
|
||||
await Promise.race([projectLocked, lockHolder.then(() => { throw new Error("The project lock holder ended before acquiring its lock."); })]);
|
||||
let syncFinished = false;
|
||||
sync = syncProjectWorkGraph(slug).then((result) => { syncFinished = true; return result; });
|
||||
const waitDeadline = Date.now() + 5000;
|
||||
let waitingTransactions: DatabaseLockWaiter[] = [];
|
||||
while (!waitingTransactions.length && Date.now() < waitDeadline) {
|
||||
waitingTransactions = (await readDatabaseLockWaiters()).filter((row) => row.blockingPids.includes(blockerPid));
|
||||
if (!waitingTransactions.length) await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
expect(waitingTransactions.length).toBeGreaterThan(0);
|
||||
expect(syncFinished).toBe(false);
|
||||
const duringSync = await prisma.workGraphDependency.findMany({ where: { projectId: project.id, createdBy: "autonomy-sync" } });
|
||||
expect(duringSync.map((edge) => [edge.nodeId, edge.dependsOnNodeId])).toEqual([[taskA.id, taskB.id]]);
|
||||
} finally {
|
||||
releaseProjectLock();
|
||||
await lockHolder.catch(() => undefined);
|
||||
if (sync) await sync;
|
||||
}
|
||||
|
||||
const afterSync = await prisma.workGraphDependency.findMany({ where: { projectId: project.id, createdBy: "autonomy-sync" } });
|
||||
expect(afterSync.map((edge) => [edge.nodeId, edge.dependsOnNodeId])).toEqual([[taskA.id, taskC.id]]);
|
||||
}, 15_000);
|
||||
|
||||
it("rechecks prerequisites before treating an expired lease as reclaimable", async () => {
|
||||
const { slug, project, graph } = await createWorkGraphProject("## Work\n- Dependent task\n- Prerequisite task");
|
||||
const task = nodeByTitle(graph, "Dependent task");
|
||||
const prerequisite = nodeByTitle(graph, "Prerequisite task");
|
||||
await updateWorkDependencies(slug, task.id, [prerequisite.id], "database-test");
|
||||
await prisma.workGraphNode.update({ where: { id: task.id }, data: {
|
||||
owner: "expired-worker",
|
||||
leaseToken: randomUUID(),
|
||||
leaseExpiresAt: new Date(Date.now() - 60_000),
|
||||
executionEligibility: "running",
|
||||
canonicalLifecycle: "running",
|
||||
} });
|
||||
|
||||
const snapshot = await getProjectWorkGraph(slug);
|
||||
expect(nodeByTitle(snapshot, "Dependent task").executionEligibility).toBe("waiting");
|
||||
expect(nodeByTitle(snapshot, "Dependent task").dependencies.some((edge) => edge.dependsOnNodeId === prerequisite.id)).toBe(true);
|
||||
const claim = await claimReadyWork(slug, "claimant");
|
||||
expect(claim?.node.id).toBe(prerequisite.id);
|
||||
expect(claim?.node.id).not.toBe(task.id);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user