diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 078772388..4cb1ee31e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,6 +106,26 @@ Keep changes reviewable and include the exact commands and limitations of your verification. Never include credentials, private transcripts or local runtime state in a contribution. +## Workflow history fixtures + +The replay determinism gate in `apps/worker/tests/replay-gate.test.ts` replays +recorded Temporal histories from +`apps/worker/tests/fixtures/workflow-histories/`. The test only records when a +fixture is missing (`existsSync` guard), so re-recording means deleting the file +first: + +```sh +rm apps/worker/tests/fixtures/workflow-histories/dot-goal.json +npm test -w @spaceapp/worker +``` + +The suite records the missing history and then replays it; a green run proves +the fresh history is replay-safe. Inspect the generated JSON, commit it together +with the workflow change that required it, and never commit a history recorded +with mocked activities. See +`apps/worker/tests/fixtures/workflow-histories/README.md` for the full +procedure, including adding a fixture for a new workflow. + ## License of contributions SpaceApp is licensed under the [MIT License](LICENSE). Contributions are diff --git a/apps/worker/package.json b/apps/worker/package.json index ec58125fa..615c0e68d 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -10,7 +10,8 @@ "start": "node dist/worker.js", "build": "tsc -p tsconfig.json", "check": "tsc -p tsconfig.json --noEmit", - "test": "vitest run" + "test": "vitest run", + "pretest": "npm run build" }, "dependencies": { "@space/codex-app-server": "0.2.5", @@ -26,6 +27,7 @@ "undici": "8.12.3" }, "devDependencies": { + "@temporalio/testing": "1.24.0", "typescript": "^5.9.3", "vitest": "^4.1.12" }, diff --git a/apps/worker/tests/fixtures/replay-negative-workflows.js b/apps/worker/tests/fixtures/replay-negative-workflows.js new file mode 100644 index 000000000..2cd54c164 --- /dev/null +++ b/apps/worker/tests/fixtures/replay-negative-workflows.js @@ -0,0 +1,16 @@ +// Test-only workflow bundle for the replay gate's negative control. +// +// It exports the same workflow type as the recorded fixture (`dotGoalWorkflow`) but emits a +// different first command: a timer instead of an activity. Replaying a history recorded by the +// real bundle against this one must be rejected as nondeterministic - that is the property the +// gate is supposed to guarantee (issue #60). +import { proxyActivities, sleep } from "@temporalio/workflow"; + +const { dotGoalStep } = proxyActivities({ startToCloseTimeout: "1 minute" }); + +export async function dotGoalWorkflow(input) { + // Divergence: the real workflow schedules an activity first; this variant starts a timer. + await sleep("1 second"); + await dotGoalStep(input.goalId); + return { goalStatus: "COMPLETED" }; +} diff --git a/apps/worker/tests/fixtures/workflow-histories/README.md b/apps/worker/tests/fixtures/workflow-histories/README.md new file mode 100644 index 000000000..8717504ed --- /dev/null +++ b/apps/worker/tests/fixtures/workflow-histories/README.md @@ -0,0 +1,51 @@ +# Workflow history fixtures + +This directory holds recorded Temporal event histories used by the replay +determinism gate (`apps/worker/tests/replay-gate.test.ts`). + +- `dot-goal.json` — event history for `dotGoalWorkflow`, recorded once against + the time-skipping test server and replayed in CI. If a workflow change makes + the recorded history replay to a different command sequence, the gate fails + and the change cannot land silently. + +## How to re-record a fixture + +The replay test only records when the fixture is missing: it checks the file +with `existsSync` and skips the record step when the file already exists, so +you have to delete the fixture to regenerate it. + +```sh +# 1. Remove the fixture you want to re-record +rm apps/worker/tests/fixtures/workflow-histories/dot-goal.json + +# 2. Run the worker suite; the gate records the missing fixture first and then +# replays it, so a green run means the fresh history is replay-safe +npm test -w @spaceapp/worker +``` + +The test writes `{ "workflowId": ..., "historyBase64": ... }` with two-space +JSON indentation and a trailing newline. Keep that shape: the file is read back +verbatim, and a hand-edited or re-serialised history no longer proves the same +execution. + +## What to commit + +1. Inspect `git diff --stat apps/worker/tests/fixtures/workflow-histories/`. + A changed history means real workflow behaviour changed — read the diff, do + not commit it blindly. +2. Commit the regenerated fixture **together with** the workflow change that + required it, in the same pull request, so reviewers can see the behaviour + change next to the new history. +3. Never commit a fixture recorded with mocked activities or a shortened + timeout: the gate must replay the same command sequence the production + worker would produce. If a change is intentional and cannot be replayed, + guard it with `patched()` and record a new fixture that covers the new + branch. + +## Adding a fixture for a new workflow + +Copy the pattern in `replay-gate.test.ts`: start a time-skipping +`TestWorkflowEnvironment`, run the workflow against test activities, capture +`env.getWorkflowHistory(...)` / `handle.fetchHistory()`, and store the base64 +history under this directory. Add the replay assertion for the new file to the +same test so the gate covers it. diff --git a/apps/worker/tests/fixtures/workflow-histories/dot-goal.json b/apps/worker/tests/fixtures/workflow-histories/dot-goal.json new file mode 100644 index 000000000..40ca17214 --- /dev/null +++ b/apps/worker/tests/fixtures/workflow-histories/dot-goal.json @@ -0,0 +1,4 @@ +{ + "workflowId": "recorded-dot-goal", + "historyBase64": "Cv4BCAESDAjzxfjWBhCA/+O9ARgBMukBChEKD2RvdEdvYWxXb3JrZmxvdyoPCgtyZXBsYXktZ2F0ZRABMkEKPwoWCghlbmNvZGluZxIKanNvbi9wbGFpbhIleyJnb2FsSWQiOiJjbW1nb2FsMDAwMDAwMDAwMDAwMDAwMDEifToGCICGsJYBQgYIgIawlgFKAggKciRlMzRlMzFlMC1mMGE5LTRlNmMtYTA3Ni04ZDRkNDg0YmUzM2N6FjIzMjY0NEBWTS00LTEyOS11YnVudHWCASRlMzRlMzFlMC1mMGE5LTRlNmMtYTA3Ni04ZDRkNDg0YmUzM2OQAQGqAQDKAQAKKwgCEgwI88X41gYQgP/jvQEYBVIXCg8KC3JlcGxheS1nYXRlEAESAggKGAEKMQgDEgwI88X41gYQgP/jvQEYBlodCAISFjIzMjY0NEBWTS00LTEyOS11YnVudHUoqQIKsQEIBBIMCPTF+NYGEICjzMEBGAdinAEIAhoWMjMyNjQ0QFZNLTQtMTI5LXVidW50dSJaQHRlbXBvcmFsaW8vd29ya2VyQDEuMjQuMCswNmM2M2U0Njk0NmY3ZDJhNDExMmQzM2RhOWU0MDM3NDk4Njk1YWZiZGQ3ZjU2NzUwNDQxYTUzM2Q1YjUzN2JhMiIKAwMBAhoTdGVtcG9yYWwtdHlwZXNjcmlwdCIGMS4yNC4wagAKnwEIBRIMCPTF+NYGEICjzMEBGAp6igEKATESDQoLZG90R29hbFN0ZXAiDwoLcmVwbGF5LWdhdGUQASoAMjYKNAoWCghlbmNvZGluZxIKanNvbi9wbGFpbhIaImNtbWdvYWwwMDAwMDAwMDAwMDAwMDAwMSI6BgiAhrCWAUIGCICGsJYBSgIIeFIAWANiEwoCCAURAAAAAAAAAEAaAgh4IBQKMQgGEgwI9MX41gYQgKPMwQEYC4IBHAgFEhYyMzI2NDRAVk0tNC0xMjktdWJ1bnR1IAEKhgEIBxIMCPTF+NYGEMDm38gBGAyKAXEKUwpRChYKCGVuY29kaW5nEgpqc29uL3BsYWluEjd7InN0YXRlIjoiV0FJVElORyIsInJ1bklkIjpudWxsLCJnb2FsU3RhdHVzIjoiUlVOTklORyJ9EAUYBiIWMjMyNjQ0QFZNLTQtMTI5LXVidW50dQorCAgSDAj0xfjWBhDA5t/IARgFUhcKDwoLcmVwbGF5LWdhdGUQARICCAoYAQoxCAkSDAj0xfjWBhDA5t/IARgGWh0ICBIWMjMyNjQ0QFZNLTQtMTI5LXVidW50dSiMBwqOAQgKEgwI9MX41gYQwMme0wEYB2J6CAgaFjIzMjY0NEBWTS00LTEyOS11YnVudHUiWkB0ZW1wb3JhbGlvL3dvcmtlckAxLjI0LjArMDZjNjNlNDY5NDZmN2QyYTQxMTJkMzNkYTllNDAzNzQ5ODY5NWFmYmRkN2Y1Njc1MDQ0MWE1MzNkNWI1MzdiYTIAagAKHggLEgwI9MX41gYQwMme0wEYEaIBCQoBMRICCB4YCQoaCAwSDAiSxvjWBhDAyZ7TARgSqgEFCgExEAsKKwgNEgwIksb41gYQwMme0wEYBVIXCg8KC3JlcGxheS1nYXRlEAESAggKGAEKMQgOEgwIksb41gYQwMme0wEYBlodCA0SFjIzMjY0NEBWTS00LTEyOS11YnVudHUorgkKjgEIDxIMCJLG+NYGEID7vdgBGAdieggNGhYyMzI2NDRAVk0tNC0xMjktdWJ1bnR1IlpAdGVtcG9yYWxpby93b3JrZXJAMS4yNC4wKzA2YzYzZTQ2OTQ2ZjdkMmE0MTEyZDMzZGE5ZTQwMzc0OTg2OTVhZmJkZDdmNTY3NTA0NDFhNTMzZDViNTM3YmEyAGoACp8BCBASDAiSxvjWBhCA+73YARgKeooBCgEyEg0KC2RvdEdvYWxTdGVwIg8KC3JlcGxheS1nYXRlEAEqADI2CjQKFgoIZW5jb2RpbmcSCmpzb24vcGxhaW4SGiJjbW1nb2FsMDAwMDAwMDAwMDAwMDAwMDEiOgYIgIawlgFCBgiAhrCWAUoCCHhSAFgOYhMKAggFEQAAAAAAAABAGgIIeCAUCjEIERIMCJLG+NYGEID7vdgBGAuCARwIEBIWMjMyNjQ0QFZNLTQtMTI5LXVidW50dSABCoUBCBISDAiSxvjWBhDAtdfeARgMigFwClIKUAoWCghlbmNvZGluZxIKanNvbi9wbGFpbhI2eyJzdGF0ZSI6IkRPTkUiLCJydW5JZCI6bnVsbCwiZ29hbFN0YXR1cyI6IkNPTVBMRVRFRCJ9EBAYESIWMjMyNjQ0QFZNLTQtMTI5LXVidW50dQorCBMSDAiSxvjWBhDAtdfeARgFUhcKDwoLcmVwbGF5LWdhdGUQARICCAoYAQoxCBQSDAiSxvjWBhDAtdfeARgGWh0IExIWMjMyNjQ0QFZNLTQtMTI5LXVidW50dSjtDQqOAQgVEgwIksb41gYQwNm/4gEYB2J6CBMaFjIzMjY0NEBWTS00LTEyOS11YnVudHUiWkB0ZW1wb3JhbGlvL3dvcmtlckAxLjI0LjArMDZjNjNlNDY5NDZmN2QyYTQxMTJkMzNkYTllNDAzNzQ5ODY5NWFmYmRkN2Y1Njc1MDQ0MWE1MzNkNWI1MzdiYTIAagAKTggWEgwIksb41gYQwNm/4gEYAjo6CjYKNAoWCghlbmNvZGluZxIKanNvbi9wbGFpbhIaeyJnb2FsU3RhdHVzIjoiQ09NUExFVEVEIn0QFA==" +} diff --git a/apps/worker/tests/replay-gate.test.ts b/apps/worker/tests/replay-gate.test.ts new file mode 100644 index 000000000..93e29d9ea --- /dev/null +++ b/apps/worker/tests/replay-gate.test.ts @@ -0,0 +1,123 @@ +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { Worker } from "@temporalio/worker"; +import protoPkg from "@temporalio/proto"; +import fs from "node:fs"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { DeterminismViolationError } from "@temporalio/workflow"; + +const temporal = (protoPkg as unknown as { temporal?: typeof protoPkg }).temporal ?? protoPkg; + +const MAIN = path.join(__dirname, "..", "dist", "workflows.js"); +const MUTATION_BUNDLE = path.join(__dirname, "..", "dist", "memory-mutation-workflows.js"); +const FIXTURE = path.join(__dirname, "fixtures", "workflow-histories", "dot-goal.json"); +// Test-only bundle that exports dotGoalWorkflow with a divergent command sequence. Used as the +// negative control: the gate must reject it, otherwise a determinism break would pass silently. +const NEGATIVE_BUNDLE = path.join(__dirname, "fixtures", "replay-negative-workflows.js"); +const QUEUE = "replay-gate"; + +let env: TestWorkflowEnvironment; +beforeAll(async () => { env = await TestWorkflowEnvironment.createTimeSkipping(); }, 240_000); +afterAll(async () => { await env?.teardown(); }); + +type Recorded = { workflowId: string; historyBase64: string }; + +let recordedStepCalls = 0; +const dotGoalActivities = { + async dotGoalStep() { + recordedStepCalls += 1; + return recordedStepCalls === 1 + ? { state: "WAITING", runId: null, goalStatus: "RUNNING" } + : { state: "DONE", runId: null, goalStatus: "COMPLETED" }; + }, + async dotGoalObserve() { return { run: "SUCCEEDED", goalStatus: "COMPLETED" }; }, +}; + +async function replayDotGoal(recorded: Recorded) { + const history = temporal.api.history.v1.History.decode(Buffer.from(recorded.historyBase64, "base64")); + const results: Array<{ error?: unknown }> = []; + for await (const r of Worker.runReplayHistories( + { connection: env.nativeConnection, taskQueue: QUEUE, workflowsPath: MAIN }, + [{ workflowId: recorded.workflowId, history }], + )) { + results.push({ error: r.error }); + } + return results; +} + +// The replay gate replays a committed history, not one recorded in the same run: replaying +// a freshly recorded history would use the changed code for both sides and pass even when +// an existing execution can no longer replay. +describe("replay gate", () => { + test("replays the committed dotGoalWorkflow history against the current bundle", async () => { + if (!fs.existsSync(FIXTURE)) { + // One-off recording; the file is committed and replayed from then on. + const worker = await Worker.create({ + connection: env.nativeConnection, taskQueue: QUEUE, workflowsPath: MAIN, activities: dotGoalActivities, + }); + await worker.runUntil(async () => { + const h = await env.client.workflow.start("dotGoalWorkflow", { + taskQueue: QUEUE, workflowId: "recorded-dot-goal", args: [{ goalId: "cmmgoal00000000000000001" }], + }); + await h.result(); + }); + const history = await env.client.workflow.getHandle("recorded-dot-goal").fetchHistory(); + const historyBase64 = Buffer.from( + temporal.api.history.v1.History.encode(history).finish(), + ).toString("base64"); + fs.mkdirSync(path.dirname(FIXTURE), { recursive: true }); + fs.writeFileSync(FIXTURE, JSON.stringify({ workflowId: "recorded-dot-goal", historyBase64 }, null, 2) + "\n"); + } + + const recorded = JSON.parse(fs.readFileSync(FIXTURE, "utf8")) as Recorded; + expect(recorded.historyBase64.length).toBeGreaterThan(0); + const results = await replayDotGoal(recorded); + expect(results).toHaveLength(1); + expect(results[0].error).toBeUndefined(); + }, 240_000); + + // Negative control for issue #60: the gate has to catch a determinism change, not just pass. + test("rejects the committed dotGoalWorkflow history when the bundle diverges", async () => { + const recorded = JSON.parse(fs.readFileSync(FIXTURE, "utf8")) as Recorded; + const results = await replayDotGoalForBundle(NEGATIVE_BUNDLE, recorded, QUEUE); + expect(results).toHaveLength(1); + const error = results[0].error; + expect(error).toBeDefined(); + expect(error instanceof DeterminismViolationError || (error as Error)?.name?.includes("Determinism")).toBe(true); + }, 240_000); + + test("memoryMutationWorkflow replays its own recorded history", async () => { + const activities = { + async executePersistedMemoryChangeSet(input: { changeSetId: string }) { + return { changeSetId: input.changeSetId, applied: 1 }; + }, + }; + const worker = await Worker.create({ + connection: env.nativeConnection, taskQueue: QUEUE + "-mut", workflowsPath: MUTATION_BUNDLE, activities, + }); + await worker.runUntil(async () => { + const h = await env.client.workflow.start("memoryMutationWorkflow", { + taskQueue: QUEUE + "-mut", workflowId: "replay-mutation", args: [{ changeSetId: "cmmcs000000000000000001" }], + }); + await h.result(); + }); + const history = await env.client.workflow.getHandle("replay-mutation").fetchHistory(); + const historyBase64 = Buffer.from( + temporal.api.history.v1.History.encode(history).finish(), + ).toString("base64"); + const results = await replayDotGoalForBundle(MUTATION_BUNDLE, { workflowId: "replay-mutation", historyBase64 }); + expect(results[0].error).toBeUndefined(); + }, 240_000); +}); + +async function replayDotGoalForBundle(bundle: string, recorded: Recorded, taskQueue = QUEUE + "-mut") { + const history = temporal.api.history.v1.History.decode(Buffer.from(recorded.historyBase64, "base64")); + const results: Array<{ error?: unknown }> = []; + for await (const r of Worker.runReplayHistories( + { connection: env.nativeConnection, taskQueue, workflowsPath: bundle }, + [{ workflowId: recorded.workflowId, history }], + )) { + results.push({ error: r.error }); + } + return results; +} diff --git a/apps/worker/tests/workflow-sandbox-2.test.ts b/apps/worker/tests/workflow-sandbox-2.test.ts new file mode 100644 index 000000000..934deb3e9 --- /dev/null +++ b/apps/worker/tests/workflow-sandbox-2.test.ts @@ -0,0 +1,73 @@ +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { Worker } from "@temporalio/worker"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import path from "node:path"; + +const dist = (f: string) => path.join(__dirname, "..", "dist", f); +const MAIN = dist("workflows.js"); +const MUTATION = dist("memory-mutation-workflows.js"); + +let env: TestWorkflowEnvironment; +beforeAll(async () => { env = await TestWorkflowEnvironment.createTimeSkipping(); }, 240_000); +afterAll(async () => { await env?.teardown(); }); + +describe("memoryMutationWorkflow", () => { + test("proxies the change-set executor and returns its result", async () => { + const seen: unknown[] = []; + const activities = { + async executePersistedMemoryChangeSet(input: unknown) { + seen.push(input); + return { applied: 2, failed: 0 }; + }, + }; + const worker = await Worker.create({ + connection: env.nativeConnection, + taskQueue: "mem-mutation", + workflowsPath: MUTATION, + activities, + }); + const result = await worker.runUntil(async () => { + const h = await env.client.workflow.start("memoryMutationWorkflow", { + taskQueue: "mem-mutation", + workflowId: "mem-mutation-1", + args: [{ changeSetId: "cs-1", traceId: "t-1" }], + }); + return h.result(); + }); + expect(result).toEqual({ applied: 2, failed: 0 }); + expect(seen).toHaveLength(1); + }, 240_000); +}); + +describe("roomAgentActionWorkflow failure path", () => { + test("reports the failure through failRoomAgentBackgroundAction and rethrows", async () => { + const reported: Record[] = []; + const activities = { + async runRoomAgentBackgroundAction() { throw new Error("bridge exploded"); }, + async failRoomAgentBackgroundAction(input: Record) { + reported.push(input); + return null; + }, + }; + const worker = await Worker.create({ + connection: env.nativeConnection, + taskQueue: "room-action", + workflowsPath: MAIN, + activities, + }); + const error = await worker.runUntil(async () => { + const h = await env.client.workflow.start("roomAgentActionWorkflow", { + taskQueue: "room-action", + workflowId: "space-room-action:act-9", + args: [{ bridge: { roomId: "room-1", missionId: "mis-1" }, action: {} }], + }); + try { await h.result(); return null; } catch (e) { return e as Error; } + }); + expect(error).toBeInstanceOf(Error); + expect(reported).toHaveLength(1); + expect(reported[0].roomId).toBe("room-1"); + expect(reported[0].missionId).toBe("mis-1"); + expect(reported[0].actionId).toBe("act-9"); + expect(reported[0].cancelled).toBe(false); + }, 240_000); +}); diff --git a/apps/worker/tests/workflow-sandbox-3.test.ts b/apps/worker/tests/workflow-sandbox-3.test.ts new file mode 100644 index 000000000..def71fc11 --- /dev/null +++ b/apps/worker/tests/workflow-sandbox-3.test.ts @@ -0,0 +1,55 @@ +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { Worker } from "@temporalio/worker"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import path from "node:path"; + +const dist = (f: string) => path.join(__dirname, "..", "dist", f); +const MAIN = dist("workflows.js"); +const MAINT = dist("memory-maintenance-workflows.js"); + +let env: TestWorkflowEnvironment; +beforeAll(async () => { env = await TestWorkflowEnvironment.createTimeSkipping(); }, 240_000); +afterAll(async () => { await env?.teardown(); }); + +describe("dummyTurnWorkflow", () => { + test("records start then completion through its two activities", async () => { + const order: string[] = []; + const activities = { + async recordDummyTurnStarted(i: unknown) { order.push("started:" + JSON.stringify(i)); return null; }, + async recordDummyTurnCompleted(i: unknown) { order.push("completed"); return { status: "COMPLETED", turnId: "t-1" }; }, + }; + const worker = await Worker.create({ connection: env.nativeConnection, taskQueue: "dummy-turn", workflowsPath: MAIN, activities }); + const result = await worker.runUntil(async () => { + const h = await env.client.workflow.start("dummyTurnWorkflow", { + taskQueue: "dummy-turn", workflowId: "dummy-1", + args: [{ roomId: "room-1", turnId: "t-1", providerId: "opencode" }], + }); + return h.result(); + }); + expect(result).toEqual({ status: "COMPLETED", turnId: "t-1" }); + expect(order[0]?.startsWith("started:")).toBe(true); + expect(order[1]).toBe("completed"); + }, 240_000); +}); + +describe("codexAppServerTurnWorkflow routing", () => { + test("providerId opencode goes to the opencode activity only", async () => { + const calls: string[] = []; + const activities = { + async runOpenCodeAgentTurn() { calls.push("opencode"); return { status: "COMPLETED", turnId: "t-2" }; }, + async runCliAgentTurn() { calls.push("cli"); return { status: "COMPLETED", turnId: "t-2" }; }, + async runCodexAppServerTurn() { calls.push("codex"); return { status: "COMPLETED", turnId: "t-2" }; }, + }; + const worker = await Worker.create({ connection: env.nativeConnection, taskQueue: "turn-route", workflowsPath: MAIN, activities }); + const result = await worker.runUntil(async () => { + const h = await env.client.workflow.start("codexAppServerTurnWorkflow", { + taskQueue: "turn-route", workflowId: "turn-route-1", + args: [{ roomId: "room-1", turnId: "t-2", providerId: "opencode" }], + }); + return h.result(); + }); + expect(result).toEqual({ status: "COMPLETED", turnId: "t-2" }); + expect(calls).toEqual(["opencode"]); + }, 240_000); +}); + diff --git a/apps/worker/tests/workflow-sandbox-4.test.ts b/apps/worker/tests/workflow-sandbox-4.test.ts new file mode 100644 index 000000000..33cfa12c0 --- /dev/null +++ b/apps/worker/tests/workflow-sandbox-4.test.ts @@ -0,0 +1,49 @@ +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { Worker } from "@temporalio/worker"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import path from "node:path"; + +const MAINT = path.join(__dirname, "..", "dist", "memory-maintenance-workflows.js"); +// Must match MEMORY_MAINTENANCE_TASK_QUEUE in src/memory-maintenance-config.ts, +// otherwise the child consolidation workflow is started on a queue no worker polls. +const QUEUE = "space-memory-maintenance"; + +let env: TestWorkflowEnvironment; +beforeAll(async () => { env = await TestWorkflowEnvironment.createTimeSkipping(); }, 240_000); +afterAll(async () => { await env?.teardown(); }); + +describe("memoryMaintenanceWorkflow", () => { + test("continues when refresh and reconciliation fail, then runs consolidation as a child", async () => { + const seen: string[] = []; + const activities = { + async refreshMemoryGraphSnapshot() { seen.push("refresh"); return { sourceHash: "h0" }; }, + async refreshAllMonthsMemoryGraphSnapshot() { seen.push("refresh-all"); throw new Error("all-months down"); }, + async reconcileStaleSpaceAgentSessions() { seen.push("reconcile"); throw new Error("reconcile down"); }, + async createScheduledMemoryAudit() { seen.push("audit"); return { id: "run-1", workflowId: "wf-consolidation-1", sourceHash: "h" }; }, + async prepareMemoryConsolidation() { seen.push("child:prepare"); return { changeSetId: null, run: { id: "run-1" } }; }, + async executePersistedMemoryChangeSet() { seen.push("child:execute"); return {}; }, + async finalizeMemoryRepair() { seen.push("child:finalize"); return {}; }, + }; + const worker = await Worker.create({ + connection: env.nativeConnection, + taskQueue: QUEUE, + workflowsPath: MAINT, + activities, + }); + const result = await worker.runUntil(async () => { + const h = await env.client.workflow.start("memoryMaintenanceWorkflow", { + taskQueue: QUEUE, + workflowId: "mem-maint-1", + args: [{ traceId: "trace-1" }], + }); + return h.result(); + }); + expect(seen[0]).toBe("refresh"); + expect(seen).toContain("audit"); + expect(seen.at(-1)).toBe("child:prepare"); + // each failing activity is retried 3 times (maximumAttempts) before the workflow swallows it + expect(seen.filter((s) => s === "refresh-all")).toHaveLength(3); + expect(seen.filter((s) => s === "reconcile")).toHaveLength(3); + expect(result).toBeTruthy(); + }, 240_000); +}); diff --git a/apps/worker/tests/workflow-sandbox-5.test.ts b/apps/worker/tests/workflow-sandbox-5.test.ts new file mode 100644 index 000000000..cb1e6990a --- /dev/null +++ b/apps/worker/tests/workflow-sandbox-5.test.ts @@ -0,0 +1,71 @@ +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { Worker } from "@temporalio/worker"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import path from "node:path"; + +const MAINT = path.join(__dirname, "..", "dist", "memory-maintenance-workflows.js"); +const MUTATION_BUNDLE = path.join(__dirname, "..", "dist", "memory-mutation-workflows.js"); +// From @space/contracts/src/workflows.ts — plural "mutations", otherwise the +// mutation activity is scheduled on a queue no worker polls and the test hangs. +const MAINT_QUEUE = "space-memory-maintenance"; +const MUTATION_QUEUE = "space-memory-mutations"; + +let env: TestWorkflowEnvironment; +beforeAll(async () => { env = await TestWorkflowEnvironment.createTimeSkipping(); }, 240_000); +afterAll(async () => { await env?.teardown(); }); + +describe("memoryConsolidationWorkflow", () => { + test("returns the prepared run and skips the mutation when there is no change set", async () => { + const seen: string[] = []; + const activities = { + async prepareMemoryConsolidation() { seen.push("prepare"); return { changeSetId: null, run: { id: "run-7" } }; }, + async executePersistedMemoryChangeSet() { seen.push("execute"); return {}; }, + async finalizeMemoryRepair() { seen.push("finalize"); return {}; }, + }; + const worker = await Worker.create({ connection: env.nativeConnection, taskQueue: MAINT_QUEUE, workflowsPath: MAINT, activities }); + const result = await worker.runUntil(async () => { + const h = await env.client.workflow.start("memoryConsolidationWorkflow", { + taskQueue: MAINT_QUEUE, workflowId: "mem-consol-empty", args: [{ runId: "run-7", traceId: "tr-7" }], + }); + return h.result(); + }); + expect(result).toEqual({ id: "run-7" }); + expect(seen).toEqual(["prepare"]); + }, 240_000); + + test("runs the change set on the mutation queue and finalizes the repair", async () => { + const seen: string[] = []; + let finishMutation: () => void = () => {}; + const mutationDone = new Promise((r) => { finishMutation = r; }); + + const maintActivities = { + async prepareMemoryConsolidation() { seen.push("prepare"); return { changeSetId: "cs-9", run: { id: "run-9" } }; }, + async finalizeMemoryRepair(input: { runId: string; changeSetId: string }) { seen.push("finalize:" + input.changeSetId); return { id: input.runId, repaired: true }; }, + async executePersistedMemoryChangeSet() { seen.push("execute-on-wrong-worker"); return {}; }, + }; + const mutationActivities = { + async executePersistedMemoryChangeSet(input: { changeSetId: string }) { seen.push("execute:" + input.changeSetId); return { applied: 1, failed: 0 }; }, + }; + const maintWorker = await Worker.create({ connection: env.nativeConnection, taskQueue: MAINT_QUEUE, workflowsPath: MAINT, activities: maintActivities }); + const mutationWorker = await Worker.create({ connection: env.nativeConnection, taskQueue: MUTATION_QUEUE, workflowsPath: MUTATION_BUNDLE, activities: mutationActivities }); + + const mutationJob = mutationWorker.runUntil(async () => { await mutationDone; }); + let result: unknown; + try { + result = await maintWorker.runUntil(async () => { + const h = await env.client.workflow.start("memoryConsolidationWorkflow", { + taskQueue: MAINT_QUEUE, workflowId: "mem-consol-mut", args: [{ runId: "run-9", traceId: "tr-9" }], + }); + return h.result(); + }); + } finally { + // Release the mutation worker even when h.result() rejects, otherwise the test + // reports a timeout instead of the workflow failure. + finishMutation(); + await mutationJob; + } + + expect(result).toEqual({ id: "run-9", repaired: true }); + expect(seen).toEqual(["prepare", "execute:cs-9", "finalize:cs-9"]); + }, 240_000); +}); diff --git a/apps/worker/tests/workflow-sandbox-6.test.ts b/apps/worker/tests/workflow-sandbox-6.test.ts new file mode 100644 index 000000000..3de319845 --- /dev/null +++ b/apps/worker/tests/workflow-sandbox-6.test.ts @@ -0,0 +1,62 @@ +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { Worker } from "@temporalio/worker"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import path from "node:path"; + +const MAIN = path.join(__dirname, "..", "dist", "workflows.js"); +const QUEUE = "room-supervisor-sandbox"; + +let env: TestWorkflowEnvironment; +beforeAll(async () => { env = await TestWorkflowEnvironment.createTimeSkipping(); }, 240_000); +afterAll(async () => { await env?.teardown(); }); + +function stubs(seen: string[]) { + const rec = (name: string, value: unknown = null) => async () => { seen.push(name); return value; }; + return { + checkRoomAgentSupervisorTurn: rec("check", { action: "RUN", reason: null }), + finishRoomAgentSupervisorTurn: async (input: { status?: string } | undefined) => { + seen.push("finish:" + String(input?.status)); + return null; + }, + discardRoomAgentSupervisorTurn: rec("discard"), + prepareRoomAgentBackgroundContinuation: rec("prepareContinuation", { pending: [], monitoring: [] }), + failRoomAgentBackgroundAction: rec("failAction"), + runCodexAppServerTurn: rec("turn", { + status: "COMPLETED", + turnId: "turn-1", + roomAgentOutcome: { status: "VERIFIED", statusReason: "verified" }, + }), + runRoomAgentBackgroundAction: rec("backgroundAction"), + settleRoomAgentBackgroundMission: rec("settle", true), + }; +} + +// The supervisor returns Promise and polls with a 1 second condition, so under +// time skipping it never ends on its own: terminate() is the only way out of a test. +describe("roomAgentSupervisorWorkflow (current branch)", () => { + test("runs a pending mission through the supervisor activities, then can be terminated", async () => { + const seen: string[] = []; + const worker = await Worker.create({ + connection: env.nativeConnection, + taskQueue: QUEUE, + workflowsPath: MAIN, + activities: stubs(seen) as never, + }); + await worker.runUntil(async () => { + const h = await env.client.workflow.start("roomAgentSupervisorWorkflow", { + taskQueue: QUEUE, + workflowId: "supervisor-1", + args: [{ + roomId: "cmroom0000000000000000001", + pending: [{ missionId: "cmmission000000000000001", turn: { roomId: "cmroom0000000000000000001", paneId: "cmpane0000000000000000001", prompt: "do the thing", traceId: "trace-0000000001" } }], + processedCount: 0, + }], + }); + await env.sleep(4_000); + await h.terminate(); + }); + // The pending mission must reach the turn and be reported as completed, not just + // pass through the admission check. + expect(seen).toEqual(["check", "turn", "finish:COMPLETED"]); + }, 120_000); +}); diff --git a/apps/worker/tests/workflow-sandbox-7.test.ts b/apps/worker/tests/workflow-sandbox-7.test.ts new file mode 100644 index 000000000..87622635d --- /dev/null +++ b/apps/worker/tests/workflow-sandbox-7.test.ts @@ -0,0 +1,140 @@ +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { Worker } from "@temporalio/worker"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import path from "node:path"; +import { roomAgentSupervisorQueueItemSchema } from "@space/contracts"; + +const MAIN = path.join(__dirname, "..", "dist", "workflows.js"); +const QUEUE = "room-supervisor-signals"; +const ENQUEUE = "enqueueRoomAgentMission"; + +let env: TestWorkflowEnvironment; +beforeAll(async () => { env = await TestWorkflowEnvironment.createTimeSkipping(); }, 240_000); +afterAll(async () => { await env?.teardown(); }); + +// continuingRoomAgentSupervisorWorkflow registers its own handler, and it drops an item +// whose turn.agentRunId matches the active mission or anything already pending. Two +// payloads without agentRunId both read as `undefined`, so the guard swallows the signal. +function stubs(seen: string[], gate?: Promise) { + const rec = (name: string, value: unknown = null) => async () => { seen.push(name); return value; }; + return { + markRoomAgentMissionStarted: rec("started"), + markRoomAgentMissionContinued: rec("continued"), + markRoomAgentMissionFinished: rec("finished"), + checkRoomAgentSupervisorTurn: rec("check", { action: "RUN", reason: null }), + finishRoomAgentSupervisorTurn: async (input: { status?: string } | undefined) => { seen.push("finish:" + String(input?.status)); return null; }, + discardRoomAgentSupervisorTurn: rec("discard"), + prepareRoomAgentBackgroundContinuation: rec("prepareContinuation", { pending: [], monitoring: [] }), + failRoomAgentBackgroundAction: rec("failAction"), + runCodexAppServerTurn: async () => { + seen.push("turn"); + // Hold the turn open when a gate is supplied: a duplicate only counts as arriving + // while the first mission is in flight if the first one is still running. + if (gate) await gate; + return { + status: "COMPLETED", + turnId: "turn-1", + roomAgentOutcome: { status: "VERIFIED", statusReason: "verified" }, + }; + }, + runRoomAgentBackgroundAction: rec("backgroundAction"), + settleRoomAgentBackgroundMission: rec("settle", true), + }; +} + +const ROOM = "cmroom0000000000000000001"; + +const mission = (args: { missionId: string; runId: string; roomId?: string }) => + roomAgentSupervisorQueueItemSchema.parse({ + missionId: args.missionId, + turn: { + roomId: args.roomId ?? ROOM, + paneId: "cmpane0000000000000000001", + prompt: "do the thing", + traceId: "trace-0000000001", + agentRunId: args.runId, + }, + }); + +describe("roomAgentSupervisorWorkflow signals", () => { + test("an enqueued mission runs check, turn and finish in that order", async () => { + const seen: string[] = []; + const worker = await Worker.create({ + connection: env.nativeConnection, + taskQueue: QUEUE, + workflowsPath: MAIN, + activities: stubs(seen) as never, + }); + await worker.runUntil(async () => { + const h = await env.client.workflow.start("roomAgentSupervisorWorkflow", { + taskQueue: QUEUE, + workflowId: "supervisor-signal-1", + args: [{ roomId: ROOM, pending: [], processedCount: 0 }], + }); + await env.sleep(1_000); + expect(seen).toHaveLength(0); + await h.signal(ENQUEUE, mission({ missionId: "cmmision0000000000000001", runId: "cmrun0000000000000000001" })); + await env.sleep(8_000); + await h.terminate(); + }); + expect(seen).toEqual(["check", "turn", "finish:COMPLETED"]); + }, 120_000); + + test("a duplicate agentRunId arriving while the first mission is in flight is dropped", async () => { + const seen: string[] = []; + let releaseTurn: () => void = () => {}; + const turnGate = new Promise((resolve) => { + releaseTurn = resolve; + }); + const worker = await Worker.create({ + connection: env.nativeConnection, + taskQueue: QUEUE + "-dup", + workflowsPath: MAIN, + activities: stubs(seen, turnGate) as never, + }); + await worker.runUntil(async () => { + const h = await env.client.workflow.start("roomAgentSupervisorWorkflow", { + taskQueue: QUEUE + "-dup", + workflowId: "supervisor-signal-dup", + args: [{ roomId: ROOM, pending: [], processedCount: 0 }], + }); + await env.sleep(1_000); + // Both arrive while the first mission is still in flight: the dedupe guard compares + // against active and pending only, so a duplicate sent after it finishes is accepted. + await h.signal(ENQUEUE, mission({ missionId: "cmmision0000000000000003", runId: "cmrun0000000000000000003" })); + // Wait until the first mission actually owns the turn before sending the duplicate. + // Poll the virtual clock instead of one fixed sleep: the child turn workflow + // needs several task cycles before the activity is dispatched. + for (let i = 0; i < 40 && !seen.includes("turn"); i += 1) await env.sleep(500); + expect(seen).toContain("turn"); + await h.signal(ENQUEUE, mission({ missionId: "cmmision0000000000000004", runId: "cmrun0000000000000000003" })); + await env.sleep(2_000); + releaseTurn(); + await env.sleep(2_000); + await h.terminate(); + }); + expect(seen.filter((s) => s === "check")).toHaveLength(1); + }, 120_000); + + test("a mission enqueued for another room is ignored", async () => { + const seen: string[] = []; + const worker = await Worker.create({ + connection: env.nativeConnection, + taskQueue: QUEUE + "-x", + workflowsPath: MAIN, + activities: stubs(seen) as never, + }); + await worker.runUntil(async () => { + const h = await env.client.workflow.start("roomAgentSupervisorWorkflow", { + taskQueue: QUEUE + "-x", + workflowId: "supervisor-signal-2", + args: [{ roomId: ROOM, pending: [], processedCount: 0 }], + }); + await env.sleep(1_000); + await h.signal(ENQUEUE, mission({ missionId: "cmmision0000000000000002", runId: "cmrun0000000000000000002", roomId: "cmroom0000000000000000002" })); + await env.sleep(5_000); + await h.terminate(); + }); + expect(seen).toHaveLength(0); + }, 120_000); +}); diff --git a/apps/worker/tests/workflow-sandbox.test.ts b/apps/worker/tests/workflow-sandbox.test.ts new file mode 100644 index 000000000..3843949c6 --- /dev/null +++ b/apps/worker/tests/workflow-sandbox.test.ts @@ -0,0 +1,87 @@ +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { Worker } from "@temporalio/worker"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import path from "node:path"; + +const WORKFLOWS = path.join(__dirname, "..", "dist", "workflows.js"); +const DOT_GOAL_SIGNAL = "dot-goal-wake"; + +let env: TestWorkflowEnvironment; + +beforeAll(async () => { + env = await TestWorkflowEnvironment.createTimeSkipping(); +}, 240_000); + +afterAll(async () => { + await env?.teardown(); +}); + +type StepResult = { state: string; runId: string | null; goalStatus: string }; + +function makeActivities(script: StepResult[], seen: { stepMs: number[] } = { stepMs: [] }) { + let i = 0; + const activities = { + async dotGoalStep() { + const next = script[Math.min(i, script.length - 1)]; + i += 1; + return next; + }, + async dotGoalObserve() { + return { run: "SUCCEEDED", goalStatus: "COMPLETED" }; + }, + }; + return { activities, calls: () => i }; +} + +describe("dotGoalWorkflow in the real workflow sandbox", () => { + test("waits on the 30 second timer for a WAITING step, then finishes on DONE", async () => { + const { activities, calls } = makeActivities([ + { state: "WAITING", runId: null, goalStatus: "RUNNING" }, + { state: "DONE", runId: null, goalStatus: "COMPLETED" }, + ]); + const worker = await Worker.create({ + connection: env.nativeConnection, + taskQueue: "dot-goal-sandbox", + workflowsPath: WORKFLOWS, + activities, + }); + const startedAt = await env.currentTimeMs(); + const result = await worker.runUntil(async () => { + const handle = await env.client.workflow.start("dotGoalWorkflow", { + taskQueue: "dot-goal-sandbox", + workflowId: "dot-goal-timeout", + args: [{ goalId: "goal-1" }], + }); + return handle.result(); + }); + expect(result).toEqual({ goalStatus: "COMPLETED" }); + // The WAITING step must actually burn the 30 second timer. + expect((await env.currentTimeMs()) - startedAt).toBeGreaterThanOrEqual(30_000); + expect(calls()).toBe(2); + }, 240_000); + + test("a wake signal ends the wait early instead of sleeping the full timeout", async () => { + const { activities } = makeActivities([ + { state: "WAITING", runId: null, goalStatus: "RUNNING" }, + { state: "WAITING", runId: null, goalStatus: "RUNNING" }, + { state: "DONE", runId: null, goalStatus: "COMPLETED" }, + ]); + const worker = await Worker.create({ + connection: env.nativeConnection, + taskQueue: "dot-goal-sandbox-wake", + workflowsPath: WORKFLOWS, + activities, + }); + const result = await worker.runUntil(async () => { + const handle = await env.client.workflow.start("dotGoalWorkflow", { + taskQueue: "dot-goal-sandbox-wake", + workflowId: "dot-goal-wake", + args: [{ goalId: "goal-2" }], + }); + await env.sleep(1000); + await handle.signal(DOT_GOAL_SIGNAL); + return handle.result(); + }); + expect(result).toEqual({ goalStatus: "COMPLETED" }); + }, 240_000); +}); diff --git a/package-lock.json b/package-lock.json index 04bf09e25..7c94206f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -192,6 +192,7 @@ "undici": "8.12.3" }, "devDependencies": { + "@temporalio/testing": "1.24.0", "typescript": "^5.9.3", "vitest": "^4.1.12" } @@ -3674,6 +3675,25 @@ "node": ">=12.0.0" } }, + "node_modules/@temporalio/testing": { + "version": "1.24.0", + "resolved": "http://mirrors.tencentyun.com/npm/@temporalio/testing/-/testing-1.24.0.tgz", + "integrity": "sha512-XKCuvgb7smKR3oXy5nDgwYSFX/hmg9zTJ2rZQQsVwJzFkga1TbnWDiuT5YMU0INKjoLhnM6c5YA/uLKlp8sdfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@temporalio/activity": "1.24.0", + "@temporalio/client": "1.24.0", + "@temporalio/common": "1.24.0", + "@temporalio/core-bridge": "1.24.0", + "@temporalio/proto": "1.24.0", + "@temporalio/worker": "1.24.0", + "@temporalio/workflow": "1.24.0" + }, + "engines": { + "node": ">= 20.3.0" + } + }, "node_modules/@temporalio/worker": { "version": "1.24.0", "resolved": "https://registry.npmjs.org/@temporalio/worker/-/worker-1.24.0.tgz", @@ -11641,12 +11661,12 @@ } }, "packages/run-spaceapp": { - "version": "1.0.35", + "version": "1.1.8", "cpu": [ "x64", "arm64" ], - "license": "Apache-2.0", + "license": "MIT", "os": [ "darwin", "linux",