test(worker): sandbox tests for every Temporal workflow

CONTRIBUTING.md

@@ -106,6 +106,26 @@ Keep changes reviewable and include the exact commands and limitations of your
 verification. Never include credentials, private transcripts or local runtime
 state in a contribution.
 
+## Workflow history fixtures
+
+The replay determinism gate in `apps/worker/tests/replay-gate.test.ts` replays
+recorded Temporal histories from
+`apps/worker/tests/fixtures/workflow-histories/`. The test only records when a
+fixture is missing (`existsSync` guard), so re-recording means deleting the file
+first:
+
+```sh
+rm apps/worker/tests/fixtures/workflow-histories/dot-goal.json
+npm test -w @spaceapp/worker
+```
+
+The suite records the missing history and then replays it; a green run proves
+the fresh history is replay-safe. Inspect the generated JSON, commit it together
+with the workflow change that required it, and never commit a history recorded
+with mocked activities. See
+`apps/worker/tests/fixtures/workflow-histories/README.md` for the full
+procedure, including adding a fixture for a new workflow.
+
 ## License of contributions
 
 SpaceApp is licensed under the [MIT License](LICENSE). Contributions are

apps/worker/package.json

@@ -10,7 +10,8 @@
     "start": "node dist/worker.js",
     "build": "tsc -p tsconfig.json",
     "check": "tsc -p tsconfig.json --noEmit",
-    "test": "vitest run"
+    "test": "vitest run",
+    "pretest": "npm run build"
   },
   "dependencies": {
     "@space/codex-app-server": "0.2.5",
@@ -26,6 +27,7 @@
     "undici": "8.12.3"
   },
   "devDependencies": {
+    "@temporalio/testing": "1.24.0",
     "typescript": "^5.9.3",
     "vitest": "^4.1.12"
   },

apps/worker/tests/fixtures/replay-negative-workflows.js

@@ -0,0 +1,16 @@
+// Test-only workflow bundle for the replay gate's negative control.
+//
+// It exports the same workflow type as the recorded fixture (`dotGoalWorkflow`) but emits a
+// different first command: a timer instead of an activity. Replaying a history recorded by the
+// real bundle against this one must be rejected as nondeterministic - that is the property the
+// gate is supposed to guarantee (issue #60).
+import { proxyActivities, sleep } from "@temporalio/workflow";
+
+const { dotGoalStep } = proxyActivities({ startToCloseTimeout: "1 minute" });
+
+export async function dotGoalWorkflow(input) {
+  // Divergence: the real workflow schedules an activity first; this variant starts a timer.
+  await sleep("1 second");
+  await dotGoalStep(input.goalId);
+  return { goalStatus: "COMPLETED" };
+}

apps/worker/tests/fixtures/workflow-histories/README.md

@@ -0,0 +1,51 @@
+# Workflow history fixtures
+
+This directory holds recorded Temporal event histories used by the replay
+determinism gate (`apps/worker/tests/replay-gate.test.ts`).
+
+- `dot-goal.json` — event history for `dotGoalWorkflow`, recorded once against
+  the time-skipping test server and replayed in CI. If a workflow change makes
+  the recorded history replay to a different command sequence, the gate fails
+  and the change cannot land silently.
+
+## How to re-record a fixture
+
+The replay test only records when the fixture is missing: it checks the file
+with `existsSync` and skips the record step when the file already exists, so
+you have to delete the fixture to regenerate it.
+
+```sh
+# 1. Remove the fixture you want to re-record
+rm apps/worker/tests/fixtures/workflow-histories/dot-goal.json
+
+# 2. Run the worker suite; the gate records the missing fixture first and then
+#    replays it, so a green run means the fresh history is replay-safe
+npm test -w @spaceapp/worker
+```
+
+The test writes `{ "workflowId": ..., "historyBase64": ... }` with two-space
+JSON indentation and a trailing newline. Keep that shape: the file is read back
+verbatim, and a hand-edited or re-serialised history no longer proves the same
+execution.
+
+## What to commit
+
+1. Inspect `git diff --stat apps/worker/tests/fixtures/workflow-histories/`.
+   A changed history means real workflow behaviour changed — read the diff, do
+   not commit it blindly.
+2. Commit the regenerated fixture **together with** the workflow change that
+   required it, in the same pull request, so reviewers can see the behaviour
+   change next to the new history.
+3. Never commit a fixture recorded with mocked activities or a shortened
+   timeout: the gate must replay the same command sequence the production
+   worker would produce. If a change is intentional and cannot be replayed,
+   guard it with `patched()` and record a new fixture that covers the new
+   branch.
+
+## Adding a fixture for a new workflow
+
+Copy the pattern in `replay-gate.test.ts`: start a time-skipping
+`TestWorkflowEnvironment`, run the workflow against test activities, capture
+`env.getWorkflowHistory(...)` / `handle.fetchHistory()`, and store the base64
+history under this directory. Add the replay assertion for the new file to the
+same test so the gate covers it.

apps/worker/tests/fixtures/workflow-histories/dot-goal.json

@@ -0,0 +1,4 @@
+{
+  "workflowId": "recorded-dot-goal",
+  "historyBase64": "Cv4BCAESDAjzxfjWBhCA/+O9ARgBMukBChEKD2RvdEdvYWxXb3JrZmxvdyoPCgtyZXBsYXktZ2F0ZRABMkEKPwoWCghlbmNvZGluZxIKanNvbi9wbGFpbhIleyJnb2FsSWQiOiJjbW1nb2FsMDAwMDAwMDAwMDAwMDAwMDEifToGCICGsJYBQgYIgIawlgFKAggKciRlMzRlMzFlMC1mMGE5LTRlNmMtYTA3Ni04ZDRkNDg0YmUzM2N6FjIzMjY0NEBWTS00LTEyOS11YnVudHWCASRlMzRlMzFlMC1mMGE5LTRlNmMtYTA3Ni04ZDRkNDg0YmUzM2OQAQGqAQDKAQAKKwgCEgwI88X41gYQgP/jvQEYBVIXCg8KC3JlcGxheS1nYXRlEAESAggKGAEKMQgDEgwI88X41gYQgP/jvQEYBlodCAISFjIzMjY0NEBWTS00LTEyOS11YnVudHUoqQIKsQEIBBIMCPTF+NYGEICjzMEBGAdinAEIAhoWMjMyNjQ0QFZNLTQtMTI5LXVidW50dSJaQHRlbXBvcmFsaW8vd29ya2VyQDEuMjQuMCswNmM2M2U0Njk0NmY3ZDJhNDExMmQzM2RhOWU0MDM3NDk4Njk1YWZiZGQ3ZjU2NzUwNDQxYTUzM2Q1YjUzN2JhMiIKAwMBAhoTdGVtcG9yYWwtdHlwZXNjcmlwdCIGMS4yNC4wagAKnwEIBRIMCPTF+NYGEICjzMEBGAp6igEKATESDQoLZG90R29hbFN0ZXAiDwoLcmVwbGF5LWdhdGUQASoAMjYKNAoWCghlbmNvZGluZxIKanNvbi9wbGFpbhIaImNtbWdvYWwwMDAwMDAwMDAwMDAwMDAwMSI6BgiAhrCWAUIGCICGsJYBSgIIeFIAWANiEwoCCAURAAAAAAAAAEAaAgh4IBQKMQgGEgwI9MX41gYQgKPMwQEYC4IBHAgFEhYyMzI2NDRAVk0tNC0xMjktdWJ1bnR1IAEKhgEIBxIMCPTF+NYGEMDm38gBGAyKAXEKUwpRChYKCGVuY29kaW5nEgpqc29uL3BsYWluEjd7InN0YXRlIjoiV0FJVElORyIsInJ1bklkIjpudWxsLCJnb2FsU3RhdHVzIjoiUlVOTklORyJ9EAUYBiIWMjMyNjQ0QFZNLTQtMTI5LXVidW50dQorCAgSDAj0xfjWBhDA5t/IARgFUhcKDwoLcmVwbGF5LWdhdGUQARICCAoYAQoxCAkSDAj0xfjWBhDA5t/IARgGWh0ICBIWMjMyNjQ0QFZNLTQtMTI5LXVidW50dSiMBwqOAQgKEgwI9MX41gYQwMme0wEYB2J6CAgaFjIzMjY0NEBWTS00LTEyOS11YnVudHUiWkB0ZW1wb3JhbGlvL3dvcmtlckAxLjI0LjArMDZjNjNlNDY5NDZmN2QyYTQxMTJkMzNkYTllNDAzNzQ5ODY5NWFmYmRkN2Y1Njc1MDQ0MWE1MzNkNWI1MzdiYTIAagAKHggLEgwI9MX41gYQwMme0wEYEaIBCQoBMRICCB4YCQoaCAwSDAiSxvjWBhDAyZ7TARgSqgEFCgExEAsKKwgNEgwIksb41gYQwMme0wEYBVIXCg8KC3JlcGxheS1nYXRlEAESAggKGAEKMQgOEgwIksb41gYQwMme0wEYBlodCA0SFjIzMjY0NEBWTS00LTEyOS11YnVudHUorgkKjgEIDxIMCJLG+NYGEID7vdgBGAdieggNGhYyMzI2NDRAVk0tNC0xMjktdWJ1bnR1IlpAdGVtcG9yYWxpby93b3JrZXJAMS4yNC4wKzA2YzYzZTQ2OTQ2ZjdkMmE0MTEyZDMzZGE5ZTQwMzc0OTg2OTVhZmJkZDdmNTY3NTA0NDFhNTMzZDViNTM3YmEyAGoACp8BCBASDAiSxvjWBhCA+73YARgKeooBCgEyEg0KC2RvdEdvYWxTdGVwIg8KC3JlcGxheS1nYXRlEAEqADI2CjQKFgoIZW5jb2RpbmcSCmpzb24vcGxhaW4SGiJjbW1nb2FsMDAwMDAwMDAwMDAwMDAwMDEiOgYIgIawlgFCBgiAhrCWAUoCCHhSAFgOYhMKAggFEQAAAAAAAABAGgIIeCAUCjEIERIMCJLG+NYGEID7vdgBGAuCARwIEBIWMjMyNjQ0QFZNLTQtMTI5LXVidW50dSABCoUBCBISDAiSxvjWBhDAtdfeARgMigFwClIKUAoWCghlbmNvZGluZxIKanNvbi9wbGFpbhI2eyJzdGF0ZSI6IkRPTkUiLCJydW5JZCI6bnVsbCwiZ29hbFN0YXR1cyI6IkNPTVBMRVRFRCJ9EBAYESIWMjMyNjQ0QFZNLTQtMTI5LXVidW50dQorCBMSDAiSxvjWBhDAtdfeARgFUhcKDwoLcmVwbGF5LWdhdGUQARICCAoYAQoxCBQSDAiSxvjWBhDAtdfeARgGWh0IExIWMjMyNjQ0QFZNLTQtMTI5LXVidW50dSjtDQqOAQgVEgwIksb41gYQwNm/4gEYB2J6CBMaFjIzMjY0NEBWTS00LTEyOS11YnVudHUiWkB0ZW1wb3JhbGlvL3dvcmtlckAxLjI0LjArMDZjNjNlNDY5NDZmN2QyYTQxMTJkMzNkYTllNDAzNzQ5ODY5NWFmYmRkN2Y1Njc1MDQ0MWE1MzNkNWI1MzdiYTIAagAKTggWEgwIksb41gYQwNm/4gEYAjo6CjYKNAoWCghlbmNvZGluZxIKanNvbi9wbGFpbhIaeyJnb2FsU3RhdHVzIjoiQ09NUExFVEVEIn0QFA=="
+}

apps/worker/tests/replay-gate.test.ts

@@ -0,0 +1,123 @@
+import { TestWorkflowEnvironment } from "@temporalio/testing";
+import { Worker } from "@temporalio/worker";
+import protoPkg from "@temporalio/proto";
+import fs from "node:fs";
+import path from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { DeterminismViolationError } from "@temporalio/workflow";
+
+const temporal = (protoPkg as unknown as { temporal?: typeof protoPkg }).temporal ?? protoPkg;
+
+const MAIN = path.join(__dirname, "..", "dist", "workflows.js");
+const MUTATION_BUNDLE = path.join(__dirname, "..", "dist", "memory-mutation-workflows.js");
+const FIXTURE = path.join(__dirname, "fixtures", "workflow-histories", "dot-goal.json");
+// Test-only bundle that exports dotGoalWorkflow with a divergent command sequence. Used as the
+// negative control: the gate must reject it, otherwise a determinism break would pass silently.
+const NEGATIVE_BUNDLE = path.join(__dirname, "fixtures", "replay-negative-workflows.js");
+const QUEUE = "replay-gate";
+
+let env: TestWorkflowEnvironment;
+beforeAll(async () => { env = await TestWorkflowEnvironment.createTimeSkipping(); }, 240_000);
+afterAll(async () => { await env?.teardown(); });
+
+type Recorded = { workflowId: string; historyBase64: string };
+
+let recordedStepCalls = 0;
+const dotGoalActivities = {
+  async dotGoalStep() {
+    recordedStepCalls += 1;
+    return recordedStepCalls === 1
+      ? { state: "WAITING", runId: null, goalStatus: "RUNNING" }
+      : { state: "DONE", runId: null, goalStatus: "COMPLETED" };
+  },
+  async dotGoalObserve() { return { run: "SUCCEEDED", goalStatus: "COMPLETED" }; },
+};
+
+async function replayDotGoal(recorded: Recorded) {
+  const history = temporal.api.history.v1.History.decode(Buffer.from(recorded.historyBase64, "base64"));
+  const results: Array<{ error?: unknown }> = [];
+  for await (const r of Worker.runReplayHistories(
+    { connection: env.nativeConnection, taskQueue: QUEUE, workflowsPath: MAIN },
+    [{ workflowId: recorded.workflowId, history }],
+  )) {
+    results.push({ error: r.error });
+  }
+  return results;
+}
+
+// The replay gate replays a committed history, not one recorded in the same run: replaying
+// a freshly recorded history would use the changed code for both sides and pass even when
+// an existing execution can no longer replay.
+describe("replay gate", () => {
+  test("replays the committed dotGoalWorkflow history against the current bundle", async () => {
+    if (!fs.existsSync(FIXTURE)) {
+      // One-off recording; the file is committed and replayed from then on.
+      const worker = await Worker.create({
+        connection: env.nativeConnection, taskQueue: QUEUE, workflowsPath: MAIN, activities: dotGoalActivities,
+      });
+      await worker.runUntil(async () => {
+        const h = await env.client.workflow.start("dotGoalWorkflow", {
+          taskQueue: QUEUE, workflowId: "recorded-dot-goal", args: [{ goalId: "cmmgoal00000000000000001" }],
+        });
+        await h.result();
+      });
+      const history = await env.client.workflow.getHandle("recorded-dot-goal").fetchHistory();
+      const historyBase64 = Buffer.from(
+        temporal.api.history.v1.History.encode(history).finish(),
+      ).toString("base64");
+      fs.mkdirSync(path.dirname(FIXTURE), { recursive: true });
+      fs.writeFileSync(FIXTURE, JSON.stringify({ workflowId: "recorded-dot-goal", historyBase64 }, null, 2) + "\n");
+    }
+
+    const recorded = JSON.parse(fs.readFileSync(FIXTURE, "utf8")) as Recorded;
+    expect(recorded.historyBase64.length).toBeGreaterThan(0);
+    const results = await replayDotGoal(recorded);
+    expect(results).toHaveLength(1);
+    expect(results[0].error).toBeUndefined();
+  }, 240_000);
+
+  // Negative control for issue #60: the gate has to catch a determinism change, not just pass.
+  test("rejects the committed dotGoalWorkflow history when the bundle diverges", async () => {
+    const recorded = JSON.parse(fs.readFileSync(FIXTURE, "utf8")) as Recorded;
+    const results = await replayDotGoalForBundle(NEGATIVE_BUNDLE, recorded, QUEUE);
+    expect(results).toHaveLength(1);
+    const error = results[0].error;
+    expect(error).toBeDefined();
+    expect(error instanceof DeterminismViolationError || (error as Error)?.name?.includes("Determinism")).toBe(true);
+  }, 240_000);
+
+  test("memoryMutationWorkflow replays its own recorded history", async () => {
+    const activities = {
+      async executePersistedMemoryChangeSet(input: { changeSetId: string }) {
+        return { changeSetId: input.changeSetId, applied: 1 };
+      },
+    };
+    const worker = await Worker.create({
+      connection: env.nativeConnection, taskQueue: QUEUE + "-mut", workflowsPath: MUTATION_BUNDLE, activities,
+    });
+    await worker.runUntil(async () => {
+      const h = await env.client.workflow.start("memoryMutationWorkflow", {
+        taskQueue: QUEUE + "-mut", workflowId: "replay-mutation", args: [{ changeSetId: "cmmcs000000000000000001" }],
+      });
+      await h.result();
+    });
+    const history = await env.client.workflow.getHandle("replay-mutation").fetchHistory();
+    const historyBase64 = Buffer.from(
+      temporal.api.history.v1.History.encode(history).finish(),
+    ).toString("base64");
+    const results = await replayDotGoalForBundle(MUTATION_BUNDLE, { workflowId: "replay-mutation", historyBase64 });
+    expect(results[0].error).toBeUndefined();
+  }, 240_000);
+});
+
+async function replayDotGoalForBundle(bundle: string, recorded: Recorded, taskQueue = QUEUE + "-mut") {
+  const history = temporal.api.history.v1.History.decode(Buffer.from(recorded.historyBase64, "base64"));
+  const results: Array<{ error?: unknown }> = [];
+  for await (const r of Worker.runReplayHistories(
+    { connection: env.nativeConnection, taskQueue, workflowsPath: bundle },
+    [{ workflowId: recorded.workflowId, history }],
+  )) {
+    results.push({ error: r.error });
+  }
+  return results;
+}

apps/worker/tests/workflow-sandbox-2.test.ts

@@ -0,0 +1,73 @@
+import { TestWorkflowEnvironment } from "@temporalio/testing";
+import { Worker } from "@temporalio/worker";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import path from "node:path";
+
+const dist = (f: string) => path.join(__dirname, "..", "dist", f);
+const MAIN = dist("workflows.js");
+const MUTATION = dist("memory-mutation-workflows.js");
+
+let env: TestWorkflowEnvironment;
+beforeAll(async () => { env = await TestWorkflowEnvironment.createTimeSkipping(); }, 240_000);
+afterAll(async () => { await env?.teardown(); });
+
+describe("memoryMutationWorkflow", () => {
+  test("proxies the change-set executor and returns its result", async () => {
+    const seen: unknown[] = [];
+    const activities = {
+      async executePersistedMemoryChangeSet(input: unknown) {
+        seen.push(input);
+        return { applied: 2, failed: 0 };
+      },
+    };
+    const worker = await Worker.create({
+      connection: env.nativeConnection,
+      taskQueue: "mem-mutation",
+      workflowsPath: MUTATION,
+      activities,
+    });
+    const result = await worker.runUntil(async () => {
+      const h = await env.client.workflow.start("memoryMutationWorkflow", {
+        taskQueue: "mem-mutation",
+        workflowId: "mem-mutation-1",
+        args: [{ changeSetId: "cs-1", traceId: "t-1" }],
+      });
+      return h.result();
+    });
+    expect(result).toEqual({ applied: 2, failed: 0 });
+    expect(seen).toHaveLength(1);
+  }, 240_000);
+});
+
+describe("roomAgentActionWorkflow failure path", () => {
+  test("reports the failure through failRoomAgentBackgroundAction and rethrows", async () => {
+    const reported: Record<string, unknown>[] = [];
+    const activities = {
+      async runRoomAgentBackgroundAction() { throw new Error("bridge exploded"); },
+      async failRoomAgentBackgroundAction(input: Record<string, unknown>) {
+        reported.push(input);
+        return null;
+      },
+    };
+    const worker = await Worker.create({
+      connection: env.nativeConnection,
+      taskQueue: "room-action",
+      workflowsPath: MAIN,
+      activities,
+    });
+    const error = await worker.runUntil(async () => {
+      const h = await env.client.workflow.start("roomAgentActionWorkflow", {
+        taskQueue: "room-action",
+        workflowId: "space-room-action:act-9",
+        args: [{ bridge: { roomId: "room-1", missionId: "mis-1" }, action: {} }],
+      });
+      try { await h.result(); return null; } catch (e) { return e as Error; }
+    });
+    expect(error).toBeInstanceOf(Error);
+    expect(reported).toHaveLength(1);
+    expect(reported[0].roomId).toBe("room-1");
+    expect(reported[0].missionId).toBe("mis-1");
+    expect(reported[0].actionId).toBe("act-9");
+    expect(reported[0].cancelled).toBe(false);
+  }, 240_000);
+});

apps/worker/tests/workflow-sandbox-3.test.ts

@@ -0,0 +1,55 @@
+import { TestWorkflowEnvironment } from "@temporalio/testing";
+import { Worker } from "@temporalio/worker";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import path from "node:path";
+
+const dist = (f: string) => path.join(__dirname, "..", "dist", f);
+const MAIN = dist("workflows.js");
+const MAINT = dist("memory-maintenance-workflows.js");
+
+let env: TestWorkflowEnvironment;
+beforeAll(async () => { env = await TestWorkflowEnvironment.createTimeSkipping(); }, 240_000);
+afterAll(async () => { await env?.teardown(); });
+
+describe("dummyTurnWorkflow", () => {
+  test("records start then completion through its two activities", async () => {
+    const order: string[] = [];
+    const activities = {
+      async recordDummyTurnStarted(i: unknown) { order.push("started:" + JSON.stringify(i)); return null; },
+      async recordDummyTurnCompleted(i: unknown) { order.push("completed"); return { status: "COMPLETED", turnId: "t-1" }; },
+    };
+    const worker = await Worker.create({ connection: env.nativeConnection, taskQueue: "dummy-turn", workflowsPath: MAIN, activities });
+    const result = await worker.runUntil(async () => {
+      const h = await env.client.workflow.start("dummyTurnWorkflow", {
+        taskQueue: "dummy-turn", workflowId: "dummy-1",
+        args: [{ roomId: "room-1", turnId: "t-1", providerId: "opencode" }],
+      });
+      return h.result();
+    });
+    expect(result).toEqual({ status: "COMPLETED", turnId: "t-1" });
+    expect(order[0]?.startsWith("started:")).toBe(true);
+    expect(order[1]).toBe("completed");
+  }, 240_000);
+});
+
+describe("codexAppServerTurnWorkflow routing", () => {
+  test("providerId opencode goes to the opencode activity only", async () => {
+    const calls: string[] = [];
+    const activities = {
+      async runOpenCodeAgentTurn() { calls.push("opencode"); return { status: "COMPLETED", turnId: "t-2" }; },
+      async runCliAgentTurn() { calls.push("cli"); return { status: "COMPLETED", turnId: "t-2" }; },
+      async runCodexAppServerTurn() { calls.push("codex"); return { status: "COMPLETED", turnId: "t-2" }; },
+    };
+    const worker = await Worker.create({ connection: env.nativeConnection, taskQueue: "turn-route", workflowsPath: MAIN, activities });
+    const result = await worker.runUntil(async () => {
+      const h = await env.client.workflow.start("codexAppServerTurnWorkflow", {
+        taskQueue: "turn-route", workflowId: "turn-route-1",
+        args: [{ roomId: "room-1", turnId: "t-2", providerId: "opencode" }],
+      });
+      return h.result();
+    });
+    expect(result).toEqual({ status: "COMPLETED", turnId: "t-2" });
+    expect(calls).toEqual(["opencode"]);
+  }, 240_000);
+});
+