test(worker): add Temporal sandbox and recorded-history replay coverage

apps/worker/package.json

@@ -10,7 +10,8 @@
     "start": "node dist/worker.js",
     "build": "tsc -p tsconfig.json",
     "check": "tsc -p tsconfig.json --noEmit",
-    "test": "vitest run"
+    "test": "vitest run",
+    "temporal:record-histories": "node scripts/record-workflow-histories.mjs"
   },
   "dependencies": {
     "@space/codex-app-server": "0.2.5",
@@ -26,6 +27,9 @@
     "undici": "8.10.2"
   },
   "devDependencies": {
+    "@temporalio/common": "1.24.0",
+    "@temporalio/proto": "1.24.0",
+    "@temporalio/testing": "1.24.0",
     "typescript": "^5.9.3",
     "vitest": "^4.1.12"
   },

apps/worker/scripts/record-workflow-histories.mjs

@@ -0,0 +1,63 @@
+// Usage: npm run temporal:record-histories -w @space/worker [-- --patched-change]
+import { mkdir, mkdtemp, open, readFile, readdir, rename, rm } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+const replace = process.argv.slice(2).includes("--patched-change");
+if (process.argv.slice(2).some((arg) => arg !== "--patched-change")) throw new Error("Unknown argument");
+const directory = new URL("../tests/fixtures/workflow-histories/", import.meta.url);
+const files = await readdir(directory).catch((error) => { if (error.code === "ENOENT") return []; throw error; });
+if (files.some((file) => file.endsWith(".json")) && !replace) {
+  throw new Error("Existing histories are compatibility contracts. Use --patched-change only with a reviewed replay-safe patched() change; never to silence a replay failure.");
+}
+const require = createRequire(import.meta.url);
+const cli = join(dirname(require.resolve("vitest/package.json")), "vitest.mjs");
+const target = fileURLToPath(directory);
+await mkdir(target, { recursive: true });
+const staging = await mkdtemp(join(target, ".record-"));
+if (dirname(staging) !== target.replace(/[\\/]$/, "")) throw new Error("Unexpected staging directory");
+let finished = false;
+try {
+  const result = spawnSync(process.execPath, [cli, "run", "tests/temporal-turn-goal.test.ts", "tests/temporal-memory.test.ts", "tests/temporal-supervisor.test.ts"], {
+    cwd: fileURLToPath(new URL("..", import.meta.url)),
+    env: { ...process.env, SPACE_RECORD_WORKFLOW_HISTORIES: "1", SPACE_WORKFLOW_HISTORY_OUTPUT: staging, SPACE_REPLACE_PATCHED_HISTORIES: "0" },
+    stdio: "inherit"
+  });
+  if (result.error) throw result.error;
+  process.exitCode = result.status ?? 1;
+  if (process.exitCode === 0) {
+    const expected = ["dotGoalWorkflow", "dummyTurnWorkflow", "codexAppServerTurnWorkflow", "roomAgentSupervisorWorkflow", "roomAgentSupervisorWorkflow-legacy", "roomAgentActionWorkflow", "memoryMaintenanceWorkflow", "memoryConsolidationWorkflow", "memoryMutationWorkflow"].map((name) => `${name}.json`);
+    const recorded = await readdir(staging);
+    for (const file of expected) if (!recorded.includes(file)) throw new Error(`Recorder did not produce ${file}`);
+    for (const file of expected) {
+      let archive;
+      let archiveHandle;
+      try {
+        if (files.includes(file)) {
+          const bytes = await readFile(join(target, file));
+          const candidate = join(target, `${file.slice(0, -5)}-pre-${randomUUID()}.json`);
+          archiveHandle = await open(candidate, "wx");
+          // Only a successful exclusive open gives us an archive to clean up.
+          archive = candidate;
+          await archiveHandle.writeFile(bytes);
+          await archiveHandle.close();
+          archiveHandle = undefined;
+        }
+        await rename(join(staging, file), join(target, file));
+      } catch (error) {
+        if (archiveHandle) await archiveHandle.close().catch(() => undefined);
+        if (archive) await rm(archive, { force: true }).catch(() => undefined);
+        throw error;
+      }
+    }
+  }
+  finished = true;
+} finally {
+  const cleanup = rm(staging, { recursive: true, force: true });
+  // Keep the original recording or publication error if cleanup also fails.
+  if (finished) await cleanup;
+  else await cleanup.catch(() => undefined);
+}

apps/worker/tests/fixtures/workflow-histories/README.md

@@ -0,0 +1,53 @@
+# Temporal replay histories
+
+These JSON fixtures are captured from synthetic executions on Temporal's real
+time-skipping test server. They contain no installation data, provider calls,
+database records or credentials. The normal worker test command replays every
+history against the current production workflows and also checks that adding an
+extra timer to `dotGoalWorkflow` produces `DeterminismViolationError`.
+
+From a clean checkout, install dependencies and build the internal packages:
+
+```sh
+ELECTRON_SKIP_BINARY_DOWNLOAD=1 npm ci
+npm run check
+npm test -w @space/worker
+```
+
+The recorder runs the same real-sandbox scenarios as the default test suite and
+serializes the server's history using the Temporal SDK. Each fixture retains its
+original synthetic `workflowId` alongside the SDK `history`, so code that derives
+activity arguments from a workflow ID sees the same ID during replay. Host identity
+strings and the checkout prefix in failure stack traces are normalized; recorded
+events, commands, workflow IDs and payloads remain intact. Scenario failures discard
+their staged histories and leave existing fixtures alone before publication.
+The legacy supervisor fixture comes from executing its pre-patch branch with a test-only bundle; it is
+then replayed against the unmodified current production workflow.
+
+Existing histories must remain useful as compatibility contracts. Re-recording
+is allowed **only together with an intentional, reviewed `patched()` change**,
+never to make a failing replay pass. Inspect the workflow change and history diff
+before using the explicit `-- --patched-change` argument. Preserve coverage of
+old executions when adding new patch histories. Ordinary tests never write or
+replace these fixtures.
+
+Before publishing a replacement, `--patched-change` copies the existing fixture
+unchanged to `<workflow>-pre-<unique-id>.json` in this directory. Every archived
+history remains in the default replay discovery, with its original workflow ID.
+Archives are never overwritten, including on repeated recordings. Duplicate
+workflow IDs across current and archived fixtures still fail replay validation.
+
+For an approved patch change, run:
+
+```sh
+npm run temporal:record-histories -w @space/worker -- --patched-change
+npm test -w @space/worker
+```
+
+The command without `--patched-change` records an initial set only when no JSON
+histories exist; it refuses to overwrite the committed fixtures.
+
+On Windows, a deeply nested temporary directory can prevent the native test
+server from creating its local sockets. For such hosts, select an existing short
+temporary directory with `TEMP` and `TMP` for the test command. Linux is the
+reference CI environment; tests never start Docker or an installed application.

apps/worker/tests/fixtures/workflow-histories/codexAppServerTurnWorkflow.json

@@ -0,0 +1,258 @@
+{
+  "workflowId": "63203fc0-61ce-41ea-9dd4-5b231aebd9d2",
+  "history": {
+    "events": [
+      {
+        "eventId": "1",
+        "eventTime": {
+          "seconds": "2423480225",
+          "nanos": 120000000
+        },
+        "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_STARTED",
+        "workflowExecutionStartedEventAttributes": {
+          "workflowType": {
+            "name": "codexAppServerTurnWorkflow"
+          },
+          "taskQueue": {
+            "name": "turn-goal-3b69f379-f28f-4800-bef7-4b239d887ce7",
+            "kind": "TASK_QUEUE_KIND_NORMAL"
+          },
+          "input": {
+            "payloads": [
+              {
+                "metadata": {
+                  "encoding": "anNvbi9wbGFpbg=="
+                },
+                "data": "eyJyb29tSWQiOiJyb29tOnRlc3QtdHVybiIsInBhbmVJZCI6InBhbmU6dGVzdC10dXJuIiwidHJhY2VJZCI6InRyYWNlOnRlc3QtdHVybiIsInByb21wdCI6IkEgc3ludGhldGljIHR1cm47IG5vIHByb3ZpZGVyIGlzIGNhbGxlZCIsImFydGlmYWN0SWRzIjpbXSwicGVybWlzc2lvbk1vZGUiOm51bGwsImNvbGxhYm9yYXRpb25Nb2RlIjoiZGVmYXVsdCIsInJvb21BZ2VudE1pc3Npb25JZCI6Im1pc3Npb246dGVzdCJ9"
+              }
+            ]
+          },
+          "workflowExecutionTimeout": {
+            "seconds": "315360000"
+          },
+          "workflowRunTimeout": {
+            "seconds": "315360000"
+          },
+          "workflowTaskTimeout": {
+            "seconds": "10"
+          },
+          "originalExecutionRunId": "0136d8c8-e58f-4f41-858c-4c5285a8a9fb",
+          "identity": "synthetic-test-worker",
+          "firstExecutionRunId": "0136d8c8-e58f-4f41-858c-4c5285a8a9fb",
+          "attempt": 1,
+          "firstWorkflowTaskBackoff": {},
+          "header": {}
+        }
+      },
+      {
+        "eventId": "2",
+        "eventTime": {
+          "seconds": "2423480225",
+          "nanos": 120000000
+        },
+        "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED",
+        "workflowTaskScheduledEventAttributes": {
+          "taskQueue": {
+            "name": "turn-goal-3b69f379-f28f-4800-bef7-4b239d887ce7",
+            "kind": "TASK_QUEUE_KIND_NORMAL"
+          },
+          "startToCloseTimeout": {
+            "seconds": "10"
+          },
+          "attempt": 1
+        }
+      },
+      {
+        "eventId": "3",
+        "eventTime": {
+          "seconds": "2423480225",
+          "nanos": 120000000
+        },
+        "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED",
+        "workflowTaskStartedEventAttributes": {
+          "scheduledEventId": "2",
+          "identity": "synthetic-test-worker",
+          "historySizeBytes": "568"
+        }
+      },
+      {
+        "eventId": "4",
+        "eventTime": {
+          "seconds": "2423480226",
+          "nanos": 790000000
+        },
+        "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED",
+        "workflowTaskCompletedEventAttributes": {
+          "scheduledEventId": "2",
+          "identity": "synthetic-test-worker",
+          "binaryChecksum": "@temporalio/worker@1.24.0+450de7a749e60979087b19188b0216faa0a22315f7f8422a97516cee8d828146",
+          "sdkMetadata": {
+            "coreUsedFlags": [
+              2,
+              1,
+              3
+            ],
+            "sdkName": "temporal-typescript",
+            "sdkVersion": "1.24.0"
+          },
+          "meteringMetadata": {}
+        }
+      },
+      {
+        "eventId": "5",
+        "eventTime": {
+          "seconds": "2423480226",
+          "nanos": 790000000
+        },
+        "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED",
+        "activityTaskScheduledEventAttributes": {
+          "activityId": "1",
+          "activityType": {
+            "name": "runCodexAppServerTurn"
+          },
+          "taskQueue": {
+            "name": "turn-goal-3b69f379-f28f-4800-bef7-4b239d887ce7",
+            "kind": "TASK_QUEUE_KIND_NORMAL"
+          },
+          "header": {},
+          "input": {
+            "payloads": [
+              {
+                "metadata": {
+                  "encoding": "anNvbi9wbGFpbg=="
+                },
+                "data": "eyJyb29tSWQiOiJyb29tOnRlc3QtdHVybiIsInBhbmVJZCI6InBhbmU6dGVzdC10dXJuIiwidHJhY2VJZCI6InRyYWNlOnRlc3QtdHVybiIsInByb21wdCI6IkEgc3ludGhldGljIHR1cm47IG5vIHByb3ZpZGVyIGlzIGNhbGxlZCIsImFydGlmYWN0SWRzIjpbXSwicGVybWlzc2lvbk1vZGUiOm51bGwsImNvbGxhYm9yYXRpb25Nb2RlIjoiZGVmYXVsdCIsInJvb21BZ2VudE1pc3Npb25JZCI6Im1pc3Npb246dGVzdCJ9"
+              }
+            ]
+          },
+          "scheduleToCloseTimeout": {
+            "seconds": "315360000"
+          },
+          "scheduleToStartTimeout": {
+            "seconds": "315360000"
+          },
+          "startToCloseTimeout": {
+            "seconds": "86400"
+          },
+          "heartbeatTimeout": {
+            "seconds": "15"
+          },
+          "workflowTaskCompletedEventId": "3",
+          "retryPolicy": {
+            "initialInterval": {
+              "seconds": "1"
+            },
+            "backoffCoefficient": 2,
+            "maximumInterval": {
+              "seconds": "100"
+            },
+            "maximumAttempts": 5
+          }
+        }
+      },
+      {
+        "eventId": "6",
+        "eventTime": {
+          "seconds": "2423480226",
+          "nanos": 790000000
+        },
+        "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED",
+        "activityTaskStartedEventAttributes": {
+          "scheduledEventId": "5",
+          "identity": "synthetic-test-worker",
+          "attempt": 1
+        }
+      },
+      {
+        "eventId": "7",
+        "eventTime": {
+          "seconds": "2423480226",
+          "nanos": 795000000
+        },
+        "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED",
+        "activityTaskCompletedEventAttributes": {
+          "result": {
+            "payloads": [
+              {
+                "metadata": {
+                  "encoding": "anNvbi9wbGFpbg=="
+                },
+                "data": "eyJ3b3JrZmxvd0lkIjoid29ya2Zsb3c6dGVzdC10dXJuIiwicm9vbUlkIjoicm9vbTp0ZXN0LXR1cm4iLCJwYW5lSWQiOiJwYW5lOnRlc3QtdHVybiIsInRyYWNlSWQiOiJ0cmFjZTp0ZXN0LXR1cm4iLCJzdGF0dXMiOiJDT01QTEVURUQiLCJtZXNzYWdlIjoiU3ludGhldGljIGFjdGl2aXR5IGNvbXBsZXRlZCJ9"
+              }
+            ]
+          },
+          "scheduledEventId": "5",
+          "startedEventId": "6",
+          "identity": "synthetic-test-worker"
+        }
+      },
+      {
+        "eventId": "8",
+        "eventTime": {
+          "seconds": "2423480226",
+          "nanos": 795000000
+        },
+        "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED",
+        "workflowTaskScheduledEventAttributes": {
+          "taskQueue": {
+            "name": "turn-goal-3b69f379-f28f-4800-bef7-4b239d887ce7",
+            "kind": "TASK_QUEUE_KIND_NORMAL"
+          },
+          "startToCloseTimeout": {
+            "seconds": "10"
+          },
+          "attempt": 1
+        }
+      },
+      {
+        "eventId": "9",
+        "eventTime": {
+          "seconds": "2423480226",
+          "nanos": 796000000
+        },
+        "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED",
+        "workflowTaskStartedEventAttributes": {
+          "scheduledEventId": "8",
+          "identity": "synthetic-test-worker",
+          "historySizeBytes": "1558"
+        }
+      },
+      {
+        "eventId": "10",
+        "eventTime": {
+          "seconds": "2423480226",
+          "nanos": 808000000
+        },
+        "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED",
+        "workflowTaskCompletedEventAttributes": {
+          "scheduledEventId": "8",
+          "identity": "synthetic-test-worker",
+          "binaryChecksum": "@temporalio/worker@1.24.0+450de7a749e60979087b19188b0216faa0a22315f7f8422a97516cee8d828146",
+          "sdkMetadata": {},
+          "meteringMetadata": {}
+        }
+      },
+      {
+        "eventId": "11",
+        "eventTime": {
+          "seconds": "2423480226",
+          "nanos": 808000000
+        },
+        "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED",
+        "workflowExecutionCompletedEventAttributes": {
+          "result": {
+            "payloads": [
+              {
+                "metadata": {
+                  "encoding": "anNvbi9wbGFpbg=="
+                },
+                "data": "eyJ3b3JrZmxvd0lkIjoid29ya2Zsb3c6dGVzdC10dXJuIiwicm9vbUlkIjoicm9vbTp0ZXN0LXR1cm4iLCJwYW5lSWQiOiJwYW5lOnRlc3QtdHVybiIsInRyYWNlSWQiOiJ0cmFjZTp0ZXN0LXR1cm4iLCJzdGF0dXMiOiJDT01QTEVURUQiLCJtZXNzYWdlIjoiU3ludGhldGljIGFjdGl2aXR5IGNvbXBsZXRlZCJ9"
+              }
+            ]
+          },
+          "workflowTaskCompletedEventId": "9"
+        }
+      }
+    ]
+  }
+}