diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e9ac6e8a..7e400599 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -165,7 +165,9 @@ Execution remains attached to its foreground request. Process crashes can leave ## Known hosted runtime limits -Sandbox identity includes the configured provider connection and authenticated principal. Each process reserves at most 25 leases before provisioning, protects active runs, and expires workspaces 15 minutes after the final run closes. Reported deletion failures retain their lease for retry. Cleanup remains best effort: abrupt process termination loses these records, and the installed Docker, Daytona, and Vercel adapters suppress deletion errors. Guaranteed eventual cleanup would require provider expiry or external reconciliation. +Sandbox identity includes the authenticated principal, provider connection, and configuration revision. Each process reserves at most 25 chat leases before provisioning, protects active runs, and expires workspaces 15 minutes after the final run closes. A separate `SandboxResources` service records each named allocation before contacting its provider, including connection tests. It renews durable ownership once a minute with a 15-minute deadline. Every tool acquisition checks ownership, and expired ownership cannot be revived. The existing Effect `ClusterCron` claims expired allocations in bounded batches and retries failed cleanup after five minutes. Local cleanup keeps a workspace fenced while deletion is still running. + +Cleanup records retain encrypted, identity-bound configuration snapshots so provider edits or deletion do not discard the original cleanup credentials. Organization deletion purges its sandboxes before deleting these records. If a process dies before recording a creation outcome, or creation returns a transport failure, cleanup reconciles the reserved name for a day after retirement to catch late results. This can delay Organization deletion. Before retaining a new allocation intent, one provider lookup of its freshly generated name checks access without resuming an existing workspace. This extra request prevents rejected credentials from leaving an unreapable intent. Provider outages or credentials revoked after creation retain failed cleanup records for later recovery. Local patches to the maintained TanStack adapters preserve names, distinguish absence from outages, and delete Vercel sandbox entities rather than only stopping their sessions. Managed sandboxes expose filesystem, process, environment, and port operations. Forks and snapshots remain disabled until their additional resources have durable ownership too. Sandbox files and command output are buffered before size checks or truncation, and wrapper timeouts may not stop transfers. Sparse XLSX worksheets are padded and rendered synchronously before text truncation, without a document-wide work budget. Both paths can exhaust replica resources. diff --git a/bun.lock b/bun.lock index ae6ec967..a55e5965 100644 --- a/bun.lock +++ b/bun.lock @@ -320,6 +320,9 @@ ], "patchedDependencies": { "nitro@3.0.260903-beta": "patches/nitro@3.0.260903-beta.patch", + "@tanstack/ai-sandbox-daytona@0.3.4": "patches/@tanstack%2Fai-sandbox-daytona@0.3.4.patch", + "@tanstack/ai-sandbox-docker@0.3.2": "patches/@tanstack%2Fai-sandbox-docker@0.3.2.patch", + "@tanstack/ai-sandbox-vercel@0.2.5": "patches/@tanstack%2Fai-sandbox-vercel@0.2.5.patch", }, "packages": { "@ag-ui/core": ["@ag-ui/core@1.0.0", "", { "peerDependencies": { "zod": "^3.25.18 || ^4.0.0" }, "optionalPeers": ["zod"] }, "sha512-yCRhsQvb4+lmGMrDsi4TOJxqKpm/kb3H64wwL+Z90jNVrkhy17RErfzsNgybfbjNKLM0nenEDYDVJsZHJJSigQ=="], diff --git a/package.json b/package.json index 85c6ca81..f55851c0 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,10 @@ }, "packageManager": "bun@1.4.4", "patchedDependencies": { - "nitro@3.0.260903-beta": "patches/nitro@3.0.260903-beta.patch" + "nitro@3.0.260903-beta": "patches/nitro@3.0.260903-beta.patch", + "@tanstack/ai-sandbox-docker@0.3.2": "patches/@tanstack%2Fai-sandbox-docker@0.3.2.patch", + "@tanstack/ai-sandbox-daytona@0.3.4": "patches/@tanstack%2Fai-sandbox-daytona@0.3.4.patch", + "@tanstack/ai-sandbox-vercel@0.2.5": "patches/@tanstack%2Fai-sandbox-vercel@0.2.5.patch" }, "trustedDependencies": [ "esbuild", diff --git a/patches/@tanstack%2Fai-sandbox-daytona@0.3.4.patch b/patches/@tanstack%2Fai-sandbox-daytona@0.3.4.patch new file mode 100644 index 00000000..2e039304 --- /dev/null +++ b/patches/@tanstack%2Fai-sandbox-daytona@0.3.4.patch @@ -0,0 +1,33 @@ +diff --git a/dist/esm/provider.js b/dist/esm/provider.js +index 1449826665afc6cc02e77ce9191a73641d1292f2..01b5211b672811ff82ebb168dd03da15c4796dd2 100644 +--- a/dist/esm/provider.js ++++ b/dist/esm/provider.js +@@ -3,1 +3,1 @@ +-import { Daytona, DaytonaConflictError } from "@daytona/sdk"; ++import { Daytona, DaytonaConflictError, DaytonaNotFoundError } from "@daytona/sdk"; +@@ -118,1 +118,2 @@ +- } catch { ++ } catch (error) { ++ if (!(error instanceof DaytonaNotFoundError)) throw error; +@@ -126,1 +127,3 @@ +- } catch {} ++ } catch (error) { ++ if (!(error instanceof DaytonaNotFoundError)) throw error; ++ } +diff --git a/src/provider.ts b/src/provider.ts +index 1c13ed38933fc9f678a70b080a7de442972ce9cc..73d0eb16a5693a6e1bbfc212be2a7e06d6f44555 100644 +--- a/src/provider.ts ++++ b/src/provider.ts +@@ -2,1 +2,1 @@ +-import { Daytona, DaytonaConflictError } from '@daytona/sdk' ++import { Daytona, DaytonaConflictError, DaytonaNotFoundError } from '@daytona/sdk' +@@ -197,2 +197,2 @@ +- } catch { +- // Gone / not found. ++ } catch (error) { ++ if (!(error instanceof DaytonaNotFoundError)) throw error +@@ -207,2 +207,2 @@ +- } catch { +- // Already deleted / gone. ++ } catch (error) { ++ if (!(error instanceof DaytonaNotFoundError)) throw error diff --git a/patches/@tanstack%2Fai-sandbox-docker@0.3.2.patch b/patches/@tanstack%2Fai-sandbox-docker@0.3.2.patch new file mode 100644 index 00000000..9e9237b5 --- /dev/null +++ b/patches/@tanstack%2Fai-sandbox-docker@0.3.2.patch @@ -0,0 +1,60 @@ +diff --git a/dist/esm/provider.js b/dist/esm/provider.js +index 6bc1a9d6355d8e62488c5c1b555133460e96ce19..1984978f2735dab87f1fa207868f0bd581ff3387 100644 +--- a/dist/esm/provider.js ++++ b/dist/esm/provider.js +@@ -37,1 +37,1 @@ +- async startContainer(image, env) { ++ async startContainer(image, env, name) { +@@ -44,0 +45,1 @@ ++ name, +@@ -84,1 +85,1 @@ +- return this.startContainer(this.config.image, input.env); ++ return this.startContainer(this.config.image, input.env, input.id); +@@ -91,1 +92,2 @@ +- } catch { ++ } catch (error) { ++ if (error.statusCode !== 404) throw error; +@@ -96,1 +98,2 @@ +- } catch { ++ } catch (error) { ++ if (error.statusCode !== 404) throw error; +@@ -115,1 +118,3 @@ +- } catch {} ++ } catch (error) { ++ if (error.statusCode !== 304 && error.statusCode !== 404) throw error; ++ } +@@ -121,1 +126,3 @@ +- } catch {} ++ } catch (error) { ++ if (error.statusCode !== 404) throw error; ++ } +diff --git a/src/provider.ts b/src/provider.ts +index 2eba55337bedb778404a042347c0b7811f91c595..079adaa61fc1c171715dfab499e49a541ee21e87 100644 +--- a/src/provider.ts ++++ b/src/provider.ts +@@ -92,0 +93,1 @@ ++ name?: string, +@@ -101,0 +103,1 @@ ++ name, +@@ -145,1 +147,1 @@ +- return this.startContainer(this.config.image, input.env) ++ return this.startContainer(this.config.image, input.env, input.id) +@@ -153,1 +155,2 @@ +- } catch { ++ } catch (error) { ++ if ((error as { statusCode?: number }).statusCode !== 404) throw error +@@ -159,1 +162,2 @@ +- } catch { ++ } catch (error) { ++ if ((error as { statusCode?: number }).statusCode !== 404) throw error +@@ -181,2 +185,3 @@ +- } catch { +- // already stopped / gone ++ } catch (error) { ++ const status = (error as { statusCode?: number }).statusCode ++ if (status !== 304 && status !== 404) throw error +@@ -187,2 +192,2 @@ +- } catch { +- // already removed ++ } catch (error) { ++ if ((error as { statusCode?: number }).statusCode !== 404) throw error diff --git a/patches/@tanstack%2Fai-sandbox-vercel@0.2.5.patch b/patches/@tanstack%2Fai-sandbox-vercel@0.2.5.patch new file mode 100644 index 00000000..628a6951 --- /dev/null +++ b/patches/@tanstack%2Fai-sandbox-vercel@0.2.5.patch @@ -0,0 +1,37 @@ +diff --git a/dist/esm/provider.js b/dist/esm/provider.js +index dde5f06611d2a38ff477e84cb631588f9d7209c7..0bdc855bedef9c0ea8c22413e18b4ce4cb5fd3c5 100644 +--- a/dist/esm/provider.js ++++ b/dist/esm/provider.js +@@ -60,0 +61,2 @@ ++ name: input.id, ++ signal: input.signal, +@@ -90,1 +92,2 @@ +- } catch { ++ } catch (error) { ++ if (!(error instanceof APIError) || error.response.status !== 404) throw error; +@@ -100,2 +103,4 @@ +- })).stop(); +- } catch {} ++ })).delete({ signal: input.signal }); ++ } catch (error) { ++ if (!(error instanceof APIError) || error.response.status !== 404) throw error; ++ } +diff --git a/src/provider.ts b/src/provider.ts +index fb4be83951bc85f8c61fca0c6a228ce8af759d58..ec4f555373c786b33c251461062c42ae1149aac1 100644 +--- a/src/provider.ts ++++ b/src/provider.ts +@@ -121,0 +122,2 @@ ++ name: input.id, ++ signal: input.signal, +@@ -164,2 +166,2 @@ +- } catch { +- // Gone / not found / expired. ++ } catch (error) { ++ if (!(error instanceof APIError) || error.response.status !== 404) throw error +@@ -177,3 +179,3 @@ +- await sandbox.stop() +- } catch { +- // Already stopped / gone. ++ await sandbox.delete({ signal: input.signal }) ++ } catch (error) { ++ if (!(error instanceof APIError) || error.response.status !== 404) throw error diff --git a/platform/src/db/migrations/20261022141912_sandbox_resources/migration.sql b/platform/src/db/migrations/20261022141912_sandbox_resources/migration.sql new file mode 100644 index 00000000..d874e8f6 --- /dev/null +++ b/platform/src/db/migrations/20261022141912_sandbox_resources/migration.sql @@ -0,0 +1,18 @@ +CREATE TABLE "sandbox_resource" ( + "organization_id" uuid, + "id" uuid DEFAULT uuidv7(), + "owner_id" uuid NOT NULL, + "provider_sandbox_id" text NOT NULL, + "configuration" text NOT NULL, + "settled" boolean DEFAULT false NOT NULL, + "retired_at" timestamp with time zone, + "expires_at" timestamp with time zone NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "sandbox_resource_pkey" PRIMARY KEY("organization_id","id") +); +--> statement-breakpoint +CREATE UNIQUE INDEX "sandbox_resource_provider_id_uidx" ON "sandbox_resource" ("organization_id","provider_sandbox_id");--> statement-breakpoint +CREATE INDEX "sandbox_resource_expires_at_idx" ON "sandbox_resource" ("expires_at");--> statement-breakpoint +CREATE INDEX "sandbox_resource_owner_id_idx" ON "sandbox_resource" ("owner_id");--> statement-breakpoint +ALTER TABLE "sandbox_resource" ADD CONSTRAINT "sandbox_resource_organization_id_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organization"("id") DEFERRABLE INITIALLY IMMEDIATE; diff --git a/platform/src/db/migrations/20261022141912_sandbox_resources/snapshot.json b/platform/src/db/migrations/20261022141912_sandbox_resources/snapshot.json new file mode 100644 index 00000000..a325c4f6 --- /dev/null +++ b/platform/src/db/migrations/20261022141912_sandbox_resources/snapshot.json @@ -0,0 +1,398 @@ +{ + "type": "diff", + "version": "9", + "dialect": "postgres", + "id": "9d36923e-f04e-4d3c-9761-d7521ce62500", + "prevIds": [ + "0183fddd-426f-44fa-9670-3fc9cd3249cc" + ], + "footprints": [ + { + "statementType": "create_table", + "statementTarget": [ + "table", + "public", + "sandbox_resource" + ], + "writes": [ + "s|tbl|public.sandbox_resource", + "n|tbl|public.sandbox_resource", + "e|tbl|public.sandbox_resource", + "n|rel|public.sandbox_resource", + "n|type|public.sandbox_resource", + "n|rel|public.sandbox_resource_pkey" + ], + "reads": [ + "n|sch|public" + ] + }, + { + "statementType": "create_index", + "statementTarget": [ + "index", + "public", + "sandbox_resource", + "sandbox_resource_provider_id_uidx" + ], + "writes": [ + "s|idx|public.sandbox_resource_provider_id_uidx", + "n|idx|public.sandbox_resource_provider_id_uidx", + "e|idx|public.sandbox_resource_provider_id_uidx", + "n|rel|public.sandbox_resource_provider_id_uidx" + ], + "reads": [ + "n|tbl|public.sandbox_resource", + "n|sch|public", + "n|col|public.sandbox_resource.organization_id", + "n|col|public.sandbox_resource.provider_sandbox_id" + ] + }, + { + "statementType": "create_index", + "statementTarget": [ + "index", + "public", + "sandbox_resource", + "sandbox_resource_expires_at_idx" + ], + "writes": [ + "s|idx|public.sandbox_resource_expires_at_idx", + "n|idx|public.sandbox_resource_expires_at_idx", + "e|idx|public.sandbox_resource_expires_at_idx", + "n|rel|public.sandbox_resource_expires_at_idx" + ], + "reads": [ + "n|tbl|public.sandbox_resource", + "n|sch|public", + "n|col|public.sandbox_resource.expires_at" + ] + }, + { + "statementType": "create_index", + "statementTarget": [ + "index", + "public", + "sandbox_resource", + "sandbox_resource_owner_id_idx" + ], + "writes": [ + "s|idx|public.sandbox_resource_owner_id_idx", + "n|idx|public.sandbox_resource_owner_id_idx", + "e|idx|public.sandbox_resource_owner_id_idx", + "n|rel|public.sandbox_resource_owner_id_idx" + ], + "reads": [ + "n|tbl|public.sandbox_resource", + "n|sch|public", + "n|col|public.sandbox_resource.owner_id" + ] + }, + { + "statementType": "create_fk", + "statementTarget": [ + "table", + "public", + "sandbox_resource" + ], + "writes": [ + "s|fk|public.sandbox_resource.sandbox_resource_organization_id_organization_id_fkey", + "n|fk|public.sandbox_resource.sandbox_resource_organization_id_organization_id_fkey", + "e|fk|public.sandbox_resource.sandbox_resource_organization_id_organization_id_fkey", + "n|con|public.sandbox_resource.sandbox_resource_organization_id_organization_id_fkey" + ], + "reads": [ + "n|tbl|public.sandbox_resource", + "n|sch|public", + "n|tbl|public.organization", + "n|sch|public", + "n|col|public.sandbox_resource.organization_id", + "n|col|public.organization.id", + "e|uniqueness|public.organization.id" + ] + } + ], + "deltas": [ + { + "op": "create", + "row": { + "schema": "public", + "name": "sandbox_resource", + "isRlsEnabled": false, + "entityType": "tables" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "organization_id", + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "id", + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "uuidv7()", + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "owner_id", + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "provider_sandbox_id", + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "configuration", + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "settled", + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "retired_at", + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "expires_at", + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "created_at", + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "updated_at", + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "sandbox_resource_provider_id_uidx", + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "provider_sandbox_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "entityType": "indexes" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "sandbox_resource_expires_at_idx", + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "entityType": "indexes" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "sandbox_resource_owner_id_idx", + "nameExplicit": true, + "columns": [ + { + "value": "owner_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "entityType": "indexes" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "sandbox_resource_organization_id_organization_id_fkey", + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "NO ACTION", + "entityType": "fks" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "sandbox_resource", + "name": "sandbox_resource_pkey", + "columns": [ + "organization_id", + "id" + ], + "nameExplicit": false, + "entityType": "pks" + } + } + ] +} \ No newline at end of file diff --git a/platform/src/db/schema/sandboxes.ts b/platform/src/db/schema/sandboxes.ts new file mode 100644 index 00000000..dfa72f28 --- /dev/null +++ b/platform/src/db/schema/sandboxes.ts @@ -0,0 +1,48 @@ +import { boolean, index, primaryKey, snakeCase, text, uniqueIndex, uuid } from "drizzle-orm/pg-core" +import { Schema } from "effect" + +import { + encryptedJson, + deferrableForeignKey, + timestamps, + timestampWithTimeZone, + uuidV7, +} from "../lib/columns" +import { SandboxProviderConfigurationSchema } from "@/lib/sandboxes/schemas" +import { UuidV7Schema } from "@/lib/schemas" +import { organization } from "./organizations" + +const SandboxResourceConfigurationSchema = Schema.Struct({ + organizationId: UuidV7Schema, + resourceId: UuidV7Schema, + providerSandboxId: Schema.String, + configuration: SandboxProviderConfigurationSchema, +}) + +export const sandboxResource = snakeCase.table( + "sandbox_resource", + { + organizationId: uuid().notNull(), + id: uuidV7(), + ownerId: uuid().notNull(), + providerSandboxId: text().notNull(), + configuration: encryptedJson({ schema: SandboxResourceConfigurationSchema }).notNull(), + settled: boolean().notNull().default(false), + retiredAt: timestampWithTimeZone(), + expiresAt: timestampWithTimeZone().notNull(), + ...timestamps(), + }, + (table) => [ + primaryKey({ columns: [table.organizationId, table.id] }), + uniqueIndex("sandbox_resource_provider_id_uidx").on( + table.organizationId, + table.providerSandboxId, + ), + index("sandbox_resource_expires_at_idx").on(table.expiresAt), + index("sandbox_resource_owner_id_idx").on(table.ownerId), + deferrableForeignKey({ + columns: [table.organizationId], + foreignColumns: [organization.id], + }).onDelete("no action"), + ], +) diff --git a/platform/src/db/schema/tables.ts b/platform/src/db/schema/tables.ts index a265bd08..3f9026c0 100644 --- a/platform/src/db/schema/tables.ts +++ b/platform/src/db/schema/tables.ts @@ -25,5 +25,6 @@ export { } from "./chat.ts" export { rateLimit } from "./rate-limit.ts" export { fileObject } from "./files.ts" +export { sandboxResource } from "./sandboxes.ts" export { mcpServer, mcpConnection } from "./mcp.ts" export { usageLimit, usageLimitCounter } from "./usage-limits.ts" diff --git a/platform/src/lib/chat/sandbox/errors.ts b/platform/src/lib/chat/sandbox/errors.ts index 84e60254..b398fcbd 100644 --- a/platform/src/lib/chat/sandbox/errors.ts +++ b/platform/src/lib/chat/sandbox/errors.ts @@ -8,16 +8,6 @@ export class ChatSandboxUnavailable extends Schema.TaggedError()( - "ChatSandboxOperationFailed", - { timedOut: Schema.Boolean, cause: Schema.Unknown }, -) { - override readonly message: string = this.timedOut - ? "The sandbox did not respond in time" - : "The sandbox operation failed" -} - /** The agent's provider configuration is gone or unreadable, so the run goes without a sandbox. */ export class ChatSandboxConfigurationUnreadable extends Schema.TaggedError()( "ChatSandboxConfigurationUnreadable", diff --git a/platform/src/lib/chat/sandbox/leases.ts b/platform/src/lib/chat/sandbox/leases.ts index ea829fbc..36e89216 100644 --- a/platform/src/lib/chat/sandbox/leases.ts +++ b/platform/src/lib/chat/sandbox/leases.ts @@ -15,7 +15,7 @@ import { CHAT_SANDBOX_SWEEP_INTERVAL_MS, } from "./constants" import { ChatSandboxUnavailable } from "./errors" -import { chatSandboxCall, logChatSandboxFailure } from "./operations" +import { sandboxCall, logSandboxFailure } from "@/lib/sandboxes/operations.server" interface ChatSandboxLease { record: SandboxInstanceRecord | undefined @@ -37,7 +37,7 @@ export const makeChatSandboxLeases = Effect.gen(function* () { const destroyLease = (key: string, lease: ChatSandboxLease, timeout: Duration.Input) => (lease.cleanup ??= runCleanup( - chatSandboxCall(async (signal) => { + sandboxCall(async (signal) => { try { if (lease.record) { await lease.provider.destroy({ id: lease.record.providerSandboxId, signal }) @@ -50,7 +50,7 @@ export const makeChatSandboxLeases = Effect.gen(function* () { } }, timeout).pipe( Effect.as(true), - Effect.catch((error) => logChatSandboxFailure("destroy", error).pipe(Effect.as(false))), + Effect.catch((error) => logSandboxFailure("destroy", error).pipe(Effect.as(false))), ), )) @@ -137,7 +137,7 @@ export const makeChatSandboxLeases = Effect.gen(function* () { reservation = yield* reserveLease(key, definition.provider) } const lease = reservation - return yield* chatSandboxCall(async (signal) => { + return yield* sandboxCall(async (signal) => { // Shutdown can run before this callback starts. Check and reserve without yielding. if (closed || scope.state._tag === "Closed") throw new ChatSandboxUnavailable() lease.pending += 1 diff --git a/platform/src/lib/chat/sandbox/operations.ts b/platform/src/lib/chat/sandbox/operations.ts deleted file mode 100644 index 5ba526a6..00000000 --- a/platform/src/lib/chat/sandbox/operations.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { type Duration, Effect } from "effect" - -import { errorReason } from "@/lib/runtime/failure-report.server" -import { ChatSandboxOperationFailed } from "./errors" - -/** Wraps one vendor sandbox call: a timeout interrupts it, and its error never reaches the agent. */ -export function chatSandboxCall( - call: (signal: AbortSignal) => PromiseLike, - timeout: Duration.Input, -) { - return Effect.tryPromise({ - try: call, - catch: (cause) => new ChatSandboxOperationFailed({ timedOut: false, cause }), - }).pipe( - Effect.timeout(timeout), - Effect.catchTag("TimeoutError", (cause) => - Effect.fail(new ChatSandboxOperationFailed({ timedOut: true, cause })), - ), - ) -} - -/** Logs why a sandbox call failed with its vendor code or type only. */ -export function logChatSandboxFailure(operation: string, error: ChatSandboxOperationFailed) { - return Effect.logWarning("Chat sandbox operation failed").pipe( - Effect.annotateLogs({ operation, timedOut: error.timedOut, reason: errorReason(error.cause) }), - ) -} diff --git a/platform/src/lib/chat/sandbox/sandbox.test.ts b/platform/src/lib/chat/sandbox/sandbox.test.ts index 38da3c8b..436ae5e8 100644 --- a/platform/src/lib/chat/sandbox/sandbox.test.ts +++ b/platform/src/lib/chat/sandbox/sandbox.test.ts @@ -4,7 +4,7 @@ import { Context, Deferred, Effect, Exit, Fiber, Layer, Scheduler, Scope } from import { TestClock } from "effect/testing" import { beforeEach } from "vitest" -import { SandboxProviderFactory } from "@/lib/sandboxes/factory.server" +import { SandboxResources } from "@/lib/sandboxes/resources.server" import { SandboxProviders } from "@/lib/sandboxes/providers.server" import { CHAT_SANDBOX_FILE_TIMEOUT_MS, @@ -19,6 +19,7 @@ function freshChatSandboxTestState() { return { created: 0, createFails: false, + configurationVersion: 0, destroyFails: false, destroyed: [] as string[], createGate: undefined as Promise | undefined, @@ -59,9 +60,18 @@ const sandboxesLayer = ChatSandboxes.layerNoDeps.pipe( Layer.provide([ Layer.succeed(SandboxProviders, { resolveConfiguration: () => - Effect.succeed({ name: "Local", provider: "docker", options: {}, credentials: {} }), + Effect.succeed({ + name: "Local", + lockVersion: sandboxTest.configurationVersion, + provider: "docker", + options: {}, + credentials: {}, + }), } as unknown as SandboxProviders["Service"]), - Layer.succeed(SandboxProviderFactory, { create: () => Effect.succeed(fakeSandboxProvider) }), + Layer.succeed(SandboxResources, { + provider: () => Effect.succeed(fakeSandboxProvider), + touch: () => Effect.void, + } as unknown as SandboxResources["Service"]), ]), ) @@ -305,7 +315,9 @@ describe("ChatSandboxes", () => { ]) { assert.notStrictEqual((yield* acquire(input)).id, "sandbox-1") } - assert.strictEqual(sandboxTest.created, 5) + sandboxTest.configurationVersion += 1 + assert.notStrictEqual((yield* acquire()).id, "sandbox-1") + assert.strictEqual(sandboxTest.created, 6) }).pipe(Effect.provide(sandboxesLayer)), ) }) diff --git a/platform/src/lib/chat/sandbox/sandbox.ts b/platform/src/lib/chat/sandbox/sandbox.ts index e1089a3e..9bbdc1a9 100644 --- a/platform/src/lib/chat/sandbox/sandbox.ts +++ b/platform/src/lib/chat/sandbox/sandbox.ts @@ -3,12 +3,12 @@ import { Context, Deferred, Effect, Exit, Layer, Option, Ref, Scope } from "effe import { APP_HANDLE } from "@/lib/constants" import { errorReason } from "@/lib/runtime/failure-report.server" -import { SandboxProviderFactory } from "@/lib/sandboxes/factory.server" +import { SandboxResources } from "@/lib/sandboxes/resources.server" import { SandboxProviders } from "@/lib/sandboxes/providers.server" import { CHAT_ATTACHMENT_UPLOAD_DIRECTORY } from "../attachments/constants" import { CHAT_SANDBOX_FILE_TIMEOUT_MS } from "./constants" import { makeChatSandboxLeases } from "./leases" -import { chatSandboxCall, logChatSandboxFailure } from "./operations" +import { sandboxCall, logSandboxFailure } from "@/lib/sandboxes/operations.server" import { ChatSandboxConfigurationUnreadable, ChatSandboxUnavailable } from "./errors.ts" import { chatPrincipalScope } from "../identity" import type { ChatAttachmentFile } from "../attachments/types" @@ -37,11 +37,11 @@ const writeChatSandboxUploads = Effect.fn("writeChatSandboxUploads")(function* ( if (uploads.length === 0) return const directory = `${resolveHarnessCwd(handle)}/${CHAT_ATTACHMENT_UPLOAD_DIRECTORY}` // One level below a workspace that already exists, so this needs no recursive `mkdir`. - yield* chatSandboxCall(() => handle.fs.mkdir(directory), CHAT_SANDBOX_FILE_TIMEOUT_MS) + yield* sandboxCall(() => handle.fs.mkdir(directory), CHAT_SANDBOX_FILE_TIMEOUT_MS) yield* Effect.forEach( uploads, (upload) => - chatSandboxCall( + sandboxCall( () => handle.fs.write(`${directory}/${upload.handle}`, upload.bytes), CHAT_SANDBOX_FILE_TIMEOUT_MS, ), @@ -71,7 +71,7 @@ export class ChatSandboxes extends Context.Service< ChatSandboxes, Effect.gen(function* () { const providers = yield* SandboxProviders - const factory = yield* SandboxProviderFactory + const resources = yield* SandboxResources const acquireSandbox = yield* makeChatSandboxLeases const session = Effect.fn("ChatSandboxes.session")( @@ -81,10 +81,10 @@ export class ChatSandboxes extends Context.Service< organizationId, id: input.sandboxProviderId, }) - const provider = yield* factory.create(configuration.provider, configuration) + const provider = yield* resources.provider({ ...configuration, organizationId }) const scope = yield* Scope.fork(yield* Effect.scope) const definition = defineSandbox({ - id: `${APP_HANDLE}-chat-${input.agentId}-${input.sandboxProviderId}`, + id: `${APP_HANDLE}-chat-${input.agentId}-${input.sandboxProviderId}-${configuration.lockVersion}`, provider, // No snapshot: the workspace is empty until the agent writes to it, so an after-setup // snapshot would cost a round trip per create and restore nothing worth restoring. @@ -118,8 +118,8 @@ export class ChatSandboxes extends Context.Service< return handle }).pipe( Effect.onError(() => Scope.close(scope, Exit.void)), - Effect.tapErrorTag("ChatSandboxOperationFailed", (error) => - logChatSandboxFailure("start", error), + Effect.tapErrorTag("SandboxOperationFailed", (error) => + logSandboxFailure("start", error), ), Effect.tapError(() => report({ state: "error" })), Effect.mapError(() => new ChatSandboxUnavailable()), @@ -131,6 +131,9 @@ export class ChatSandboxes extends Context.Service< alreadyClaimed ? Deferred.await(started) : start(report).pipe(Effect.onExit((exit) => Deferred.done(started, exit))), + ).pipe( + Effect.tap((handle) => resources.touch(organizationId, handle.id)), + Effect.mapError(() => new ChatSandboxUnavailable()), ), prepareUploads: (files) => Effect.gen(function* () { @@ -138,6 +141,7 @@ export class ChatSandboxes extends Context.Service< const handle = yield* Deferred.poll(started) if (Option.isNone(handle)) return const running = yield* handle.value + yield* resources.touch(organizationId, running.id) const additions = files.filter((file) => !uploaded.has(file.handle)) yield* writeChatSandboxUploads(running, additions) for (const file of additions) uploaded.add(file.handle) @@ -157,6 +161,6 @@ export class ChatSandboxes extends Context.Service< ) static readonly layer = ChatSandboxes.layerNoDeps.pipe( - Layer.provide([SandboxProviders.layer, SandboxProviderFactory.layer]), + Layer.provide([SandboxProviders.layer, SandboxResources.layer]), ) } diff --git a/platform/src/lib/chat/sandbox/tools.ts b/platform/src/lib/chat/sandbox/tools.ts index 2b0be150..d4186328 100644 --- a/platform/src/lib/chat/sandbox/tools.ts +++ b/platform/src/lib/chat/sandbox/tools.ts @@ -15,8 +15,9 @@ import { CHAT_SANDBOX_MAX_WRITE_CHARACTERS, CHAT_SANDBOX_STATUS_EVENT, } from "./constants" -import { ChatSandboxOperationFailed, type ChatSandboxUnavailable } from "./errors.ts" -import { chatSandboxCall, logChatSandboxFailure } from "./operations" +import type { ChatSandboxUnavailable } from "./errors.ts" +import { SandboxOperationFailed } from "@/lib/sandboxes/errors" +import { sandboxCall, logSandboxFailure } from "@/lib/sandboxes/operations.server" import type { ChatSandboxSession, ChatSandboxStatus } from "./sandbox" import { resolveSandboxPath } from "./paths" import { chatToolInputSchema } from "../tool-schema" @@ -78,7 +79,7 @@ interface SandboxToolContext { abortSignal?: AbortSignal | undefined } -type SandboxToolEffect = Effect.Effect +type SandboxToolEffect = Effect.Effect /** The sandbox tools, which run here rather than in the host page. A refused path, a failed * command and a timeout are ordinary results the agent can act on, and only a broken sandbox fails. */ @@ -109,11 +110,9 @@ export function createChatSandboxTools(input: { .pipe( Effect.flatMap(effect), Effect.catchDefect((cause) => - Effect.fail(new ChatSandboxOperationFailed({ timedOut: false, cause })), - ), - Effect.tapErrorTag("ChatSandboxOperationFailed", (error) => - logChatSandboxFailure(tool, error), + Effect.fail(new SandboxOperationFailed({ timedOut: false, cause })), ), + Effect.tapErrorTag("SandboxOperationFailed", (error) => logSandboxFailure(tool, error)), ), { signal: context?.abortSignal }, ) @@ -139,7 +138,7 @@ export function createChatSandboxTools(input: { const directory = resolved.path.slice(0, resolved.path.lastIndexOf("/")) // `mkdir -p` through exec rather than `fs.mkdir`, which no provider promises is recursive. if (directory.length > 0) { - const created = yield* chatSandboxCall( + const created = yield* sandboxCall( (signal) => handle.process.exec(`mkdir -p ${quoteSandboxArgument(directory)}`, { signal }), CHAT_SANDBOX_FILE_TIMEOUT_MS, @@ -149,7 +148,7 @@ export function createChatSandboxTools(input: { return { refusal: `The directory ${directory} could not be created: ${stderr.text}` } } } - yield* chatSandboxCall( + yield* sandboxCall( () => handle.fs.write(resolved.path, content), CHAT_SANDBOX_FILE_TIMEOUT_MS, ) @@ -172,7 +171,7 @@ export function createChatSandboxTools(input: { Effect.gen(function* () { const resolved = resolveSandboxPath(resolveHarnessCwd(handle), path) if ("refusal" in resolved) return resolved - const content = yield* chatSandboxCall( + const content = yield* sandboxCall( () => handle.fs.read(resolved.path), CHAT_SANDBOX_FILE_TIMEOUT_MS, ) @@ -197,7 +196,7 @@ export function createChatSandboxTools(input: { const root = resolveHarnessCwd(handle) const resolved = resolveSandboxPath(root, path ?? root) if ("refusal" in resolved) return resolved - const entries = yield* chatSandboxCall( + const entries = yield* sandboxCall( () => handle.fs.list(resolved.path), CHAT_SANDBOX_FILE_TIMEOUT_MS, ) @@ -230,7 +229,7 @@ export function createChatSandboxTools(input: { // The timeout interrupts the call, aborting the vendor's command through its signal, so // a hung command reports a timeout to the agent instead of holding the run open. const outcome = yield* Effect.result( - chatSandboxCall( + sandboxCall( (signal) => handle.process.exec(command, { cwd: resolved.path, signal }), CHAT_SANDBOX_COMMAND_TIMEOUT_MS, ), @@ -240,7 +239,7 @@ export function createChatSandboxTools(input: { return { command, cwd: resolved.path, timedOut: true, durationMs } } // A command that cannot launch is the agent's problem to fix, not a broken run. - if (Result.isFailure(outcome)) yield* logChatSandboxFailure("run_command", outcome.failure) + if (Result.isFailure(outcome)) yield* logSandboxFailure("run_command", outcome.failure) const result = Result.getOrElse(outcome, () => ({ stdout: "", stderr: "The command could not be run.", @@ -274,11 +273,11 @@ export function createChatSandboxTools(input: { const resolved = resolveSandboxPath(resolveHarnessCwd(handle), path) if ("refusal" in resolved) return resolved const read = yield* Effect.result( - chatSandboxCall(() => handle.fs.readBytes(resolved.path), CHAT_SANDBOX_FILE_TIMEOUT_MS), + sandboxCall(() => handle.fs.readBytes(resolved.path), CHAT_SANDBOX_FILE_TIMEOUT_MS), ) // The agent only learns the path did not work, and the reason goes to the log. if (Result.isFailure(read)) { - yield* logChatSandboxFailure("publish_artifact", read.failure) + yield* logSandboxFailure("publish_artifact", read.failure) return { refusal: "The file could not be read. Check the path with sandbox_list_files." } } const bytes = read.success diff --git a/platform/src/lib/runtime/runtime.server.ts b/platform/src/lib/runtime/runtime.server.ts index 48b12a89..3e031121 100644 --- a/platform/src/lib/runtime/runtime.server.ts +++ b/platform/src/lib/runtime/runtime.server.ts @@ -22,6 +22,7 @@ import { McpTools } from "@/lib/mcp/tools.server" import { ModelProviders } from "@/lib/model-providers/model-providers.server" import { Organizations } from "@/lib/organizations/organizations.server" import { SandboxProviders } from "@/lib/sandboxes/providers.server" +import { SandboxResources } from "@/lib/sandboxes/resources.server" import { TenantUsers } from "@/lib/tenants/tenant-users" import { Tenants } from "@/lib/tenants/tenants" import { StoredFiles } from "@/lib/storage/stored-files.server" @@ -57,6 +58,7 @@ function makeAppLayer() { McpAuthorization.layer, McpTools.layer, SandboxProviders.layer, + SandboxResources.layer, Tenants.layer, TenantUsers.layer, UsageLimits.layer, diff --git a/platform/src/lib/sandboxes/connection-test.server.test.ts b/platform/src/lib/sandboxes/connection-test.server.test.ts index 870970ff..d093069d 100644 --- a/platform/src/lib/sandboxes/connection-test.server.test.ts +++ b/platform/src/lib/sandboxes/connection-test.server.test.ts @@ -1,7 +1,8 @@ import { assert, describe, it } from "@effect/vitest" -import { Deferred, Effect, Fiber } from "effect" +import { Deferred, Effect, Fiber, Layer } from "effect" import { TestClock } from "effect/testing" import { vi } from "vitest" +import { SandboxResources } from "./resources.server" const connectionTestSandbox = vi.hoisted(() => ({ createError: undefined as Error | undefined, @@ -10,8 +11,8 @@ const connectionTestSandbox = vi.hoisted(() => ({ destroy: vi.fn(() => Promise.resolve()), })) -vi.mock("./factory.server.ts", () => ({ - createSandboxProvider: () => +const testResources = Layer.succeed(SandboxResources, { + provider: () => Effect.succeed({ create: () => connectionTestSandbox.pendingCreate ?? @@ -22,11 +23,17 @@ vi.mock("./factory.server.ts", () => ({ }) : Promise.reject(connectionTestSandbox.createError)), }), -})) +} as unknown as SandboxResources["Service"]) import { runSandboxConnectionTest } from "./connection-test.server.ts" -const dockerTest = { provider: "docker", options: { image: "node:22" }, credentials: {} } as const +const dockerTest = { + organizationId: "01992a80-1d71-7f24-a150-f1177e3f6419", + provider: "docker", + options: { image: "node:22" }, + credentials: {}, +} as const +const runTest = () => runSandboxConnectionTest(dockerTest).pipe(Effect.provide(testResources)) describe("runSandboxConnectionTest", () => { it.effect("destroys a created sandbox when the test is interrupted", () => @@ -37,7 +44,7 @@ describe("runSandboxConnectionTest", () => { Deferred.doneUnsafe(started, Effect.void) return new Promise(() => {}) }) - const fiber = yield* Effect.forkChild(runSandboxConnectionTest(dockerTest)) + const fiber = yield* Effect.forkChild(runTest()) yield* Deferred.await(started) yield* Fiber.interrupt(fiber) assert.strictEqual(connectionTestSandbox.destroy.mock.calls.length, 1) @@ -48,7 +55,7 @@ describe("runSandboxConnectionTest", () => { Effect.gen(function* () { connectionTestSandbox.destroy.mockClear() connectionTestSandbox.exec.mockImplementation(() => new Promise(() => {})) - const fiber = yield* Effect.forkChild(runSandboxConnectionTest(dockerTest)) + const fiber = yield* Effect.forkChild(runTest()) yield* TestClock.adjust("15 seconds") const result = yield* Fiber.join(fiber) assert.deepInclude(result, { status: "failure", errorCode: "timeout" }) @@ -63,7 +70,7 @@ describe("runSandboxConnectionTest", () => { connectionTestSandbox.destroy.mockClear() const late = Promise.withResolvers() connectionTestSandbox.pendingCreate = late.promise - const fiber = yield* Effect.forkChild(runSandboxConnectionTest(dockerTest)) + const fiber = yield* Effect.forkChild(runTest()) if (end === "timeout") { yield* TestClock.adjust("30 seconds") assert.deepInclude(yield* Fiber.join(fiber), { status: "failure", errorCode: "timeout" }) @@ -88,7 +95,7 @@ describe("runSandboxConnectionTest", () => { const sprites = new Error("Sprites API POST https://api.sprites.dev failed: 403 Forbidden") for (const error of [vercel, sprites]) { connectionTestSandbox.createError = error - const result = yield* runSandboxConnectionTest(dockerTest) + const result = yield* runTest() assert.deepInclude(result, { status: "failure", errorCode: "authentication" }) } connectionTestSandbox.createError = undefined diff --git a/platform/src/lib/sandboxes/connection-test.server.ts b/platform/src/lib/sandboxes/connection-test.server.ts index 781d9000..35f2c810 100644 --- a/platform/src/lib/sandboxes/connection-test.server.ts +++ b/platform/src/lib/sandboxes/connection-test.server.ts @@ -1,57 +1,39 @@ import type { SandboxHandle } from "@tanstack/ai-sandbox" -import { Cause, DateTime, Effect, Exit, Predicate, Schema } from "effect" +import { Cause, DateTime, Effect, Exit, Predicate } from "effect" -import { createSandboxProvider } from "./factory.server.ts" -import type { - SandboxConnectionErrorCode, - SandboxProviderId, - SandboxTestMetadata, -} from "./schemas.ts" +import { SandboxResources } from "./resources.server" +import { SandboxOperationFailed } from "./errors" +import { sandboxCall, sandboxErrorCode } from "./operations.server" +import type { SandboxProviderId, SandboxTestMetadata } from "./schemas.ts" type SandboxTestAction = "create-provider" | "create-sandbox" | "execute-test" | "destroy-sandbox" -/** A vendor call failed or timed out, carrying its cause only for classification. */ -class SandboxOperationFailed extends Schema.TaggedError()( - "SandboxOperationFailed", - { cause: Schema.Defect() }, -) {} - -function sandboxOperation( - operation: (signal: AbortSignal) => Promise, - timeout: `${number} seconds`, -) { - return Effect.tryPromise({ - try: operation, - catch: (cause) => new SandboxOperationFailed({ cause }), - }).pipe( - Effect.timeout(timeout), - Effect.catchTag("TimeoutError", (cause) => Effect.fail(new SandboxOperationFailed({ cause }))), - ) -} - const createSandbox = (provider: { create: (options: { signal: AbortSignal }) => Promise }) => { let pending: Promise | undefined // Docker's create ignores the signal, so a timed-out or interrupted create destroys a late handle. - return sandboxOperation((signal) => (pending = provider.create({ signal })), "30 seconds").pipe( + return sandboxCall((signal) => (pending = provider.create({ signal })), "30 seconds").pipe( Effect.onError(() => Effect.sync(() => void pending?.then((handle) => handle.destroy()).catch(() => undefined)), ), ) } -const destroySandbox = (handle: SandboxHandle) => - sandboxOperation(() => handle.destroy(), "15 seconds") +const destroySandbox = (handle: SandboxHandle) => sandboxCall(() => handle.destroy(), "15 seconds") const execSandboxTest = (handle: SandboxHandle) => - sandboxOperation( + sandboxCall( (signal) => handle.process.exec("printf sandbox-connection-ok", { signal }), "15 seconds", ).pipe( Effect.filterOrFail( (result) => result.exitCode === 0 && result.stdout === "sandbox-connection-ok", - () => new SandboxOperationFailed({ cause: new Error("Sandbox connection command failed") }), + () => + new SandboxOperationFailed({ + timedOut: false, + cause: new Error("Sandbox connection command failed"), + }), ), ) @@ -61,6 +43,7 @@ const execSandboxTest = (handle: SandboxHandle) => */ export const runSandboxConnectionTest = Effect.fn("runSandboxConnectionTest")(function* (input: { readonly provider: SandboxProviderId + readonly organizationId: string readonly options: unknown readonly credentials: unknown }) { @@ -75,7 +58,8 @@ export const runSandboxConnectionTest = Effect.fn("runSandboxConnectionTest")(fu }), Effect.as({ status: "failure", testedAt, errorCode }), ) - const provider = yield* Effect.exit(createSandboxProvider(input.provider, input)) + const resources = yield* SandboxResources + const provider = yield* Effect.exit(resources.provider(input)) if (Exit.isFailure(provider)) return yield* failed("create-provider", Cause.squash(provider.cause)) return yield* Effect.acquireUseRelease( @@ -100,42 +84,3 @@ function errorType(cause: unknown): string { if (Predicate.isTagged(error, "TimeoutError")) return "TimeoutError" return error instanceof Error ? error.name : "UnknownError" } - -function sandboxErrorCode(cause: unknown): SandboxConnectionErrorCode { - const error = cause instanceof SandboxOperationFailed ? cause.cause : cause - if (Cause.isTimeoutError(error)) return "timeout" - if (error instanceof DOMException && error.name === "AbortError") return "cancelled" - if (!(error instanceof Error)) return "provider_error" - let current: unknown = error - const seen = new Set() - while (Predicate.isObject(current) && !seen.has(current)) { - seen.add(current) - // Vercel's APIError keeps the status on its `response`. - const response = Predicate.hasProperty(current, "response") ? current.response : undefined - const status = Predicate.hasProperty(current, "status") - ? current.status - : Predicate.hasProperty(current, "statusCode") - ? current.statusCode - : Predicate.hasProperty(response, "status") - ? response.status - : undefined - if (status === 401 || status === 403) return "authentication" - if (status === 404) return "not_found" - if (status === 429) return "quota" - current = Predicate.hasProperty(current, "cause") ? current.cause : undefined - } - // Sprites throws a plain Error naming the HTTP status, so read it from the message. - // https://github.com/TanStack/ai/blob/main/packages/ai-sandbox-sprites/src/client.ts - const httpStatus = / failed: (\d{3}) /.exec(error.message)?.[1] - if (httpStatus === "401" || httpStatus === "403") return "authentication" - if (httpStatus === "404") return "not_found" - if (httpStatus === "429") return "quota" - const code = - Predicate.hasProperty(error, "code") && Predicate.isString(error.code) ? error.code : "" - if (/timeout/i.test(error.name) || /timed?out/i.test(code)) return "timeout" - if (/auth|token/i.test(error.name)) return "authentication" - if (/quota|rate/i.test(error.name)) return "quota" - if (/notfound/i.test(error.name)) return "not_found" - if (/cleanup/i.test(error.name)) return "cleanup_failed" - return "provider_error" -} diff --git a/platform/src/lib/sandboxes/errors.ts b/platform/src/lib/sandboxes/errors.ts index 06b80c89..ac632163 100644 --- a/platform/src/lib/sandboxes/errors.ts +++ b/platform/src/lib/sandboxes/errors.ts @@ -2,6 +2,21 @@ import { Schema } from "effect" import { type SandboxConnectionErrorCode, SandboxConnectionErrorCodeSchema } from "./schemas.ts" +/** A vendor sandbox call that failed or timed out. The cause stays server-side for diagnostics. */ +export class SandboxOperationFailed extends Schema.TaggedError()( + "SandboxOperationFailed", + { timedOut: Schema.Boolean, cause: Schema.Unknown }, +) { + override readonly message: string = this.timedOut + ? "The sandbox did not respond in time" + : "The sandbox operation failed" +} + +export class SandboxResourceUnavailable extends Schema.TaggedError()( + "SandboxResourceUnavailable", + {}, +) {} + // Each message is written for the member who sees it, so handlers may expose it verbatim. export class SandboxProviderNotFound extends Schema.TaggedError()( diff --git a/platform/src/lib/sandboxes/factory.server.ts b/platform/src/lib/sandboxes/factory.server.ts index 042d5255..9f721b52 100644 --- a/platform/src/lib/sandboxes/factory.server.ts +++ b/platform/src/lib/sandboxes/factory.server.ts @@ -13,6 +13,9 @@ import { // `tail` as PID 1 ignores SIGTERM, so without the trap every destroy waits out Docker's stop timeout. const DOCKER_KEEP_ALIVE_COMMAND = ["sh", "-c", "trap 'exit 0' TERM; tail -f /dev/null & wait"] +// Local adapter patches preserve named creation and surface cleanup failures for durable retries. +// https://github.com/TanStack/ai/tree/main/packages + type ProviderConfiguration = { options: SandboxProviderOptions[Provider] credentials: SandboxProviderCredentials[Provider] diff --git a/platform/src/lib/sandboxes/operations.server.ts b/platform/src/lib/sandboxes/operations.server.ts new file mode 100644 index 00000000..bded1bd2 --- /dev/null +++ b/platform/src/lib/sandboxes/operations.server.ts @@ -0,0 +1,67 @@ +import { Cause, type Duration, Effect, Predicate } from "effect" + +import { errorReason } from "@/lib/runtime/failure-report.server" +import { SandboxOperationFailed } from "./errors" +import type { SandboxConnectionErrorCode } from "./schemas" + +/** Wraps one vendor sandbox call: a timeout interrupts it, and its error never reaches the agent. */ +export function sandboxCall( + call: (signal: AbortSignal) => PromiseLike, + timeout: Duration.Input, +) { + return Effect.tryPromise({ + try: call, + catch: (cause) => new SandboxOperationFailed({ timedOut: false, cause }), + }).pipe( + Effect.timeout(timeout), + Effect.catchTag("TimeoutError", (cause) => + Effect.fail(new SandboxOperationFailed({ timedOut: true, cause })), + ), + ) +} + +/** Logs why a sandbox call failed with its vendor code or type only. */ +export function logSandboxFailure(operation: string, error: SandboxOperationFailed) { + return Effect.logWarning("Sandbox operation failed").pipe( + Effect.annotateLogs({ operation, timedOut: error.timedOut, reason: errorReason(error.cause) }), + ) +} + +export function sandboxErrorCode(cause: unknown): SandboxConnectionErrorCode { + const error = cause instanceof SandboxOperationFailed ? cause.cause : cause + if (Cause.isTimeoutError(error)) return "timeout" + if (error instanceof DOMException && error.name === "AbortError") return "cancelled" + if (!(error instanceof Error)) return "provider_error" + let current: unknown = error + const seen = new Set() + while (Predicate.isObject(current) && !seen.has(current)) { + seen.add(current) + // Vercel's APIError keeps the status on its `response`. + const response = Predicate.hasProperty(current, "response") ? current.response : undefined + const status = Predicate.hasProperty(current, "status") + ? current.status + : Predicate.hasProperty(current, "statusCode") + ? current.statusCode + : Predicate.hasProperty(response, "status") + ? response.status + : undefined + if (status === 401 || status === 403) return "authentication" + if (status === 404) return "not_found" + if (status === 429) return "quota" + current = Predicate.hasProperty(current, "cause") ? current.cause : undefined + } + // Sprites throws a plain Error naming the HTTP status, so read it from the message. + // https://github.com/TanStack/ai/blob/main/packages/ai-sandbox-sprites/src/client.ts + const httpStatus = / failed: (\d{3}) /.exec(error.message)?.[1] + if (httpStatus === "401" || httpStatus === "403") return "authentication" + if (httpStatus === "404") return "not_found" + if (httpStatus === "429") return "quota" + const code = + Predicate.hasProperty(error, "code") && Predicate.isString(error.code) ? error.code : "" + if (/timeout/i.test(error.name) || /timed?out/i.test(code)) return "timeout" + if (/auth|token/i.test(error.name)) return "authentication" + if (/quota|rate/i.test(error.name)) return "quota" + if (/notfound/i.test(error.name)) return "not_found" + if (/cleanup/i.test(error.name)) return "cleanup_failed" + return "provider_error" +} diff --git a/platform/src/lib/sandboxes/provider-cleanup.test.ts b/platform/src/lib/sandboxes/provider-cleanup.test.ts new file mode 100644 index 00000000..692f9a08 --- /dev/null +++ b/platform/src/lib/sandboxes/provider-cleanup.test.ts @@ -0,0 +1,67 @@ +import { createServer } from "node:http" + +import { daytonaSandbox } from "@tanstack/ai-sandbox-daytona" +import { dockerSandbox } from "@tanstack/ai-sandbox-docker" +import { vercelSandbox } from "@tanstack/ai-sandbox-vercel" +import { expect, test, vi } from "vitest" + +test("patched adapters preserve ownership names and distinguish missing resources from outages", async () => { + let status = 403 + const requests: { path: string; body: string }[] = [] + const server = createServer((request, response) => { + let body = "" + request.on("data", (chunk: unknown) => { + body += String(chunk) + }) + request.on("end", () => { + requests.push({ path: request.url ?? "", body }) + response.writeHead(request.url?.startsWith("/images/") ? 200 : status, { + "content-type": "application/json", + }) + response.end( + JSON.stringify({ error: { code: "test_error", message: "test provider response" } }), + ) + }) + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + const address = server.address() + if (!address || typeof address === "string") throw new Error("Expected TCP listener") + const origin = `http://127.0.0.1:${address.port}` + const originalFetch = globalThis.fetch + vi.stubGlobal("fetch", (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(typeof input === "string" || input instanceof URL ? input : input.url) + return originalFetch(`${origin}${url.pathname}${url.search}`, init) + }) + try { + const docker = dockerSandbox({ + image: "node:22", + dockerodeOptions: { host: "127.0.0.1", port: address.port, protocol: "http" }, + }) + const vercel = vercelSandbox({ + token: "test-token", + teamId: "test-team", + projectId: "test-project", + }) + const providers = [docker, vercel, daytonaSandbox({ apiKey: "test-key", apiUrl: origin })] + for (const provider of providers) { + await expect(provider.destroy({ id: "owned-sandbox" })).rejects.toThrow() + await expect(provider.resume({ id: "owned-sandbox" })).rejects.toThrow() + } + await expect(docker.create({ id: "owned-sandbox" })).rejects.toThrow() + await expect(vercel.create({ id: "owned-sandbox" })).rejects.toThrow() + expect( + requests.some(({ path }) => path.includes("/containers/create?name=owned-sandbox")), + ).toBe(true) + expect(requests.some(({ body }) => body.includes('"name":"owned-sandbox"'))).toBe(true) + status = 404 + for (const provider of providers) { + await expect(provider.destroy({ id: "owned-sandbox" })).resolves.toBeUndefined() + await expect(provider.resume({ id: "owned-sandbox" })).resolves.toBeNull() + } + } finally { + vi.unstubAllGlobals() + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ) + } +}) diff --git a/platform/src/lib/sandboxes/providers.server.test.ts b/platform/src/lib/sandboxes/providers.server.test.ts index 2439c38b..5b4049f6 100644 --- a/platform/src/lib/sandboxes/providers.server.test.ts +++ b/platform/src/lib/sandboxes/providers.server.test.ts @@ -7,6 +7,7 @@ import { parseDatabaseEncryptionKeyring } from "@/db/lib/database-credentials" import { encryptDatabaseValue } from "@/db/lib/encryption" import { SandboxProviderCredentialsPayloadSchema } from "@/db/schema/organizations" import { SandboxProviders } from "./providers.server.ts" +import { SandboxResources } from "./resources.server" const ORGANIZATION_ID = "01992a80-1d71-7f24-a150-f1177e3f6419" const SANDBOX_PROVIDER_ID = "01992a80-1d71-7f24-a150-f1177e3f6420" @@ -43,7 +44,12 @@ function providersOver(row: { id: string; storedCredentials: string | null }) { const database = { select: () => ({ from: () => ({ where: () => ({ limit: () => Effect.succeed([stored]) }) }) }), } as unknown as EffectDatabase - return SandboxProviders.layerNoDeps.pipe(Layer.provide(Layer.succeed(Database, database))) + return SandboxProviders.layerNoDeps.pipe( + Layer.provide([ + Layer.succeed(Database, database), + Layer.succeed(SandboxResources, {} as SandboxResources["Service"]), + ]), + ) } const resolveConfiguration = (id: string) => diff --git a/platform/src/lib/sandboxes/providers.server.ts b/platform/src/lib/sandboxes/providers.server.ts index b0449f87..aa219ac0 100644 --- a/platform/src/lib/sandboxes/providers.server.ts +++ b/platform/src/lib/sandboxes/providers.server.ts @@ -10,6 +10,7 @@ import { mapDatabaseErrors } from "@/db/lib/sqlstate" import { sandboxProvider, SandboxProviderCredentialsPayloadSchema } from "@/db/schema/organizations" import { LockVersionSchema, UuidV7Schema } from "@/lib/schemas" import { runSandboxConnectionTest } from "./connection-test.server.ts" +import { SandboxResources } from "./resources.server" import { SandboxCleanupFailed, SandboxConnectionFailed, @@ -72,6 +73,7 @@ export type OrganizationSandboxProvider = SandboxProviderRow & { export interface SandboxProviderConfiguration { readonly name: string + readonly lockVersion: number readonly provider: SandboxProviderId readonly options: SandboxProviderOptions[SandboxProviderId] readonly credentials: SandboxProviderCredentials[SandboxProviderId] @@ -189,6 +191,9 @@ export class SandboxProviders extends Context.Service< SandboxProviders, Effect.gen(function* () { const db = yield* Database + const resources = yield* SandboxResources + const testProvider = (input: Parameters[0]) => + runSandboxConnectionTest(input).pipe(Effect.provideService(SandboxResources, resources)) const readRow = Effect.fnUntraced(function* (organizationId: string, id: string) { const [stored] = yield* db @@ -314,7 +319,8 @@ export class SandboxProviders extends Context.Service< const requiresTest = credentialsChanged || !Equal.equals(existing.row.options, input.options) const tested = requiresTest - ? yield* runSandboxConnectionTest({ + ? yield* testProvider({ + organizationId: input.organizationId, provider: input.providerType, options: input.options, credentials: input.credentials, @@ -361,7 +367,8 @@ export class SandboxProviders extends Context.Service< const found = yield* readRow(input.organizationId, input.id) if (found?.row.lockVersion !== input.lockVersion) return yield* new SandboxProviderChanged() if (Option.isNone(found.credentials)) return yield* new SandboxProviderUnreadable() - const result = yield* runSandboxConnectionTest({ + const result = yield* testProvider({ + organizationId: input.organizationId, provider: found.row.providerType, options: found.row.options, credentials: found.credentials.value, @@ -403,6 +410,7 @@ export class SandboxProviders extends Context.Service< if (Option.isNone(found.credentials)) return yield* new SandboxProviderUnreadable() return { name: found.row.name, + lockVersion: found.row.lockVersion, provider: found.row.providerType, options: found.row.options, credentials: found.credentials.value, @@ -421,5 +429,7 @@ export class SandboxProviders extends Context.Service< }), ) - static readonly layer = SandboxProviders.layerNoDeps.pipe(Layer.provide(Database.layer)) + static readonly layer = SandboxProviders.layerNoDeps.pipe( + Layer.provide([Database.layer, SandboxResources.layer]), + ) } diff --git a/platform/src/lib/sandboxes/resources.integration.test.ts b/platform/src/lib/sandboxes/resources.integration.test.ts new file mode 100644 index 00000000..219d0cac --- /dev/null +++ b/platform/src/lib/sandboxes/resources.integration.test.ts @@ -0,0 +1,214 @@ +import type { SandboxHandle, SandboxProvider } from "@tanstack/ai-sandbox" +import { eq, sql } from "drizzle-orm" +import { Effect, Layer, Scope, Exit, Context } from "effect" +import { beforeEach, describe, expect, test } from "vitest" +import { TestClock } from "effect/testing" + +import { Database, getAuthDatabase } from "@/db/database" +import { organization } from "@/db/schema/organizations" +import { sandboxResource } from "@/db/schema/sandboxes" +import { SandboxProviderFactory } from "./factory.server" +import { SandboxResources } from "./resources.server" + +const url = process.env.DATABASE_URL +const enabled = !!url && url !== "postgres://test:test@127.0.0.1:5432/test" +if ( + enabled && + (new URL(url).hostname !== "127.0.0.1" || !new URL(url).pathname.endsWith("_test")) +) { + throw new Error("Use a disposable loopback database ending in _test") +} +const remote = new Set() +let failDelete = false +let createGate: Promise | undefined +let created: (() => void) | undefined +let createFailure: Error | undefined +let rejectedBeforeCreation = false +const adapter = { + name: "daytona", + capabilities: () => ({}), + create: async ({ id }: { id: string }) => { + expect( + await getAuthDatabase() + .select() + .from(sandboxResource) + .where(eq(sandboxResource.providerSandboxId, id)), + ).toHaveLength(1) + remote.add(id) + created?.() + await createGate + if (createFailure) throw createFailure + return { id, provider: "daytona" } as SandboxHandle + }, + resume: ({ id }: { id: string }) => + rejectedBeforeCreation && createFailure + ? Promise.reject(createFailure) + : Promise.resolve(remote.has(id) ? ({ id } as SandboxHandle) : null), + destroy: ({ id }: { id: string }) => { + if (rejectedBeforeCreation && createFailure) return Promise.reject(createFailure) + if (failDelete) return Promise.reject(new Error("provider offline")) + remote.delete(id) + return Promise.resolve() + }, +} as unknown as SandboxProvider +const layer = SandboxResources.layerNoDeps.pipe( + Layer.provide([ + Database.layer, + Layer.succeed(SandboxProviderFactory, { create: () => Effect.succeed(adapter) }), + ]), +) +const run = (effect: Effect.Effect) => + Effect.runPromise(effect.pipe(Effect.provide(layer))) + +describe.skipIf(!enabled)("durable sandbox ownership", () => { + const db = getAuthDatabase() + let organizationId: string + const input = () => ({ + organizationId, + provider: "daytona" as const, + options: { target: "us", snapshot: "test" }, + credentials: { apiKey: "original-credential" }, + }) + beforeEach(async () => { + await db.execute(sql`truncate organization cascade`) + const [row] = await db + .insert(organization) + .values({ name: "Sandbox recovery", slug: "sandbox-recovery" }) + .returning() + organizationId = row!.id + remote.clear() + failDelete = false + createGate = undefined + created = undefined + createFailure = undefined + rejectedBeforeCreation = false + }) + + test("another owner retries cleanup after expiry and cannot revive a retiring workspace", async () => { + const id = await run( + Effect.gen(function* () { + const resources = yield* SandboxResources + const provider = yield* resources.provider(input()) + const handle = yield* Effect.promise(() => provider.create({})) + yield* resources.touch(organizationId, handle.id) + yield* resources.reap() + expect(remote.has(handle.id)).toBe(true) + failDelete = true + yield* Effect.tryPromise(() => handle.destroy()).pipe(Effect.exit) + expect((yield* Effect.flip(resources.touch(organizationId, handle.id)))._tag).toBe( + "SandboxResourceUnavailable", + ) + return handle.id + }), + ) + const stored = ( + await db.execute<{ stored: string }>( + sql`select configuration as stored from sandbox_resource`, + ) + ).rows + expect(stored[0]!.stored).not.toContain("original-credential") + await db.update(sandboxResource).set({ expiresAt: sql`now() - interval '1 minute'` }) + failDelete = true + await run(Effect.flatMap(SandboxResources, (resources) => resources.reap())) + expect(remote.has(id)).toBe(true) + expect((await db.select().from(sandboxResource))[0]?.retiredAt).toBeInstanceOf(Date) + await expect( + run(Effect.flatMap(SandboxResources, (resources) => resources.touch(organizationId, id))), + ).rejects.toThrow() + failDelete = false + await db.update(sandboxResource).set({ expiresAt: sql`now() - interval '1 minute'` }) + await run(Effect.flatMap(SandboxResources, (resources) => resources.reap())) + expect(remote.size).toBe(0) + expect(await db.select().from(sandboxResource)).toEqual([]) + }) + + test("retains an uncertain creation intent and cleans a late result after ownership is lost", async () => { + const gate = Promise.withResolvers() + const started = Promise.withResolvers() + createGate = gate.promise + created = started.resolve + const scope = Effect.runSync(Scope.make()) + const context = await Effect.runPromise(Layer.buildWithScope(layer, scope)) + const resources = Context.get(context, SandboxResources) + const provider = await Effect.runPromise(resources.provider(input())) + const pending = provider.create({}).then( + () => true, + () => false, + ) + try { + await started.promise + await Effect.runPromise(Scope.close(scope, Exit.void)) + await db.update(sandboxResource).set({ expiresAt: sql`now() - interval '1 minute'` }) + await run(Effect.flatMap(SandboxResources, (nextOwner) => nextOwner.reap())) + expect(remote.size).toBe(0) + const rows = await db.select().from(sandboxResource) + expect(rows).toHaveLength(1) + expect(rows[0]?.settled).toBe(false) + expect(rows[0]?.retiredAt).toBeInstanceOf(Date) + gate.resolve() + expect(await pending).toBe(false) + expect(await db.select().from(sandboxResource)).toEqual([]) + } finally { + gate.resolve() + await Effect.runPromise(Scope.close(scope, Exit.void)) + await pending + } + }) + + test("does not renew an orphan when saving its creation outcome fails", async () => { + await db.execute(sql`create function fail_sandbox_settlement() returns trigger language plpgsql as $$ + begin if new.settled then raise exception 'injected settlement failure'; end if; return new; end $$`) + await db.execute(sql`create trigger fail_sandbox_settlement before update on sandbox_resource + for each row execute function fail_sandbox_settlement()`) + try { + await Effect.runPromise( + Effect.gen(function* () { + const resources = yield* SandboxResources + const provider = yield* resources.provider(input()) + yield* Effect.tryPromise(() => provider.create({})).pipe(Effect.flip) + yield* Effect.promise(() => + db.execute(sql`drop trigger fail_sandbox_settlement on sandbox_resource`), + ) + yield* Effect.promise(() => + db.update(sandboxResource).set({ expiresAt: sql`now() + interval '10 minutes'` }), + ) + const [before] = yield* Effect.promise(() => db.select().from(sandboxResource)) + yield* TestClock.adjust("1 minute") + const [after] = yield* Effect.promise(() => db.select().from(sandboxResource)) + expect(after!.expiresAt).toEqual(before!.expiresAt) + expect(after!.settled).toBe(false) + }).pipe(Effect.provide(layer), Effect.provide(TestClock.layer())), + ) + } finally { + await db.execute(sql`drop trigger if exists fail_sandbox_settlement on sandbox_resource`) + await db.execute(sql`drop function fail_sandbox_settlement()`) + } + }) + + test("discards rejected authentication but retains a lost creation response for reconciliation", async () => { + const attempt = () => + run( + Effect.gen(function* () { + const resources = yield* SandboxResources + const provider = yield* resources.provider(input()) + yield* Effect.tryPromise(() => provider.create({})).pipe(Effect.flip) + }), + ) + rejectedBeforeCreation = true + createFailure = Object.assign(new Error("Unauthorized"), { status: 401 }) + await attempt() + expect(await db.select().from(sandboxResource)).toEqual([]) + rejectedBeforeCreation = false + createFailure = new Error("lost creation response") + await attempt() + const rows = await db.select().from(sandboxResource) + expect(rows).toHaveLength(1) + expect(rows[0]!.settled).toBe(false) + await db.update(sandboxResource).set({ + retiredAt: sql`now() - interval '25 hours'`, + expiresAt: sql`now() - interval '1 minute'`, + }) + await run(Effect.flatMap(SandboxResources, (resources) => resources.reap())) + expect(await db.select().from(sandboxResource)).toEqual([]) + }) +}) diff --git a/platform/src/lib/sandboxes/resources.server.ts b/platform/src/lib/sandboxes/resources.server.ts new file mode 100644 index 00000000..a58d09d5 --- /dev/null +++ b/platform/src/lib/sandboxes/resources.server.ts @@ -0,0 +1,322 @@ +import type { SandboxHandle, SandboxProvider } from "@tanstack/ai-sandbox" +import { and, eq, gt, inArray, isNull, or, sql } from "drizzle-orm" +import { Context, Effect, Layer, Result, Schema } from "effect" + +import { Database } from "@/db/database" +import { sandboxResource } from "@/db/schema/sandboxes" +import { mapDatabaseErrors } from "@/db/lib/sqlstate" +import { APP_HANDLE } from "@/lib/constants" +import { sandboxCall, logSandboxFailure, sandboxErrorCode } from "./operations.server" +import { + SandboxCleanupFailed, + SandboxResourceUnavailable, + SandboxProviderUnavailable, +} from "./errors" +import { SandboxProviderFactory } from "./factory.server" +import { SandboxProviderConfigurationSchema, type SandboxProviderId } from "./schemas" + +const expiry = sql`now() + interval '15 minutes'` +const retryAt = sql`now() + interval '5 minutes'` +const retiredAt = sql`coalesce(${sandboxResource.retiredAt}, now())` +type ResourceIdentity = { organizationId: string; id: string } +interface SandboxResourceInput { + organizationId: string + provider: SandboxProviderId + options: unknown + credentials: unknown +} +const resourceWhere = ({ organizationId, id }: ResourceIdentity) => + and(eq(sandboxResource.organizationId, organizationId), eq(sandboxResource.id, id)) + +/** Durable ownership survives callers, provider edits, and process loss. */ +export class SandboxResources extends Context.Service< + SandboxResources, + { + readonly provider: ( + input: SandboxResourceInput, + ) => Effect.Effect + readonly touch: ( + organizationId: string, + providerSandboxId: string, + ) => Effect.Effect + readonly reap: (organizationId?: string) => Effect.Effect + readonly purgeOrganization: ( + organizationId: string, + ) => Effect.Effect + } +>()("astralbeam/sandboxes/SandboxResources") { + static readonly layerNoDeps = Layer.effect( + SandboxResources, + Effect.gen(function* () { + const db = yield* Database + const factory = yield* SandboxProviderFactory + const run = Effect.runPromiseWith(yield* Effect.context()) + const ownerId = crypto.randomUUID() + const owned = new Map() + let closed = false + yield* Effect.addFinalizer(() => + Effect.sync(() => { + closed = true + }), + ) + + yield* Effect.suspend(() => + owned.size === 0 + ? Effect.void + : db + .update(sandboxResource) + .set({ expiresAt: expiry }) + .where( + and( + eq(sandboxResource.ownerId, ownerId), + inArray(sandboxResource.id, [...owned.values()]), + isNull(sandboxResource.retiredAt), + gt(sandboxResource.expiresAt, sql`now()`), + ), + ) + .pipe(Effect.asVoid), + ).pipe( + Effect.catchCause(() => Effect.logWarning("Sandbox ownership renewal failed")), + Effect.delay("1 minute"), + Effect.forever, + Effect.forkScoped, + ) + + const touch = Effect.fn("SandboxResources.touch")(function* ( + organizationId: string, + providerSandboxId: string, + ) { + if (closed) return yield* new SandboxResourceUnavailable() + const renewed = yield* db + .update(sandboxResource) + .set({ expiresAt: expiry }) + .where( + and( + eq(sandboxResource.organizationId, organizationId), + eq(sandboxResource.providerSandboxId, providerSandboxId), + eq(sandboxResource.ownerId, ownerId), + isNull(sandboxResource.retiredAt), + gt(sandboxResource.expiresAt, sql`now()`), + ), + ) + .returning({ id: sandboxResource.id }) + .pipe(Effect.catchCause(() => Effect.fail(new SandboxResourceUnavailable()))) + if (renewed.length === 0) { + owned.delete(providerSandboxId) + return yield* new SandboxResourceUnavailable() + } + }) + + const destroy = Effect.fnUntraced(function* (identity: ResourceIdentity) { + const [row] = yield* db + .update(sandboxResource) + .set({ retiredAt, expiresAt: retryAt }) + .where(resourceWhere(identity)) + .returning() + if (!row) return + owned.delete(row.providerSandboxId) + const payload = row.configuration + if ( + payload.organizationId !== row.organizationId || + payload.resourceId !== row.id || + payload.providerSandboxId !== row.providerSandboxId + ) { + return yield* Effect.die(new Error("Sandbox cleanup configuration identity mismatch")) + } + const provider = yield* factory.create( + payload.configuration.providerType, + payload.configuration, + ) + yield* sandboxCall( + (signal) => provider.destroy({ id: row.providerSandboxId, signal }), + "30 seconds", + ) + // A crashed create may still finish remotely. Reconcile its name for a day before retiring the intent. + yield* db + .delete(sandboxResource) + .where( + and( + resourceWhere(identity), + or( + eq(sandboxResource.settled, true), + sql`${sandboxResource.retiredAt} < now() - interval '1 day'`, + ), + ), + ) + }) + + const reap = Effect.fn("SandboxResources.reap")(function* ( + organizationId: string | undefined, + ) { + const rows = yield* db + .update(sandboxResource) + .set({ retiredAt, expiresAt: retryAt }) + .where(sql`(${sandboxResource.organizationId}, ${sandboxResource.id}) in ( + select organization_id, id from sandbox_resource + where ${organizationId ? sql`organization_id = ${organizationId}` : sql`true`} + and ${organizationId ? sql`true` : sql`expires_at <= now()`} + order by expires_at, id limit 25 for update skip locked + )`) + .returning({ organizationId: sandboxResource.organizationId, id: sandboxResource.id }) + yield* Effect.forEach( + rows, + (row) => + destroy(row).pipe( + Effect.catchTag("SandboxOperationFailed", (error) => + logSandboxFailure("reconcile", error), + ), + Effect.catchCause(() => Effect.logWarning("Sandbox cleanup will retry")), + Effect.annotateLogs({ organizationId: row.organizationId, resourceId: row.id }), + ), + { concurrency: 4, discard: true }, + ) + }, Effect.orDie) + + const purgeOrganization = Effect.fn("SandboxResources.purgeOrganization")(function* ( + organizationId: string, + ) { + yield* reap(organizationId) + const remaining = yield* db + .select({ id: sandboxResource.id }) + .from(sandboxResource) + .where(eq(sandboxResource.organizationId, organizationId)) + .limit(1) + .pipe(mapDatabaseErrors()) + if (remaining.length > 0) return yield* new SandboxCleanupFailed() + }) + + const provider = Effect.fn("SandboxResources.provider")(function* ( + input: SandboxResourceInput, + ) { + const adapter = yield* factory.create(input.provider, input) + const configuration = yield* Schema.decodeUnknownEffect(SandboxProviderConfigurationSchema)( + { + providerType: input.provider, + options: input.options, + credentials: input.credentials, + }, + ).pipe(Effect.mapError((cause) => new SandboxProviderUnavailable({ cause }))) + const remove = async (id: string) => { + owned.delete(id) + const rows = await run( + db + .select({ id: sandboxResource.id, organizationId: sandboxResource.organizationId }) + .from(sandboxResource) + .where( + and( + eq(sandboxResource.organizationId, input.organizationId), + eq(sandboxResource.ownerId, ownerId), + eq(sandboxResource.providerSandboxId, id), + ), + ), + ) + for (const row of rows) await run(destroy(row)) + } + const wrap = (handle: SandboxHandle, id: string): SandboxHandle => ({ + id, + provider: handle.provider, + ...(handle.workspaceRoot === undefined ? {} : { workspaceRoot: handle.workspaceRoot }), + capabilities: { ...handle.capabilities, snapshots: false, fork: false }, + fs: handle.fs, + git: handle.git, + process: handle.process, + ports: handle.ports, + env: handle.env, + destroy: () => remove(id), + }) + return { + name: adapter.name, + capabilities: () => ({ ...adapter.capabilities(), snapshots: false, fork: false }), + create: async (options) => { + if (closed) throw new SandboxResourceUnavailable() + const { + rows: [identity], + } = await run(db.execute<{ id: string }>(sql`select uuidv7() as id`)) + const id = identity!.id + const providerSandboxId = `${APP_HANDLE}-${id}` + const row = { organizationId: input.organizationId, id } + // Probe a fresh name before retaining credentials for an allocation that may never be accepted. + if ( + await adapter.resume({ + id: providerSandboxId, + ...(options.signal ? { signal: options.signal } : {}), + }) + ) { + throw new SandboxResourceUnavailable() + } + options.signal?.throwIfAborted() + await run( + db.insert(sandboxResource).values({ + ...row, + ownerId, + providerSandboxId, + expiresAt: expiry, + configuration: { + organizationId: input.organizationId, + resourceId: id, + providerSandboxId, + configuration, + }, + }), + ) + owned.set(providerSandboxId, id) + let succeeded = false + let settled = true + try { + if (closed) throw new SandboxResourceUnavailable() + options.signal?.throwIfAborted() + settled = false + const handle = await adapter + .create({ ...options, id: providerSandboxId }) + .catch((cause: unknown) => { + const code = sandboxErrorCode(cause) + settled = code !== "timeout" && code !== "cancelled" && code !== "provider_error" + throw cause + }) + settled = true + await run(touch(input.organizationId, providerSandboxId)) + options.signal?.throwIfAborted() + succeeded = true + return wrap(handle, providerSandboxId) + } finally { + if (!succeeded) owned.delete(providerSandboxId) + await run( + db + .update(sandboxResource) + .set({ + settled, + ...(!succeeded ? { retiredAt, expiresAt: sql`now()` } : {}), + }) + .where(resourceWhere(row)), + ).catch((cause: unknown) => { + owned.delete(providerSandboxId) + throw cause + }) + if (!succeeded) + await run( + destroy(row).pipe( + Effect.catchCause(() => Effect.logWarning("Sandbox cleanup will retry")), + ), + ) + } + }, + resume: async ({ id, signal }) => { + const ownership = await run(touch(input.organizationId, id).pipe(Effect.result)) + if (Result.isFailure(ownership)) { + await remove(id) + return null + } + const handle = await adapter.resume({ id, ...(signal ? { signal } : {}) }) + if (!handle) await remove(id) + return handle ? wrap(handle, id) : null + }, + destroy: ({ id }) => remove(id), + } satisfies SandboxProvider + }) + return { provider, touch, reap, purgeOrganization } + }), + ) + static readonly layer = SandboxResources.layerNoDeps.pipe( + Layer.provide([Database.layer, SandboxProviderFactory.layer]), + ) +} diff --git a/platform/src/lib/workflows/cron.ts b/platform/src/lib/workflows/cron.ts index 5da7641d..3b72baf2 100644 --- a/platform/src/lib/workflows/cron.ts +++ b/platform/src/lib/workflows/cron.ts @@ -4,8 +4,16 @@ import { ClusterCron } from "effect/cluster" import fileMaintenance from "./file-maintenance.ts" import expiredCacheCleanup from "./expired-cache-cleanup.ts" import modelPriceCatalogRefresh from "./model-price-catalog-refresh.ts" +import sandboxMaintenance from "./sandbox-maintenance.ts" export const scheduledWorkflowsLayer = Layer.mergeAll( + ClusterCron.make({ + name: "SandboxMaintenance/v1", + cron: Cron.parseUnsafe("0 * * * * *", "UTC"), + calculateNextRunFromPrevious: false, + skipIfOlderThan: "10 minutes", + execute: sandboxMaintenance, + }), ClusterCron.make({ name: "ExpiredCacheCleanup/v1", cron: Cron.parseUnsafe("0 */5 * * * *", "UTC"), diff --git a/platform/src/lib/workflows/delete-organization.integration.test.ts b/platform/src/lib/workflows/delete-organization.integration.test.ts index c599284c..c619209e 100644 --- a/platform/src/lib/workflows/delete-organization.integration.test.ts +++ b/platform/src/lib/workflows/delete-organization.integration.test.ts @@ -4,6 +4,7 @@ import { Cause, Effect, Exit, Fiber, Layer } from "effect" import { SqlClient } from "effect/sql" import { TestClock } from "effect/testing" import { WorkflowEngine } from "effect/workflow" +import type { SandboxProvider } from "@tanstack/ai-sandbox" import { beforeEach, describe, expect, test, vi } from "vitest" const deleteOrganizationIntegration = vi.hoisted(() => { @@ -55,6 +56,12 @@ import { user, } from "@/db/schema" import { ObjectStorage } from "@/lib/storage/object-storage.server" +import { SandboxResources } from "@/lib/sandboxes/resources.server" +import { SandboxProviderFactory } from "@/lib/sandboxes/factory.server" +import { sandboxResource } from "@/db/schema/sandboxes" +import { APP_HANDLE } from "@/lib/constants" + +const removedOrganizationSandboxes: string[] = [] const removedOrganizationFiles: string[] = [] @@ -290,6 +297,23 @@ describe.skipIf(!deleteOrganizationIntegration.url)("organization deletion workf removedOrganizationFiles.push(key) }), } as unknown as ObjectStorage["Service"]), + Effect.provide( + SandboxResources.layerNoDeps.pipe( + Layer.provide([ + Database.layerNoDeps, + Layer.succeed(SandboxProviderFactory, { + create: () => + Effect.succeed({ + destroy: ({ id }: { id: string }) => { + removedOrganizationSandboxes.push(id) + return Promise.resolve() + }, + } as unknown as SandboxProvider), + }), + ]), + Layer.fresh, + ), + ), ) } @@ -781,6 +805,26 @@ describe.skipIf(!deleteOrganizationIntegration.url)("organization deletion workf }, ]) + for (const organizationId of [deletedId, keptId]) { + const { + rows: [identity], + } = await db.execute<{ id: string }>(sql`select uuidv7() as id`) + const id = identity!.id + await db.insert(sandboxResource).values({ + organizationId, + id, + ownerId: crypto.randomUUID(), + providerSandboxId: `${APP_HANDLE}-${id}`, + configuration: { + organizationId, + resourceId: id, + providerSandboxId: `${APP_HANDLE}-${id}`, + configuration: { providerType: "docker", options: { image: "node:22" }, credentials: {} }, + }, + settled: true, + expiresAt: new Date(Date.now() + 15 * 60_000), + }) + } expect(await runAppEffect(revokeOrganizationAccess(deletedId))).toEqual([ownerId]) expect(await db.select().from(member).where(eq(member.organizationId, deletedId))).toEqual([]) @@ -810,6 +854,10 @@ describe.skipIf(!deleteOrganizationIntegration.url)("organization deletion workf ]) const remaining = await db.select({ id: organization.id }).from(organization) expect(remaining).toEqual([{ id: keptId }]) + const sandboxes = await db.select().from(sandboxResource) + expect(sandboxes.map((row) => row.organizationId)).toEqual([keptId]) + expect(removedOrganizationSandboxes).not.toContain(sandboxes[0]!.providerSandboxId) + expect(removedOrganizationSandboxes.length).toBeGreaterThan(0) const tenantUsers = await db .select({ organizationId: tenantUser.organizationId }) .from(tenantUser) diff --git a/platform/src/lib/workflows/delete-organization.ts b/platform/src/lib/workflows/delete-organization.ts index 4c0098bd..60eb0a6e 100644 --- a/platform/src/lib/workflows/delete-organization.ts +++ b/platform/src/lib/workflows/delete-organization.ts @@ -3,6 +3,7 @@ import { SqlError } from "effect/sql" import { Activity, Workflow } from "effect/workflow" import { Mailer } from "../email/email.server.ts" +import { SandboxResources } from "../sandboxes/resources.server" import { deleteOrganization as deleteOrganizationFn, deleteOrganizationTenantBatch, @@ -72,6 +73,7 @@ const notifyOrganizationOwners = Effect.fn("notifyOrganizationOwners")(function* export const deleteOrganizationWorkflowLayer = deleteOrganization.toLayer((payload) => Effect.gen(function* () { const { organizationId } = payload + const sandboxes = yield* SandboxResources yield* purgeOrganizationStep( "DeleteTenants", drainOrganizationBatches(deleteOrganizationTenantBatch(organizationId)), @@ -80,6 +82,7 @@ export const deleteOrganizationWorkflowLayer = deleteOrganization.toLayer((paylo "DeleteFiles", drainOrganizationBatches(deleteOrganizationFileBatch(organizationId)), ) + yield* purgeOrganizationStep("DeleteSandboxes", sandboxes.purgeOrganization(organizationId)) yield* purgeOrganizationStep("DeleteOrganization", deleteOrganizationFn(organizationId)) yield* Effect.logInfo("Organization deleted", { organizationId }) yield* Activity.make({ name: "NotifyOwners", execute: notifyOrganizationOwners(payload) }) diff --git a/platform/src/lib/workflows/sandbox-maintenance.ts b/platform/src/lib/workflows/sandbox-maintenance.ts new file mode 100644 index 00000000..c8ca1efa --- /dev/null +++ b/platform/src/lib/workflows/sandbox-maintenance.ts @@ -0,0 +1,7 @@ +import { Effect } from "effect" + +import { SandboxResources } from "../sandboxes/resources.server" + +const sandboxMaintenance = Effect.flatMap(SandboxResources, (resources) => resources.reap()) + +export default sandboxMaintenance