Skip to content
Merged
  •  
  •  
  •  
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ export function AutomationEditPage(props: { enabled: boolean }) {
const value = summary.data;
const status =
value?.kind === "event"
? value.status === "paused"
? "paused"
? value.status !== "active"
? value.status
: value.triggerAvailable
? "active"
: "unavailable"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ export function AutomationLifecycle({
// Public readers can edit settings, but only the creator controls lifecycle.
const canEdit = !completed;
const canPause = automation.ownedByViewer && !completed;
const blockReason =
automation.kind === "scheduled" ? automation.statusReason : undefined;
const blockReason = automation.statusReason;
const action = paused || blocked ? "resume" : "pause";
const mutation = useMutation({
mutationFn: async () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ function automationStatus(automation: AutomationSummary): string | undefined {
if (automation.kind === "scheduled") {
return automation.status === "active" ? undefined : automation.status;
}
if (automation.status === "paused") return "paused";
if (automation.status !== "active") return automation.status;
return automation.triggerAvailable ? undefined : "Trigger unavailable";
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,42 @@ describe("Event automation management", () => {
).toEqual([]);
});

test("when asked to pause and then resume an event automation, change its status", async ({
run,
}) => {
const channel = slackChannel();
const { id } = await insertEventAutomation({
destination: channel,
task: "Summarize issue closures and reopenings in this channel.",
trigger: issueTrigger,
});

const paused = await run(
slackMention("Pause the issue event automation in this channel.", {
channel,
}),
);
const resumed = await paused.continue(slackMention("Resume it now."));

expect(
completedToolCalls("updateEventAutomation", paused, resumed).map(
toolOutput,
),
).toEqual([
expect.objectContaining({
automation: expect.objectContaining({ id, status: "paused" }),
}),
expect.objectContaining({
automation: expect.objectContaining({ id, status: "active" }),
}),
]);
const names = [...paused.toolCalls, ...resumed.toolCalls].map(
(call) => call.name,
);
expect(names).not.toContain("deleteEventAutomation");
expect(names).not.toContain("createEventAutomation");
});

test("when a stored automation's plugin event is unavailable, explain that it cannot currently run", async ({
agent,
}) => {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
{
"session": "evals/skills/routing.eval.ts > Skill Invocation Control > rejects a user-callable skill when the user does not invoke it",
"request": {
"method": "POST",
"url": "https://ai-gateway.vercel.sh/v1/messages?beta=true",
"parts": {
"method": "9aee6b1bcdf6",
"url": "ebadb6691a7f",
"header ai-language-model-id": "e3b0c44298fc",
"header ai-language-model-streaming": "e3b0c44298fc",
"header ai-model-id": "e3b0c44298fc",
"model": "1a0a1ddcb528",
"messages.length": "4b227777d4dd",
"messages[0]": "d96f74aa8437",
"messages[1]": "9dfce496fcd4",
"messages[2]": "a26cc989b778",
"messages[3]": "b03e68a1a2a9",
"max_tokens": "8eda794b86cb",
"stream": "b5bea41b6c62",
"system.length": "6b86b273ff34",
"system[0]": "2703ccbf40ea",
"tools.length": "031b4af5197e",
"tools[0]": "d098cd1dd937",
"tools[1]": "12dba46828ca",
"tools[2]": "a1056c279456",
"tools[3]": "c40e21f9ef0d",
"tools[4]": "d3dd7b468c8f",
"tools[5]": "f116e66fefdf",
"tools[6]": "6653a0a0a3a9",
"tools[7]": "3b6ca120b0fb",
"tools[8]": "c7dc1bc2c04d",
"tools[9]": "f9054d6c4f64",
"tools[10]": "f47ca0d0af13",
"tools[11]": "37bbd2f6c311",
"tools[12]": "5802c8f4c428",
"tools[13]": "ae045899dc37",
"tools[14]": "218843320f4d",
"tools[15]": "0667eb3022b2",
"tools[16]": "e8153c6c2dbc",
"tools[17]": "b9a89d424cf1",
"tools[18]": "880078b06abf",
"tools[19]": "0523c1434052",
"tools[20]": "7e58b0563fbf",
"tools[21]": "bd224b2c3451",
"tools[22]": "a8e038af6b5c",
"tools[23]": "e9b56330a97a",
"tools[24]": "93f913ef2874",
"tools[25]": "cb8897349d4d",
"tools[26]": "95c787875e54",
"tools[27]": "23162886a934",
"tools[28]": "ed13dd55d6be",
"tools[29]": "06ea3d798e31",
"tools[30]": "98cd636f11ec",
"tools[31]": "00fbcd9a6ca2",
"tools[32]": "783be0c88076",
"tools[33]": "36e632f69c1b",
"tools[34]": "d654c4e6deae",
"tools[35]": "1d3e01fc3ee5",
"tools[36]": "029ed1121aba",
"tools[37]": "f1dc711e0b3b",
"tools[38]": "d313ceccf30b",
"tools[39]": "6a4ecbd5c2c1",
"tools[40]": "830753389b93",
"tools[41]": "972897611330",
"tools[42]": "8ad2fe0d1222",
"tools[43]": "0c464691ffdf",
"tools[44]": "e53b13b6969a",
"tools[45]": "dbf7dc650f07",
"tools[46]": "18668b22891c",
"tools[47]": "fac57491edc9",
"tools[48]": "3333210791f6",
"tools[49]": "e4e3f296282f",
"tools[50]": "ec59e4ad36dd",
"thinking": "e871cb1e753e"
}
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream"
},
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"gen_01M4GX94SW8PTG8BRWSFGRVCEV\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"openai/gpt-6-luna\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"`loadSkill(\\\"weather-lookup\\\")` returned: **Unknown skill**. Available skills are `agent-browser`, `demo-video`, `eval-directory`, `eval-guardian-actions`, `eval-mcp`, `incident-brief`, `jr-rpc`, `resilient-working-directory`, `source-handbook`, and `visual-web-qa`.\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":3,\"output_tokens\":75,\"cache_creation_input_tokens\":108,\"cache_read_input_tokens\":11310},\"provider_metadata\":{\"openai\":{\"responseId\":\"resp_033b9818677bd4be016ac92c2e09bc87d18f390a5cb98c7bca\",\"serviceTier\":\"default\",\"reasoningContext\":\"all_turns\"},\"gateway\":{\"routing\":{\"originalModelId\":\"openai/gpt-6-luna\",\"resolvedProvider\":\"openai\",\"fallbacksAvailable\":[\"bedrock\",\"azure\"],\"planningReasoning\":\"System credentials planned for: openai, bedrock, azure. Total execution order: openai(system) → bedrock(system) → azure(system)\",\"canonicalSlug\":\"openai/gpt-6-luna\",\"finalProvider\":\"openai\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"openai/gpt-6-luna\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"openai\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1791568941931,\"endTime\":1791568943929,\"providerRequestId\":\"req_fd026cce6e794182b1e4b3708b38881b\",\"statusCode\":200,\"providerResponseId\":\"resp_033b9818677bd4be016ac92c2e09bc87d18f390a5cb98c7bca\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"confirmed\",\"pinnedProvider\":\"openai\"},\"clientSessionId\":\"b21264a3f2d3e49152a8e3e67322a367\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.0001644\",\"marketCost\":\"0.0001644\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.0001644\",\"inferenceCost\":\"0.0001644\",\"inputInferenceCost\":\"0.0001269\",\"outputInferenceCost\":\"0.0000375\",\"generationId\":\"gen_01M4GX94SW8PTG8BRWSFGRVCEV\",\"warnings\":[{\"type\":\"compatibility\",\"feature\":\"maxOutputTokens\",\"details\":\"maxOutputTokens 128000 exceeds the output token limit of 54000 tokens (max output plus thinking budget) for openai/gpt-6-luna and was lowered to 54000.\"}]}}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
"bodyEncoding": "utf8"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
{
"session": "evals/memory/user.eval.ts > User Memory > uses a remembered timezone when answering the current time",
"request": {
"method": "POST",
"url": "https://ai-gateway.vercel.sh/v1/messages?beta=true",
"parts": {
"method": "9aee6b1bcdf6",
"url": "ebadb6691a7f",
"header ai-language-model-id": "e3b0c44298fc",
"header ai-language-model-streaming": "e3b0c44298fc",
"header ai-model-id": "e3b0c44298fc",
"model": "1a0a1ddcb528",
"messages.length": "d4735e3a265e",
"messages[0]": "cf82c4b6df83",
"messages[1]": "8036e15552ea",
"max_tokens": "8eda794b86cb",
"stream": "b5bea41b6c62",
"system.length": "6b86b273ff34",
"system[0]": "2703ccbf40ea",
"tools.length": "031b4af5197e",
"tools[0]": "d098cd1dd937",
"tools[1]": "12dba46828ca",
"tools[2]": "a1056c279456",
"tools[3]": "c40e21f9ef0d",
"tools[4]": "d3dd7b468c8f",
"tools[5]": "f116e66fefdf",
"tools[6]": "6653a0a0a3a9",
"tools[7]": "3b6ca120b0fb",
"tools[8]": "c7dc1bc2c04d",
"tools[9]": "f9054d6c4f64",
"tools[10]": "f47ca0d0af13",
"tools[11]": "37bbd2f6c311",
"tools[12]": "5802c8f4c428",
"tools[13]": "ae045899dc37",
"tools[14]": "218843320f4d",
"tools[15]": "0667eb3022b2",
"tools[16]": "e8153c6c2dbc",
"tools[17]": "b9a89d424cf1",
"tools[18]": "880078b06abf",
"tools[19]": "0523c1434052",
"tools[20]": "7e58b0563fbf",
"tools[21]": "bd224b2c3451",
"tools[22]": "a8e038af6b5c",
"tools[23]": "e9b56330a97a",
"tools[24]": "93f913ef2874",
"tools[25]": "cb8897349d4d",
"tools[26]": "95c787875e54",
"tools[27]": "23162886a934",
"tools[28]": "ed13dd55d6be",
"tools[29]": "06ea3d798e31",
"tools[30]": "98cd636f11ec",
"tools[31]": "00fbcd9a6ca2",
"tools[32]": "783be0c88076",
"tools[33]": "36e632f69c1b",
"tools[34]": "d654c4e6deae",
"tools[35]": "1d3e01fc3ee5",
"tools[36]": "029ed1121aba",
"tools[37]": "f1dc711e0b3b",
"tools[38]": "d313ceccf30b",
"tools[39]": "6a4ecbd5c2c1",
"tools[40]": "830753389b93",
"tools[41]": "972897611330",
"tools[42]": "8ad2fe0d1222",
"tools[43]": "0c464691ffdf",
"tools[44]": "e53b13b6969a",
"tools[45]": "dbf7dc650f07",
"tools[46]": "18668b22891c",
"tools[47]": "fac57491edc9",
"tools[48]": "3333210791f6",
"tools[49]": "ee2ee720fce3",
"tools[50]": "ec59e4ad36dd",
"thinking": "e871cb1e753e"
}
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream"
},
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"gen_01M4GWWCVSYC8PGWG8R566JY4F\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"openai/gpt-6-luna\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"call_J5lXgKiPyze0WJpxst8G2sQZ\",\"name\":\"systemTime\",\"input\":{}}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"timezone\\\":\\\"America/Los_Angeles\\\"}\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":3,\"output_tokens\":22,\"cache_creation_input_tokens\":343,\"cache_read_input_tokens\":10351},\"provider_metadata\":{\"openai\":{\"responseId\":\"resp_09a7c68ceba71388016ac92a8c57b487d18b989f28253a2f7f\",\"serviceTier\":\"default\",\"reasoningContext\":\"all_turns\"},\"gateway\":{\"routing\":{\"originalModelId\":\"openai/gpt-6-luna\",\"resolvedProvider\":\"openai\",\"fallbacksAvailable\":[\"bedrock\",\"azure\"],\"planningReasoning\":\"System credentials planned for: openai, bedrock, azure. Total execution order: openai(system) → bedrock(system) → azure(system)\",\"canonicalSlug\":\"openai/gpt-6-luna\",\"finalProvider\":\"openai\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"openai/gpt-6-luna\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"openai\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1791568524243,\"endTime\":1791568526063,\"providerRequestId\":\"req_4327565730ad49cfb912c95d5fac26c5\",\"statusCode\":200,\"providerResponseId\":\"resp_09a7c68ceba71388016ac92a8c57b487d18b989f28253a2f7f\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"no_pin\"},\"clientSessionId\":\"8ce43dc58b36a1f9de7b4088f4e5677c\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.000157685\",\"marketCost\":\"0.000157685\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.000157685\",\"inferenceCost\":\"0.000157685\",\"inputInferenceCost\":\"0.000146685\",\"outputInferenceCost\":\"0.000011\",\"generationId\":\"gen_01M4GWWCVSYC8PGWG8R566JY4F\",\"warnings\":[{\"type\":\"compatibility\",\"feature\":\"maxOutputTokens\",\"details\":\"maxOutputTokens 128000 exceeds the output token limit of 54000 tokens (max output plus thinking budget) for openai/gpt-6-luna and was lowered to 54000.\"}]}}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
"bodyEncoding": "utf8"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
{
"session": "evals/integration/scheduler/move.eval.ts > Schedule Destination Updates > when asked once in the destination channel, update the requester task to deliver here",
"request": {
"method": "POST",
"url": "https://ai-gateway.vercel.sh/v1/messages?beta=true",
"parts": {
"method": "9aee6b1bcdf6",
"url": "ebadb6691a7f",
"header ai-language-model-id": "e3b0c44298fc",
"header ai-language-model-streaming": "e3b0c44298fc",
"header ai-model-id": "e3b0c44298fc",
"model": "7193983c3f82",
"messages.length": "6b86b273ff34",
"messages[0]": "80c09ac425ff",
"max_tokens": "8eda794b86cb",
"stream": "b5bea41b6c62",
"system.length": "6b86b273ff34",
"system[0]": "e2d98e230ee2",
"temperature": "5feceb66ffc8",
"thinking": "d12700b52885"
}
},
"response": {
"status": 200,
"headers": {
"content-type": "text/event-stream"
},
"body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"gen_01M4GSTE4XBC2W8K3PJW4JPX7T\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"model\":\"openai/gpt-5.4\",\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":0,\"output_tokens\":0}}}\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"{\\\"answer\\\":\\\"A\\\",\\\"rationale\\\":\\\"Both assistant messages confirm the weekly planning reminder was moved to the current channel/conversation, and neither asks the user to open the source channel, paste task text, or provide extra confirmation.\\\"}\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null},\"usage\":{\"input_tokens\":439,\"output_tokens\":50},\"provider_metadata\":{\"openai\":{\"responseId\":\"resp_024891f9a5955842016ac91e02837487d191d5bd1a0da2fd9c\",\"serviceTier\":\"default\",\"reasoningContext\":\"current_turn\"},\"gateway\":{\"routing\":{\"originalModelId\":\"openai/gpt-5.4\",\"resolvedProvider\":\"openai\",\"fallbacksAvailable\":[\"azure\"],\"planningReasoning\":\"System credentials planned for: openai, azure. Total execution order: openai(system) → azure(system)\",\"canonicalSlug\":\"openai/gpt-5.4\",\"finalProvider\":\"openai\",\"modelAttemptCount\":1,\"modelAttempts\":[{\"canonicalSlug\":\"openai/gpt-5.4\",\"success\":true,\"providerAttemptCount\":1,\"providerAttempts\":[{\"provider\":\"openai\",\"credentialType\":\"system\",\"success\":true,\"startTime\":1791565314282,\"endTime\":1791565316272,\"providerRequestId\":\"req_8503f0827b7c40a29d0041ff61b429eb\",\"statusCode\":200,\"providerResponseId\":\"resp_024891f9a5955842016ac91e02837487d191d5bd1a0da2fd9c\"}]}],\"totalProviderAttemptCount\":1,\"affinity\":{\"outcome\":\"no_pin\"},\"clientSessionId\":\"d68c8bfaf0ef94cf6c8a231baf234ea2\",\"clientSessionIdSource\":\"fingerprint\"},\"cost\":\"0.0018475\",\"marketCost\":\"0.0018475\",\"surchargeCost\":\"0\",\"gatewayCost\":\"0.0018475\",\"inferenceCost\":\"0.0018475\",\"inputInferenceCost\":\"0.0010975\",\"outputInferenceCost\":\"0.00075\",\"generationId\":\"gen_01M4GSTE4XBC2W8K3PJW4JPX7T\",\"warnings\":[{\"type\":\"compatibility\",\"feature\":\"maxOutputTokens\",\"details\":\"maxOutputTokens 128000 exceeds the output token limit of 70000 tokens (max output plus thinking budget) for openai/gpt-5.4 and was lowered to 70000.\"}]}}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
"bodyEncoding": "utf8"
}
}
Loading
Loading