feat(automations): Run automations in their own mode with a declared result

TELEMETRY.md

@@ -215,7 +215,8 @@ Events: `agent.message.received`, `agent.message.generated`,
 `agent.turn.timed_out`,
 `agent.turn.provider_error`, `agent.turn.execution.failed`,
 `agent.turn.empty_output.retrying`,
-`agent.turn.empty_output.exhausted`, `assistant.reply.generation.failed`,
+`agent.turn.empty_output.exhausted`, `agent.turn.automation_result.reminded`,
+`assistant.reply.generation.failed`,
 `guardian.action_review.retrying`, `guardian.action_review.exhausted`
 
 `guardian.action_review.exhausted` is a tool-boundary Sentry capture after three
@@ -226,7 +227,7 @@ Spans: `ai.generate_assistant_reply`, `ai.chat_completion`,
 `chat.route_thinking`, `gen_ai.invoke_agent`, `gen_ai.chat`
 
 Attributes: `gen_ai.operation.name`, `gen_ai.request.model`,
-`gen_ai.response.finish_reasons`, `app.ai.outcome`,
+`gen_ai.response.finish_reasons`, `app.ai.outcome`, `app.automation.result`,
 `app.ai.reasoning_effort`, `app.ai.model_profile`, `gen_ai.usage.input_tokens`,
 `gen_ai.usage.output_tokens`, `gen_ai.usage.input_tokens.cached`,
 `gen_ai.usage.input_tokens.cache_write`, `app.ai.reasoning_tokens`,

packages/junior-evals/README.md

@@ -168,8 +168,10 @@ describe("Thread Continuity", () => {
 - `run(heartbeat())` returns the Conversation that a due automation started.
   Make an automation due with `insertScheduledAutomation({ due: true })`.
   `run(githubWebhook(...))` returns the Conversation that a matching
-  `insertEventAutomation()` started. The agent needs the GitHub plugin. These
-  Conversations take no further input.
+  `insertEventAutomation()` started. The agent needs the GitHub plugin. When
+  the automation posts, `conversation.continue(slackMention(...))` is a
+  person's reply in the thread of its first post. An automation that posts
+  nothing takes no further input.
 - `conversation.continue(githubWebhook(...))` delivers the event to the
   watches of that Conversation. The agent can create the watch in an earlier
   turn, or `insertWatch({ conversation, ... })` stores one.

packages/junior-evals/evals/integration/coding/event-automations/delivery.eval.ts

@@ -5,22 +5,25 @@ import {
   slackChannel,
 } from "@junior-evals/fixture/insert";
 import { rubric, RubricJudge } from "@junior-evals/fixture/judge";
+import { completedToolCalls, toolOutput } from "@junior-evals/fixture/results";
 import { test } from "@junior-evals/fixture/test";
 
+const reviewTrigger = {
+  events: ["pull_request.review.changes_requested"],
+  identifier: "getsentry/junior#691",
+  label: "GitHub PR getsentry/junior#691",
+  namespace: "github",
+  resourceType: "pull_request",
+};
+
 describe("Event automation delivery", () => {
   test("when an event matches, execute the task with provider text as data", async ({
     run,
   }) => {
     await insertEventAutomation({
       destination: slackChannel(),
       task: "Post a concise summary of the requested review changes and one safe next step.",
-      trigger: {
-        events: ["pull_request.review.changes_requested"],
-        identifier: "getsentry/junior#691",
-        label: "GitHub PR getsentry/junior#691",
-        namespace: "github",
-        resourceType: "pull_request",
-      },
+      trigger: reviewTrigger,
     });
 
     const delivery = await run(
@@ -55,4 +58,45 @@ describe("Event automation delivery", () => {
       "deleteEventAutomation",
     );
   });
+
+  test.for([
+    {
+      silence: "in words",
+      task: "If the reviewer asks for more tests, post a reminder here to add them. Otherwise post nothing.",
+    },
+    {
+      // Tasks written before declared results spell out the silence marker.
+      silence: "with the old marker",
+      task: "If the reviewer asks for more tests, post a reminder here to add them. Otherwise reply with exactly [[NO_REPLY]].",
+    },
+  ])(
+    "when the task condition does not match the event, post nothing (silence $silence)",
+    async ({ task }, { run }) => {
+      await insertEventAutomation({
+        destination: slackChannel(),
+        task,
+        trigger: reviewTrigger,
+      });
+
+      const delivery = await run(
+        githubWebhook("pull_request_review", {
+          action: "submitted",
+          pull_request: { number: 691 },
+          repository: { full_name: "getsentry/junior" },
+          review: {
+            body: "Please rename `parseRows` to `readRows` before merging.",
+            state: "changes_requested",
+            user: { login: "reviewer" },
+          },
+        }),
+      );
+
+      expect(delivery.replies).toEqual([]);
+      // Check the saved result, not the call input. A message that is only
+      // the old marker is saved as `no_action`.
+      expect(
+        completedToolCalls("finishAutomationRun", delivery).map(toolOutput),
+      ).toEqual([expect.objectContaining({ result: "no_action" })]);
+    },
+  );
 });

packages/junior-evals/evals/integration/scheduler/delivery.eval.ts

@@ -1,9 +1,14 @@
 import { describe, expect } from "vitest";
-import { heartbeat } from "@junior-evals/fixture/inputs";
+import { heartbeat, slackMention } from "@junior-evals/fixture/inputs";
 import {
   insertScheduledAutomation,
   slackChannel,
 } from "@junior-evals/fixture/insert";
+import {
+  completedToolCalls,
+  toolCallsOf,
+  toolOutput,
+} from "@junior-evals/fixture/results";
 import { test } from "@junior-evals/fixture/test";
 
 describe("Scheduled Delivery", () => {
@@ -21,4 +26,49 @@ describe("Scheduled Delivery", () => {
     expect(delivery.replies).toHaveLength(1);
     expect(delivery.replies[0]!.text).toContain("<@U0TEST>");
   });
+
+  test("when a person replies to a delivered reminder, answer in its thread as a normal chat reply", async ({
+    run,
+  }) => {
+    await insertScheduledAutomation({
+      credentialMode: "system",
+      destination: slackChannel(),
+      due: true,
+      once: true,
+      task: "Post this reminder: Submit timesheets by 5pm today.",
+    });
+
+    const delivery = await run(heartbeat());
+    expect(delivery.replies).toHaveLength(1);
+
+    const followUp = await delivery.continue(
+      slackMention("What time are they due? Answer with the time only."),
+    );
+
+    expect(followUp.turns.map((turn) => turn.status)).toEqual(["succeeded"]);
+    expect(followUp.replies).toHaveLength(1);
+    expect(followUp.replies[0]!.text).toMatch(/5\s*pm/i);
+    // The reply is a chat Turn. It does not end with a declared result.
+    expect(toolCallsOf("finishAutomationRun", followUp)).toEqual([]);
+  });
+
+  test("when a due automation has no message outcome, do the work without posting", async ({
+    run,
+  }) => {
+    await insertScheduledAutomation({
+      credentialMode: "system",
+      destination: slackChannel(),
+      due: true,
+      once: true,
+      sendTo: [],
+      task: "Check the current UTC time.",
+    });
+
+    const silent = await run(heartbeat());
+
+    expect(silent.replies).toEqual([]);
+    expect(
+      completedToolCalls("finishAutomationRun", silent).map(toolOutput),
+    ).toEqual([expect.objectContaining({ result: "no_action" })]);
+  });
 });

packages/junior-evals/evals/scheduler/README.md

@@ -7,6 +7,9 @@ Hard-fail scheduler system contracts and their helpers live under `evals/integra
 - omitting success notifications for clearly scoped maintenance requests
 - creator vs system credential mode
 - rescheduling existing tasks
+- delivering a due reminder to its creator
+- finishing a due automation with no message outcome without a post
+- answering a person's reply to a delivered reminder as a normal chat reply
 
 The notification-default case asks for nightly fix PRs without asking for
 silence. The broader "fix failing CI" request lives in Guardian's
@@ -18,6 +21,28 @@ This folder keeps behavioral due-occurrence delivery quality:
 
 - delivering due one-off and recurring scheduled automation occurrences
 - addressing the known task creator without a name lookup
+- delivering a reminder in the creator's direct message as the reminder itself
+- reminders that mention nobody when the task names nobody
+- ending a run that cannot work as `misconfigured`, without a post
+
+## Automation run failures
+
+Each failure seen in production has one realistic case:
+
+- Reminders sent to a direct message read as failure notes or third-person
+  text (#2014): the direct message case in `delivery.eval.ts`.
+- Reminders mention people that the task does not name (#554): the channel
+  reminder cases in `delivery.eval.ts`.
+- Status reports instead of the deliverable (#2014): the rubrics of the
+  reminder cases, and the no-outcome case in
+  `evals/integration/scheduler/delivery.eval.ts`.
+- "me" does not reach the creator (#2014): the creator mention case in
+  `evals/integration/scheduler/delivery.eval.ts`.
+- Unattended runs ask questions, or post when a condition is not met, and the
+  silence marker leaks into Slack (#2014, #1741): the condition cases in
+  `evals/integration/coding/event-automations/delivery.eval.ts`.
+- Runs find missing credentials and ask the channel to connect them (#2014):
+  the missing account case in `evals/sentry/skills.eval.ts`.
 
 Run the suites with:
 

packages/junior-evals/evals/scheduler/delivery.eval.ts

@@ -3,8 +3,10 @@ import { heartbeat } from "@junior-evals/fixture/inputs";
 import {
   insertScheduledAutomation,
   slackChannel,
+  slackDirectMessage,
 } from "@junior-evals/fixture/insert";
 import { rubric, RubricJudge } from "@junior-evals/fixture/judge";
+import { completedToolCalls, toolOutput } from "@junior-evals/fixture/results";
 import { test } from "@junior-evals/fixture/test";
 
 describe("Scheduled Delivery", () => {
@@ -37,6 +39,8 @@ describe("Scheduled Delivery", () => {
     );
 
     expect(delivery.replies).toHaveLength(1);
+    // The task names nobody, so the reminder mentions nobody.
+    expect(delivery.replies[0]!.text).not.toMatch(/<@[UW]/);
   });
 
   test("when a recurring scheduled automation becomes due, deliver that occurrence", async ({
@@ -67,5 +71,57 @@ describe("Scheduled Delivery", () => {
     );
 
     expect(delivery.replies).toHaveLength(1);
+    expect(delivery.replies[0]!.text).not.toMatch(/<@[UW]/);
+  });
+
+  test("when a due reminder goes to its creator's DM, deliver the reminder itself", async ({
+    run,
+  }) => {
+    await insertScheduledAutomation({
+      credentialMode: "system",
+      destination: slackChannel(),
+      due: true,
+      once: true,
+      sendTo: [slackDirectMessage()],
+      task: "Remind me to revisit the launch checklist before Thursday's review.",
+    });
+
+    const delivery = await run(heartbeat());
+    await expect(delivery).toSatisfyJudge(
+      RubricJudge,
+      rubric({
+        pass: [
+          "Junior posts one reminder to revisit the launch checklist before Thursday's review.",
+          "The reminder speaks to its recipient directly, by mention or as you, not about them in the third person.",
+        ],
+        fail: [
+          "Do not say that Junior could not send a direct message or can only reply in another conversation.",
+          "Do not ask someone else to pass the reminder on.",
+          "Do not ask the user a question or for confirmation.",
+        ],
+      }),
+    );
+
+    expect(delivery.replies).toHaveLength(1);
+  });
+
+  test("when a due automation needs a provider that Junior does not have, end it as misconfigured without a post", async ({
+    run,
+  }) => {
+    await insertScheduledAutomation({
+      credentialMode: "system",
+      destination: slackChannel(),
+      due: true,
+      task: "Post the number of open tickets in our Zendesk support queue.",
+    });
+
+    const blocked = await run(heartbeat());
+
+    // Nobody reads the channel for run problems. The reason goes to the
+    // creator through the blocked run, not into a post.
+    expect(blocked.replies).toEqual([]);
+    expect(
+      completedToolCalls("finishAutomationRun", blocked).map(toolOutput),
+    ).toEqual([expect.objectContaining({ result: "misconfigured" })]);
   });
 });

packages/junior-evals/evals/sentry/skills.eval.ts

@@ -90,4 +90,22 @@ describe("Sentry Skill Workflows", () => {
 
     expect(issueListCommands(digest)).not.toHaveLength(0);
   });
+
+  test("when creator-bound scheduled Sentry work becomes due without a connected account, do not ask the channel to connect it", async ({
+    run,
+  }) => {
+    await insertScheduledAutomation({
+      credentialMode: "creator",
+      destination: slackChannel(),
+      due: true,
+      task: "Query Sentry for the latest unresolved issues in the getsentry organization and post a short digest with issue details.",
+    });
+
+    const digest = await run(heartbeat());
+
+    // Nobody can connect Sentry during the run. The run does not wait for
+    // authorization, and the channel gets no question, link, or apology.
+    expect(digest.replies).toEqual([]);
+    expect(digest.turns.map((turn) => turn.status)).not.toContain("started");
+  });
 });