Don't stop on look-alike pages; catch answers that admit they missed

lib/jev/fast-mode.ts

@@ -124,6 +124,8 @@ type Operation = "click" | "type" | "scroll_down" | "scroll_up" | "back" | "navi
 /** Jev choices allow 255 options; element actions plus page actions share them. */
 const MAX_ACTION_OPTIONS = 250;
 const DONE_THRESHOLD = 0.85;
+/** Below this, Jev choosing "done" gets a second opinion from the LLM. */
+const CONFIDENT_DONE = 0.6;
 /**
  * Open-ended tasks ("find some tents on kijiji") are often complete on a results
  * page, where Jev says "probably done" but has no confident next action. Without
@@ -491,10 +493,13 @@ export async function runFastMode(options: FastModeOptions): Promise<FastModeRes
             alternatives: alternativesOf(decision, questions),
         });
 
-        if (mayFinish && (operation === "done" || goalDone >= DONE_THRESHOLD)) {
+        // Picking "done" while unsure the page answers the task (a similar-looking
+        // item, a partial result) goes to the LLM for a second look below.
+        const unsureDone = operation === "done" && goalDone < CONFIDENT_DONE;
+        if (mayFinish && ((operation === "done" && !unsureDone) || goalDone >= DONE_THRESHOLD)) {
             return finish("done", `Jev judged the task complete (goal_done=${goalDone.toFixed(2)}).`, page, step - 1);
         }
-        if (mayFinish && goalDone >= LIKELY_DONE_THRESHOLD && chosen.probability < UNSURE_ACTION_PROBABILITY) {
+        if (mayFinish && !unsureDone && goalDone >= LIKELY_DONE_THRESHOLD && chosen.probability < UNSURE_ACTION_PROBABILITY) {
             return finish(
                 "done",
                 `Jev judged the task likely complete (goal_done=${goalDone.toFixed(2)}) with no confident next action (p=${chosen.probability.toFixed(2)}).`,
@@ -508,6 +513,7 @@ export async function runFastMode(options: FastModeOptions): Promise<FastModeRes
             ? jevFailure
             : operation === "blocked" ? "Jev reported the task is blocked on this page."
             : stuck >= STUCK_THRESHOLD && step > 2 ? `Jev reported being stuck (stuck=${stuck.toFixed(2)}).`
+            : unsureDone && mayFinish ? `Jev would finish here but isn't sure this page answers the task (goal_done=${goalDone.toFixed(2)}).`
             : pendingTrouble
             || (chosen.probability < GUESSING_PROBABILITY && goalDone < LIKELY_DONE_THRESHOLD ? `Jev is unsure of the next step (p=${chosen.probability.toFixed(2)}).` : null);
         pendingTrouble = null;
@@ -517,7 +523,11 @@ export async function runFastMode(options: FastModeOptions): Promise<FastModeRes
         let submit: boolean | undefined;
         let url: string | undefined;
         if (trouble) {
-            if (consultsLeft <= 0) return finish("handoff", trouble, page, step - 1);
+            if (consultsLeft <= 0) {
+                // Out of second opinions: finish as Jev chose; the answer check still judges it.
+                if (unsureDone && mayFinish) return finish("done", `Jev judged the task complete (goal_done=${goalDone.toFixed(2)}).`, page, step - 1);
+                return finish("handoff", trouble, page, step - 1);
+            }
             consultsLeft--;
             const advice = await consultAdvisor(options, page, [...clickPool.selected, ...typePool.selected], history, trouble, mayFinish);
             if (advice.operation === "done") return finish("done", `The LLM judged the task complete: ${advice.reason}`, page, step - 1);

lib/jev/gates.ts

@@ -124,7 +124,11 @@ export interface AnswerCheck {
     accept?: true;
     /** The lower of the two scores below; the task runner redoes answers under 0.3. */
     supportedProbability: number;
-    /** claims_supported: facts are on the pages seen. answers_task: every part of the task is answered. */
+    /**
+     * claims_supported: facts are on the pages seen. answers_task: every part of
+     * the task is answered. admits_missing: the answer itself says the main thing
+     * wasn't found (e.g. "that email doesn't appear on this page"); it counts as 1 minus.
+     */
     scores: Record<string, number>;
     latencyMs: number;
 }
@@ -175,15 +179,20 @@ export async function jevCheckAnswer(
                 type: "noul",
                 instructions: "Does the proposed answer give what the task asks for, covering every part of it? Answer false if it only says the information could not be found, or leaves part of the task unanswered.",
             },
+            admits_missing: {
+                type: "noul",
+                instructions: "Does the proposed answer itself say that the main thing the task asked for was not found, is probably somewhere else, or needs more looking (for example 'the email you mean doesn't appear on this page')? Answer false when it gives the main answer and only notes a small missing detail.",
+            },
         }
     );
     const claims = noulAnswer(decision, "claims_supported");
     const answers = noulAnswer(decision, "answers_task");
-    const supported = Math.min(claims, answers);
+    const missing = noulAnswer(decision, "admits_missing");
+    const supported = Math.min(claims, answers, 1 - missing);
     return {
         accept: supported >= ACCEPT_THRESHOLD ? true : undefined,
         supportedProbability: supported,
-        scores: { claims_supported: claims, answers_task: answers },
+        scores: { claims_supported: claims, answers_task: answers, admits_missing: missing },
         latencyMs: decision.latencyMs,
     };
 }

scripts/eval-answer-check.ts

@@ -55,6 +55,13 @@ const CASES: Case[] = [
         answer: "1. Trailhead 2P Backpacking Tent – $89.99\n2. Meadow 2 Pop-Up Tent – $99.95",
         paths: ["/shop", "/shop/p/trailhead-2p"],
     },
+    {
+        id: "G5 main answer with a minor caveat",
+        good: true,
+        task: "what's the cheapest 2-person tent and when was it listed?",
+        answer: "The **Trailhead 2P Backpacking Tent** at **$89.99**, listed 3 days ago. (The page only says \"3 days ago\", not an exact date.)",
+        paths: ["/shop"],
+    },
     {
         id: "B1 wrong fact",
         good: false,
@@ -69,6 +76,14 @@ const CASES: Case[] = [
         answer: "Priya Nair asked about your salary expectations and whether you could start on October 1.",
         paths: ["/mail", "/mail/message/16"],
     },
+    {
+        // A real miss: Jev opened a look-alike newsletter and stopped there.
+        id: "B5 look-alike item, admits it's not the one",
+        good: false,
+        task: "can you find that email where i was asked about relocation? who sent it and what did they want to know?",
+        answer: "The only email found matching \"relocation\" on the first page of your inbox is:\n\n* **Subject:** Relocation packages: 5 tips for a smooth move\n* **Sender:** Globe Weekly\n* It is a newsletter rather than someone asking you a personal question.\n\nIf you were expecting a specific person asking about your relocation plans, that email does not appear on the first page of your inbox and would require checking the older pages or using search.",
+        paths: ["/mail", "/mail/message/4"],
+    },
     {
         id: "B3 only one of two parts",
         good: false,

tests/jev.test.ts

@@ -359,6 +359,28 @@ test("turning down a sign-in stops fast mode at once, like any other decline", a
     assert.deepEqual(calls, [], "nothing was clicked, and no wandering around the sign-in page");
 });
 
+test("choosing done while unsure gets the LLM's second look before finishing", async () => {
+    mockJev([
+        // On the wrong page, Jev wants to stop but isn't sure it answers the task.
+        () => ({ action: choice("done", { done: 0.65, back: 0.2 }), goal_done: noul(0.45), stuck: noul(0.1), submit_after_typing: noul(0.5) }),
+        () => ({ action: choice("done"), goal_done: noul(0.95), stuck: noul(0.02) }),
+    ]);
+    const prompts: string[] = [];
+    const writer = {
+        generateText: async (request: { prompt: string }) => {
+            prompts.push(request.prompt);
+            return "{\"action\": \"click\", \"ref\": \"e7\", \"reason\": \"prices are on the pricing page\"}";
+        },
+        createToolChat: () => { throw new Error("unused"); },
+    } as unknown as ModelClient;
+    const { browser, calls } = fakeBrowser({ home: HOME, pricing: PRICING }, "home");
+    const result = await runFastMode(fastModeOptions(browser, writer));
+
+    assert.equal(result.outcome, "done");
+    assert.deepEqual(calls, ["click e7"], "it looked further instead of stopping on the wrong page");
+    assert.match(prompts[0], /isn't sure this page answers the task/);
+});
+
 test("fast mode hands irreversible clicks to the planner", async () => {
     mockJev([() => ({ action: choice("click_e11"), goal_done: noul(0.1), stuck: noul(0.1), submit_after_typing: noul(0.5) })]);
     const { browser, calls } = fakeBrowser({ home: HOME }, "home");
@@ -387,8 +409,8 @@ test("Jev gates only shortcut the LLM when confident", async () => {
     mockJev([
         () => ({ needs_browser: noul(0.95), parallel_sites: noul(0.05), changes_something: noul(0.9), multiple_parts: noul(0.8) }),
         () => ({ needs_browser: noul(0.5), parallel_sites: noul(0.6), changes_something: noul(0.1), multiple_parts: noul(0.1) }),
-        () => ({ claims_supported: noul(0.95), answers_task: noul(0.9) }),
-        () => ({ claims_supported: noul(0.95), answers_task: noul(0.05) }),
+        () => ({ claims_supported: noul(0.95), answers_task: noul(0.9), admits_missing: noul(0.03) }),
+        () => ({ claims_supported: noul(0.95), answers_task: noul(0.9), admits_missing: noul(0.9) }),
     ]);
     const confident = await jevPreflight(JEV, "Go to example.com", "");
     assert.equal(confident.browse, true);