Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion extension/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ Fast mode (Jev) is on by default.
- Pages often navigate a moment after an action (a search that submits once its suggestions load). WebPilot notices when the page starts changing after it last looked, and looks again instead of clicking on the page being left.
- Answers stream into the panel as they are written, including the planner's (it replies in plain text rather than through a tool call, whose arguments only arrive at the end); Jev then checks them against every page it saw (are the facts on those pages, and is every part of the request answered?), and an answer that fails is redone (first by Jev looking closer, then by the model).
- Requests that only find or read information never send, buy or delete anything: those controls aren't even considered.
- When a request does ask for something irreversible (buy, place order, checkout, send, delete, publish, unsubscribe...), the panel shows **"Hold on — this can't be undone"** before each such click, with Allow and Skip. Skip leaves that action undone (it isn't asked about again during the task) and WebPilot carries on with the rest, saying in the answer what it skipped. The stop button ends the whole task.
- When a request does ask for something irreversible (buy, place order, checkout, send, delete, publish, unsubscribe...), the panel shows **"Hold on — this can't be undone"** before each such click, with Allow and Skip. Skip leaves that action undone (it isn't asked about again during the task) and WebPilot carries on with the rest, saying in the answer what it skipped. Skipping a sign-in stops work on that site at once (nothing behind it can be reached): WebPilot finishes any other part of the request and tells you to sign in and ask again. The stop button ends the whole task.
- Sites are picked for your country, from your browser's language and time zone (walmart.ca rather than walmart.com in Canada).
- WebPilot never types into password, one-time-code or card fields and never makes up usernames or other personal details. On a sign-in page your password manager has already filled in, clicking "Log in" asks first (**"Hold on — sign in?"**); otherwise it asks you to sign in yourself and then continue.
- A follow-up like "no, it's on Gmail" or "try again" continues a request that didn't finish.
Expand Down
2 changes: 1 addition & 1 deletion extension/src/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ export class Engine {
settled = true;
session.confirmResolve = undefined;
session.state = { ...session.state, pending: null };
addStep({ lane: "note", label: allowed ? "allowed" : "cancelled", detail: action });
addStep({ lane: "note", label: allowed ? "allowed" : "skipped", detail: action });
resolve(allowed);
};
session.confirmResolve = settle;
Expand Down
17 changes: 14 additions & 3 deletions lib/core/run-task.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

import { createModelClient, resolveJevConfig, type RuntimeModelConfig, type ToolResponsePart } from "../model-client";
import { extractExplicitUrls } from "../goal-urls";
import { DECLINED_OUTCOME, NOTHING_TO_TYPE_OUTCOME, runFastMode, summarizeFastModeForPlanner, writeFastModeAnswer, type FastModeBrowser, type FastModeResult, type FastModeStep } from "../jev/fast-mode";
import { DECLINED_OUTCOME, NOTHING_TO_TYPE_OUTCOME, SIGN_IN_DECLINED_PREFIX, runFastMode, summarizeFastModeForPlanner, writeFastModeAnswer, type FastModeBrowser, type FastModeResult, type FastModeStep } from "../jev/fast-mode";
import { jevCheckAnswer, jevPreflight } from "../jev/gates";
import { describeElement, parsePage, selectCandidates, type PageModel } from "../jev/page";
import { CREDENTIALS_RULE, UNTRUSTED_CONTENT_RULE, irreversibleAction, isSecretField, signInAction } from "../safety";
Expand Down Expand Up @@ -110,6 +110,9 @@ const FAST_ANSWER_ACCEPT = 0.8;
* article section, so it reads instead of scrolling (fast mode's Jev state stays small).
*/
const PLANNER_TEXT_CHARS = 8000;
/** For the planner once the user turns down signing in to a site. */
const SIGN_IN_DECLINED_GUIDANCE = "Don't try to sign in again or work around it. If some other part of the task doesn't need that site, do it; then answer right away: say what couldn't be done without signing in, and that they can sign in in this tab (or allow it when asked) and ask again.";

/** A delegated sub-goal is short legwork: a search, opening an item, a few pages. */
const DELEGATE_MAX_STEPS = 10;
const DELEGATE_TIME_BUDGET_MS = 40_000;
Expand Down Expand Up @@ -486,6 +489,7 @@ When an earlier request did not finish, or the user corrects or adds to one ("no
emit({ type: "handoff", reason: `Fast mode's answer fell short: ${problem}.` });
} else {
handoff = summarizeFastModeForPlanner(fast);
if (fast.reason.startsWith(SIGN_IN_DECLINED_PREFIX)) handoff += `\n${SIGN_IN_DECLINED_GUIDANCE}`;
emit({ type: "handoff", reason: fast.reason });
}
break;
Expand Down Expand Up @@ -555,7 +559,12 @@ Rules:${jev ? `
result = { ok: false, error: `The user only asked to find or read information, so WebPilot won't click ${irreversible}: it would change something for them. Carry on without it.` };
} else if (risky && !await confirmAction(risky, lastPage.url)) {
stepStatus = "declined";
result = { ok: false, error: "The user declined this, so it was not done. Don't try it or anything equivalent again; carry on with any other part of the task, or answer with what was done and what was skipped." };
result = {
ok: false,
error: irreversible
? "The user declined this, so it was not done. Don't try it or anything equivalent again; carry on with any other part of the task, or answer with what was done and what was skipped."
: `The user chose not to sign in here. ${SIGN_IN_DECLINED_GUIDANCE}`,
};
} else {
await browser.click(ref, String(args.element || ""));
result = { ok: true };
Expand Down Expand Up @@ -583,7 +592,9 @@ Rules:${jev ? `
lastPage = parsePage(await browser.snapshot(), PLANNER_TEXT_CHARS);
result = {
ok: delegated.outcome === "done",
outcome: delegated.outcome === "done" ? "Jev finished the sub-goal." : `Jev stopped: ${delegated.reason}`,
outcome: delegated.outcome === "done"
? "Jev finished the sub-goal."
: `Jev stopped: ${delegated.reason}${delegated.reason.startsWith(SIGN_IN_DECLINED_PREFIX) ? ` ${SIGN_IN_DECLINED_GUIDANCE}` : ""}`,
steps: delegated.history.map((entry) => `${entry.action} -> ${entry.outcome}`),
page: plannerObservation(lastPage, browserGoal),
};
Expand Down
21 changes: 20 additions & 1 deletion lib/jev/fast-mode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,17 @@ const DEFAULT_MAX_CONSULTS = 2;
/** The outcome recorded for a step the user declined; Jev reads it in its history. */
export const DECLINED_OUTCOME = "declined by the user: not done, and not to be tried again";

/** Starts the handoff reason when the user turned down signing in. */
export const SIGN_IN_DECLINED_PREFIX = "The user chose not to sign in to";

function hostOf(url: string): string {
try {
return new URL(url).host;
} catch {
return url;
}
}

/** The outcome recorded for a field the task gives nothing to type into. */
export const NOTHING_TO_TYPE_OUTCOME = "skipped: the task gives nothing to type here";

Expand Down Expand Up @@ -548,7 +559,15 @@ export async function runFastMode(options: FastModeOptions): Promise<FastModeRes
if (!options.confirm || (irreversible && options.allowIrreversible === false)) {
return finish("handoff", `Next click looks irreversible (${description}); handing to the planner.`, page, step - 1);
}
if (!await options.confirm(risky, page.url)) throw new DeclinedByUser(risky);
if (!await options.confirm(risky, page.url)) {
// Nothing behind a sign-in the user turned down can be reached:
// stop working on this site now instead of wandering the sign-in page.
if (!irreversible && signIn) {
history.push({ step, action: actionLabel, outcome: DECLINED_OUTCOME });
return finish("handoff", `${SIGN_IN_DECLINED_PREFIX} ${hostOf(page.url)}.`, page, step - 1);
}
throw new DeclinedByUser(risky);
}
}
resultText = await options.browser.click(target.ref, description);
} else if (operation === "type" && target) {
Expand Down
6 changes: 4 additions & 2 deletions scripts/extension-smoke.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,11 @@ const CASES: Case[] = [
start: (base) => `${base}/portal`,
request: "check my student portal for my next appointment",
onConfirm: "skip",
check: (_answer, actions, confirmed) => {
check: (answer, actions, confirmed) => {
if (actions.some((action) => action.type === "login")) return `signed in without asking: ${JSON.stringify(actions)}`;
return confirmed ? null : "never asked before signing in";
if (!confirmed) return "never asked before signing in";
// Skipping the sign-in ends work there, and the answer says why and what to do.
return /sign(ed)? in|log(ged)? in/i.test(answer) ? null : "the answer doesn't mention signing in";
},
},
{
Expand Down
19 changes: 19 additions & 0 deletions tests/jev.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,25 @@ test("a field the task gives nothing to type into is skipped, and Jev carries on
assert.ok(!("type_e8" in (requests[1].body.questions.action.criteria || {})), "the field isn't offered again on that page");
});

test("turning down a sign-in stops work on that site right away", async () => {
const login = snapshot("https://portal.example/login", "Sign in", [
"- main [ref=e1]:",
" - textbox \"Login ID\" [ref=e2]: student42",
" - textbox \"Password:\" [ref=e3]: ••••••",
" - button \"LOG IN\" [ref=e4] [cursor=pointer]",
" - link \"Forgot password?\" [ref=e5] [cursor=pointer]:",
" - /url: /forgot",
].join("\n"));
mockJev([() => ({ action: choice("click_e4"), goal_done: noul(0.05), stuck: noul(0.1), submit_after_typing: noul(0.5) })]);
const calls: string[] = [];
const browser: FastModeBrowser = { snapshot: async () => login, click: async (ref) => { calls.push(ref); return ""; }, type: async () => "", pressKey: async () => "", back: async () => "" };
const result = await runFastMode({ ...fastModeOptions(browser), confirm: async () => false });

assert.equal(result.outcome, "handoff");
assert.match(result.reason, /chose not to sign in to portal\.example/);
assert.deepEqual(calls, [], "nothing was clicked, and no wandering around the sign-in page");
});

test("fast mode hands irreversible clicks to the planner", async () => {
mockJev([() => ({ action: choice("click_e11"), goal_done: noul(0.1), stuck: noul(0.1), submit_after_typing: noul(0.5) })]);
const { browser, calls } = fakeBrowser({ home: HOME }, "home");
Expand Down
Loading