From 99edb0d6289dd0dad891d43ec5bcbef6fe96bacd Mon Sep 17 00:00:00 2001 From: leeguooooo Date: Tue, 6 Oct 2026 18:10:53 +0900 Subject: [PATCH] feat(observe): act tools observe by default; the settled screen is captured, kept in memory and sent only when asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit leo: after every step take the screenshot and read the tree so the AI has the data ready and makes fewer calls — but do not hand it the image (tokens); give it only when the AI wants to look, and leave no junk behind. - MCP single-step act tools now default observe:true and name the last snapshot they saw as the baseline (?since=), so the result is the settled change, not a whole tree; observe:false keeps the bare action. - The daemon keeps the settled screen in memory: the frame the settle check already took when the screen was stable, else one captured in the background right after the response. GET /agent/screenshot (and phone_screenshot) returns it with no capture (X-Screenshot-Source: settled-after-action) while nothing was sent since and it is under 30 s old; ?fresh=1 always captures. - Nothing touches disk; the next screen-changing POST drops the frame. --- crates/mcp/src/client.rs | 65 +++++++++++-- crates/mcp/src/server.rs | 82 ++++++++++------ crates/mcp/tests/observe_over_stdio.rs | 7 +- crates/server/src/http.rs | 110 ++++++++++++++++++---- crates/server/src/wda.rs | 20 ++++ crates/server/tests/agent_input_settle.rs | 86 +++++++++++++++++ docs/agent-reference.md | 9 +- skills/iphone-use/SKILL.md | 6 +- 8 files changed, 329 insertions(+), 56 deletions(-) diff --git a/crates/mcp/src/client.rs b/crates/mcp/src/client.rs index 5bea174..1ab5ace 100644 --- a/crates/mcp/src/client.rs +++ b/crates/mcp/src/client.rs @@ -37,6 +37,10 @@ pub struct DaemonClient { /// Sent as `X-Phone-Owner` on every control request so the daemon can /// refuse a second session that tries to drive the same phone (#72). owner: String, + /// The last element snapshot this client was handed (an elements read or + /// an observed action). An observed action names it as its baseline, so + /// the daemon answers with what changed instead of the whole tree. + last_snapshot: std::sync::Arc>>, } #[derive(Debug, serde::Deserialize)] @@ -135,6 +139,35 @@ impl DaemonClient { .map(|value| value.trim().to_string()) .filter(|value| !value.is_empty()) .unwrap_or_else(|| format!("mcp-{}", std::process::id())), + last_snapshot: std::sync::Arc::new(std::sync::Mutex::new(None)), + } + } + + fn remember_snapshot(&self, json: Option<&serde_json::Value>) { + if let Some(snapshot) = json + .and_then(|json| json.get("snapshot")) + .and_then(serde_json::Value::as_str) + .filter(|snapshot| !snapshot.is_empty()) + { + *self.last_snapshot.lock().unwrap_or_else(|e| e.into_inner()) = + Some(snapshot.to_string()); + } + } + + /// `/agent/input?return=delta`, with the last snapshot as the baseline + /// when there is one. + fn observed_input_path(&self) -> String { + match self + .last_snapshot + .lock() + .unwrap_or_else(|e| e.into_inner()) + .as_deref() + { + Some(since) => format!( + "/agent/input?return=delta&since={}", + url_query_escape(since) + ), + None => "/agent/input?return=delta".to_string(), } } @@ -227,12 +260,12 @@ impl DaemonClient { observe: bool, ) -> anyhow::Result { let path = if observe { - "/agent/input?return=delta" + self.observed_input_path() } else { - "/agent/input" + "/agent/input".to_string() }; let mut req = self - .auth(self.client.post(self.url(path))) + .auth(self.client.post(self.url(&path))) .header("x-phone-control", "1") .header("x-phone-owner", &self.owner) .header(header::CONTENT_TYPE, "application/json") @@ -240,7 +273,9 @@ impl DaemonClient { if observe { req = req.timeout(OBSERVE_TIMEOUT); } - read_response(req.send().await?).await + let response = read_response(req.send().await?).await?; + self.remember_snapshot(response.json.as_ref()); + Ok(response) } @@ -333,7 +368,9 @@ impl DaemonClient { .timeout(ELEMENTS_TIMEOUT); let resp = req.send().await?; let resp = check_status(resp).await?; - Ok(resp.text().await?) + let body = resp.text().await?; + self.remember_snapshot(serde_json::from_str(&body).ok().as_ref()); + Ok(body) } /// `POST /agent/mode {"mode":"agent"}` — reconnect the configured, @@ -391,7 +428,9 @@ impl DaemonClient { // the ordinary 30s is how a caller stops knowing what happened. req = req.timeout(OBSERVE_TIMEOUT); } - read_response(req.send().await?).await + let response = read_response(req.send().await?).await?; + self.remember_snapshot(response.json.as_ref()); + Ok(response) } @@ -657,6 +696,20 @@ async fn check_status(resp: reqwest::Response) -> anyhow::Result String { + value + .bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + (b as char).to_string() + } + _ => format!("%{b:02X}"), + }) + .collect() +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs index 17dabc6..8715b19 100644 --- a/crates/mcp/src/server.rs +++ b/crates/mcp/src/server.rs @@ -36,9 +36,12 @@ pub struct TapParams { pub x: f64, /// Vertical position, normalized 0–1 (0 = top edge, 1 = bottom edge). pub y: f64, - /// Ask the daemon to observe the screen after the action and return what - /// settled (`settle`, `snapshot`, `delta`). Costs extra latency, so it is - /// off unless you need to know what the tap produced. + /// On by default: the result carries what the screen settled to — what + /// changed since your last look (`delta`), or the whole tree the first + /// time — so you rarely need `phone_elements` afterwards. The settled + /// screen is also captured but NOT sent: `phone_screenshot` returns it at + /// once if you need to see it. `false` skips both for the fastest bare + /// action. #[serde(default)] pub observe: Option, } @@ -55,9 +58,12 @@ pub struct ScrollParams { /// Vertical scroll delta. **Positive dy reveals content farther down**; /// negative dy reveals content above. ~80 ≈ 15% of a screen, ~400 ≈ 75%. pub dy: f64, - /// Ask the daemon to observe the screen after the action and return what - /// settled (`settle`, `snapshot`, `delta`). Costs extra latency, so it is - /// off unless you need to know what the action produced. + /// On by default: the result carries what the screen settled to — what + /// changed since your last look (`delta`), or the whole tree the first + /// time — so you rarely need `phone_elements` afterwards. The settled + /// screen is also captured but NOT sent: `phone_screenshot` returns it at + /// once if you need to see it. `false` skips both for the fastest bare + /// action. #[serde(default)] pub observe: Option, } @@ -68,9 +74,12 @@ pub struct TypeParams { /// Unicode text to send through the device-side input service. Focus the /// intended field and verify it before typing. pub text: String, - /// Ask the daemon to observe the screen after the action and return what - /// settled (`settle`, `snapshot`, `delta`). Costs extra latency, so it is - /// off unless you need to know what the action produced. + /// On by default: the result carries what the screen settled to — what + /// changed since your last look (`delta`), or the whole tree the first + /// time — so you rarely need `phone_elements` afterwards. The settled + /// screen is also captured but NOT sent: `phone_screenshot` returns it at + /// once if you need to see it. `false` skips both for the fastest bare + /// action. #[serde(default)] pub observe: Option, } @@ -81,9 +90,12 @@ pub struct TapLabelParams { /// The element's visible accessibility label, exactly as shown by /// `phone_elements` (e.g. "新备忘录", "Connect"). pub label: String, - /// Ask the daemon to observe the screen after the action and return what - /// settled (`settle`, `snapshot`, `delta`). Costs extra latency, so it is - /// off unless you need to know what the action produced. + /// On by default: the result carries what the screen settled to — what + /// changed since your last look (`delta`), or the whole tree the first + /// time — so you rarely need `phone_elements` afterwards. The settled + /// screen is also captured but NOT sent: `phone_screenshot` returns it at + /// once if you need to see it. `false` skips both for the fastest bare + /// action. #[serde(default)] pub observe: Option, } @@ -95,9 +107,12 @@ pub struct TapElementParams { pub element: usize, /// Snapshot token from the same `phone_elements` response. pub snapshot: String, - /// Ask the daemon to observe the screen after the action and return what - /// settled (`settle`, `snapshot`, `delta`). Costs extra latency, so it is - /// off unless you need to know what the action produced. + /// On by default: the result carries what the screen settled to — what + /// changed since your last look (`delta`), or the whole tree the first + /// time — so you rarely need `phone_elements` afterwards. The settled + /// screen is also captured but NOT sent: `phone_screenshot` returns it at + /// once if you need to see it. `false` skips both for the fastest bare + /// action. #[serde(default)] pub observe: Option, } @@ -122,9 +137,12 @@ pub struct KeyParams { /// Supported names: `return`/`enter`, `escape`, `space`, `tab`, /// `delete`/`backspace`, `up`, `down`, `left`, `right`. pub name: String, - /// Ask the daemon to observe the screen after the action and return what - /// settled (`settle`, `snapshot`, `delta`). Costs extra latency, so it is - /// off unless you need to know what the action produced. + /// On by default: the result carries what the screen settled to — what + /// changed since your last look (`delta`), or the whole tree the first + /// time — so you rarely need `phone_elements` afterwards. The settled + /// screen is also captured but NOT sent: `phone_screenshot` returns it at + /// once if you need to see it. `false` skips both for the fastest bare + /// action. #[serde(default)] pub observe: Option, } @@ -135,9 +153,12 @@ pub struct ShortcutParams { /// Supported names: `home` (Home Screen) and `spotlight` (search). /// App Switcher is unsupported by the Direct/WDA backend. pub name: String, - /// Ask the daemon to observe the screen after the action and return what - /// settled (`settle`, `snapshot`, `delta`). Costs extra latency, so it is - /// off unless you need to know what the action produced. + /// On by default: the result carries what the screen settled to — what + /// changed since your last look (`delta`), or the whole tree the first + /// time — so you rarely need `phone_elements` afterwards. The settled + /// screen is also captured but NOT sent: `phone_screenshot` returns it at + /// once if you need to see it. `false` skips both for the fastest bare + /// action. #[serde(default)] pub observe: Option, } @@ -525,7 +546,9 @@ impl PhoneHandler { #[tool( description = "Capture the current iPhone screen through WDA and return it as \ an image/png content block (1200 px on the long side unless max_side says \ - otherwise). Capture only when a current \ + otherwise). Right after an observed action this is instant: the screen as \ + it settled was already captured and is returned without a new capture \ + (header source settled-after-action). Capture only when a current \ user-requested task needs phone pixels; do not capture or reconnect for \ initialization, health checks, or to keep the phone ready. Idle release is \ intentional. If that task cannot proceed because Direct is released/offline, \ @@ -815,7 +838,7 @@ impl PhoneHandler { observe, }): Parameters, ) -> CallToolResult { - let observe = observe.unwrap_or(false); + let observe = observe.unwrap_or(true); // Refused here, before anything is sent: this is the one case where a // retry is provably safe, so it is reported as such rather than as an // unknown outcome. @@ -861,7 +884,7 @@ impl PhoneHandler { &self, Parameters(TapLabelParams { label, observe }): Parameters, ) -> CallToolResult { - let observe = observe.unwrap_or(false); + let observe = observe.unwrap_or(true); // The snapshot comes from the element read this call performs — never // a cached or borrowed baseline. match self.daemon.tap_label_observed(&label, observe).await { @@ -1945,7 +1968,7 @@ async fn send_input_observed( msg: &InputMsg, observe: Option, ) -> CallToolResult { - let observe = observe.unwrap_or(false); + let observe = observe.unwrap_or(true); match daemon.input_observed(msg, observe).await { Ok(response) => daemon_action_result(&response, observe, "ok"), // The request may well have reached the phone before the transport @@ -2061,8 +2084,8 @@ mod tests { assert_eq!(structured["snapshot"], "snap-1"); } - /// Without `observe` the result stays the short string callers already - /// parse, and the request must not have asked for a delta. + /// With `observe:false` the result stays the short string callers + /// already parse, and the request must not have asked for a delta. #[test] fn an_unobserved_tap_keeps_its_plain_result() { let (url, task) = scripted_daemon( @@ -2074,7 +2097,7 @@ mod tests { let result = block(handler.phone_tap(Parameters(TapParams { x: 0.5, y: 0.5, - observe: None, + observe: Some(false), }))); task.join().unwrap(); @@ -2298,7 +2321,8 @@ mod tests { capabilities.input_schema ); - // `observe` is opt-in on every single-step UI tool: in the schema, + // `observe` is accepted (and optional — it defaults on) by every + // single-step UI tool: in the schema, // never required, so calls written before it keep working. for name in [ "phone_tap", diff --git a/crates/mcp/tests/observe_over_stdio.rs b/crates/mcp/tests/observe_over_stdio.rs index 4131ccf..a3eb4d6 100644 --- a/crates/mcp/tests/observe_over_stdio.rs +++ b/crates/mcp/tests/observe_over_stdio.rs @@ -205,7 +205,12 @@ fn an_unobserved_tap_is_unchanged_on_the_wire() { let daemon = ScriptedDaemon::start("200 OK", br#"{"ok":true,"transport":"wda"}"#.to_vec()); let mut mcp = McpChild::start(&daemon.url); - let reply = mcp.call_tool(2, "phone_tap", serde_json::json!({ "x": 0.5, "y": 0.5 })); + // Observation is on by default; `observe:false` keeps the bare action. + let reply = mcp.call_tool( + 2, + "phone_tap", + serde_json::json!({ "x": 0.5, "y": 0.5, "observe": false }), + ); let result = &reply["result"]; assert_ne!(result["isError"], true, "{reply}"); diff --git a/crates/server/src/http.rs b/crates/server/src/http.rs index 6986996..971b6e4 100644 --- a/crates/server/src/http.rs +++ b/crates/server/src/http.rs @@ -9110,6 +9110,35 @@ async fn settle_frame( frame } +/// How long the settled screen answers `/agent/screenshot` with no new +/// capture, when nothing was sent since. Long enough for an agent to read an +/// observation and decide to look; short enough that a screen changing on its +/// own (a message arriving) is not served stale for long. `?fresh=1` always +/// captures. +const SETTLED_FRAME_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(30); + +/// Capture the settled screen in the background, unless something was sent +/// in the meantime (`post_mark` is the last POST the observation saw) — then +/// that screen is gone and there is nothing to keep. +fn capture_settled_frame_later( + wda: Arc>, + post_mark: Option, +) { + tokio::spawn(async move { + let mut w = wda.lock().await; + if w.last_post() != post_mark || w.settled_frame(SETTLED_FRAME_MAX_AGE).is_some() { + return; + } + if let Ok(Ok(png)) = + tokio::time::timeout(std::time::Duration::from_secs(3), w.screenshot_png()).await + { + if w.last_post() == post_mark && is_valid_png(&png) { + w.remember_settled_frame(png); + } + } + }); +} + async fn settle_and_read_elements( w: &mut crate::wda::WdaClient, budget: std::time::Duration, @@ -9160,6 +9189,11 @@ async fn settle_and_read_elements( if !typing && !report.sparse && tokio::time::Instant::now() < deadline { let after = settle_frame(w, deadline).await; if after.as_ref() == Some(before) { + // That frame IS the settled screen: keep it for a screenshot + // request, which then costs no capture. + if let Some(after) = after { + w.remember_settled_frame(after); + } report.settled = true; report.reason = SettleReason::Stable; report.waited_ms = started.elapsed().as_millis() as u64; @@ -9612,7 +9646,16 @@ async fn agent_input( // deadline and the observation budget. alert = probe_alert(&mut client).await; } + // The settled screen is ready for a screenshot request without a capture: + // the settle check kept its frame, or — when it judged by the tree (a + // focused text field, a screen that kept moving) — one is taken now, after + // the response is on its way, while the agent reads it. + let frame_needed = settled.is_some() && client.settled_frame(SETTLED_FRAME_MAX_AGE).is_none(); + let post_mark = client.last_post(); drop(client); + if frame_needed { + capture_settled_frame_later(Arc::clone(wda), post_mark); + } match outcome { WdaControlOutcome::Applied => { let body = match settled { @@ -10555,6 +10598,37 @@ struct ScreenshotQuery { /// ~0.9k at 1200. Asking for a size also lets a live frame answer, when /// someone is watching and it is current, instead of a WDA capture. max_side: Option, + /// `fresh=1`: capture now, even when the screen as it settled after the + /// last action is already in memory. + fresh: Option, +} + +/// A valid capture, finished for the caller: the wireframe when the app hid +/// its screen from capture, else the capture shrunk to `max_side` (`raw` +/// skips both). +async fn finish_screenshot( + wda: &Arc>, + bytes: Vec, + raw: bool, + max_side: Option, + source: Option<&'static str>, +) -> Response { + if !raw { + let wireframe = tokio::time::timeout( + std::time::Duration::from_secs(15), + redacted_capture_wireframe(wda, &bytes), + ) + .await + .ok() + .flatten(); + if let Some(wireframe) = wireframe { + let wireframe = fit_png(wireframe, max_side).await; + return png_response(wireframe, Some("accessibility-wireframe"), true); + } + } + // `raw` is WDA's capture untouched, size included. + let max_side = if raw { None } else { max_side }; + png_response(fit_png(bytes, max_side).await, source, false) } /// Shrink a PNG to `max_side` off the async runtime; the original when it @@ -10706,6 +10780,25 @@ async fn agent_screenshot( .as_deref() .is_some_and(|v| v == "1" || v == "true"); let max_side = query.max_side.map(|side| side.clamp(200, 4000)); + let fresh = query + .fresh + .as_deref() + .is_some_and(|v| v == "1" || v == "true"); + // The screen as it settled after the last action was captured then (see + // `SETTLED_FRAME_MAX_AGE`); nothing was sent since, so it is this screen. + if !raw && !fresh { + let settled = wda.lock().await.settled_frame(SETTLED_FRAME_MAX_AGE); + if let Some(png) = settled { + return finish_screenshot( + wda, + (*png).clone(), + false, + max_side, + Some("settled-after-action"), + ) + .await; + } + } // Someone is watching, so the stream is already producing this screen: // its newest frame answers in milliseconds instead of a ~0.5–1.5 s // capture. Only for a sized request (the stream is half resolution) and @@ -10737,22 +10830,7 @@ async fn agent_screenshot( .await { Ok(Ok(bytes)) if is_valid_png(&bytes) => { - if !raw { - let wireframe = tokio::time::timeout( - std::time::Duration::from_secs(15), - redacted_capture_wireframe(wda, &bytes), - ) - .await - .ok() - .flatten(); - if let Some(wireframe) = wireframe { - let wireframe = fit_png(wireframe, max_side).await; - return png_response(wireframe, Some("accessibility-wireframe"), true); - } - } - // `raw` is WDA's capture untouched, size included. - let max_side = if raw { None } else { max_side }; - png_response(fit_png(bytes, max_side).await, None, false) + finish_screenshot(wda, bytes, raw, max_side, None).await } Ok(Ok(bytes)) => { tracing::warn!( diff --git a/crates/server/src/wda.rs b/crates/server/src/wda.rs index dea173b..5aac206 100644 --- a/crates/server/src/wda.rs +++ b/crates/server/src/wda.rs @@ -48,6 +48,10 @@ pub struct WdaClient { /// The last tree read and when: snapshot-bound actions reuse it instead /// of reading the whole tree again (see [`Self::recent_tree`]). last_tree: Option<(std::sync::Arc>, std::time::Instant)>, + /// The screen as it stood after the last action settled (PNG), kept in + /// memory only so `/agent/screenshot` can answer without a new capture. + /// Dropped by the next screen-changing POST. + settled_frame: Option<(std::sync::Arc>, std::time::Instant)>, /// Until when screenshots are not used to judge "settled" (a frame was /// slow or failed; see `settle_frame` in http.rs). settle_frames_paused_until: Option, @@ -93,6 +97,7 @@ impl WdaClient { posted_at: None, no_alert_at: None, last_tree: None, + settled_frame: None, settle_frames_paused_until: None, actionability_budget: ACTIONABILITY_PROBE_BUDGET, }) @@ -1331,6 +1336,19 @@ impl WdaClient { self.settle_frames_paused_until = Some(std::time::Instant::now() + for_how_long); } + /// Keep `png` as the settled screen (see `settled_frame`). + pub fn remember_settled_frame(&mut self, png: Vec) { + self.settled_frame = Some((std::sync::Arc::new(png), std::time::Instant::now())); + } + + /// The settled screen, when it is younger than `max_age` and nothing that + /// changes the screen was sent since it was captured. + pub fn settled_frame(&self, max_age: Duration) -> Option>> { + let (png, at) = self.settled_frame.as_ref()?; + let untouched = self.posted_at.is_none_or(|posted| posted < *at); + (untouched && at.elapsed() < max_age).then(|| png.clone()) + } + /// Drop the reusable tree so the next snapshot-bound action reads afresh. pub fn forget_tree(&mut self) { self.last_tree = None; @@ -1359,6 +1377,8 @@ impl WdaClient { || path.ends_with("/element"); if !read_only { self.posted_at = Some(std::time::Instant::now()); + // Shows a screen that is about to change; free it now. + self.settled_frame = None; } self.http.post(url) } diff --git a/crates/server/tests/agent_input_settle.rs b/crates/server/tests/agent_input_settle.rs index 2d5dd97..67bec60 100644 --- a/crates/server/tests/agent_input_settle.rs +++ b/crates/server/tests/agent_input_settle.rs @@ -923,3 +923,89 @@ fn a_focused_text_field_never_settles_on_frames() { "a blinking caret keeps the tree check: {json}" ); } + +/// After an observed action the settled screen is already in memory: the +/// next screenshot costs no capture. Anything sent in between makes it stale, +/// and `?fresh=1` always captures. +#[test] +fn the_settled_screen_answers_the_next_screenshot_without_a_capture() { + use base64::Engine as _; + block(async { + let png = server::redaction::encode_png(&server::redaction::Image { + width: 4, + height: 8, + rgba: (0..4 * 8 * 4).map(|i| (i * 7 % 251) as u8).collect(), + }) + .unwrap(); + let encoded = base64::engine::general_purpose::STANDARD.encode(&png); + let frames = Arc::new(AtomicUsize::new(0)); + let seen_frames = frames.clone(); + let wda = mock_wda(move |request, _| { + if is_session(request) { + return Some((Duration::ZERO, SESSION.to_string())); + } + if is_mutation(request) { + return Some((Duration::ZERO, r#"{"value":null}"#.to_string())); + } + if is_source(request) { + return Some((Duration::ZERO, simple_tree("搜索"))); + } + if request.contains("/screenshot") { + seen_frames.fetch_add(1, Ordering::AcqRel); + return Some((Duration::ZERO, format!(r#"{{"value":"{encoded}"}}"#))); + } + Some(( + Duration::ZERO, + r#"{"value":{"error":"no such alert","message":"no alert"}}"#.to_string(), + )) + }); + let state = build_state_with_wda(wda.url()); + let screenshot = |uri: &'static str| { + let state = state.clone(); + async move { + let response = server::http::router(state) + .oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + let source = response + .headers() + .get("x-screenshot-source") + .map(|v| v.to_str().unwrap().to_string()); + (response.status(), source) + } + }; + + let (status, json, _) = request_json( + &state, + "POST", + "/agent/input?return=delta", + Some(r#"{"type":"home"}"#), + ) + .await; + assert_eq!(status, StatusCode::OK, "{json}"); + assert_eq!(json["settle"]["reason"], "stable", "{json}"); + let after_action = frames.load(Ordering::Acquire); + + let (status, source) = screenshot("/agent/screenshot").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(source.as_deref(), Some("settled-after-action")); + assert_eq!( + frames.load(Ordering::Acquire), + after_action, + "no new capture" + ); + + let (_, source) = screenshot("/agent/screenshot?fresh=1").await; + assert_eq!(source, None, "fresh=1 captures"); + assert_eq!(frames.load(Ordering::Acquire), after_action + 1); + + // An action without observation changes the screen: the kept frame + // no longer describes it. + let (status, _, _) = + request_json(&state, "POST", "/agent/input", Some(r#"{"type":"home"}"#)).await; + assert_eq!(status, StatusCode::OK); + let (_, source) = screenshot("/agent/screenshot").await; + assert_eq!(source, None); + assert_eq!(frames.load(Ordering::Acquire), after_action + 2); + }); +} diff --git a/docs/agent-reference.md b/docs/agent-reference.md index 1bd728c..73bf82b 100644 --- a/docs/agent-reference.md +++ b/docs/agent-reference.md @@ -180,7 +180,7 @@ one-shot `batch_hint`; a batch, or a pause over a minute, resets the count. ## Reading results: settle, delta, wait_for -**`?return=delta`** (MCP `observe:true`) settles after an applied action and +**`?return=delta`** (MCP act tools: on by default, `observe:false` to skip) settles after an applied action and returns `{ok, snapshot, baseline, delta}` (full `elements` when the baseline is no longer cached). The action result and the observation are separate facts: `ok:true` stands even when the observation fails. @@ -266,7 +266,12 @@ run / draft / update / publish / report`. - Act tools and `phone_capabilities` return JSON in `structuredContent`; the text block is a preview trimmed at 8 KiB. `phone_run_steps`, `phone_elements` and `phone_flow_*` return complete JSON as text; `phone_screenshot` an image. -- Single-step act tools take `observe` (= `?return=delta`). +- Single-step act tools observe by default (= `?return=delta`, baseline = the + last snapshot this client saw, so only the change comes back); `observe:false` + skips it. The settled screen is captured in memory, not sent: the next + `phone_screenshot` / `GET /agent/screenshot` returns it without a capture + (`X-Screenshot-Source: settled-after-action`; `?fresh=1` forces one). It is + dropped by the next action and never written to disk. - Decide by `retry_safe`, not `outcome`: `outcome:"unknown"` with `retry_safe:false` may have reached the phone. `not_sent` on one step never makes a whole batch replayable. diff --git a/skills/iphone-use/SKILL.md b/skills/iphone-use/SKILL.md index 9cd0977..c2d06c2 100644 --- a/skills/iphone-use/SKILL.md +++ b/skills/iphone-use/SKILL.md @@ -57,8 +57,10 @@ curl -s -H "$AUTH" "$HOST/agent/status" # probe first; on failure st locator) are for exploring a screen you have not read yet; after three in a row the daemon says so (`batch_hint`). -5. **Verify** each step against your postcondition (`?return=delta` / - `observe:true` returns the settled change in the same call). `ok:true` means +5. **Verify** each step against your postcondition: MCP act tools observe by + default (HTTP: `?return=delta`) and return the settled change in the same + call; the settled screen is captured too, and `phone_screenshot` returns it + instantly — look only when the text is not enough. `ok:true` means the action was sent, not that it achieved anything. → reference: *Reading results*