diff --git a/Cargo.lock b/Cargo.lock index 2c515319..d0cd11ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11908,6 +11908,7 @@ dependencies = [ name = "octosense-shell" version = "0.1.0" dependencies = [ + "base64 0.22.1", "chrono", "chrono-tz", "crossbeam-queue", diff --git a/crates/shell/Cargo.toml b/crates/shell/Cargo.toml index 2950de5c..aa701ffb 100644 --- a/crates/shell/Cargo.toml +++ b/crates/shell/Cargo.toml @@ -48,6 +48,7 @@ libc = "0.2" sha2 = "0.10" # The hub's per-launch connect secrets (hub.rs): v4 UUIDs from the OS CSPRNG. uuid = { workspace = true } +base64 = "0.22" serde = { workspace = true } serde_json = { workspace = true } chrono = { workspace = true } diff --git a/crates/shell/src/apps.rs b/crates/shell/src/apps.rs index 5a25f64a..edf71ad9 100644 --- a/crates/shell/src/apps.rs +++ b/crates/shell/src/apps.rs @@ -252,6 +252,27 @@ pub fn register_mail_services() { }); } +#[cfg(any(feature = "app-hub", native_mobile))] +/// The `model` service's grant: an app as itself, by its host-side +/// admission, and the system agent through the studio's image tool while +/// developer mode covers it (ADR 0006 slice 2). +pub(crate) fn model_grant(app: &str, host_dir: &std::path::Path) -> bool { + crate::host_tools::studio::system_caller_admitted(app) + || crate::host_tools::script_apps::admitted_host(app, host_dir).is_ok() +} + +/// The budget scope of a `model` call: the app's contained account, and +/// the device for the system agent's studio calls, in a build that has the +/// studio. +#[cfg(any(feature = "app-hub", native_mobile))] +pub(crate) fn model_scope(app: &str) -> Option { + if crate::host_tools::studio::SUPPORTED && app == crate::host_tools::SYSTEM { + return Some("device".into()); + } + crate::app_storage::lifecycle::contained_account(app) + .map(|account| crate::app_storage::account_hash(&account)) +} + #[cfg(any(feature = "app-hub", native_mobile))] pub fn register_host_services() { register_mail_services(); @@ -259,16 +280,11 @@ pub fn register_host_services() { ONCE.call_once(|| { // Capability declarations describe intended use. Admission still binds // every call to a verified app and the current host profile. - crate::ai_host::set_model_grants(|app, host_dir| { - crate::host_tools::script_apps::admitted_host(app, host_dir).is_ok() - }); + crate::ai_host::set_model_grants(model_grant); crate::ai_host::contained::set_caller_admitted(|app, host_dir| { crate::host_tools::script_apps::admitted_host(app, host_dir).is_ok() }); - crate::ai_host::set_model_scope(|app, _| { - crate::app_storage::lifecycle::contained_account(app) - .map(|account| crate::app_storage::account_hash(&account)) - }); + crate::ai_host::set_model_scope(|app, _| model_scope(app)); crate::platform_services::register(); crate::device_calendar::register(|app| { crate::app_storage::lifecycle::contained_account(app) diff --git a/crates/shell/src/host_tools/mod.rs b/crates/shell/src/host_tools/mod.rs index 215c74e3..5dc23b3a 100644 --- a/crates/shell/src/host_tools/mod.rs +++ b/crates/shell/src/host_tools/mod.rs @@ -43,6 +43,8 @@ pub mod schema; pub mod studio; #[cfg(all(unix, any(feature = "app-hub", native_mobile)))] pub mod studio_bundles; +#[cfg(all(unix, any(feature = "app-hub", native_mobile)))] +pub mod studio_image; #[cfg(any(feature = "app-hub", native_mobile))] pub(crate) mod admission; #[cfg(feature = "toolbox-peers")] @@ -127,6 +129,10 @@ pub fn pump() { // Components' calls to their apps' host services (ADR 0014). #[cfg(wasm_functions)] crate::wasm_service::pump_host_calls(); + // The studio's image generations (ADR 0006 slice 2): `model.image` + // calls under the calling agent's identity. + #[cfg(all(unix, any(feature = "app-hub", native_mobile)))] + studio_image::pump(); for _ in 0..8 { let events = std::mem::take(&mut *INBOX.lock().unwrap_or_else(|e| e.into_inner())); if events.is_empty() { diff --git a/crates/shell/src/host_tools/relay.rs b/crates/shell/src/host_tools/relay.rs index b8ddff7d..7e32fc4f 100644 --- a/crates/shell/src/host_tools/relay.rs +++ b/crates/shell/src/host_tools/relay.rs @@ -673,7 +673,8 @@ impl Relay { _ if super::studio::is_tool(&tool) => ( if call.caller_kind == CallerKind::System { Caller::SystemAgent } else { Caller::OwnAgent { client: call.client.clone() } }, super::studio::SUPPORTED && calling == owner && env.grants_all(&calling) - && (call.caller_kind == CallerKind::AppPeer || calling == SYSTEM), + && (call.caller_kind == CallerKind::AppPeer || calling == SYSTEM) + && super::studio::granted_to(&calling, &tool), ), // The shell's own: only the app's own agent, on its own peer; // `dev.run` only while developer mode covers the app. diff --git a/crates/shell/src/host_tools/studio.rs b/crates/shell/src/host_tools/studio.rs index 9d69c8c6..fc981bb0 100644 --- a/crates/shell/src/host_tools/studio.rs +++ b/crates/shell/src/host_tools/studio.rs @@ -21,6 +21,7 @@ pub const APP_TOOLS: &[&str] = &[ "studio.close", "studio.install", "studio.uninstall", + "studio.image", ]; /// A development build only (`cfg(dev_mode)`: a debug build, or the /// `dev-mode` feature): a release build has no studio, whatever it is @@ -36,10 +37,33 @@ pub fn declarations(app: &str) -> Vec { } let mut out = vec![declaration(app)]; if cfg!(all(unix, any(feature = "app-hub", native_mobile))) { - out.extend(APP_TOOLS.iter().map(|name| app_declaration(app, name))); + out.extend( + APP_TOOLS + .iter() + .filter(|name| app == SYSTEM || !SYSTEM_ONLY.contains(*name)) + .map(|name| tool_declaration(app, name)), + ); } out } +/// The tools of [`APP_TOOLS`] the system agent alone gets: `studio.image` +/// spends the person's image budget, so an app's agent is not offered it +/// in this slice (ADR 0006). +const SYSTEM_ONLY: &[&str] = &["studio.image"]; +/// Whether `tool` is a studio tool that `app`'s agent may be granted at +/// all: every one for the system agent, all but [`SYSTEM_ONLY`] for an +/// app's own agent. The relay's grant and the executor both ask. +pub fn granted_to(app: &str, tool: &str) -> bool { + is_tool(tool) && (app == SYSTEM || !SYSTEM_ONLY.contains(&tool)) +} +/// The declaration of one of [`APP_TOOLS`]. +fn tool_declaration(app: &str, name: &str) -> Value { + #[cfg(all(unix, any(feature = "app-hub", native_mobile)))] + if name == super::studio_image::TOOL { + return super::studio_image::declaration(app); + } + app_declaration(app, name) +} fn app_declaration(app: &str, name: &str) -> Value { let (description,properties,required)=match name { "studio.bundle_check" => ("Stage and validate an offline main.splash developer bundle from your conversation workspace. Computes and stamps the digest only on a private copy; never modifies source, signs a publisher release, or installs.",json!({"bundle_path":{"type":"string","minLength":1,"maxLength":1024}}),vec!["bundle_path"]), @@ -54,6 +78,16 @@ fn app_declaration(app: &str, name: &str) -> Value { "input_schema":{"type":"object","properties":properties,"required":required,"additionalProperties":false}, "output_schema":{"type":"object"},"risk":if name=="studio.install" || name=="studio.uninstall" || name=="studio.input" || name=="studio.open" {"act"} else {"read"},"background":false,"shareable":false}) } +/// Whether the system agent may use the `model` service as itself: only +/// while developer mode covers it, for the studio's image tool +/// (`apps.rs` consults this when it registers the model service). +pub fn system_caller_admitted(app: &str) -> bool { + system_caller_admitted_given(app, crate::dev_mode::grants_all(SYSTEM)) +} +/// [`system_caller_admitted`] with the live grant answered by the caller. +pub fn system_caller_admitted_given(app: &str, system_granted_all: bool) -> bool { + SUPPORTED && app == SYSTEM && system_granted_all +} /// Whether this build has the studio at all: a development build on a unix /// target. Release builds answer no tool, module, launcher row or fixture. pub const SUPPORTED: bool = cfg!(all(unix, dev_mode)); @@ -113,7 +147,7 @@ fn workspace(call: &HostToolCall) -> Result { .ok_or_else(|| "the broker has not confirmed this peer workspace".into()) } -fn app_available(call: &HostToolCall) -> Result<(), String> { +pub(super) fn app_available(call: &HostToolCall) -> Result<(), String> { let account = call .account .as_deref() @@ -129,7 +163,7 @@ fn app_available(call: &HostToolCall) -> Result<(), String> { Ok(()) } -fn authorized(app: &str, tag: &crate::dev_mode::DevTag) -> bool { +pub(super) fn authorized(app: &str, tag: &crate::dev_mode::DevTag) -> bool { crate::dev_mode::tag_valid(tag) && crate::dev_mode::grants_all(app) } @@ -179,10 +213,17 @@ impl StudioExecutor { )); return; } + if app != SYSTEM && SYSTEM_ONLY.contains(&call.name.as_str()) { + reply.finish(ToolOutcome::error( + "not_granted", + "studio.image is the system agent's tool", + )); + return; + } let decl = if call.name == RENDER { declaration(&app) } else { - app_declaration(&app, &call.name) + tool_declaration(&app, &call.name) }; if let Err(why) = super::schema::check(&decl["input_schema"], &call.args) { reply.finish(ToolOutcome::error("invalid_args", why)); @@ -218,7 +259,9 @@ impl StudioExecutor { let running = self.running.clone(); std::thread::spawn(move || { #[cfg(all(unix, any(feature = "app-hub", native_mobile)))] - let outcome = if call.name != RENDER { + let outcome = if call.name == super::studio_image::TOOL { + super::studio_image::execute(&call, &root, &app, &tag, &flag, &reply) + } else if call.name != RENDER { super::studio_bundles::execute(&call, &root, &tag, &flag, &reply) } else { render_call(&call, &root, &app, &tag, &flag, &reply) @@ -455,7 +498,9 @@ pub fn test_flow(spec_path: &str) -> Result<(), String> { }; seen.insert(name.clone()); let tool = request["name"].as_str().unwrap_or(""); - if !is_tool(tool) { + // A fixture drives the free tools only: `studio.image` + // spends the person's image budget. + if !is_tool(tool) || SYSTEM_ONLY.contains(&tool) { continue; } let mut call = match HostToolCall::parse( @@ -703,6 +748,13 @@ pub(super) mod scoped { pub fn output_json(&self) -> Result { self.output_extension("json") } + /// An image file, in the formats the `model` service returns. + pub fn output_image(&self, format: &str) -> Result { + if !matches!(format, "png" | "jpg") { + return Err(format!("unexpected image format {format}")); + } + self.output_extension(format) + } fn output_extension(&self, extension: &str) -> Result { let name = format!(".studio-{}.{}", uuid::Uuid::new_v4(), extension); let fd = openat( diff --git a/crates/shell/src/host_tools/studio_image.rs b/crates/shell/src/host_tools/studio_image.rs new file mode 100644 index 00000000..7896e2ce --- /dev/null +++ b/crates/shell/src/host_tools/studio_image.rs @@ -0,0 +1,1237 @@ +//! `studio.image`: an image from the host's `model` service, written into +//! the system agent's workspace (ADR 0006, slice 2 of the accepted route). +//! +//! The call runs `model.image` under the system agent's own identity, so the +//! person's configured image provider, the daily pixel budget and the +//! service's byte limits all apply as they do to any app; developer mode +//! admits the system agent to that service (`apps.rs`) and, as for every +//! studio tool, answers the approval itself. This slice offers the tool to +//! the system agent only: an app's agent would spend its app's budget. +//! +//! A generation can outlast one tool call, so the tool returns a job id when +//! the provider has not answered by the call's deadline, and the same tool +//! with `job` polls it; the image is written when a call sees the job done, +//! into that call's workspace, as `.studio-.png` (or `.jpg`), the way +//! `studio.inspect` writes its PNG. Every job carries the developer-mode tag +//! of the call that started it: once the tag no longer stands, a queued call +//! is not sent, a call in flight is cancelled, an answer is dropped, a +//! finished image is forgotten, and nothing is written. +//! +//! Threads: the service call is queued for the UI thread ([`pump`]), like a +//! component's host calls, and its reply comes back through App Hub's reply +//! queue. The UI thread owns only that queue, the waits and the cancel +//! list; it never takes the job table or the developer-mode lock. Each job +//! has a watcher thread that parses the answer, notices developer mode +//! moving, and ends the job when it expires; the executor's thread only +//! polls the job table. +use super::studio::scoped; +use crate::ai_host::app_peers::host_tools::{CallerKind, HostToolCall, ToolOutcome, ToolReply}; +use crate::dev_mode::DevTag; +use base64::Engine as _; +use octosense_appstore::services::{ServiceCall, ServiceHost}; +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +pub const TOOL: &str = "studio.image"; +/// Generations in flight at once: the model service's own bound per app +/// (`media::dispatch`), which the system agent is here. +const MAX_RUNNING: usize = 2; +/// Jobs of any state held at once, on the device and per conversation: a +/// finished, uncollected image is up to 2 MiB. +const MAX_JOBS: usize = 4; +const MAX_JOBS_PER_OWNER: usize = 2; +/// A finished job waits this long to be collected. +const JOB_TTL: Duration = Duration::from_secs(600); +/// How long a job waits for the service at most; the service's own timeout +/// covers every provider attempt and is shorter in practice. +const SERVICE_WAIT: Duration = Duration::from_secs(240); +/// A queued call the UI thread picks up with less than this left is not +/// sent: a provider may need two minutes, and a late answer would be thrown +/// away after it was paid for. +const MIN_SERVICE_TIME: Duration = Duration::from_secs(130); +/// Poll interval while a call waits for its job. +const POLL: Duration = Duration::from_millis(150); +/// How often a job's watcher looks for developer mode moving while it +/// waits for the answer, and then while it holds the finished image. +const WATCH_TICK: Duration = Duration::from_millis(250); +const HOLD_TICK: Duration = Duration::from_secs(1); +/// The sizes `model.image` accepts. +pub const SIZES: [&str; 4] = ["512x512", "1024x1024", "1536x1024", "1024x1536"]; +const SERVICE: &str = "model"; +const REVOKED: &str = "developer mode ended; the image job was dropped and nothing was written"; +const CHANGED: &str = "developer mode changed while the call waited; call again"; +const CANCELLED: &str = "the generation was cancelled"; +const TIMEOUT: &str = "the image provider did not answer in time"; +const NO_SUCH_JOB: &str = "no such image job in this conversation"; + +pub fn declaration(app: &str) -> Value { + json!({ + "name": TOOL, "app": app, + "description": "Developer mode only; the system agent's tool. Generate one image with the person's configured image provider and write it into your workspace, for an app's artwork or a design to build from. Give prompt (and optionally size, seed, class) to start; the reply has path, width, height and format, or, when the provider needs longer than this call, a job id: call again with only that job until state is done. Each generation spends the daily image budget and is approved by developer mode without asking; some providers refuse seed or 512x512; the image never leaves the workspace by itself.", + "input_schema": {"type":"object","properties":{ + "prompt":{"type":"string","minLength":1,"maxLength":4096}, + "size":{"type":"string","enum":SIZES}, + "seed":{"type":"integer","minimum":0,"maximum":4294967295u64}, + "class":{"type":"string","enum":["fast","strong"]}, + "job":{"type":"string","minLength":1,"maxLength":100} + },"additionalProperties":false}, + "output_schema": {"type":"object","properties":{ + "job":{"type":"string"},"state":{"type":"string","enum":["running","done"]}, + "path":{"type":"string"},"width":{"type":"integer","minimum":1}, + "height":{"type":"integer","minimum":1},"format":{"type":"string"} + },"required":["job","state"],"additionalProperties":false}, + "risk":"act","background":false,"shareable":false + }) +} + +/// Whether `tag` still stands and developer mode still covers `app`. Takes +/// the developer-mode lock: never called on the UI thread. +fn authorized(app: &str, tag: &DevTag) -> bool { + #[cfg(test)] + if let Some(answer) = tests::auth_override() { + return answer(app, tag); + } + super::studio::authorized(app, tag) +} + +/// The developer-mode generation, without its lock: it moves whenever a +/// grant may have changed. +fn generation() -> u64 { + #[cfg(test)] + if let Some(generation) = tests::generation_override() { + return generation; + } + crate::dev_mode::generation_relaxed() +} + +/// The host service family the generation is asked of. +fn family() -> &'static str { + #[cfg(test)] + if let Some(family) = tests::family_override() { + return family; + } + SERVICE +} + +// ---------------------------------------------------------------- the UI thread's state + +struct Queued { + key: usize, + call: ServiceCall, + /// The developer-mode generation when the call was queued. + generation: u64, + deadline: Instant, + done: Sender>, +} +struct Waiting { + done: Sender>, + deadline: Instant, +} +static QUEUE: Mutex> = Mutex::new(Vec::new()); +static WAITING: Mutex>> = Mutex::new(None); +/// Keys of calls to cancel: queued ones are not sent, sent ones are +/// cancelled at App Hub, by the UI thread, which owns App Hub's listeners. +static CANCELS: Mutex> = Mutex::new(Vec::new()); +/// Heap keys for these calls, in a range no isolate, script tool or +/// component uses. +static NEXT_KEY: AtomicUsize = AtomicUsize::new(1 << 52); + +/// A tool call never raises a sheet: the person is not in the app. +struct NoSheet; +impl ServiceHost for NoSheet { + fn open_sheet(&mut self, _body: String) {} + fn close_sheet(&mut self) {} +} + +/// Asks the UI thread to cancel the call with `key`, wherever it is. +fn request_cancel(key: usize) { + CANCELS.lock().unwrap_or_else(|e| e.into_inner()).push(key); + makepad_widgets::makepad_platform::thread::SignalToUI::set_ui_signal(); +} + +/// Whether a queued call may still be sent at `now`, with developer mode at +/// `generation`: a move since it was queued may have ended its grant, and +/// the job's watcher, not this thread, is the one that can check. +fn admit(queued: &Queued, now: Instant, generation: u64) -> Result<(), String> { + if queued.generation != generation { + return Err(CHANGED.into()); + } + if queued.deadline.saturating_duration_since(now) < MIN_SERVICE_TIME { + return Err("the host was not ready in time to start the generation".into()); + } + Ok(()) +} + +/// Answers every wait past its deadline and returns their keys, for the +/// caller to cancel at App Hub. +fn expire(waiting: &mut HashMap, now: Instant) -> Vec { + let expired: Vec = waiting + .iter() + .filter(|(_, wait)| wait.deadline <= now) + .map(|(key, _)| *key) + .collect(); + for key in &expired { + if let Some(wait) = waiting.remove(key) { + let _ = wait.done.send(Err(TIMEOUT.into())); + } + } + expired +} + +/// Dispatch the queued `model.image` calls, cancel what was asked, and +/// deliver the answers; on the UI thread ([`super::pump`]). The answer is +/// passed on as the text App Hub returned; the job's own thread parses it. +pub fn pump() { + let generation = generation(); + let queued = std::mem::take(&mut *QUEUE.lock().unwrap_or_else(|e| e.into_inner())); + let cancels = std::mem::take(&mut *CANCELS.lock().unwrap_or_else(|e| e.into_inner())); + let mut waiting = WAITING.lock().unwrap_or_else(|e| e.into_inner()); + let waiting = waiting.get_or_insert_with(HashMap::new); + let now = Instant::now(); + for queued in queued { + if cancels.contains(&queued.key) { + let _ = queued.done.send(Err(CANCELLED.into())); + continue; + } + if let Err(why) = admit(&queued, now, generation) { + let _ = queued.done.send(Err(why)); + continue; + } + waiting.insert( + queued.key, + Waiting { + done: queued.done, + deadline: queued.deadline, + }, + ); + octosense_appstore::services::dispatch(queued.call, queued.key, 1, &mut NoSheet); + } + for key in cancels { + if let Some(wait) = waiting.remove(&key) { + let _ = wait.done.send(Err(CANCELLED.into())); + octosense_appstore::services::cancel_heap(key); + } + } + if waiting.is_empty() { + return; + } + let keys: Vec = waiting.keys().copied().collect(); + for (key, _, result) in octosense_appstore::services::take_replies_for(&keys) { + if let Some(wait) = waiting.remove(&key) { + let _ = wait.done.send(result); + } + } + for key in expire(waiting, Instant::now()) { + octosense_appstore::services::cancel_heap(key); + } +} + +// ---------------------------------------------------------------- jobs + +/// The agent and conversation a job belongs to: another conversation never +/// sees it, as with the other studio tools' instances. +#[derive(Clone, PartialEq, Eq)] +struct Owner { + app: String, + account: Option, + session: String, + context: Option, +} +impl Owner { + fn of(app: &str, call: &HostToolCall) -> Owner { + Owner { + app: app.to_string(), + account: call.account.clone(), + session: call.session_id.clone(), + context: call.context_id.clone(), + } + } +} + +enum State { + Running, + /// The decoded image, kept until a call writes it into its workspace. + Done { + bytes: Vec, + width: u64, + height: u64, + format: String, + }, + /// A call is writing the image now. + Collecting, + Failed(String), +} +struct Job { + owner: Owner, + app: String, + tag: DevTag, + /// The heap key of its service call, to cancel it at App Hub. + key: usize, + started: Instant, + finished: Option, + state: State, +} +static JOBS: Mutex>> = Mutex::new(None); + +fn with_jobs(f: impl FnOnce(&mut HashMap) -> R) -> R { + let mut jobs = JOBS.lock().unwrap_or_else(|e| e.into_inner()); + let jobs = jobs.get_or_insert_with(HashMap::new); + let now = Instant::now(); + // Each job's watcher ends it on time; this is the backstop, and the only + // end of a job whose watcher never answered. + jobs.retain(|_, job| match job.finished { + Some(at) => now.duration_since(at) < JOB_TTL + HOLD_TICK, + None => now.duration_since(job.started) < SERVICE_WAIT + Duration::from_secs(60), + }); + f(jobs) +} + +/// Admits `job` into `jobs` under the bounds and returns its id, or why not. +fn register(jobs: &mut HashMap, job: Job) -> Result { + if jobs.values().filter(|j| matches!(j.state, State::Running)).count() >= MAX_RUNNING { + return Err("the studio is already generating two images; wait for one to finish".into()); + } + if jobs.len() >= MAX_JOBS { + return Err("the studio holds as many images as it can; collect one with its job id, or let one expire".into()); + } + if jobs.values().filter(|j| j.owner == job.owner).count() >= MAX_JOBS_PER_OWNER { + return Err("this conversation already holds two image jobs; collect them with their job ids first".into()); + } + let id = uuid::Uuid::new_v4().to_string(); + jobs.insert(id.clone(), job); + Ok(id) +} + +/// What `model.image` answered, decoded. +fn decode(reply: Result) -> State { + let value = match reply.and_then(|text| { + serde_json::from_str::(&text).map_err(|e| format!("invalid service reply: {e}")) + }) { + Ok(value) => value, + Err(why) => return State::Failed(why), + }; + let encoded = value["b64_json"].as_str().unwrap_or(""); + let format = value["format"].as_str().unwrap_or("png").to_string(); + let (width, height) = ( + value["width"].as_u64().unwrap_or(0), + value["height"].as_u64().unwrap_or(0), + ); + match base64::engine::general_purpose::STANDARD.decode(encoded) { + Ok(bytes) if !bytes.is_empty() && width > 0 && height > 0 => State::Done { + bytes, + width, + height, + format, + }, + _ => State::Failed("the image service returned no usable image".into()), + } +} + +/// Records a job's outcome, once; false when the job is already gone. +fn finish(id: &str, outcome: State) -> bool { + with_jobs(|jobs| match jobs.get_mut(id).filter(|job| job.finished.is_none()) { + Some(job) => { + job.state = outcome; + job.finished = Some(Instant::now()); + true + } + None => false, + }) +} + +/// Whether developer mode moved since `seen` and no longer covers the job. +fn revoked(app: &str, tag: &DevTag, seen: &mut u64) -> bool { + let now = generation(); + if now == *seen { + return false; + } + *seen = now; + !authorized(app, tag) +} + +/// Waits for the service's answer on a thread of the job's own, records +/// it, then holds the job until it is collected, abandoned, revoked or +/// expired. Started before the call is queued, so a request nobody waits +/// for is never sent; `generation` is the developer-mode generation the +/// queued call carries too, so a move in between fails one or the other. +/// An answer after developer mode ended is dropped with the job: nothing +/// is written later, and no thread waits on. +fn watch( + id: String, + app: String, + tag: DevTag, + key: usize, + generation: u64, + answer: Receiver>, +) -> Result<(), String> { + std::thread::Builder::new() + .name("studio-image".into()) + .spawn(move || { + let deadline = Instant::now() + SERVICE_WAIT; + let mut seen = generation; + let outcome = loop { + match answer.recv_timeout(WATCH_TICK) { + Ok(reply) => break decode(reply), + Err(RecvTimeoutError::Timeout) => { + if Instant::now() >= deadline { + break State::Failed(TIMEOUT.into()); + } + if revoked(&app, &tag, &mut seen) { + request_cancel(key); + break State::Failed(REVOKED.into()); + } + } + Err(RecvTimeoutError::Disconnected) => { + break State::Failed("the image service went away".into()) + } + } + }; + let revoked = matches!(&outcome, State::Failed(why) if why == REVOKED) || !authorized(&app, &tag); + if revoked { + with_jobs(|jobs| { + jobs.remove(&id); + }); + } else if finish(&id, outcome) { + hold(&id, &app, &tag, seen); + } + }) + .map(|_| ()) + .map_err(|e| format!("the studio could not start a thread to wait for the image: {e}")) +} + +/// Keeps a finished job for its collection window, and drops it when +/// developer mode no longer covers it. +fn hold(id: &str, app: &str, tag: &DevTag, mut seen: u64) { + let expiry = Instant::now() + JOB_TTL; + loop { + std::thread::sleep(HOLD_TICK); + if with_jobs(|jobs| !jobs.contains_key(id)) { + return; + } + if Instant::now() >= expiry || revoked(app, tag, &mut seen) { + with_jobs(|jobs| { + jobs.remove(id); + }); + return; + } + } +} + +/// Forgets a job whose first call was cancelled before the agent learned +/// its id; the UI thread keeps its call from going out, or cancels it. +fn abandon(id: &str, key: usize) { + request_cancel(key); + with_jobs(|jobs| { + jobs.remove(id); + }); +} + +/// The directory App Hub hands every host service, where the `model` +/// service keeps its budgets. +fn host_dir() -> Result { + Ok(octosense_appstore::data_root_if_set() + .ok_or("App Hub has no apps root yet")? + .join(".host")) +} + +/// The service call for `args`, under the calling agent's identity. +fn service_call(app: &str, args: &Value) -> Result { + Ok(service_call_in(app, args, host_dir()?, family())) +} + +/// The `.image` request `args` ask for, one image, as an agent's +/// tool call rather than the app in the foreground. +fn service_call_in(app: &str, args: &Value, host_dir: PathBuf, family: &str) -> ServiceCall { + let mut request = json!({"prompt": args["prompt"], "n": 1}); + for key in ["size", "seed", "class"] { + if !args[key].is_null() { + request[key] = args[key].clone(); + } + } + ServiceCall { + app_id: app.to_string(), + service: format!("{family}.image"), + args: request, + from_sheet: false, + may_prompt: false, + host_dir, + } +} + +/// One look at a job by the call polling it. +enum Step { + Wait, + Busy, + Failed(String), + Revoked(usize), + Image { + bytes: Vec, + width: u64, + height: u64, + format: String, + }, +} + +/// The job's state for `owner`, taking a finished image out for writing. +fn step(id: &str, owner: &Owner) -> Result { + let (app, tag, key) = with_jobs(|jobs| { + jobs.get(id) + .filter(|job| &job.owner == owner) + .map(|job| (job.app.clone(), job.tag.clone(), job.key)) + }) + .ok_or_else(|| NO_SUCH_JOB.to_string())?; + // Outside the job lock: this takes the developer-mode lock. + if !authorized(&app, &tag) { + with_jobs(|jobs| { + jobs.remove(id); + }); + return Ok(Step::Revoked(key)); + } + with_jobs(|jobs| { + let Some(job) = jobs.get_mut(id) else { + return Err(NO_SUCH_JOB.to_string()); + }; + Ok(match std::mem::replace(&mut job.state, State::Collecting) { + State::Running => { + job.state = State::Running; + Step::Wait + } + State::Collecting => Step::Busy, + State::Failed(why) => { + jobs.remove(id); + Step::Failed(why) + } + State::Done { + bytes, + width, + height, + format, + } => Step::Image { + bytes, + width, + height, + format, + }, + }) + }) +} + +/// Writes a finished job's image into the call's workspace and describes +/// it; on failure the image goes back to the job for a later call. +fn collect( + scope: &scoped::Workspace, + root: &Path, + call: &HostToolCall, + id: &str, + bytes: Vec, + width: u64, + height: u64, + format: String, +) -> Result { + match write_image(scope, root, call, &bytes, &format) { + Ok(path) => { + with_jobs(|jobs| { + jobs.remove(id); + }); + Ok(json!({"job": id, "state": "done", "path": path, "width": width, "height": height, "format": format})) + } + Err(why) => { + with_jobs(|jobs| { + if let Some(job) = jobs.get_mut(id) { + job.state = State::Done { + bytes, + width, + height, + format, + }; + } + }); + Err(why) + } + } +} + +fn write_image( + scope: &scoped::Workspace, + root: &Path, + call: &HostToolCall, + bytes: &[u8], + format: &str, +) -> Result { + use std::io::Write; + if call.caller_kind == CallerKind::AppPeer { + super::studio::app_available(call)?; + } + scope.verify(root, call.context_id.as_deref())?; + let output = scope.output_image(format)?; + (&output.file) + .write_all(bytes) + .and_then(|()| output.file.sync_all()) + .map_err(|e| format!("could not write the image: {e}"))?; + Ok(output.keep()) +} + +/// `studio.image`, on the executor's thread: `root` is the caller's +/// kernel-confirmed workspace, `app` the calling agent's app and `tag` the +/// developer-mode tag that admitted the call. +pub fn execute( + call: &HostToolCall, + root: &Path, + app: &str, + tag: &DevTag, + cancel: &AtomicBool, + reply: &ToolReply, +) -> ToolOutcome { + let run = || -> Result { + let scope = scoped::Workspace::open(root, call.context_id.as_deref())?; + let owner = Owner::of(app, call); + let (id, started) = match (call.args["job"].as_str(), call.args["prompt"].as_str()) { + (Some(job), None) => (job.to_string(), None), + (None, Some(_)) => { + // Built before a slot is taken: an invalid request holds none. + let service_call = service_call(app, &call.args)?; + let key = NEXT_KEY.fetch_add(1, Ordering::Relaxed); + let id = with_jobs(|jobs| { + register( + jobs, + Job { + owner: owner.clone(), + app: app.to_string(), + tag: tag.clone(), + key, + started: Instant::now(), + finished: None, + state: State::Running, + }, + ) + })?; + let (done, answer) = channel(); + let generation = generation(); + if let Err(why) = watch(id.clone(), app.to_string(), tag.clone(), key, generation, answer) { + with_jobs(|jobs| { + jobs.remove(&id); + }); + return Err(why); + } + QUEUE.lock().unwrap_or_else(|e| e.into_inner()).push(Queued { + key, + call: service_call, + generation, + deadline: Instant::now() + SERVICE_WAIT, + done, + }); + makepad_widgets::makepad_platform::thread::SignalToUI::set_ui_signal(); + (id, Some(key)) + } + _ => return Err("give prompt to start an image, or job alone to poll one".into()), + }; + // Wait for this call's share of the deadline; a running job outlives + // the call and is polled with its id. + let deadline = Instant::now() + + Duration::from_millis(call.timeout_ms.saturating_sub(1_500).max(500)).min(SERVICE_WAIT); + loop { + if cancel.load(Ordering::SeqCst) || !reply.is_open() { + // The agent never learns a cancelled first call's job id, so + // the job is not left to run for nobody. + return Err(match started { + Some(key) => { + abandon(&id, key); + "cancelled; the generation was abandoned".into() + } + None => "cancelled".into(), + }); + } + match step(&id, &owner)? { + Step::Wait => {} + Step::Busy => return Err("this image is being written by another call".into()), + Step::Failed(why) => return Err(why), + Step::Revoked(key) => { + request_cancel(key); + return Err(REVOKED.into()); + } + Step::Image { + bytes, + width, + height, + format, + } => return collect(&scope, root, call, &id, bytes, width, height, format), + } + if Instant::now() >= deadline { + return Ok(json!({"job": id, "state": "running"})); + } + std::thread::sleep(POLL); + } + }; + match run() { + Ok(value) => ToolOutcome::Ok(value), + Err(why) => ToolOutcome::error("studio_image_failed", why), + } +} + +#[cfg(test)] +mod tests { + use super::super::relay::SYSTEM; + use super::super::studio; + use super::*; + use crate::ai_host::app_peers::host_tools::ToolExecutor; + use octosense_appstore::services::{HostService, Replier}; + use std::sync::atomic::AtomicU64; + use std::sync::Arc; + + // ------------------------------------------------ seams for these tests + // + // Developer mode is a process-wide controller, and the `model` family + // may be the real service in this test process, so most tests answer + // the authorization and the generation themselves and register their + // fake under a family of their own. The two tests that need the real + // controller run in a child process of this test binary. + static DEV_ON: AtomicBool = AtomicBool::new(true); + static GENERATION: AtomicU64 = AtomicU64::new(1); + static SEAMS: AtomicBool = AtomicBool::new(false); + pub(super) fn auth_override() -> Option bool> { + SEAMS.load(Ordering::SeqCst).then_some(dev_flag as fn(&str, &DevTag) -> bool) + } + pub(super) fn generation_override() -> Option { + SEAMS.load(Ordering::SeqCst).then(|| GENERATION.load(Ordering::SeqCst)) + } + pub(super) fn family_override() -> Option<&'static str> { + SEAMS.load(Ordering::SeqCst).then_some(TEST_FAMILY) + } + fn dev_flag(_app: &str, _tag: &DevTag) -> bool { + DEV_ON.load(Ordering::SeqCst) + } + /// Developer mode "moves": the next pump refuses what was queued before, + /// and a watcher re-checks its job. + fn move_generation() { + GENERATION.fetch_add(1, Ordering::SeqCst); + } + const TEST_FAMILY: &str = "studio_image_test"; + + static CALLS: AtomicUsize = AtomicUsize::new(0); + static DELAY_MS: AtomicUsize = AtomicUsize::new(1500); + static REPLIERS: Mutex> = Mutex::new(Vec::new()); + static SETUP: std::sync::Once = std::sync::Once::new(); + /// The fake service and the job table are shared: one test at a time. + static SERIAL: Mutex<()> = Mutex::new(()); + + /// Answers `.image` like the real service, from a thread, after + /// `DELAY_MS`; keeps each replier so a test can see whether App Hub still + /// has the request pending. + struct FakeModel(&'static str); + impl HostService for FakeModel { + fn family(&self) -> &'static str { + self.0 + } + fn timeout(&self, _call: &ServiceCall) -> Duration { + Duration::from_secs(270) + } + fn call(&mut self, call: ServiceCall, reply: Replier, _host: &mut dyn ServiceHost) { + CALLS.fetch_add(1, Ordering::SeqCst); + assert!(call.service.ends_with(".image"), "{}", call.service); + assert!(!call.may_prompt && !call.from_sheet); + assert_eq!(call.app_id, SYSTEM); + REPLIERS.lock().unwrap().push(reply.clone()); + let delay = Duration::from_millis(DELAY_MS.load(Ordering::SeqCst) as u64); + std::thread::spawn(move || { + std::thread::sleep(delay); + reply.send(Ok(png_reply(1024, 1024))); + }); + } + } + + fn png_bytes() -> Vec { + vec![0x89, b'P', b'N', b'G', 1, 2, 3, 4] + } + fn png_reply(width: u64, height: u64) -> Value { + json!({"b64_json": base64::engine::general_purpose::STANDARD.encode(png_bytes()), + "format": "png", "mime": "image/png", "width": width, "height": height}) + } + fn tag() -> DevTag { + DevTag { + profile_id: "test-profile".into(), + since: 1, + } + } + fn job(owner: &str, state: State) -> Job { + Job { + owner: Owner { + app: SYSTEM.into(), + account: None, + session: owner.into(), + context: None, + }, + app: SYSTEM.into(), + tag: tag(), + key: 0, + started: Instant::now(), + finished: if matches!(state, State::Running) { None } else { Some(Instant::now()) }, + state, + } + } + + /// A workspace of its own for one test, under a data root the fake + /// service never reads; the seams on. + fn setup(test: &str) -> PathBuf { + let root = std::env::temp_dir().join(format!("studio-image-{}", std::process::id())); + SETUP.call_once(|| { + std::fs::create_dir_all(root.join("apps/.host")).unwrap(); + octosense_appstore::set_data_root(root.join("apps")); + octosense_appstore::services::register_host_service(Box::new(FakeModel(TEST_FAMILY))); + SEAMS.store(true, Ordering::SeqCst); + }); + let workspace = root.join(test); + let _ = std::fs::remove_dir_all(&workspace); + std::fs::create_dir_all(&workspace).unwrap(); + DEV_ON.store(true, Ordering::SeqCst); + with_jobs(|jobs| jobs.clear()); + workspace + } + + fn call(id: &str, session: &str, args: Value, timeout_ms: u64) -> HostToolCall { + let mut call = HostToolCall::parse(&json!({"call_id": id, "name": TOOL, "app": SYSTEM, + "session_id": session, "caller": {"kind": "system"}, "args": args, + "timeout_ms": timeout_ms, "risk": "act"})) + .unwrap(); + call.calling_app = SYSTEM.into(); + call + } + + fn data(outcome: ToolOutcome) -> Result { + match outcome { + ToolOutcome::Ok(value) => Ok(value), + ToolOutcome::Error { kind, message } => Err(format!("{kind}: {message}")), + } + } + + /// Runs `execute` as the executor does, on its own thread, while this + /// thread pumps as the UI thread would; `between` runs once the call + /// is queued, before the first pump. + fn run( + call: HostToolCall, + workspace: &Path, + cancel: Arc, + between: impl FnOnce(), + ) -> Result { + let workspace = workspace.to_path_buf(); + let worker = std::thread::spawn(move || { + let reply = ToolReply::new(call.call_id.clone(), |_| {}); + let outcome = execute(&call, &workspace, SYSTEM, &tag(), &cancel, &reply); + reply.finish(outcome.clone()); + outcome + }); + std::thread::sleep(Duration::from_millis(50)); + between(); + while !worker.is_finished() { + pump(); + std::thread::sleep(Duration::from_millis(10)); + } + data(worker.join().unwrap()) + } + + /// Pumps for `for_` so late answers and cancels are handled. + fn pump_for(for_: Duration) { + let until = Instant::now() + for_; + while Instant::now() < until { + pump(); + std::thread::sleep(Duration::from_millis(10)); + } + } + + fn jobs_of(session: &str) -> usize { + with_jobs(|jobs| jobs.values().filter(|job| job.owner.session == session).count()) + } + + fn state_of(job: &str) -> Option { + with_jobs(|jobs| { + jobs.get(job).map(|job| match &job.state { + State::Running => "running".to_string(), + State::Done { .. } => "done".to_string(), + State::Collecting => "collecting".to_string(), + State::Failed(why) => format!("failed: {why}"), + }) + }) + } + + fn files_in(workspace: &Path) -> usize { + std::fs::read_dir(workspace).unwrap().count() + } + + fn last_replier() -> Replier { + REPLIERS.lock().unwrap().last().cloned().expect("a service call reached the fake") + } + + // ------------------------------------------------ the real executor + + #[test] + fn a_start_returns_running_and_a_poll_collects_the_image_once() { + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let workspace = setup("happy"); + DELAY_MS.store(1500, Ordering::SeqCst); + let calls = CALLS.load(Ordering::SeqCst); + let cancel = Arc::new(AtomicBool::new(false)); + let first = run(call("h1", "sys", json!({"prompt": "a red square"}), 2000), &workspace, cancel.clone(), || {}).unwrap(); + assert_eq!(first["state"], "running"); + let job = first["job"].as_str().unwrap().to_string(); + // Another conversation cannot collect it. + let other = run(call("o1", "other", json!({"job": job}), 2000), &workspace, cancel.clone(), || {}); + assert!(other.unwrap_err().contains(NO_SUCH_JOB)); + let second = run(call("h2", "sys", json!({"job": job}), 6000), &workspace, cancel.clone(), || {}).unwrap(); + assert_eq!(second["state"], "done"); + assert_eq!((second["width"].as_u64(), second["format"].as_str()), (Some(1024), Some("png"))); + let path = second["path"].as_str().unwrap(); + assert!(path.starts_with(".studio-") && path.ends_with(".png"), "{path}"); + assert_eq!(std::fs::read(workspace.join(path)).unwrap(), png_bytes()); + assert_eq!(CALLS.load(Ordering::SeqCst), calls + 1); + // Collected once: a second poll finds nothing. + let again = run(call("h3", "sys", json!({"job": job}), 2000), &workspace, cancel, || {}); + assert!(again.is_err()); + assert_eq!(files_in(&workspace), 1); + } + + #[test] + fn a_cancel_before_the_first_reply_abandons_the_job_and_cancels_it_at_app_hub() { + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let workspace = setup("cancel"); + DELAY_MS.store(800, Ordering::SeqCst); + let calls = CALLS.load(Ordering::SeqCst); + let cancel = Arc::new(AtomicBool::new(false)); + let flag = cancel.clone(); + let outcome = run(call("c1", "sys-cancel", json!({"prompt": "a blue square"}), 30_000), &workspace, cancel, move || { + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(150)); + flag.store(true, Ordering::SeqCst); + }); + }); + assert!(outcome.unwrap_err().contains("abandoned")); + assert_eq!(jobs_of("sys-cancel"), 0); + // The UI thread cancels the sent request at App Hub; a late answer + // finds no job and writes nothing. + pump_for(Duration::from_millis(100)); + assert_eq!(CALLS.load(Ordering::SeqCst), calls + 1, "it had been dispatched before the cancel"); + assert!(!last_replier().is_pending(), "cancelled at App Hub"); + pump_for(Duration::from_millis(1200)); + assert_eq!(jobs_of("sys-cancel"), 0); + assert_eq!(files_in(&workspace), 0); + } + + #[test] + fn ending_developer_mode_drops_a_finished_job_on_its_poll_and_writes_nothing() { + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let workspace = setup("revoke"); + DELAY_MS.store(1500, Ordering::SeqCst); + let cancel = Arc::new(AtomicBool::new(false)); + let first = run(call("r1", "sys-revoke", json!({"prompt": "a green square"}), 2000), &workspace, cancel.clone(), || {}).unwrap(); + let job = first["job"].as_str().unwrap().to_string(); + // The grant ends without the generation moving (the watcher cannot + // tell); the poll re-checks the tag before it writes. + DEV_ON.store(false, Ordering::SeqCst); + let poll = run(call("r2", "sys-revoke", json!({"job": job}), 6000), &workspace, cancel.clone(), || {}); + assert!(poll.unwrap_err().contains("developer mode ended")); + assert_eq!(jobs_of("sys-revoke"), 0); + pump_for(Duration::from_millis(1500)); + assert_eq!(files_in(&workspace), 0); + // Turned on again, the old job is still gone. + DEV_ON.store(true, Ordering::SeqCst); + let later = run(call("r3", "sys-revoke", json!({"job": job}), 2000), &workspace, cancel, || {}); + assert!(later.unwrap_err().contains(NO_SUCH_JOB)); + } + + #[test] + fn a_watcher_ends_a_running_job_when_developer_mode_moves_and_no_longer_covers_it() { + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let workspace = setup("watch"); + DELAY_MS.store(3000, Ordering::SeqCst); + let cancel = Arc::new(AtomicBool::new(false)); + let first = run(call("w1", "sys-watch", json!({"prompt": "a yellow square"}), 2000), &workspace, cancel.clone(), || {}).unwrap(); + let job = first["job"].as_str().unwrap().to_string(); + let replier = last_replier(); + assert!(replier.is_pending()); + DEV_ON.store(false, Ordering::SeqCst); + move_generation(); + // Within a watch tick the watcher drops the job and the UI thread + // cancels the request at App Hub; no poll was needed, and no + // thread waits on. + pump_for(Duration::from_millis(700)); + assert_eq!(state_of(&job), None, "dropped at once"); + assert!(!replier.is_pending(), "cancelled at App Hub"); + pump_for(Duration::from_millis(2600)); + assert_eq!(files_in(&workspace), 0, "nothing written"); + // Back on, under a new generation: the old job cannot be collected. + DEV_ON.store(true, Ordering::SeqCst); + move_generation(); + let poll = run(call("w2", "sys-watch", json!({"job": job}), 2000), &workspace, cancel, || {}); + assert!(poll.is_err()); + assert_eq!(files_in(&workspace), 0); + } + + #[test] + fn a_queued_call_is_not_sent_once_developer_mode_has_moved() { + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let workspace = setup("queued"); + let calls = CALLS.load(Ordering::SeqCst); + let cancel = Arc::new(AtomicBool::new(false)); + let outcome = run(call("q1", "sys-queued", json!({"prompt": "never sent"}), 30_000), &workspace, cancel, move_generation); + assert!(outcome.unwrap_err().contains("developer mode changed")); + assert_eq!(CALLS.load(Ordering::SeqCst), calls, "the paid call must not go out"); + assert_eq!(jobs_of("sys-queued"), 0); + } + + // ------------------------------------------------ the real developer-mode controller + // + // Two checks need developer mode on for real: that the executor refuses + // an app's agent for the system-only reason, and that turning the mode + // off ends a job in flight. The controller is process-wide, so each + // runs in a child of this test binary launched with `OCTOSENSE_DEV_MODE=all`. + + /// In the parent: runs `test` in a child with the real controller on and + /// returns true once it passed. In the child: false, run the body. + fn in_child_with_developer_mode(test: &str) -> bool { + const CHILD: &str = "OCTOSENSE_STUDIO_IMAGE_TEST_CHILD"; + if std::env::var(CHILD).as_deref() == Ok(test) { + return false; + } + let output = std::process::Command::new(std::env::current_exe().expect("the shell test binary")) + .args(["--exact", &format!("host_tools::studio_image::tests::{test}"), "--nocapture"]) + .env(CHILD, test) + .env(crate::dev_mode::ENV, "all") + .output() + .expect("start the child test"); + let (stdout, stderr) = (String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); + assert!(output.status.success(), "{stdout}\n{stderr}"); + assert!(stdout.contains("test result: ok. 1 passed;"), "the child must run the one test:\n{stdout}"); + true + } + + /// The child's world: a fresh data root, the fake under the real + /// `model` family, the real controller on, and a UI-thread stand-in. + fn real_setup(test: &str) -> PathBuf { + let root = std::env::temp_dir().join(format!("studio-image-real-{test}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("apps/.host")).unwrap(); + std::fs::create_dir_all(root.join("home")).unwrap(); + std::fs::create_dir_all(root.join("ws")).unwrap(); + octosense_appstore::set_data_root(root.join("apps")); + octosense_appstore::services::register_host_service(Box::new(FakeModel(SERVICE))); + crate::dev_mode::init(&root.join("home")); + assert!(crate::dev_mode::grants_all(SYSTEM), "developer mode is on for the system agent"); + std::thread::Builder::new() + .name("ui".into()) + .spawn(|| loop { + pump(); + std::thread::sleep(Duration::from_millis(10)); + }) + .unwrap(); + root + } + + /// Through the real executor: developer-mode tag, grants, schema and + /// the system-only rule; the one reply it sends. + fn via_executor(call: HostToolCall) -> Value { + let (tx, rx) = channel(); + let reply = ToolReply::new(call.call_id.clone(), move |value| { + let _ = tx.send(value); + }); + studio::StudioExecutor::default().execute(call, reply); + rx.recv_timeout(Duration::from_secs(20)).expect("one reply") + } + + #[test] + fn an_app_agent_naming_the_tool_is_refused_before_any_service_call() { + if !studio::SUPPORTED || in_child_with_developer_mode("an_app_agent_naming_the_tool_is_refused_before_any_service_call") { + return; + } + let root = real_setup("app"); + let workspace = std::fs::canonicalize(root.join("ws")).unwrap(); + let calls = CALLS.load(Ordering::SeqCst); + // Each of these owns the tool it names, as far as the relay's + // ownership check goes; only the system agent gets this one. + for (calling, owner) in [("card.os.news", "os.news"), ("card.org.example.notes", "org.example.notes"), ("card.os.news", SYSTEM)] { + let mut call = HostToolCall::parse(&json!({"peer": "p1", "session_id": "s#peer-p1", "call_id": format!("app-{owner}"), + "name": TOOL, "app": owner, "caller": {"kind": "app_peer"}, "args": {"prompt": "x"}, + "timeout_ms": 2000, "risk": "act"})) + .unwrap(); + call.calling_app = calling.into(); + call.account = Some("device".into()); + call.peer_workspace = Some(workspace.clone()); + let reply = via_executor(call); + assert_eq!(reply["ok"], false, "{reply}"); + assert_eq!(reply["error"]["kind"], "not_granted", "{reply}"); + } + assert_eq!(CALLS.load(Ordering::SeqCst), calls, "no service call for an app's agent"); + assert_eq!(files_in(&workspace), 0); + // The system agent itself is admitted by the same executor. + studio::system_workspace_opened(Some(workspace.to_str().unwrap())); + DELAY_MS.store(100, Ordering::SeqCst); + let mut call = call("sys-1", crate::system_chat::session::SYSTEM_SESSION, json!({"prompt": "a red square"}), 6000); + call.calling_app = SYSTEM.into(); + let reply = via_executor(call); + assert_eq!(reply["data"]["state"], "done", "{reply}"); + assert_eq!(files_in(&workspace), 1); + } + + #[test] + fn turning_developer_mode_off_ends_a_job_in_flight_without_a_poll() { + if !studio::SUPPORTED || in_child_with_developer_mode("turning_developer_mode_off_ends_a_job_in_flight_without_a_poll") { + return; + } + let root = real_setup("off"); + let workspace = root.join("ws"); + DELAY_MS.store(3000, Ordering::SeqCst); + let live = crate::dev_mode::tag().expect("a live developer-mode tag"); + let cancel = AtomicBool::new(false); + let reply = ToolReply::new("off-1", |_| {}); + let first = data(execute(&call("off-1", "sys", json!({"prompt": "a green square"}), 2000), &workspace, SYSTEM, &live, &cancel, &reply)).unwrap(); + let job = first["job"].as_str().unwrap().to_string(); + let replier = last_replier(); + assert!(replier.is_pending()); + assert!(crate::apps::model_grant(SYSTEM, &root.join("apps/.host"))); + crate::dev_mode::turn_off("studio.image test"); + assert!(!crate::apps::model_grant(SYSTEM, &root.join("apps/.host")), "the model service refuses the system agent now"); + std::thread::sleep(Duration::from_millis(700)); + assert_eq!(state_of(&job), None, "the watcher dropped it at once"); + assert!(!replier.is_pending(), "the watcher had it cancelled at App Hub"); + std::thread::sleep(Duration::from_millis(3000)); + assert_eq!(files_in(&workspace), 0, "nothing written"); + // Back on, with a new tag: the old job cannot be collected. + std::thread::sleep(Duration::from_millis(1100)); + crate::dev_mode::init(&root.join("home")); + let tag = crate::dev_mode::tag().expect("a new tag"); + assert_ne!(tag, live); + let poll = data(execute(&call("off-2", "sys", json!({"job": job}), 2000), &workspace, SYSTEM, &tag, &cancel, &reply)); + assert!(poll.is_err()); + assert_eq!(files_in(&workspace), 0); + } + + // ------------------------------------------------ the pieces + + #[test] + fn jobs_are_bounded_running_held_and_per_conversation() { + let mut jobs = HashMap::new(); + let a1 = register(&mut jobs, job("a", State::Running)).unwrap(); + let b1 = register(&mut jobs, job("b", State::Running)).unwrap(); + let third = register(&mut jobs, job("c", State::Running)).unwrap_err(); + assert!(third.contains("two images"), "{third}"); + // A finished image frees its running slot but is still held: it + // counts per conversation and in all until collected or expired. + jobs.get_mut(&a1).unwrap().state = State::Failed("x".into()); + jobs.get_mut(&b1).unwrap().state = State::Failed("x".into()); + register(&mut jobs, job("a", State::Running)).unwrap(); + let a_third = register(&mut jobs, job("a", State::Running)).unwrap_err(); + assert!(a_third.contains("this conversation"), "{a_third}"); + let d = register(&mut jobs, job("d", State::Running)).unwrap(); + jobs.get_mut(&d).unwrap().state = State::Done { + bytes: png_bytes(), + width: 1, + height: 1, + format: "png".into(), + }; + let fifth = register(&mut jobs, job("e", State::Running)).unwrap_err(); + assert!(fifth.contains("holds as many"), "{fifth}"); + jobs.remove(&a1); + register(&mut jobs, job("e", State::Running)).unwrap(); + } + + #[test] + fn a_service_reply_is_decoded_or_refused() { + match decode(Ok(png_reply(512, 512).to_string())) { + State::Done { bytes, width, height, format } => { + assert_eq!((bytes.len(), width, height, format.as_str()), (8, 512, 512, "png")); + } + _ => panic!("decoded"), + } + assert!(matches!(decode(Ok(json!({"format":"png"}).to_string())), State::Failed(_))); + assert!(matches!(decode(Ok(json!({"b64_json":"!!","width":1,"height":1}).to_string())), State::Failed(_))); + assert!(matches!(decode(Ok("not json".into())), State::Failed(w) if w.contains("invalid service reply"))); + assert!(matches!(decode(Err("budget".into())), State::Failed(w) if w == "budget")); + } + + #[test] + fn the_ui_thread_refuses_a_late_or_outdated_call_and_answers_an_expired_wait() { + let queued = |deadline: Instant, generation: u64| { + let (done, answer) = channel(); + let queued = Queued { + key: 7, + call: service_call_in(SYSTEM, &json!({"prompt": "p"}), std::env::temp_dir(), TEST_FAMILY), + generation, + deadline, + done, + }; + (queued, answer) + }; + let now = Instant::now(); + let (fresh, _) = queued(now + SERVICE_WAIT, 3); + assert!(admit(&fresh, now, 3).is_ok()); + assert!(admit(&fresh, now, 4).unwrap_err().contains("developer mode changed")); + let (late, _) = queued(now + MIN_SERVICE_TIME - Duration::from_secs(1), 3); + assert!(admit(&late, now, 3).unwrap_err().contains("not ready in time")); + + let (done, answer) = channel(); + let mut waiting = HashMap::new(); + waiting.insert( + 9usize, + Waiting { + done, + deadline: now - Duration::from_secs(1), + }, + ); + let (kept, _keep) = channel(); + waiting.insert( + 10usize, + Waiting { + done: kept, + deadline: now + Duration::from_secs(1), + }, + ); + assert_eq!(expire(&mut waiting, now), vec![9]); + assert_eq!(answer.try_recv().unwrap().unwrap_err(), TIMEOUT); + assert!(waiting.contains_key(&10) && !waiting.contains_key(&9)); + } + + #[test] + fn the_declaration_takes_a_prompt_or_a_job_and_nothing_else() { + let decl = declaration(SYSTEM); + let schema = &decl["input_schema"]; + assert!(super::super::schema::check(schema, &json!({"prompt":"a red square"})).is_ok()); + assert!(super::super::schema::check(schema, &json!({"prompt":"x","size":"512x512","seed":7,"class":"fast"})).is_ok()); + assert!(super::super::schema::check(schema, &json!({"job":"j"})).is_ok()); + assert!(super::super::schema::check(schema, &json!({"prompt":"x","size":"640x480"})).is_err()); + assert!(super::super::schema::check(schema, &json!({"prompt":"x","output_path":"a.png"})).is_err()); + assert_eq!(decl["risk"], "act"); + let request = service_call_in(SYSTEM, &json!({"prompt":"p","size":"512x512","seed":3}), std::env::temp_dir(), SERVICE); + assert_eq!(request.service, "model.image"); + assert_eq!(request.args, json!({"prompt":"p","n":1,"size":"512x512","seed":3})); + assert!(!request.may_prompt && !request.from_sheet); + } + + #[test] + fn the_tool_is_the_system_agents_and_the_model_service_admits_it_under_developer_mode_only() { + if !studio::SUPPORTED { + return; + } + assert!(studio::declarations(SYSTEM).iter().any(|d| d["name"] == TOOL)); + assert!(!studio::declarations("os.news").iter().any(|d| d["name"] == TOOL)); + assert!(studio::declarations("os.news").iter().any(|d| d["name"] == "studio.install")); + // What the relay grants, by owner and tool. + assert!(studio::granted_to(SYSTEM, TOOL)); + assert!(!studio::granted_to("os.news", TOOL)); + assert!(studio::granted_to("os.news", "studio.install")); + assert!(!studio::granted_to(SYSTEM, "studio.nothing")); + // The admission the model service asks `apps.rs` for: the system + // agent only while developer mode grants it everything, and only in + // a build that has the studio. + assert!(studio::system_caller_admitted_given(SYSTEM, true)); + assert!(!studio::system_caller_admitted_given(SYSTEM, false)); + assert!(!studio::system_caller_admitted_given("os.news", true)); + assert_eq!(studio::system_caller_admitted(SYSTEM), crate::dev_mode::grants_all(SYSTEM)); + assert_eq!(crate::apps::model_scope(SYSTEM).as_deref(), Some("device")); + // An app's scope is its contained account's hash, if it has one; + // never the device. + assert_ne!(crate::apps::model_scope("os.news").as_deref(), Some("device")); + assert!(!crate::apps::model_grant("os.news", &std::env::temp_dir())); + } +} diff --git a/crates/shell/src/host_tools/tests.rs b/crates/shell/src/host_tools/tests.rs index 13cd398a..d05231a8 100644 --- a/crates/shell/src/host_tools/tests.rs +++ b/crates/shell/src/host_tools/tests.rs @@ -753,8 +753,9 @@ fn a_peer_is_offered_exactly_its_granted_toolbox_tools_marked_with_their_owner_a let mut every = shareable_native_tools(); every.push(super::relay::DEV_RUN.to_string()); if super::studio::SUPPORTED { every.push(super::studio::RENDER.to_string()); } + // `studio.image` is the system agent's alone. #[cfg(all(unix, any(feature="app-hub", native_mobile)))] - every.extend(super::studio::APP_TOOLS.iter().map(|name| name.to_string())); + every.extend(super::studio::APP_TOOLS.iter().filter(|name| **name != "studio.image").map(|name| name.to_string())); every.sort(); assert_eq!(offered_names(&relay, "calendar", true, true), every); } @@ -1418,7 +1419,10 @@ fn studio_app_tools_are_own_caller_only_and_disappear_on_revocation() { let host=Arc::new(Exec::default()); relay.set_executor(super::relay::HOST_EXECUTOR,Some(host.clone())); let mut w=World::new(FixedDevMode::all());w.dev_all=true; - for (index,name) in super::studio::APP_TOOLS.iter().enumerate(){ + // `studio.image` is the system agent's alone: an app's agent is not + // offered it (checked in studio_image's tests). + let app_tools: Vec<&str>=super::studio::APP_TOOLS.iter().copied().filter(|name| *name!="studio.image").collect(); + for (index,name) in app_tools.iter().enumerate(){ let id=format!("studio-app-{index}"); let mut c=call(&id,name,"os.news");c.app="os.news".into(); c.args=match *name { @@ -1439,7 +1443,17 @@ fn studio_app_tools_are_own_caller_only_and_disappear_on_revocation() { assert_eq!(sent.lock().unwrap()[0]["error"]["kind"],"not_granted"); w.dev_all=true; } - assert_eq!(host.0.lock().unwrap().len(),super::studio::APP_TOOLS.len()); + // An app's agent naming `studio.image` directly is refused by the relay + // itself, before the schema check or the executor. + #[cfg(all(unix, any(feature="app-hub", native_mobile)))] + { + let mut c=call("studio-app-image","studio.image","os.news");c.app="os.news".into(); + c.args=json!({"prompt":"a red square on white"}); + let (r,sent)=reply("studio-app-image"); + relay.handle(Event::Call{call:c,reply:r},&mut w); + assert_eq!(sent.lock().unwrap()[0]["error"]["kind"],"not_granted"); + } + assert_eq!(host.0.lock().unwrap().len(),app_tools.len()); } // ------------------------------------------------------------ the engines // diff --git a/docs/adr/0006-app-studio-on-the-phone.md b/docs/adr/0006-app-studio-on-the-phone.md index f75f7bc5..fb4fcfea 100644 --- a/docs/adr/0006-app-studio-on-the-phone.md +++ b/docs/adr/0006-app-studio-on-the-phone.md @@ -373,12 +373,12 @@ tools because of what they do, not because of a declaration. A host-tool reply is capped at 256 KiB of text, so an image a tool produces goes to the workspace as a file, as section 7 already requires. -### Where things stand, 11 October 2026 +### Where things stand, 12 October 2026 | Step of the flow | State | | --- | --- | | Brief, and writing `main.splash` | Works: the system agent writes files into its workspace | -| Generated designs | Not available to the agent: octos's `image_generation` tool still has no backend (`no_backend_bound`); the `model.image` host method exists for script apps, with the model set by the host (`gpt-image-1`, `gpt-image-1-mini` or MiniMax `image-01`), and no live call is recorded | +| Generated designs | Merged, no live call recorded yet: `studio.image` (slice 2, #513), offered only to the system agent in this slice, generates one image through `model.image` under that agent's identity and writes it into the agent's workspace; the system agent is admitted to `model.image` only while covered by developer mode, and the host still sets the model (`gpt-image-1`, `gpt-image-1-mini` or MiniMax `image-01`); octos's own `image_generation` tool still has no backend (`no_backend_bound`) | | Reference images in | `files.pick_photo` on Home 0.1.0-beta.2, one image, 1 MiB; no image share target; no capture of other apps | | Measurement, OCR, mapping | Not ported; `ocrs` and `rten` are in the lock file and unused outside the PDF engine | | Review screen | Not built | @@ -397,9 +397,10 @@ Each slice is usable on its own. APK re-validated on a stock Xiaomi. Gives brief → `main.splash` → preview → install on the phone. 2. **Inputs.** An agent tool that calls `model.image` and saves the PNG into - the agent's workspace, with the provider the person chose and a model still - to be chosen (see the open questions); the first live call. Reference images - through `files.pick_photo`. + the agent's workspace, with the provider the person chose and the model the + host sets for it; which to use first is an open question (below). Merged in + #513 as `studio.image`; no live call is recorded yet. Reference + images through `files.pick_photo`. 3. **The pipeline as components.** The five crates above, developed on the desktop with OctoSense's `wasm_call` example (OctoSense #455) and App Flow's `tools/octo wasm call` (App Flow #184, merged), shipped in the studio's diff --git a/docs/architecture-walkthrough.md b/docs/architecture-walkthrough.md index b86a10f1..58d85428 100644 --- a/docs/architecture-walkthrough.md +++ b/docs/architecture-walkthrough.md @@ -291,6 +291,9 @@ An agent can write a fresh `manifest.json` and `main.splash` with its normal fil | `studio.close {instance_id}` | Closes the app and discards preview state. | | `studio.install {bundle_path}` | Records a local developer install, visible in Home. Open it with `studio.open {app_id}` or its launcher tile; its own state survives closing and reopening. | | `studio.uninstall {app_id}` | Removes one of the caller's own developer installs: receipt, snapshot, app data and owner record. Refused while an instance of it is open. | +| `studio.image {prompt, size?, seed?, class?}` or `{job}` | Generates one image with the person's configured image provider, through the `model.image` service under the calling agent's identity, and writes it into the caller's workspace: the reply has the image's `path`, `width`, `height` and `format`. When the provider needs longer than one call allows, the reply is `{job, state: "running"}`; call again with that `job` alone until `state` is `done`. | + +`studio.image` ([studio_image.rs](../crates/shell/src/host_tools/studio_image.rs)) is the one studio tool that spends money, and in this slice it is offered only to the system agent. It calls `model.image`, the same host service any app reaches, so the provider the person configured, the host's choice of model, the daily image budget and the service's byte limits apply unchanged; developer mode answers the approval request itself, without showing a sheet. The system agent is admitted to that service only while covered by developer mode. The service call runs on the UI thread with the other host calls; the tool's thread only waits, and hands back a job id if its share of the kernel's 30-second deadline runs out. A conversation holds at most two jobs, running or finished (the model service allows an app two generations at a time); a finished image waits up to ten minutes for a call from the same conversation to collect it, and is never automatically sent outside the workspace. Every job carries the developer-mode tag of the call that started it: once developer mode ends, a queued service call is not sent, one in flight is cancelled, an answer is dropped, a finished image is forgotten, and nothing is written. The first full-app path accepts offline, storage-only `main.splash` bundles under `dev.studio.*`. These apps have no agent of their own, account access or `net` module, and every `host.request` is refused; the admitted instruction budget is a declaration, while the jail, quota and memory cap are enforced. Admission also refuses a `main.splash` whose text names a URL or the `{{assets}}` route; that scan is a lint, since Splash builds strings at runtime, and the offline guarantee is the isolate itself, which runs with no network hosts and no host capabilities. A bundle may contain an original launcher icon, but in-screen resource routes are not supported yet. In [studio_bundles.rs](../crates/shell/src/host_tools/studio_bundles.rs), admission limits the bundle to 128 files/directories, eight directory levels and 2 MiB total, with at most 512 KiB per file and 64 KiB for `main.splash`. The resolved policy allows at most 1 MiB of private app storage, five million script instructions and a 16 MiB heap; a lower manifest limit remains lower. @@ -312,7 +315,7 @@ The Android HTTP remote instrument is unavailable at this pin. Studio uses its u None of this creates an agent peer or assigns one Tokio task to each app. The existing agent invokes a host tool; Rust workers handle files and encoding, and the UI thread owns widgets and GPU submission. -A fresh Task Planner authored by DeepSeek V4 Flash passed **129 tool calls on a physical OnePlus 6**: task entry/completion/filtering, disposable preview state, separate installed state, close/reopen, process restart, exact Chinese text input and scrolling. The harness did not modify app source or storage directly. Portrait app and keyboard visual review passed, with generous spacing and separate shell overlay/status-bar observations; malformed-storage/save-failure fault injection remains pending. See the [validation report](studio/oneplus6-validation.md), [ADR 0006](adr/0006-app-studio-on-the-phone.md) and the [fresh-app brief](studio/task-planner-brief.md). The `mod.studio` toolbox adapter, image generation/comparison, richer asset routes and public publishing are not implemented by this slice. After the merges of `main`, a fresh Task Planner authored by Claude passed the same acceptance flow on a stock Xiaomi (134 tool calls on the final head), and a second spool-driven check covered owned app data, `studio.uninstall` and an ended developer grant ([re-validation](studio/xiaomi-revalidation.md)). +A fresh Task Planner authored by DeepSeek V4 Flash passed **129 tool calls on a physical OnePlus 6**: task entry/completion/filtering, disposable preview state, separate installed state, close/reopen, process restart, exact Chinese text input and scrolling. The harness did not modify app source or storage directly. Portrait app and keyboard visual review passed, with generous spacing and separate shell overlay/status-bar observations; malformed-storage/save-failure fault injection remains pending. See the [validation report](studio/oneplus6-validation.md), [ADR 0006](adr/0006-app-studio-on-the-phone.md) and the [fresh-app brief](studio/task-planner-brief.md). The `mod.studio` toolbox adapter, image comparison, richer asset routes and public publishing are not implemented by this slice; image generation arrived with slice 2's `studio.image`. After the merges of `main`, a fresh Task Planner authored by Claude passed the same acceptance flow on a stock Xiaomi (134 tool calls on the final head), and a second spool-driven check covered owned app data, `studio.uninstall` and an ended developer grant ([re-validation](studio/xiaomi-revalidation.md)). ## 11. Tests diff --git a/docs/architecture-walkthrough.zh-CN.md b/docs/architecture-walkthrough.zh-CN.md index a3c044f7..9ba35d44 100644 --- a/docs/architecture-walkthrough.zh-CN.md +++ b/docs/architecture-walkthrough.zh-CN.md @@ -279,6 +279,9 @@ Agent 可先用普通文件工具从头编写 `manifest.json` 和 `main.splash` | `studio.close {instance_id}` | 关闭应用，丢弃预览状态。 | | `studio.install {bundle_path}` | 登记本地开发者安装，在 Home 中显示。随后可用 `studio.open {app_id}` 或启动器图块打开；应用自己的状态在关闭、重开后保留。 | | `studio.uninstall {app_id}` | 删除调用方自己的一个开发者安装：收据、快照、应用数据和所有者记录。该应用仍有实例打开时拒绝。 | +| `studio.image {prompt, size?, seed?, class?}` 或 `{job}` | 借助用户配置的图像提供方生成一张图：以调用方 Agent 自己的身份经 `model.image` 服务生成，写入调用方的 workspace，返回图像的 `path` 以及 `width`、`height`、`format`。若提供方在一次调用内未完成，则返回 `{job, state: "running"}`，之后只传入该 `job` 再次调用，直到 `state` 为 `done`。 | + +`studio.image`（[studio_image.rs](../crates/shell/src/host_tools/studio_image.rs)）是唯一会产生费用的 studio 工具，本阶段只提供给系统 Agent。它调用的 `model.image` 就是任何应用都能访问的那个宿主服务，因此用户配置的提供方、宿主选定的模型、每日图像预算和该服务的字节上限都原样适用；审批由开发者模式自行通过，不弹出面板。系统 Agent 只在开发者模式涵盖它时获准使用该服务。服务调用与其他宿主调用一样在 UI 线程上执行，工具线程只负责等待：若内核 30 秒期限中属于自己的份额已用完，则返回一个 job id。每个对话最多持有两个 job，无论进行中还是已完成（模型服务允许单个应用同时进行两次生成）；已生成的图像最长保留 10 分钟，等待同一对话的调用取走，且不会被自动发送到 workspace 之外。每个 job 都带有发起它的那次调用的开发者模式标记：开发者模式一旦结束，排队中的调用不再发送，进行中的调用被取消，返回的结果和已生成的图像都被丢弃，也不会写入任何文件。 首个完整应用路径接受 `dev.studio.*` 命名空间下的离线、仅存储权限 `main.splash` 包。应用本身没有 Agent、账户访问和 `net` 模块，每个 `host.request` 都会被拒绝；准入的指令预算只是声明，真正强制的是存储 jail、配额和内存上限。准入还会拒绝文本中含有 URL 或 `{{assets}}` 路径的 `main.splash`。这只是一道静态检查，因为 Splash 可以在运行时拼接出字符串；真正的离线保证来自隔离环境本身：它没有任何网络主机，也没有任何宿主能力，运行时发起的 URL 加载会被直接拒绝。包中可携带原创启动器图标，但暂不支持屏幕内的资源加载路径。[studio_bundles.rs](../crates/shell/src/host_tools/studio_bundles.rs)把包限制为 128 个文件/目录、八层目录、总计 2 MiB；单文件最多 512 KiB，`main.splash` 最多 64 KiB。解析后的策略最多允许 1 MiB 私有应用存储、五百万脚本指令和 16 MiB 堆；清单中更低的限制仍然生效。 @@ -300,7 +303,7 @@ Agent 可先用普通文件工具从头编写 `manifest.json` 和 `main.splash` 这些操作不会创建 Agent peer，也不是每个应用对应一个 Tokio task。已有 Agent 调用宿主工具，Rust 工作线程负责文件与编码，UI 线程持有控件并提交 GPU 工作。 -DeepSeek V4 Flash 从头编写的 Task Planner 已在**真实 OnePlus 6 上通过 129 次工具调用**，覆盖任务输入、完成与筛选、预览状态丢弃、独立安装状态、关闭重开、进程重启、准确的中文输入及滚动。测试工具没有直接修改应用源码或存储。竖屏应用与键盘视觉评审已通过；间距较宽，Shell 浮层和状态栏另有观察记录。损坏存储与保存失败的故障注入仍待完成，详见[验收报告](studio/oneplus6-validation.md)。见 [ADR 0006](adr/0006-app-studio-on-the-phone.md) 和[新应用需求](studio/task-planner-brief.md)。本次实现不包含 `mod.studio` 工具箱适配器、图像生成/比较、更完整的资源路径或公开发布。合并 `main` 之后，一个由 Claude 从头编写的 Task Planner 在一部普通小米手机上通过了同样的验收流程（在最终代码上共 134 次工具调用）；另一项由请求队列驱动的检查则覆盖了应用数据的所有权、`studio.uninstall` 和开发者授权结束后的行为（见[再次验证](studio/xiaomi-revalidation.md)）。 +DeepSeek V4 Flash 从头编写的 Task Planner 已在**真实 OnePlus 6 上通过 129 次工具调用**，覆盖任务输入、完成与筛选、预览状态丢弃、独立安装状态、关闭重开、进程重启、准确的中文输入及滚动。测试工具没有直接修改应用源码或存储。竖屏应用与键盘视觉评审已通过；间距较宽，Shell 浮层和状态栏另有观察记录。损坏存储与保存失败的故障注入仍待完成，详见[验收报告](studio/oneplus6-validation.md)。见 [ADR 0006](adr/0006-app-studio-on-the-phone.md) 和[新应用需求](studio/task-planner-brief.md)。本阶段不包含 `mod.studio` 工具箱适配器、图像比较、更完整的资源路径或公开发布；图像生成随第 2 阶段的 `studio.image` 加入。合并 `main` 之后，一个由 Claude 从头编写的 Task Planner 在一部普通小米手机上通过了同样的验收流程（在最终代码上共 134 次工具调用）；另一项由请求队列驱动的检查则覆盖了应用数据的所有权、`studio.uninstall` 和开发者授权结束后的行为（见[再次验证](studio/xiaomi-revalidation.md)）。 ## 11. 测试 diff --git a/phone/README.md b/phone/README.md index 4648ecac..1d2713da 100644 --- a/phone/README.md +++ b/phone/README.md @@ -113,6 +113,7 @@ developer mode on in that build (Settings → About phone, tap Build number seve times, open Developer options, type `turn on developer mode` and choose Turn on). From then on everything happens on the phone. In the system agent's chat, ask it to work on an app bundle in its workspace: with the studio tools it +generates a design image to build from (when an image provider is configured), checks the bundle, opens a live preview, taps and types into it, reads the screenshots and the widget tree, fixes the script, installs the app as a launcher tile whose state survives a restart, and uninstalls it on request diff --git a/phone/README.zh-CN.md b/phone/README.zh-CN.md index 4bcf268b..04eda219 100644 --- a/phone/README.zh-CN.md +++ b/phone/README.zh-CN.md @@ -54,7 +54,7 @@ Home 默认使用 `dev.makepad.octosense`。要在已安装的应用旁构建测 新增构建计划和回执路径已通过单元测试与 dry-run 检查；APK 和设备验证另行进行。[App Studio 里程碑 1](../docs/adr/0006-app-studio-on-the-phone.md) 已合并：开发版让系统 Agent（以及应用自己的 Agent，仅限该应用）拥有 `studio.*` 工具；完整的手机应用创作循环尚不可用。 -**目前如何使用 App Studio。** 它只存在于开发版中（发布版 Home 在编译时就去掉了它），因此手机上需要一个开发版 Home：按上文构建，用 `adb install` 把它的 Home APK 装在商店版 Home 旁边，然后在这个开发版 Home 里开启开发者模式（设置 → 关于手机，连点七次版本号，进入开发者选项，输入 `turn on developer mode` 并选择 Turn on）。此后一切都在手机上进行。在系统 Agent 的聊天中让它处理其工作区里的某个应用包：借助 studio 工具，它检查应用包、打开实时预览、在其中点按和输入、读取截图和控件树、修正脚本、把应用安装为重启后状态仍在的启动器磁贴，并按要求卸载（[代码导读](../docs/architecture-walkthrough.zh-CN.md#从-agent-调用跟到-app-studio-中可运行的应用)列出了这些工具）。只有两件事需要电脑：构建并安装那个开发版 Home，以及需要设备证据时运行 `tools/` 中的两个设备测试工具（下文的 Studio 设备探针和 `studio-flow-device-test.py`）。如果某次 Home 发布附带开发者模式的 Home APK，就可以省去构建这一步；但那仍是下载后安装，而不是更新，因为更新器拒绝测试包。目前还没有任何构建提供面向非开发者的体验：在手机上描述一个应用，然后拿到做好的应用；这是 ADR 0006 余下几个阶段要做的事。 +**目前如何使用 App Studio。** 它只存在于开发版中（发布版 Home 在编译时就去掉了它），因此手机上需要一个开发版 Home：按上文构建，用 `adb install` 把它的 Home APK 装在商店版 Home 旁边，然后在这个开发版 Home 里开启开发者模式（设置 → 关于手机，连点七次版本号，进入开发者选项，输入 `turn on developer mode` 并选择 Turn on）。此后一切都在手机上进行。在系统 Agent 的聊天中让它处理其工作区里的某个应用包：借助 studio 工具，它生成一张可据以构建的设计图（需已配置图像提供方）、检查应用包、打开实时预览、在其中点按和输入、读取截图和控件树、修正脚本、把应用安装为重启后状态仍在的启动器磁贴，并按要求卸载（[代码导读](../docs/architecture-walkthrough.zh-CN.md#从-agent-调用跟到-app-studio-中可运行的应用)列出了这些工具）。只有两件事需要电脑：构建并安装那个开发版 Home，以及需要设备证据时运行 `tools/` 中的两个设备测试工具（下文的 Studio 设备探针和 `studio-flow-device-test.py`）。如果某次 Home 发布附带开发者模式的 Home APK，就可以省去构建这一步；但那仍是下载后安装，而不是更新，因为更新器拒绝测试包。目前还没有任何构建提供面向非开发者的体验：在手机上描述一个应用，然后拿到做好的应用；这是 ADR 0006 余下几个阶段要做的事。 [Studio 设备探针](../tools/studio-device-probe.py) 在该独立测试包中检查权限拒绝、浅色/深色 PNG 输出和切入后台后的取消。它需要使用 `MAKEPAD_FORCE_DEBUGGABLE=1` 构建的 `--dev-mode` APK，让 `run-as` 能写入测试夹具。探针会恢复开发者设置并返回主屏；它不安装 APK，也不测试模型能否收到图片。运行前须先处理首次启动的权限提示；渲染不需要定位权限，可以保持拒绝。启动夹具（`studio-render:` 与 `studio-flow:` 测试动作）只接受位于该包私有目录 `files/studio-fixture/` 下的文件，探针也把夹具写在那里。这些设备检查已于 2026 年 10 月在一部普通小米手机上通过（见[证据](../docs/studio/xiaomi-revalidation.md)）。