From a131b474c2ec391bff2827da24e66265358c5cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?power=E8=83=96=E4=B8=B8?= <77043631+power246@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:54:19 -0700 Subject: [PATCH 1/3] feat: add Pantry vision bridge and Windows camera preview --- apps/ai-providers/host-service/Cargo.toml | 2 +- .../host-service/src/complete/media/mod.rs | 27 +- .../host-service/src/complete/mod.rs | 22 +- .../host-service/src/complete/vision.rs | 64 +++ .../host-service/src/complete/wire.rs | 13 +- .../host-service/tests/complete.rs | 94 +++- crates/shell/src/platform_services/mod.rs | 25 +- crates/shell/src/platform_services/tests.rs | 10 +- runtime-patches.lock.json | 11 +- .../makepad-windows-camera-preview.patch | 439 ++++++++++++++++++ tools/setup.py | 12 +- 11 files changed, 680 insertions(+), 39 deletions(-) create mode 100644 apps/ai-providers/host-service/src/complete/vision.rs create mode 100644 tools/runtime-patches/makepad-windows-camera-preview.patch diff --git a/apps/ai-providers/host-service/Cargo.toml b/apps/ai-providers/host-service/Cargo.toml index 19df95f90..0f4194f65 100644 --- a/apps/ai-providers/host-service/Cargo.toml +++ b/apps/ai-providers/host-service/Cargo.toml @@ -28,7 +28,7 @@ ureq = { version = "2", default-features = false, features = ["tls"] } # A provider QR read out of a picked or dropped image (src/image_qr.rs): PNG # and JPEG only, decoded to grey and searched with rqrr, the decoder the # phone's scanner is tested against. -image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] } rqrr = { version = "0.10", default-features = false } # The shell's octos kernel (crates/octos-core), with the `octos-core` diff --git a/apps/ai-providers/host-service/src/complete/media/mod.rs b/apps/ai-providers/host-service/src/complete/media/mod.rs index 262a1dceb..2e9056b94 100644 --- a/apps/ai-providers/host-service/src/complete/media/mod.rs +++ b/apps/ai-providers/host-service/src/complete/media/mod.rs @@ -437,11 +437,21 @@ fn run_call(context: &Context) -> Result { context.check()?; context.host.attach(&context.call.host_dir); match context.call.method() { + "vision_capability" => { + if super::vision::supported(&context.call.app_id) && context.call.args.as_object().is_some_and(|o| o.is_empty()) { + Ok(json!({"schema":1,"supported":true})) + } else { + Err(Refusal::new(Code::Capability, "Image calls are unavailable to this app on this host.")) + } + } "complete" => { - let request = super::Request::from_args(&context.call.args)?; + let mut args = context.call.args.clone(); + let image = args.as_object_mut().and_then(|o| o.remove("image")); + let request = super::Request::from_args(&args)?; + let image = image.as_ref().map(|value| super::vision::load(&context.call.app_id, &context.call.host_dir, value)).transpose()?; context .host - .complete_while(&context.call.app_id, request, || context.check().is_ok()) + .complete_while(&context.call.app_id, request, image.as_deref(), || context.check().is_ok()) .map(|completion| completion.to_reply()) } "budget" => { @@ -755,12 +765,21 @@ fn public_url(raw: &str) -> Result { } pub(super) fn catalog() -> Vec { - wire::schemas() + let mut methods: Vec<_> = wire::schemas() .into_iter() .map(|(name, summary, args, result)| { HostApiMethod::new(name, 1, "model", summary, args, result) .with_platforms(&["macos", "android", "windows", "linux", "ios", "openharmony"]) .with_agent_access(AgentAccess::Allowed) }) - .collect() + .collect(); + methods.push(HostApiMethod::new("model.vision_capability", 1, "model", + "Check whether this host offers Pantry's app-local image call", + json!({"type":"object","additionalProperties":false}), + json!({"type":"object","required":["schema","supported"], + "properties":{"schema":{"const":1},"supported":{"const":true}}, + "additionalProperties":false})) + .with_platforms(&["windows"]) + .with_agent_access(AgentAccess::Allowed)); + methods } diff --git a/apps/ai-providers/host-service/src/complete/mod.rs b/apps/ai-providers/host-service/src/complete/mod.rs index 992209f77..c9963bbb2 100644 --- a/apps/ai-providers/host-service/src/complete/mod.rs +++ b/apps/ai-providers/host-service/src/complete/mod.rs @@ -8,7 +8,8 @@ //! //! | method | args | answer | //! |---|---|---| -//! | `model.complete` | `{task, input, schema, class?, allow_urls?}` | `{output, meta: {class, requested, attempts, usage: {input_tokens, output_tokens, estimated}, budget}}` | +//! | `model.complete` | `{task, input, schema, class?, allow_urls?, image?: {app_path}}` | `{output, meta: {class, requested, attempts, usage: {input_tokens, output_tokens, estimated}, budget}}` | +//! | `model.vision_capability` | `{}` | `{schema: 1, supported: true}` for the admitted Pantry app on Windows | //! | `model.budget` | – | the caller's `budget` alone | //! //! - `task` (1 to [`TASK_MAX`] bytes): what to do, in words. @@ -70,6 +71,7 @@ pub mod ledger; pub mod media; pub mod schema; pub mod wire; +mod vision; use ledger::{Budget, Ledger, Limits, Refused}; use schema::Schema; @@ -507,10 +509,10 @@ impl ModelHost { /// One call for `app`, blocking (run it off the UI thread). The caller /// has checked that `app` may call at all. pub fn complete(&self, app: &str, request: Request) -> Result { - self.complete_while(app, request, || true) + self.complete_while(app, request, None, || true) } - fn complete_while(&self, app: &str, request: Request, active: impl Fn() -> bool) -> Result { + fn complete_while(&self, app: &str, request: Request, image: Option<&str>, active: impl Fn() -> bool) -> Result { let check = || if active() { Ok(()) } else { Err(Refusal::new(Code::Capability, "The request ended or this app no longer has model access.")) }; check()?; let schema = request.check()?; @@ -521,7 +523,13 @@ impl ModelHost { if candidates.is_empty() { return Err(Refusal::new(Code::NoProvider, "No AI provider is set up. Add one in AI providers.")); } - let ordered = order(candidates, request.class); + let mut ordered = order(candidates, request.class); + if image.is_some() { + ordered.retain(|candidate| vision::provider_allowed(&candidate.provider)); + if ordered.is_empty() { + return Err(Refusal::new(Code::NoProvider, "No configured MiniMax-M3 or kimi-k2.5 image model is available.")); + } + } let now = (self.clock)(); let admitted = self.ledger.lock().unwrap().admit(app, now, request.estimate()); if let Err(refused) = admitted { @@ -554,7 +562,7 @@ impl ModelHost { "{user}\n\nYour previous answer was refused: {why}. Answer again with only one JSON value that validates against the schema." ), }; - let text = match self.send(candidate, &system, &user) { + let text = match self.send(candidate, &system, &user, image) { Ok((text, used)) => { usage.add(used); self.ledger.lock().unwrap().charge(app, (self.clock)(), used.total()); @@ -592,9 +600,9 @@ impl ModelHost { /// One request to one provider: the reply text and its usage, or why the /// provider failed (with the key removed). - fn send(&self, c: &Candidate, system: &str, user: &str) -> Result<(String, Usage), String> { + fn send(&self, c: &Candidate, system: &str, user: &str, image: Option<&str>) -> Result<(String, Usage), String> { let key = c.key.as_deref(); - let prepared = wire::prepare(&c.provider, key, system, user)?; + let prepared = wire::prepare_with_image(&c.provider, key, system, user, image)?; let (status, body) = self.transport.post(&prepared.url, &prepared.headers, &prepared.body).map_err(|e| { format!("{}: {}", probe::transport_reason(&e), probe::describe_error(&e, key)) })?; diff --git a/apps/ai-providers/host-service/src/complete/vision.rs b/apps/ai-providers/host-service/src/complete/vision.rs new file mode 100644 index 000000000..6d1ce84b8 --- /dev/null +++ b/apps/ai-providers/host-service/src/complete/vision.rs @@ -0,0 +1,64 @@ +//! One bounded image from the calling Pantry app's own storage. +use super::{Code, Refusal}; +use base64::{engine::general_purpose::STANDARD, Engine as _}; +use image::{ImageFormat, ImageReader}; +use octosense_llm_config::{ApiType, Provider}; +use serde_json::Value; +use std::{io::Cursor, path::Path}; + +const APP: &str = "pantry-steward"; +const MAX_BYTES: u64 = 10 * 1024 * 1024; + +pub(super) fn supported(app: &str) -> bool { + cfg!(target_os = "windows") && app == APP +} + +pub(super) fn provider_allowed(provider: &Provider) -> bool { + if crate::model::effective_api_type(provider) != ApiType::OpenAi { + return false; + } + let family = super::family_id(provider); + let model = super::effective_model(provider); + matches!((family, model.as_deref()), + ("minimax" | "minimax-cn", Some("MiniMax-M3")) + | ("moonshot", Some("kimi-k2.5"))) +} + +fn invalid() -> Refusal { + Refusal::new(Code::BadRequest, "The app image is missing, invalid, or over 10 MiB.") +} + +pub(super) fn load(app: &str, host_dir: &Path, value: &Value) -> Result { + if !supported(app) { + return Err(Refusal::new(Code::Capability, "Image calls are unavailable to this app on this host.")); + } + let object = value.as_object().ok_or_else(invalid)?; + if object.len() != 1 { + return Err(invalid()); + } + let path = object.get("app_path").and_then(Value::as_str).ok_or_else(invalid)?; + let path = path.strip_prefix('/').unwrap_or(path); + if path.is_empty() || path.contains('\\') || path.contains(':') + || path.split('/').any(|part| part.is_empty() || part == "." || part == "..") + { + return Err(invalid()); + } + let root = host_dir.parent().ok_or_else(invalid)?.join(APP).canonicalize().map_err(|_| invalid())?; + let file = root.join(path).canonicalize().map_err(|_| invalid())?; + if !file.starts_with(&root) || !file.is_file() || file.metadata().map_err(|_| invalid())?.len() > MAX_BYTES { + return Err(invalid()); + } + let bytes = std::fs::read(file).map_err(|_| invalid())?; + if bytes.is_empty() || bytes.len() as u64 > MAX_BYTES { + return Err(invalid()); + } + let format = image::guess_format(&bytes).map_err(|_| invalid())?; + let mime = match format { + ImageFormat::Jpeg => "image/jpeg", + ImageFormat::Png => "image/png", + ImageFormat::WebP => "image/webp", + _ => return Err(invalid()), + }; + ImageReader::with_format(Cursor::new(&bytes), format).into_dimensions().map_err(|_| invalid())?; + Ok(format!("data:{mime};base64,{}", STANDARD.encode(bytes))) +} diff --git a/apps/ai-providers/host-service/src/complete/wire.rs b/apps/ai-providers/host-service/src/complete/wire.rs index e20a778d4..2c9b178d7 100644 --- a/apps/ai-providers/host-service/src/complete/wire.rs +++ b/apps/ai-providers/host-service/src/complete/wire.rs @@ -24,6 +24,10 @@ pub struct Prepared { /// The request for `provider`: `system` and one `user` message. pub fn prepare(provider: &Provider, key: Option<&str>, system: &str, user: &str) -> Result { + prepare_with_image(provider, key, system, user, None) +} + +pub fn prepare_with_image(provider: &Provider, key: Option<&str>, system: &str, user: &str, image: Option<&str>) -> Result { let family = registry::lookup(&provider.family).map(|f| f.id).unwrap_or(provider.family.as_str()); if family == "vertex" { return Err("Vertex AI is not supported for direct calls".into()); @@ -36,6 +40,9 @@ pub fn prepare(provider: &Provider, key: Option<&str>, system: &str, user: &str) let model = effective_model(provider).ok_or("no model")?; let mut headers = vec![("Content-Type".to_string(), "application/json".to_string())]; let api = effective_api_type(provider); + if image.is_some() && api != ApiType::OpenAi { + return Err("the selected image model requires the OpenAI chat protocol".into()); + } let (url, body) = match api { ApiType::Anthropic => { if let Some(key) = key { @@ -55,10 +62,14 @@ pub fn prepare(provider: &Provider, key: Option<&str>, system: &str, user: &str) } else { format!("{base}/chat/completions") }; + let content = match image { + Some(data) => json!([{"type":"text","text":user},{"type":"image_url","image_url":{"url":data}}]), + None => json!(user), + }; ( url, json!({"model": model, "stream": false, "messages": [ - {"role": "system", "content": system}, {"role": "user", "content": user} + {"role": "system", "content": system}, {"role": "user", "content": content} ]}), ) } diff --git a/apps/ai-providers/host-service/tests/complete.rs b/apps/ai-providers/host-service/tests/complete.rs index 938588cba..ea6a43d82 100644 --- a/apps/ai-providers/host-service/tests/complete.rs +++ b/apps/ai-providers/host-service/tests/complete.rs @@ -102,7 +102,7 @@ impl Rig { let options = complete::Options::default() .providers(Arc::new(Fixed(providers))) .transport(fake.clone()) - .grants(|app: &str, _: &Path| app == APP) + .grants(|app: &str, _: &Path| app == APP || app == "pantry-steward") .clock(move || clock.load(Ordering::SeqCst)); let host = complete::register_with(extra(options)); Rig { dir, fake, now, host, _serial: serial } @@ -463,3 +463,95 @@ fn a_host_caller_may_send_a_larger_input_than_an_app() { let huge = Request { input: json!("x".repeat(complete::HOST_INPUT_MAX + 1)), ..request(Some("s")) }; assert_eq!(rig.host.complete(APP, huge).unwrap_err().code, Code::BadRequest); } + +#[cfg(target_os = "windows")] +#[test] +fn pantry_vision_probe_is_authenticated_and_narrow() { + let rig = Rig::new("pantry-probe", vec![deepseek("deepseek-v4-flash")]); + assert_eq!(octosense_appstore::host_api::available_versions().get("model.vision_capability"), Some(&1)); + assert_eq!(rig.call("pantry-steward", "vision_capability", json!({})).unwrap(), + json!({"schema":1,"supported":true})); + assert!(rig.call(APP, "vision_capability", json!({})).unwrap_err().starts_with("capability:")); + assert!(rig.call("pantry-steward", "vision_capability", json!({"unexpected":true})) + .unwrap_err().starts_with("capability:")); + assert!(rig.fake.seen().is_empty()); +} + +#[cfg(target_os = "windows")] +#[test] +fn pantry_image_routes_to_only_the_two_approved_models() { + for (family, model, tag) in [ + ("minimax", "MiniMax-M3", "pantry-minimax"), + ("minimax-cn", "MiniMax-M3", "pantry-minimax-cn"), + ("moonshot", "kimi-k2.5", "pantry-kimi"), + ] { + let rig = Rig::new(tag, vec![deepseek("deepseek-v4-flash"), + Candidate { provider: Provider::new(family, Some(model.into())), key: Some("vision-test-key".into()) }]); + let dir = rig.dir.join("pantry-steward"); + std::fs::create_dir_all(&dir).unwrap(); + image::DynamicImage::new_rgba8(1, 1) + .save_with_format(dir.join("photo.png"), image::ImageFormat::Png).unwrap(); + rig.fake.says(GOOD); + let mut request = args(); + request["image"] = json!({"app_path":"/photo.png"}); + assert_eq!(rig.call("pantry-steward", "complete", request).unwrap()["output"]["title"], "Errands"); + let seen = rig.fake.seen(); + assert_eq!(seen.len(), 1); + let expected_host = match family { + "minimax" => "api.minimax.io", + "minimax-cn" => "api.minimaxi.com", + _ => "api.moonshot.ai", + }; + assert!(seen[0].0.starts_with(&format!("https://{expected_host}/v1/chat/completions"))); + let body = &seen[0].2; + assert_eq!(body["model"], model); + assert_eq!(body["messages"][1]["content"][0]["type"], "text"); + assert_eq!(body["messages"][1]["content"][1]["type"], "image_url"); + assert!(body["messages"][1]["content"][1]["image_url"]["url"] + .as_str().unwrap().starts_with("data:image/png;base64,")); + assert!(!body.to_string().contains("photo.png")); + } +} + +#[cfg(target_os = "windows")] +#[test] +fn pantry_image_validation_precedes_any_provider_call() { + let rig = Rig::new("pantry-image-refusal", vec![deepseek("deepseek-v4-flash")]); + let dir = rig.dir.join("pantry-steward"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("fake.png"), b"not an image").unwrap(); + std::fs::write(dir.join("large.png"), vec![0u8; 10 * 1024 * 1024 + 1]).unwrap(); + for path in ["../.host/secret", "C:/Windows/win.ini", "missing.png", "fake.png", "large.png"] { + let mut request = args(); + request["image"] = json!({"app_path":path}); + assert!(rig.call("pantry-steward", "complete", request).unwrap_err().starts_with("bad_request:"), "{path}"); + } + let mut request = args(); + request["image"] = json!({"app_path":"fake.png"}); + assert!(rig.call(APP, "complete", request).unwrap_err().starts_with("capability:")); + image::DynamicImage::new_rgba8(1, 1) + .save_with_format(dir.join("valid.png"), image::ImageFormat::Png).unwrap(); + let mut request = args(); + request["image"] = json!({"app_path":"valid.png"}); + assert!(rig.call("pantry-steward", "complete", request).unwrap_err().starts_with("no_provider:")); + assert!(rig.fake.seen().is_empty()); + std::fs::remove_file(dir.join("large.png")).unwrap(); +} + +#[cfg(target_os = "windows")] +#[test] +fn pantry_image_keeps_the_structured_output_validator() { + let rig = Rig::new("pantry-schema", vec![Candidate { + provider: Provider::new("minimax", Some("MiniMax-M3".into())), + key: Some("vision-test-key".into()), + }]); + let dir = rig.dir.join("pantry-steward"); + std::fs::create_dir_all(&dir).unwrap(); + image::DynamicImage::new_rgba8(1, 1) + .save_with_format(dir.join("valid.png"), image::ImageFormat::Png).unwrap(); + rig.fake.says(r#"{"title":5,"tags":[]}"#).says(r#"{"title":5,"tags":[]}"#); + let mut request = args(); + request["image"] = json!({"app_path":"valid.png"}); + assert!(rig.call("pantry-steward", "complete", request).unwrap_err().starts_with("invalid_output:")); + assert_eq!(rig.fake.seen().len(), 2); +} diff --git a/crates/shell/src/platform_services/mod.rs b/crates/shell/src/platform_services/mod.rs index 07309e6b3..3b2e22417 100644 --- a/crates/shell/src/platform_services/mod.rs +++ b/crates/shell/src/platform_services/mod.rs @@ -116,11 +116,12 @@ fn permission(family: &str) -> Option { _ => None, } } -fn permission_supported() -> bool { +fn permission_supported(family: &str) -> bool { cfg!(any(target_os = "android", target_os = "macos")) + || (cfg!(target_os = "windows") && family == "camera") } -fn prompt_supported() -> bool { - cfg!(any(target_os = "android", target_os = "macos")) +fn prompt_supported(family: &str) -> bool { + permission_supported(family) } fn status_name(status: PermissionStatus) -> &'static str { match status { @@ -131,10 +132,10 @@ fn status_name(status: PermissionStatus) -> &'static str { } } fn status(work: &Work, os: &str) -> Value { - json!({"capability": work.family, "supported": permission_supported(), "app_policy_granted": work.policy_allows(), + json!({"capability": work.family, "supported": permission_supported(work.family), "app_policy_granted": work.policy_allows(), "app_consent": work.current_grant().map(|g| g.allowed).unwrap_or(false), "os_permission": os, "location_read_supported": work.family == "location" && cfg!(target_os = "android"), - "location_sample_supported": work.family == "location" && permission_supported(), + "location_sample_supported": work.family == "location" && permission_supported("location"), "scope": "host_api_v1_device_access", "background_permission": false}) } fn queue(work: Work) { @@ -180,7 +181,7 @@ impl HostService for DeviceService { json!({"type":"object","required":["abi","defaults","options","library_export_supported","library_export_requires_foreground","microphone_consent_required"]})) .with_agent_access(AgentAccess::Allowed)] } else { vec![] }; - if !permission_supported() { + if !permission_supported(self.family) { return capture_intent; } let input = json!({"type":"object","additionalProperties":false}); @@ -197,7 +198,7 @@ impl HostService for DeviceService { input.clone(), output.clone(), ) - .with_platforms(&["android", "macos"]) + .with_platforms(if self.family == "camera" { &["android", "macos", "windows"] } else { &["android", "macos"] }) .with_agent_access(AgentAccess::Allowed), HostApiMethod::new( format!("{}.permission.request", self.family), @@ -207,7 +208,7 @@ impl HostService for DeviceService { input.clone(), output.clone(), ) - .with_platforms(&["android", "macos"]) + .with_platforms(if self.family == "camera" { &["android", "macos", "windows"] } else { &["android", "macos"] }) .with_agent_access(AgentAccess::ForegroundOnly), HostApiMethod::new( format!("{}.permission.revoke", self.family), @@ -218,7 +219,7 @@ impl HostService for DeviceService { json!({"type":"object","required":["app_consent","os_permission_unchanged","scope"],"properties":{ "app_consent":{"const":false},"os_permission_unchanged":{"const":true},"scope":{"const":"host_api_v1_device_access"}}}), ) - .with_platforms(&["android", "macos"]) + .with_platforms(if self.family == "camera" { &["android", "macos", "windows"] } else { &["android", "macos"] }) .with_agent_access(AgentAccess::Allowed), ]; methods.extend(capture_intent); @@ -311,7 +312,7 @@ impl HostService for DeviceService { return; } if self.family == "location" && call.method() == "sample.cancel" { - if !permission_supported() { + if !permission_supported(self.family) { reply.send(Err( "unsupported_platform: Location sampling is unavailable".into(), )); @@ -363,7 +364,7 @@ impl HostService for DeviceService { requesting: false, sample: sample_options.map(|options| location::Sample { options, fix: None }), }; - if !permission_supported() { + if !permission_supported(self.family) { if operation == Operation::Status { work.reply.clone().send(Ok(status(&work, "unsupported"))); } else { @@ -405,7 +406,7 @@ impl HostService for DeviceService { ); return; } - if !prompt_supported() { + if !prompt_supported(self.family) { work.fail( "unsupported_platform", "Trusted device consent is unavailable on this platform", diff --git a/crates/shell/src/platform_services/tests.rs b/crates/shell/src/platform_services/tests.rs index 2c35b29c5..c38099b11 100644 --- a/crates/shell/src/platform_services/tests.rs +++ b/crates/shell/src/platform_services/tests.rs @@ -141,7 +141,7 @@ fn glance_devices_use_host_consent_for_declared_and_legacy_apps() { let replies = services::take_replies_for(&[heap]); assert_eq!(replies.len(), 1); let error = replies[0].2.as_ref().unwrap_err(); - if permission_supported() { + if permission_supported("camera") { assert_eq!( error, "camera.permission.request is unavailable to agents/background surfaces" @@ -257,7 +257,7 @@ fn descriptors_do_not_claim_unimplemented_location_or_background_authorization() for family in ["camera", "microphone", "location"] { let service = DeviceService { family }; let methods = service.api_methods(); - if !permission_supported() { + if !permission_supported(family) { assert_eq!(methods.len(), usize::from(family == "camera")); continue; } @@ -266,7 +266,7 @@ fn descriptors_do_not_claim_unimplemented_location_or_background_authorization() .find(|m| m.name.ends_with("permission.request")) .unwrap(); assert_eq!(request.agent_access, services::AgentAccess::ForegroundOnly); - assert!(!request.platforms.contains(&"windows".into())); + assert_eq!(request.platforms.contains(&"windows".into()), family == "camera" && cfg!(target_os = "windows")); assert!(request.input_schema["additionalProperties"] == false); assert_eq!( methods.iter().any(|m| m.name == "location.get"), @@ -361,7 +361,7 @@ fn location_samples_validate_bounds_age_accuracy_and_platform_data() { } let methods = DeviceService { family: "location" }.api_methods(); let sample = methods.iter().find(|m| m.name == "location.sample"); - assert_eq!(sample.is_some(), permission_supported()); + assert_eq!(sample.is_some(), permission_supported("location")); if let Some(sample) = sample { assert_eq!(sample.agent_access, services::AgentAccess::ForegroundOnly); assert_eq!(sample.platforms, ["android", "macos"]); @@ -374,7 +374,7 @@ fn location_samples_validate_bounds_age_accuracy_and_platform_data() { /// events. A child process owns global registries; no OS/device is opened. #[test] fn location_sampling_broker_lifecycle() { - if !permission_supported() { + if !permission_supported("location") { return; } const CHILD: &str = "OCTOSENSE_LOCATION_TEST_CHILD"; diff --git a/runtime-patches.lock.json b/runtime-patches.lock.json index 5b9836137..a50581e9f 100644 --- a/runtime-patches.lock.json +++ b/runtime-patches.lock.json @@ -4,7 +4,7 @@ "base_revision": "d653bc4493619ab979ccff165cf48c0c63b77d19", "patch": "tools/runtime-patches/makepad-settings.patch", "sha256": "3a79cddddc89dc4d064a9f0bdd17dc8111dcab9c796a334d95f61ecb9d5011aa", - "tree": "a6fae94aa4d233503b14bc6d0a36bcbc2b264b44", + "tree": "3c19414f676789f67d65f1f7d9ec678f79665866", "product_fixes": [ "Settings integration: Android IME synchronization, accessibility and reduced motion, GLES program-cache validation, obscured-gesture cancellation and scroll-position observation.", "Contained-app isolation and secret-input restrictions come from the pinned upstream runtime; the Settings overlay preserves those gates.", @@ -14,7 +14,7 @@ "Windows frame completion: backport Makepad work c508ab0c8 GPU completion polls around native and hosted repaint so upload buffers can retire.", "Android application extensions honor the manifest's explicit class name, keeping Back and platform integration active under custom test package ids.", "Android physical touchscreen input has scoped host approval provenance; default and nested synthetic input are untrusted. Compromised OS/root impersonation is outside this app-level boundary; the subsequent desktop patch adds macOS pointer provenance.", - "macOS AppKit pointer metadata enables native approval for HID events without a posting process. Synthetic and remote dispatch remain untrusted, including nested callbacks; Windows/Linux physical approval is unsupported.", + "macOS AppKit pointer metadata enables native approval for HID events without a posting process. Synthetic and remote dispatch remain untrusted, including nested callbacks; the later Windows camera patch adds Windows pointer provenance, while Linux physical approval remains unsupported.", "Android multiline Enter key events finish the active composition before inserting a newline, retaining the last word; ordinary IME composition commits keep replacement semantics.", "Host-owned backend authentication reuses native WebViews with isolated sessions, exact callback interception, same-origin navigation and no page tool bridge; Android uses an ephemeral private process and profile.", "Android Camera waits for one permission answer and refreshes an empty camera list after grant. Video pause/resume preserves an unprepared player so first-use authorization cannot prevent its initial source assignment.", @@ -47,7 +47,8 @@ "Android Video seeks to the closest frame on API26+ instead of the previous keyframe; older backends retain a bounded legacy fallback.", "A native host may stage up to 64 MiB into an app's jail (MAX_IMPORT_BYTES; prepare_import and commit_import), so files.import can take documents above the 1 MiB script write limit. Staged bytes never enter the script heap; the jail's byte quota and entry cap still apply, and script writes, photo picks and exports keep 1 MiB.", "An OS resume restarts only the Video playback its pause stopped, including a start it cancelled; a player paused by the person, or paused or stopped by its app during the pause, stays paused (#406).", - "Camera capture and recording use explicit per-call boolean audio/library options, defaulting to silent app-local output. Permission availability never opts into recording audio or exporting captures; async microphone checks retain capture intent and background provenance. Unsupported library export fails before capture." + "Camera capture and recording use explicit per-call boolean audio/library options, defaulting to silent app-local output. Permission availability never opts into recording audio or exporting captures; async microphone checks retain capture intent and background provenance. Unsupported library export fails before capture.", + "Windows native physical pointer provenance enables host-owned camera consent. A Media Foundation camera source feeds the existing YUV video preview and still-capture path, with latest-frame buffering and cleanup; macOS retains its native AVFoundation preview." ], "stacked": [ { @@ -185,6 +186,10 @@ { "patch": "tools/runtime-patches/makepad-camera-capture-intent.patch", "sha256": "d056b76d2eb6b55aa9c96447a49fc90d32a9a235b6bc9a1ff125902a73c28208" + }, + { + "patch": "tools/runtime-patches/makepad-windows-camera-preview.patch", + "sha256": "9e0a044e2557d3d654d5988e026fc254f70dfdbbb1a188ff7e273abbb22da24c" } ] } diff --git a/tools/runtime-patches/makepad-windows-camera-preview.patch b/tools/runtime-patches/makepad-windows-camera-preview.patch new file mode 100644 index 000000000..c1b9b631e --- /dev/null +++ b/tools/runtime-patches/makepad-windows-camera-preview.patch @@ -0,0 +1,439 @@ +diff --git a/platform/src/os/windows/mod.rs b/platform/src/os/windows/mod.rs +index 4856a92..0132132 100644 +--- a/platform/src/os/windows/mod.rs ++++ b/platform/src/os/windows/mod.rs +@@ -20,6 +20,7 @@ + pub mod windows_media_engine_notify; + pub mod windows_mf_source_reader; + pub mod windows_video_playback; ++pub mod windows_camera_player; + pub mod windows_video_player; + pub mod winrt_midi; + +diff --git a/platform/src/os/windows/win32_event.rs b/platform/src/os/windows/win32_event.rs +index 8a02d73..3299138 100644 +--- a/platform/src/os/windows/win32_event.rs ++++ b/platform/src/os/windows/win32_event.rs +@@ -30,8 +30,8 @@ pub enum Win32Event { + primary: bool, + }, + +- MouseDown(MouseDownEvent), +- MouseUp(MouseUpEvent), ++ MouseDown(MouseDownEvent, bool), ++ MouseUp(MouseUpEvent, bool), + MouseMove(MouseMoveEvent), + MouseLeave(MouseLeaveEvent), + Scroll(ScrollEvent), +diff --git a/platform/src/os/windows/win32_window.rs b/platform/src/os/windows/win32_window.rs +index 01a356d..8cd0ceb 100644 +--- a/platform/src/os/windows/win32_window.rs ++++ b/platform/src/os/windows/win32_window.rs +@@ -1864,6 +1864,22 @@ pub fn send_focus_lost_event(&mut self) { + self.do_callback(Win32Event::WindowLostFocus(self.window_id)); + } + ++ fn hardware_mouse_message() -> bool { ++ // This pinned windows-rs shim lacks Win32::UI::Input's message-origin ++ // bindings. Query user32 directly while processing the native message. ++ #[repr(C)] ++ #[derive(Default)] ++ struct InputMessageSource { device_type: i32, origin_id: i32 } ++ #[link(name = "user32")] ++ unsafe extern "system" { ++ fn GetCurrentInputMessageSource(source: *mut InputMessageSource) -> i32; ++ } ++ let mut source = InputMessageSource::default(); ++ let received = unsafe { GetCurrentInputMessageSource(&mut source) != 0 }; ++ received && matches!(source.device_type, 2 | 4 | 8 | 16) // mouse, touch, pen, touchpad ++ && source.origin_id == 1 // IMO_HARDWARE; injected/post messages fail closed ++ } ++ + pub fn send_mouse_down(&mut self, button: MouseButton, modifiers: KeyModifiers) { + if self.mouse_buttons_down == 0 { + unsafe { +@@ -1878,7 +1894,7 @@ pub fn send_mouse_down(&mut self, button: MouseButton, modifiers: KeyModifiers) + abs: self.last_mouse_pos, + time: self.time_now(), + handled: Cell::new(Area::Empty), +- })); ++ }, Self::hardware_mouse_message())); + } + + pub fn send_mouse_up(&mut self, button: MouseButton, modifiers: KeyModifiers) { +@@ -1896,7 +1912,7 @@ pub fn send_mouse_up(&mut self, button: MouseButton, modifiers: KeyModifiers) { + window_id: self.window_id, + abs: self.last_mouse_pos, + time: self.time_now(), +- })); ++ }, Self::hardware_mouse_message())); + } + + pub fn send_mouse_move(&mut self, pos: Vec2d, modifiers: KeyModifiers) { +diff --git a/platform/src/os/windows/windows.rs b/platform/src/os/windows/windows.rs +index fb1dd91..5f099d8 100644 +--- a/platform/src/os/windows/windows.rs ++++ b/platform/src/os/windows/windows.rs +@@ -293,11 +293,11 @@ fn win32_event_callback( + let time_now = with_win32_app(|app| app.time_now()); + self.paint_tick(time_now, true, d3d11_cx, d3d11_windows); + } +- Win32Event::MouseDown(mut e) => { ++ Win32Event::MouseDown(mut e, trusted) => { + self.dpi_override_scale(&mut e.abs, e.window_id); + self.fingers.process_tap_count(e.abs, e.time); + self.fingers.mouse_down(e.button, e.window_id); +- self.call_event_handler(&Event::MouseDown(e.into())) ++ self.call_event_handler_with_input_trust(&Event::MouseDown(e.into()), trusted) + } + Win32Event::MouseMove(mut e) => { + self.dpi_override_scale(&mut e.abs, e.window_id); +@@ -305,10 +305,10 @@ fn win32_event_callback( + self.fingers.cycle_hover_area(live_id!(mouse).into()); + self.fingers.switch_captures(); + } +- Win32Event::MouseUp(mut e) => { ++ Win32Event::MouseUp(mut e, trusted) => { + self.dpi_override_scale(&mut e.abs, e.window_id); + let button = e.button; +- self.call_event_handler(&Event::MouseUp(e.into())); ++ self.call_event_handler_with_input_trust(&Event::MouseUp(e.into()), trusted); + self.fingers.mouse_up(button); + self.fingers.cycle_hover_area(live_id!(mouse).into()); + } +@@ -1219,6 +1219,9 @@ fn handle_platform_ops( + let tex_y_id = tex_y.texture_id(); + let tex_u_id = tex_u.texture_id(); + let tex_v_id = tex_v.texture_id(); ++ let camera_access = if matches!(&source, VideoSource::Camera(..)) { ++ Some(self.os.media.media_foundation()) ++ } else { None }; + let player = WindowsUnifiedVideoPlayer::new( + &device, + video_id, +@@ -1229,6 +1232,7 @@ fn handle_platform_ops( + source, + autoplay, + should_loop, ++ camera_access, + ); + self.os.video_players.insert(video_id, player); + // Notify widget so it can bind textures to shader slots +diff --git a/platform/src/os/windows/windows_camera_player.rs b/platform/src/os/windows/windows_camera_player.rs +new file mode 100644 +index 0000000..060c7bb +--- /dev/null ++++ b/platform/src/os/windows/windows_camera_player.rs +@@ -0,0 +1,84 @@ ++//! Media Foundation camera frames as a bounded latest-frame video source. ++use { ++ super::media_foundation::MediaFoundationAccess, ++ crate::{ ++ media_plugin::PlaybackPrepared, ++ video::{CameraFrameInputFn, CameraFrameOwned, CameraFramePool, CameraFrameRef, VideoFormatId, VideoInputId}, ++ }, ++ std::{sync::{Arc, Mutex}, time::{Duration, Instant}}, ++}; ++ ++pub struct WindowsCameraPlayer { ++ frame_pool: Arc>, ++ access: Option>>, ++ started: Instant, ++ prepared: bool, ++ playing: bool, ++} ++ ++impl WindowsCameraPlayer { ++ pub fn new(input_id: VideoInputId, format_id: VideoFormatId, ++ access: Arc>, autoplay: bool) -> Self { ++ let frame_pool = Arc::new(Mutex::new(CameraFramePool::new(4))); ++ let pending = frame_pool.clone(); ++ let callback: CameraFrameInputFn = Box::new(move |source: CameraFrameRef<'_>| { ++ let mut pool = pending.lock().unwrap(); ++ let mut frame = pool.checkout(); ++ if frame.convert_to_i420(source) { ++ pool.publish_latest(frame); ++ } else { ++ pool.recycle(frame); ++ } ++ }); ++ { ++ let mut camera = access.lock().unwrap(); ++ *camera.camera_frame_input_cb[0].lock().unwrap() = Some(callback); ++ *camera.video_input_cb[0].lock().unwrap() = None; ++ camera.use_video_input(&[(input_id, format_id)]); ++ } ++ Self { frame_pool, access: Some(access), started: Instant::now(), ++ prepared: false, playing: autoplay } ++ } ++ ++ pub fn check_prepared(&mut self) -> Option> { ++ if self.prepared { return None; } ++ let mut pool = self.frame_pool.lock().unwrap(); ++ if let Some(frame) = pool.take_latest() { ++ let (width, height) = (frame.width as u32, frame.height as u32); ++ pool.publish_latest(frame); ++ self.prepared = true; ++ return Some(Ok(PlaybackPrepared::new(width, height, 0, false, ++ vec!["camera".to_string()], vec![]))); ++ } ++ if self.started.elapsed() > Duration::from_secs(6) { ++ self.prepared = true; ++ return Some(Err("Camera did not deliver a video frame".to_string())); ++ } ++ None ++ } ++ ++ pub fn take_frame(&self) -> Option { ++ if self.playing { self.frame_pool.lock().unwrap().take_latest() } else { None } ++ } ++ ++ pub fn recycle_frame(&self, frame: CameraFrameOwned) { ++ self.frame_pool.lock().unwrap().recycle(frame); ++ } ++ ++ pub fn is_playing(&self) -> bool { self.playing } ++ pub fn keep_polling(&self) -> bool { !self.prepared || self.playing } ++ pub fn play(&mut self) { self.playing = true; } ++ pub fn pause(&mut self) { self.playing = false; } ++ pub fn cleanup(&mut self) { ++ if let Some(access) = self.access.take() { ++ let mut camera = access.lock().unwrap(); ++ camera.use_video_input(&[]); ++ *camera.camera_frame_input_cb[0].lock().unwrap() = None; ++ } ++ self.playing = false; ++ } ++} ++ ++impl Drop for WindowsCameraPlayer { ++ fn drop(&mut self) { self.cleanup(); } ++} +diff --git a/platform/src/os/windows/windows_video_player.rs b/platform/src/os/windows/windows_video_player.rs +index 9a7ac30..7604ccb 100644 +--- a/platform/src/os/windows/windows_video_player.rs ++++ b/platform/src/os/windows/windows_video_player.rs +@@ -1,5 +1,7 @@ + use { + super::{ ++ media_foundation::MediaFoundationAccess, ++ windows_camera_player::WindowsCameraPlayer, + windows_mf_source_reader::WindowsMfSourceReaderPlayer, + windows_video_playback::WindowsVideoPlayer, + }, +@@ -12,7 +14,8 @@ + CxTexturePool, TextureAlloc, TextureCategory, TextureFormat, TextureId, TexturePixel, + }, + video_decode::software_video::PlaybackSessionHandle, +- video_decode::yuv::YuvPlaneData, ++ video_decode::yuv::{YuvColorMatrix, YuvLayout, YuvPlaneData}, ++ video::CameraColorMatrix, + windows::{ + core::Interface, + Win32::Graphics::{ +@@ -25,6 +28,7 @@ + }, + }, + }, ++ std::sync::{Arc, Mutex}, + }; + + struct D3d11R8GpuPlane { +@@ -250,6 +254,7 @@ pub struct WindowsUnifiedVideoPlayer { + } + + enum WindowsPlayerMode { ++ Camera(WindowsCameraPlayer), + /// Progressive files: Source Reader + DXGI NV12 (true zero-copy when possible). + SourceReader(WindowsMfSourceReaderPlayer), + /// Adaptive / fallback: MediaEngine Transfer present. +@@ -268,9 +273,13 @@ pub fn new( + source: VideoSource, + autoplay: bool, + is_looping: bool, ++ camera_access: Option>>, + ) -> Self { + let force_software = std::env::var_os("MAKEPAD_FORCE_SOFTWARE_VIDEO").is_some(); +- let mode = if force_software { ++ let mode = if let VideoSource::Camera(input_id, format_id) = &source { ++ WindowsPlayerMode::Camera(WindowsCameraPlayer::new(*input_id, *format_id, ++ camera_access.expect("camera access required for camera source"), autoplay)) ++ } else if force_software { + crate::log!("VIDEO: MAKEPAD_FORCE_SOFTWARE_VIDEO set, using software video decoder"); + WindowsPlayerMode::Software(PlaybackSessionHandle::new( + video_id, +@@ -396,6 +405,7 @@ fn switch_to_software(&mut self, reason: &str) { + /// Pump the active native worker once per UI paint (coalesces Tick). + pub fn sync_worker(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.sync_worker(), + WindowsPlayerMode::Native(player) => player.sync_worker(), + WindowsPlayerMode::Software(_) => {} +@@ -414,11 +424,13 @@ pub fn sync_worker(&mut self) { + + pub fn check_prepared(&mut self) -> Option> { + let out = match &mut self.mode { ++ WindowsPlayerMode::Camera(player) => player.check_prepared(), + WindowsPlayerMode::SourceReader(player) => match player.check_prepared() { + Some(Err(err)) => { + self.switch_to_media_engine(&err); + self.prepare_resolved = false; + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => Some(Err(err.clone())), + WindowsPlayerMode::Native(native) => native.check_prepared(), + WindowsPlayerMode::Software(software) => software.check_prepared(), + WindowsPlayerMode::SourceReader(_) => Some(Err(err)), +@@ -455,7 +467,39 @@ pub fn check_prepared(&mut self) -> Option> { + } + + pub fn poll_frame(&mut self, textures: &mut CxTexturePool) -> bool { ++ if let WindowsPlayerMode::Camera(player) = &mut self.mode { ++ let Some(mut frame) = player.take_frame() else { return false; }; ++ if frame.width == 0 || frame.height == 0 || frame.plane_count < 3 { ++ player.recycle_frame(frame); ++ return false; ++ } ++ let matrix = match frame.matrix { ++ CameraColorMatrix::BT601 => YuvColorMatrix::BT601, ++ CameraColorMatrix::BT2020 => YuvColorMatrix::BT2020, ++ _ => YuvColorMatrix::BT709, ++ }; ++ let planes = YuvPlaneData { ++ y: std::mem::take(&mut frame.planes[0].bytes), ++ u: std::mem::take(&mut frame.planes[1].bytes), ++ v: std::mem::take(&mut frame.planes[2].bytes), ++ width: frame.width as u32, height: frame.height as u32, ++ layout: YuvLayout::I420, matrix, ++ y_stride: frame.planes[0].row_stride as u32, ++ u_stride: frame.planes[1].row_stride as u32, ++ v_stride: frame.planes[2].row_stride as u32, ++ }; ++ self.yuv_matrix = matrix.as_f32(); ++ self.yuv_biplanar = false; ++ self.yuv_array = false; ++ self.upload_yuv_to_d3d11(textures, &planes); ++ frame.planes[0].bytes = planes.y; ++ frame.planes[1].bytes = planes.u; ++ frame.planes[2].bytes = planes.v; ++ if let WindowsPlayerMode::Camera(player) = &self.mode { player.recycle_frame(frame); } ++ return true; ++ } + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => unreachable!(), + WindowsPlayerMode::SourceReader(player) => { + let ok = player.poll_frame(textures); + self.yuv_biplanar = player.presents_nv12(); +@@ -568,7 +612,7 @@ pub fn is_software_mode(&self) -> bool { + + /// YUV shader path: software decode, SourceReader DXGI NV12, or MF Transfer NV12. + pub fn uses_yuv(&self) -> bool { +- self.is_software_mode() ++ matches!(self.mode, WindowsPlayerMode::Camera(_)) || self.is_software_mode() + || matches!(self.mode, WindowsPlayerMode::SourceReader(_)) + || self.yuv_biplanar + } +@@ -591,6 +635,7 @@ pub fn yuv_full_range(&self) -> bool { + + pub fn check_eos(&mut self) -> bool { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => false, + WindowsPlayerMode::SourceReader(player) => player.check_eos(), + WindowsPlayerMode::Native(player) => player.check_eos(), + WindowsPlayerMode::Software(player) => player.check_eos(), +@@ -599,6 +644,7 @@ pub fn check_eos(&mut self) -> bool { + + pub fn is_playing(&self) -> bool { + match &self.mode { ++ WindowsPlayerMode::Camera(player) => player.is_playing(), + WindowsPlayerMode::SourceReader(player) => player.is_playing(), + WindowsPlayerMode::Native(player) => player.is_playing(), + WindowsPlayerMode::Software(player) => player.is_playing(), +@@ -608,6 +654,7 @@ pub fn is_playing(&self) -> bool { + /// Keep Poll while Media Foundation is still buffering, or while playing. + pub fn keep_polling(&self) -> bool { + match &self.mode { ++ WindowsPlayerMode::Camera(player) => player.keep_polling(), + WindowsPlayerMode::SourceReader(player) => player.keep_polling(), + WindowsPlayerMode::Native(player) => player.keep_polling(), + WindowsPlayerMode::Software(player) => !self.prepare_resolved || player.is_playing(), +@@ -616,6 +663,7 @@ pub fn keep_polling(&self) -> bool { + + pub fn play(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(player) => player.play(), + WindowsPlayerMode::SourceReader(player) => player.play(), + WindowsPlayerMode::Native(player) => player.play(), + WindowsPlayerMode::Software(player) => player.play(), +@@ -624,6 +672,7 @@ pub fn play(&mut self) { + + pub fn pause(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(player) => player.pause(), + WindowsPlayerMode::SourceReader(player) => player.pause(), + WindowsPlayerMode::Native(player) => player.pause(), + WindowsPlayerMode::Software(player) => player.pause(), +@@ -632,6 +681,7 @@ pub fn pause(&mut self) { + + pub fn resume(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(player) => player.play(), + WindowsPlayerMode::SourceReader(player) => player.resume(), + WindowsPlayerMode::Native(player) => player.resume(), + WindowsPlayerMode::Software(player) => player.resume(), +@@ -640,6 +690,7 @@ pub fn resume(&mut self) { + + pub fn mute(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.mute(), + WindowsPlayerMode::Native(player) => player.mute(), + WindowsPlayerMode::Software(player) => player.mute(), +@@ -648,6 +699,7 @@ pub fn mute(&mut self) { + + pub fn unmute(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.unmute(), + WindowsPlayerMode::Native(player) => player.unmute(), + WindowsPlayerMode::Software(player) => player.unmute(), +@@ -656,6 +708,7 @@ pub fn unmute(&mut self) { + + pub fn seek_to(&mut self, position_ms: u64) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.seek_to(position_ms), + WindowsPlayerMode::Native(player) => player.seek_to(position_ms), + WindowsPlayerMode::Software(player) => player.seek_to(position_ms), +@@ -664,6 +717,7 @@ pub fn seek_to(&mut self, position_ms: u64) { + + pub fn set_volume(&mut self, volume: f64) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.set_volume(volume), + WindowsPlayerMode::Native(player) => player.set_volume(volume), + WindowsPlayerMode::Software(player) => player.set_volume(volume), +@@ -672,6 +726,7 @@ pub fn set_volume(&mut self, volume: f64) { + + pub fn set_playback_rate(&mut self, rate: f64) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.set_playback_rate(rate), + WindowsPlayerMode::Native(player) => player.set_playback_rate(rate), + WindowsPlayerMode::Software(player) => player.set_playback_rate(rate), +@@ -680,6 +735,7 @@ pub fn set_playback_rate(&mut self, rate: f64) { + + pub fn current_position_ms(&self) -> u128 { + match &self.mode { ++ WindowsPlayerMode::Camera(_) => 0, + WindowsPlayerMode::SourceReader(player) => player.current_position_ms(), + WindowsPlayerMode::Native(player) => player.current_position_ms(), + WindowsPlayerMode::Software(player) => player.current_position_ms(), +@@ -688,6 +744,7 @@ pub fn current_position_ms(&self) -> u128 { + + pub fn cleanup(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(player) => player.cleanup(), + WindowsPlayerMode::SourceReader(player) => player.cleanup(), + WindowsPlayerMode::Native(player) => player.cleanup(), + WindowsPlayerMode::Software(player) => player.cleanup(), diff --git a/tools/setup.py b/tools/setup.py index d1da298d4..a04655a24 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -170,7 +170,9 @@ def owned_state(path, overlay): untracked = git(path, "ls-files", "--others", "--exclude-standard").stdout.split() if untracked: return f"untracked files ({', '.join(untracked[:5])})" - tree = git(path, "write-tree").stdout.strip() + # A promisor worktree may have index blobs that are not fetched locally; + # the staged tree hash still proves the exact reviewed overlay. + tree = git(path, "write-tree", "--missing-ok").stdout.strip() allowed = {git(path, "rev-parse", "HEAD^{tree}").stdout.strip()} if overlay: allowed.add(overlay["tree"]) @@ -257,7 +259,7 @@ def prepare_source(root, name, spec, args, overlay=None): current = git(path, "rev-parse", "--verify", "HEAD", check=False).stdout.strip() if current and (git(path, "diff", "--quiet", check=False).returncode or git(path, "ls-files", "--others", "--exclude-standard").stdout): raise RuntimeError(f"Preserving local changes: {path}") - tree = git(path, "write-tree").stdout.strip() + tree = git(path, "write-tree", "--missing-ok").stdout.strip() base_tree = git(path, "rev-parse", "HEAD^{tree}", check=False).stdout.strip() patches = [] if overlay: @@ -290,7 +292,7 @@ def prepare_source(root, name, spec, args, overlay=None): for patch in patches: git(path, "apply", "--check", str(patch)) git(path, "apply", "--index", str(patch)) - if git(path, "write-tree").stdout.strip() != overlay["tree"]: + if git(path, "write-tree", "--missing-ok").stdout.strip() != overlay["tree"]: raise RuntimeError(f"Runtime patch produced an unexpected tree: {path}") @@ -325,7 +327,7 @@ def verify_consumer(expected): if "Cargo.toml" not in files: continue path = Path(directory) / "Cargo.toml" - for line in path.read_text().splitlines(): + for line in path.read_text(encoding="utf-8").splitlines(): if line.lstrip().startswith("#"): continue url = re.search(r'git\s*=\s*"([^"]+)"', line) @@ -338,7 +340,7 @@ def verify_consumer(expected): def verify_cargo(root): """One Makepad (the prepared checkout), one App Hub, one octos, one Rinx.""" metadata = json.loads(subprocess.check_output( - ["cargo", "metadata", "--locked", "--format-version", "1", "--all-features"], cwd=CONSUMER, text=True)) + ["cargo", "metadata", "--locked", "--format-version", "1", "--all-features"], cwd=CONSUMER, text=True, encoding="utf-8")) seen = set() sources = {} for package in metadata["packages"]: From 684a8f9b34c98b0d39b2c656c3048d55e50c3db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?power=E8=83=96=E4=B8=B8?= <77043631+power246@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:39:44 +0800 Subject: [PATCH 2/3] feat(windows): isolate trusted physical pointer input --- crates/oauth-service/README.md | 2 +- crates/oauth-service/README.zh-CN.md | 2 +- docs/architecture.md | 4 +- docs/architecture.zh-CN.md | 4 +- docs/mail-composable-cards.md | 18 ++- docs/mail-composable-cards.zh-CN.md | 8 +- runtime-patches.lock.json | 11 +- .../makepad-windows-trusted-input.patch | 115 ++++++++++++++++++ 8 files changed, 145 insertions(+), 19 deletions(-) create mode 100644 tools/runtime-patches/makepad-windows-trusted-input.patch diff --git a/crates/oauth-service/README.md b/crates/oauth-service/README.md index 108f0a4d9..cec7b5719 100644 --- a/crates/oauth-service/README.md +++ b/crates/oauth-service/README.md @@ -42,7 +42,7 @@ Inbox Assistant and Google Calendar have not run there. | Platform | Provider authorization | Credential storage | Gmail send approval | | --- | --- | --- | --- | | macOS | GitHub device flow; Google browser/PKCE loopback | Mail's platform Keychain adapter, separate OAuth namespace | Native pointer provenance; remote clicks refused; physical acceptance unverified | -| Windows | Same desktop flows; platform execution unverified | Windows Credential Manager; unverified on Windows | Unsupported: fails closed | +| Windows | Same desktop flows; platform execution unverified | Windows Credential Manager; unverified on Windows | Physical pointer provenance is available to native review; real Gmail send remains unverified | | Linux | Protocol tests and host compilation passed on Linux; browser login and GUI unverified | Secret Service; unlocked service required, no plaintext fallback; native vault test refused the locked/unavailable build-host store | Unsupported: fails closed | | Android | GitHub flow present but unverified; **Google connection refused until its native adapter is implemented** | Mail's Android platform vault, separate namespace | Existing physical-touch provenance; this sample unverified | diff --git a/crates/oauth-service/README.zh-CN.md b/crates/oauth-service/README.zh-CN.md index a97fa43d5..fdb8317ec 100644 --- a/crates/oauth-service/README.zh-CN.md +++ b/crates/oauth-service/README.zh-CN.md @@ -11,7 +11,7 @@ Rust 授权协议、连接器、原生审阅、账户生命周期和示例界面 | 平台 | 提供商授权 | 凭据保存 | Gmail 发信审批 | | --- | --- | --- | --- | | macOS | GitHub 设备授权;Google 浏览器/PKCE/回环回调 | 复用 Mail 的 Keychain 适配器,独立 OAuth 命名空间 | 原生鼠标来源校验;宿主拒绝远程点击,亲手点按未验证 | -| Windows | 同样的桌面流程,平台运行未验证 | Windows Credential Manager;未在 Windows 验证 | 不支持,明确拒绝 | +| Windows | 同样的桌面流程,平台运行未验证 | Windows Credential Manager;未在 Windows 验证 | 原生审阅可识别物理指针输入;真实 Gmail 发信仍未验证 | | Linux | 已在 Linux 通过协议测试和主机编译;浏览器登录和 GUI 未验证 | 需要解锁 Secret Service,不回退到明文;构建主机的凭据库未解锁或不可用,原生测试遭到拒绝 | 不支持,明确拒绝 | | Android | GitHub 流程存在但未验证;**Google 原生适配器完成前拒绝连接** | Mail 的 Android 凭据库,独立命名空间 | 现有的亲手点按来源校验;本示例未验证 | diff --git a/docs/architecture.md b/docs/architecture.md index e98ce3f3d..9a9eacd09 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -362,9 +362,9 @@ flowchart TB 4. **Standing rules** on (owning app, tool), whoever calls. Runs started by incoming content or an unknown trigger skip them unless a rule opts in. 5. **A shell-drawn sheet** with the owning app, tool, exact arguments and any calling app. The system agent's approvals for one request can share one sheet. -**Mail sending** never goes through the router. Mail's own composer (`mail.review_send`) and its reply cards end in one host-owned review of the exact message, drawn by the shell inside the card (`crates/shell/src/mail_review.rs`); Mail's agent can only propose a send (`mail.propose_send`). Only a trusted press and release of the review's Approve & Send control sends, and only a physical press is trusted: a tap on Android or a click on macOS. Developer mode and standing rules cannot authorize a send, and `mail.send` answers only `approval_required`. [Composed Mail cards](mail-composable-cards.md) describes the flow. +**Mail sending** never goes through the router. Mail's own composer (`mail.review_send`) and its reply cards end in one host-owned review of the exact message, drawn by the shell inside the card (`crates/shell/src/mail_review.rs`); Mail's agent can only propose a send (`mail.propose_send`). Only a trusted press and release of the review's Approve & Send control sends, and only a physical press is trusted: a tap on Android or a click on macOS or Windows. Windows still has no Mail send adapter. Developer mode and standing rules cannot authorize a send, and `mail.send` answers only `approval_required`. [Composed Mail cards](mail-composable-cards.md) describes the flow. -Two reviewed Makepad patches establish that trust (`tools/runtime-patches/`): `makepad-trusted-user-input.patch` for Android touchscreens, and `makepad-desktop-trusted-input.patch` for macOS pointer events that come from the HID source and that no other process posted. Synthetic and remote input are refused, and so is a native callback that runs inside their dispatch. Input on Windows or Linux, and accessibility input, cannot approve a send. The macOS path is **unverified**: no real message has been sent from a Mac. +Three reviewed Makepad patches establish that trust (`tools/runtime-patches/`): `makepad-trusted-user-input.patch` for Android touchscreens, `makepad-desktop-trusted-input.patch` for macOS pointer events from the HID source without another posting process, and `makepad-windows-trusted-input.patch` for physical Windows mouse, touch, pen or touchpad messages. This Windows provenance applies to every host review using `trusted_user_input()`, including connected-account saves and sends and device consent. Injected, posted and remote input, nested callbacks inside their dispatch, accessibility input and Linux input remain untrusted. The macOS Mail path is **unverified**: no real message has been sent from a Mac. Windows provenance does not supply a Mail send adapter or prove a real connected-account write. **Connected-account writes and sends** do not go through the router either. A GitHub commit (`github.review_save`) or a Google Calendar write (`gcalendar.review_save`) opens a host review of the exact change, and only its Approve & Save control saves it. A Gmail send (`gmail.draft.review`) opens the host's review of the exact reply, and only its Approve & Send control sends. On `main`, all three reviews are native (`crates/shell/src/connected_review.rs`) and accept only a trusted physical press and release, and each approval works once (`crates/oauth-service/src/host_api.rs`, `host_inbox.rs`). `desktop-v0.1.0-beta.2` checks for a physical press only on the Gmail send; its GitHub and Calendar saves use a host sheet that does not check how Approve & Save was pressed. An agent cannot open any of these screens: its tool calls reach the services with `may_prompt: false` (`host_tools/script_apps.rs`). The platform table in the [OAuth service README](../crates/oauth-service/README.md#current-delivery-boundary) shows where the physical press is supported and what is verified. diff --git a/docs/architecture.zh-CN.md b/docs/architecture.zh-CN.md index 909061a85..a25d0d153 100644 --- a/docs/architecture.zh-CN.md +++ b/docs/architecture.zh-CN.md @@ -362,9 +362,9 @@ flowchart TB 4. **常设规则**,按(所属应用,工具)匹配,不论调用方是谁。由收到的内容或未知来源触发的运行会跳过规则,除非某条规则明确纳入。 5. 否则由 **Shell 绘制的面板**列出所属应用、工具、确切参数和调用方应用(如有)。系统 Agent 为同一请求发起的审批可以合并到一个面板中。 -**发送邮件**从不经过审批路由。邮件自己的写信界面(`mail.review_send`)和它的回复卡片,最终都进入同一个由宿主拥有的确切邮件审阅界面,由 Shell 绘制在卡片内(`crates/shell/src/mail_review.rs`);邮件的 Agent 只能提议发送(`mail.propose_send`)。只有对审阅界面上 Approve & Send 控件的一次可信按下和释放才会发送,而且只有亲手点按才可信:Android 上触摸屏幕,macOS 上用鼠标或触控板点击。开发者模式和常设规则都不能授权发送,`mail.send` 只会回答 `approval_required`。流程见[组合式邮件卡片](mail-composable-cards.zh-CN.md)。 +**发送邮件**从不经过审批路由。邮件自己的写信界面(`mail.review_send`)和它的回复卡片,最终都进入同一个由宿主拥有的确切邮件审阅界面,由 Shell 绘制在卡片内(`crates/shell/src/mail_review.rs`);邮件的 Agent 只能提议发送(`mail.propose_send`)。只有对审阅界面上 Approve & Send 控件的一次可信按下和释放才会发送,而且只有亲手点按才可信:Android 上触摸屏幕,macOS 或 Windows 上点击。Windows 仍没有 Mail 发信适配器。开发者模式和常设规则都不能授权发送,`mail.send` 只会回答 `approval_required`。流程见[组合式邮件卡片](mail-composable-cards.zh-CN.md)。 -这种信任来自两个经审查的 Makepad 补丁(`tools/runtime-patches/`):`makepad-trusted-user-input.patch` 针对 Android 触摸屏,`makepad-desktop-trusted-input.patch` 针对来自 HID 源、且不是由其他进程投递(post)的 macOS 指针事件。合成输入和远程输入都会被拒绝,在其分发过程中运行的原生回调也不例外。Windows 和 Linux 上的任何输入以及无障碍输入,都不能批准发送。macOS 路径**未验证**:还没有在 macOS 上实际发送过邮件。 +这种信任来自三个经审查的 Makepad 补丁(`tools/runtime-patches/`):`makepad-trusted-user-input.patch` 针对 Android 触摸屏,`makepad-desktop-trusted-input.patch` 针对来自 HID 源、且不是由其他进程投递的 macOS 指针事件,`makepad-windows-trusted-input.patch` 针对 Windows 的物理鼠标、触摸、笔和触控板消息。Windows 的输入来源判断作用于所有使用 `trusted_user_input()` 的宿主审阅界面,包括已连接账户的保存与发送以及设备授权。注入、投递、远程输入及其分发期间嵌套的回调、无障碍输入和 Linux 输入仍不可信。macOS Mail 路径**未验证**:还没有实际发送过邮件。Windows 输入来源判断本身不提供 Mail 发信适配器,也不证明已连接账户的真实写入。 **已连接账户的写入和发送**也不经过审批路由。GitHub 提交(`github.review_save`)或 Google Calendar 写入(`gcalendar.review_save`)会打开宿主对确切改动的审阅界面,只有其中的 Approve & Save 控件才能保存。Gmail 发信(`gmail.draft.review`)会打开宿主对确切回复的审阅界面,只有其中的 Approve & Send 控件才能发送。在 `main` 上,这三种审阅界面都是原生的(`crates/shell/src/connected_review.rs`),只接受可信的亲手点按(按下和释放都必须可信),每次批准只能用一次(`crates/oauth-service/src/host_api.rs`、`host_inbox.rs`)。`desktop-v0.1.0-beta.2` 只在 Gmail 发信时检查是否亲手点按;它的 GitHub 和 Calendar 保存使用宿主面板,不检查 Approve & Save 是怎样按下的。Agent 打不开其中任何一个界面:它的工具调用以 `may_prompt: false` 到达服务(`host_tools/script_apps.rs`)。哪些平台支持亲手点按、哪些已经验证,见 [OAuth 服务 README](../crates/oauth-service/README.zh-CN.md#当前交付边界) 中的平台表。 diff --git a/docs/mail-composable-cards.md b/docs/mail-composable-cards.md index ea4719ef2..c774c29ca 100644 --- a/docs/mail-composable-cards.md +++ b/docs/mail-composable-cards.md @@ -10,10 +10,11 @@ person approved the send with a physical press, and the threaded SMTP reply was verified. These are separate checkpoints: the new workspace tests did not send real email. -Sending needs a physical press on the host's review, either a tap on Android or -a click on macOS; synthetic and remote input are refused. The macOS path is -**unverified**: no real message has been sent from a Mac. Windows, Linux and -accessibility approval, and the complete phone UI matrix, remain deferred. +Sending needs a physical press on the host's review: a tap on Android or a +click on macOS. Windows now recognizes physical pointer input for host reviews, +but Mail has no Windows send adapter. Synthetic, remote and accessibility input +are refused. The macOS path is **unverified**: no real message has been sent +from a Mac. Linux approval and the complete phone UI matrix remain deferred. After a saved transport receipt confirms SMTP acceptance, the bound reply card and notification leave Glance. The draft and send receipt remain stored, and an @@ -330,13 +331,18 @@ null objects or Java exceptions enqueue no trusted input. A [second overlay](../tools/runtime-patches/makepad-desktop-trusted-input.patch) trusts a macOS pointer press or release only when AppKit supplies a CGEvent from the HID source with no posting process. +The [Windows overlay](../tools/runtime-patches/makepad-windows-trusted-input.patch) +trusts a physical mouse, touch, pen or touchpad message reported by user32; +posted or injected messages do not qualify. This provenance also applies to +connected-account and device-consent reviews, not just Mail. A scoped guard applies only while the native handler of a trusted touch (`TouchUpdate`) or click runs. Remote and synthetic dispatch (with any native callback nested inside it), deferred actions and script tasks do not inherit the guard. Host review requires a matching trusted press and release. `with_untrusted_input` can only remove trust. Keyboard/IME, long press, -accessibility input and any input on Windows or Linux cannot approve a send. A +accessibility input and Linux input cannot approve a send. Windows can satisfy +the input provenance gate, but its Mail send adapter remains unavailable. A compromised OS/root process impersonating hardware lies outside this application-level boundary; this is not hardware attestation. @@ -371,7 +377,7 @@ acceptance results; installing the later build does not rerun the earlier flows. | Remaining phone UI matrix | Full IME composition, cursor/selection, Unicode, themes, touch-target sizing, conflicts, account switching, expiry and retry coverage remains incomplete. Relevant local tests are not a substitute. | | Notification-toast touch route | Four new regressions passed. On 0415, a visible notification followed by an injected Android tap opened the exact bound card and retained revision 44. This proves navigation, not send authorization. | | Physical-press approval and real threaded reply | No real SMTP send performed. Approval with a physical press, provider acceptance and matching card/sheet receipts remain unverified. | -| Desktop/accessibility approval | Deferred at this checkpoint; the provenance boundary denied these routes. Later builds accept a click on macOS as a physical press (**unverified**); input on Windows or Linux, and accessibility input, still cannot approve. See [Runtime contracts and input provenance](#runtime-contracts-and-input-provenance). | +| Desktop/accessibility approval | Deferred at this historical checkpoint. Later builds accept physical pointer input on macOS (**unverified**) and Windows; Windows Mail still has no send adapter. Linux and accessibility input cannot approve. See [Runtime contracts and input provenance](#runtime-contracts-and-input-provenance). | ### Per-provider engineering review diff --git a/docs/mail-composable-cards.zh-CN.md b/docs/mail-composable-cards.zh-CN.md index ecb152696..d45907623 100644 --- a/docs/mail-composable-cards.zh-CN.md +++ b/docs/mail-composable-cards.zh-CN.md @@ -4,7 +4,7 @@ 原生 **Email / Chat** 工作区共用一份已保存草稿。在隔离的 OnePlus 6 手机测试中,实际的 `deepseek-v4-flash` 与 `MiniMax-M3.1-Flash-Preview` 都通过聊天修改了预约时间,Reply 和最终审阅界面显示完全相同的已保存正文。在较早的受控物流演示中,用户已亲手点按批准发送,真实的线程回复也已通过验证。这是不同验证阶段:本次新工作区测试没有发送真实邮件。 -发送需要用户在宿主的审阅界面上亲手点按:Android 上触摸屏幕,macOS 上用鼠标或触控板点击;合成输入和远程输入都会被拒绝。macOS 路径**未验证**:还没有在 macOS 上实际发送过邮件。Windows、Linux 和无障碍输入的审批,以及完整手机 UI 矩阵,仍然暂缓。 +发送需要用户在宿主的审阅界面上亲手点按:Android 上触摸屏幕,macOS 上用鼠标或触控板点击。Windows 现在能识别宿主审阅界面的物理指针输入,但 Mail 仍没有 Windows 发信适配器。合成、远程和无障碍输入都会被拒绝。macOS 路径**未验证**:还没有在 macOS 上实际发送过邮件。Linux 审批及完整手机 UI 矩阵仍然暂缓。 生成的卡片描述展示。Rust 宿主代码拥有账户、原邮件、已保存草稿、修订号和发送操作。编辑字段会更新持久化草稿;生成的 Review reply 按钮只打开宿主审阅界面。模型和卡片本地的 `sent` 状态都不能授权 SMTP。 @@ -108,9 +108,9 @@ Lab **0468** 还验证了拒绝首次助手同意:Enable assistant 按钮消 下文的历史真机运行对应 **0416**(`abf06d8f`),当时使用 Octoscript `9ca9545b` 和 Octoscript-Makepad `a950f7fb`。当前固定版本仅对同一份 L0 实现应用 rustfmt,并将该修订同步到包装层。较新的 0427–0432 工作区版本使用当前固定版本,其验证单独记录在上方。 -[Makepad 补丁](../tools/runtime-patches/makepad-trusted-user-input.patch)默认令 `trusted_user_input()` 为 false。Android JNI 检查正数且非虚拟的设备 ID、触摸屏来源、未遮挡标志及非取消输入;空对象或 Java 异常不会入队可信输入。[第二个补丁](../tools/runtime-patches/makepad-desktop-trusted-input.patch)只在 AppKit 提供来自 HID 源、且不是由其他进程投递(post)的 CGEvent 时,才信任 macOS 指针的按下或释放。 +[Makepad 补丁](../tools/runtime-patches/makepad-trusted-user-input.patch)默认令 `trusted_user_input()` 为 false。Android JNI 检查正数且非虚拟的设备 ID、触摸屏来源、未遮挡标志及非取消输入;空对象或 Java 异常不会入队可信输入。[第二个补丁](../tools/runtime-patches/makepad-desktop-trusted-input.patch)只在 AppKit 提供来自 HID 源、且不是由其他进程投递(post)的 CGEvent 时,才信任 macOS 指针的按下或释放。[Windows 补丁](../tools/runtime-patches/makepad-windows-trusted-input.patch)只信任 user32 报告的物理鼠标、触摸、笔或触控板消息;投递或注入的消息不可信。这项来源判断也作用于已连接账户与设备授权审阅界面。 -作用域 guard 只在可信触摸(`TouchUpdate`)或点击的原生处理器运行期间生效。远程和合成分发(包括嵌套在其中的原生回调)、延后的 action 和脚本任务都不能继承这个 guard。宿主审阅界面要求相匹配的可信按下与释放。`with_untrusted_input` 只能移除信任。键盘/输入法、长按、无障碍输入,以及 Windows 和 Linux 上的任何输入,都不能批准发送。受攻击的 OS/root 进程冒充硬件,不在这一应用级边界内;这不是硬件认证。 +作用域 guard 只在可信触摸(`TouchUpdate`)或点击的原生处理器运行期间生效。远程和合成分发(包括嵌套在其中的原生回调)、延后的 action 和脚本任务都不能继承这个 guard。宿主审阅界面要求相匹配的可信按下与释放。`with_untrusted_input` 只能移除信任。键盘/输入法、长按、无障碍输入和 Linux 输入不能批准发送。Windows 可满足输入来源条件,但仍没有 Mail 发信适配器。受攻击的 OS/root 进程冒充硬件,不在这一应用级边界内;这不是硬件认证。 @@ -131,7 +131,7 @@ Lab **0468** 还验证了拒绝首次助手同意:Enable assistant 按钮消 | 剩余手机 UI 矩阵 | 完整输入法组合输入、光标/选择、Unicode、主题、触摸目标尺寸、冲突、账户切换、到期与重试覆盖仍不完整。相关本地测试不能替代这些检查。 | | 通知浮层触摸路由 | 四项新增回归测试通过。在 0415 上,可见通知经注入的 Android 点击打开准确绑定的卡片,保留修订版 44。这证明导航有效,不是发送授权。 | | 亲手点按批准与真实线程回复 | 没有执行真实 SMTP 发送。用户亲手点按批准、提供商接受,以及卡片与审阅界面的回执是否一致,仍未验证。 | -| 桌面/无障碍审批 | 在该检查点暂缓;当时的输入来源边界拒绝这些路径。后来的构建也把 macOS 上用鼠标或触控板的点击视为亲手点按(**未验证**);Windows 和 Linux 上的任何输入以及无障碍输入仍不能批准。见[运行时契约与输入来源](#运行时契约与输入来源)。 | +| 桌面/无障碍审批 | 在该历史检查点暂缓。后来的构建接受 macOS(**未验证**)和 Windows 的物理指针输入;Windows Mail 仍没有发信适配器。Linux 和无障碍输入不能批准。见[运行时契约与输入来源](#运行时契约与输入来源)。 | ### 按提供商记录的工程审查 diff --git a/runtime-patches.lock.json b/runtime-patches.lock.json index 5b9836137..52e427dfe 100644 --- a/runtime-patches.lock.json +++ b/runtime-patches.lock.json @@ -4,7 +4,7 @@ "base_revision": "d653bc4493619ab979ccff165cf48c0c63b77d19", "patch": "tools/runtime-patches/makepad-settings.patch", "sha256": "3a79cddddc89dc4d064a9f0bdd17dc8111dcab9c796a334d95f61ecb9d5011aa", - "tree": "a6fae94aa4d233503b14bc6d0a36bcbc2b264b44", + "tree": "7d6eccdbc1626449a87d86d5bd5533a2251ae4ad", "product_fixes": [ "Settings integration: Android IME synchronization, accessibility and reduced motion, GLES program-cache validation, obscured-gesture cancellation and scroll-position observation.", "Contained-app isolation and secret-input restrictions come from the pinned upstream runtime; the Settings overlay preserves those gates.", @@ -14,7 +14,7 @@ "Windows frame completion: backport Makepad work c508ab0c8 GPU completion polls around native and hosted repaint so upload buffers can retire.", "Android application extensions honor the manifest's explicit class name, keeping Back and platform integration active under custom test package ids.", "Android physical touchscreen input has scoped host approval provenance; default and nested synthetic input are untrusted. Compromised OS/root impersonation is outside this app-level boundary; the subsequent desktop patch adds macOS pointer provenance.", - "macOS AppKit pointer metadata enables native approval for HID events without a posting process. Synthetic and remote dispatch remain untrusted, including nested callbacks; Windows/Linux physical approval is unsupported.", + "macOS AppKit pointer metadata enables native approval for HID events without a posting process. Synthetic and remote dispatch remain untrusted, including nested callbacks; the Windows patch adds pointer provenance while Linux physical approval remains unsupported.", "Android multiline Enter key events finish the active composition before inserting a newline, retaining the last word; ordinary IME composition commits keep replacement semantics.", "Host-owned backend authentication reuses native WebViews with isolated sessions, exact callback interception, same-origin navigation and no page tool bridge; Android uses an ephemeral private process and profile.", "Android Camera waits for one permission answer and refreshes an empty camera list after grant. Video pause/resume preserves an unprepared player so first-use authorization cannot prevent its initial source assignment.", @@ -47,7 +47,8 @@ "Android Video seeks to the closest frame on API26+ instead of the previous keyframe; older backends retain a bounded legacy fallback.", "A native host may stage up to 64 MiB into an app's jail (MAX_IMPORT_BYTES; prepare_import and commit_import), so files.import can take documents above the 1 MiB script write limit. Staged bytes never enter the script heap; the jail's byte quota and entry cap still apply, and script writes, photo picks and exports keep 1 MiB.", "An OS resume restarts only the Video playback its pause stopped, including a start it cancelled; a player paused by the person, or paused or stopped by its app during the pause, stays paused (#406).", - "Camera capture and recording use explicit per-call boolean audio/library options, defaulting to silent app-local output. Permission availability never opts into recording audio or exporting captures; async microphone checks retain capture intent and background provenance. Unsupported library export fails before capture." + "Camera capture and recording use explicit per-call boolean audio/library options, defaulting to silent app-local output. Permission availability never opts into recording audio or exporting captures; async microphone checks retain capture intent and background provenance. Unsupported library export fails before capture.", + "Windows physical pointer messages carry scoped input provenance to host review sheets. Injected, posted, remote and nested synthetic input remains untrusted; approval eligibility changes across host services, not only camera consent." ], "stacked": [ { @@ -185,6 +186,10 @@ { "patch": "tools/runtime-patches/makepad-camera-capture-intent.patch", "sha256": "d056b76d2eb6b55aa9c96447a49fc90d32a9a235b6bc9a1ff125902a73c28208" + }, + { + "patch": "tools/runtime-patches/makepad-windows-trusted-input.patch", + "sha256": "a5786f0121051d074813b7ad908da5388dcad3c46f804d4def57c1e1ef057ed3" } ] } diff --git a/tools/runtime-patches/makepad-windows-trusted-input.patch b/tools/runtime-patches/makepad-windows-trusted-input.patch new file mode 100644 index 000000000..9fb7bde8e --- /dev/null +++ b/tools/runtime-patches/makepad-windows-trusted-input.patch @@ -0,0 +1,115 @@ +diff --git a/platform/src/os/windows/win32_event.rs b/platform/src/os/windows/win32_event.rs +index 8a02d73..3299138 100644 +--- a/platform/src/os/windows/win32_event.rs ++++ b/platform/src/os/windows/win32_event.rs +@@ -30,8 +30,8 @@ pub enum Win32Event { + primary: bool, + }, + +- MouseDown(MouseDownEvent), +- MouseUp(MouseUpEvent), ++ MouseDown(MouseDownEvent, bool), ++ MouseUp(MouseUpEvent, bool), + MouseMove(MouseMoveEvent), + MouseLeave(MouseLeaveEvent), + Scroll(ScrollEvent), +diff --git a/platform/src/os/windows/win32_window.rs b/platform/src/os/windows/win32_window.rs +index 01a356d..3c79a78 100644 +--- a/platform/src/os/windows/win32_window.rs ++++ b/platform/src/os/windows/win32_window.rs +@@ -1864,6 +1864,26 @@ pub fn send_focus_lost_event(&mut self) { + self.do_callback(Win32Event::WindowLostFocus(self.window_id)); + } + ++ fn hardware_mouse_message() -> bool { ++ // This pinned windows-rs shim lacks Win32::UI::Input's message-origin ++ // bindings. Query user32 directly while processing the native message. ++ #[repr(C)] ++ #[derive(Default)] ++ struct InputMessageSource { device_type: i32, origin_id: i32 } ++ #[link(name = "user32")] ++ unsafe extern "system" { ++ fn GetCurrentInputMessageSource(source: *mut InputMessageSource) -> i32; ++ } ++ let mut source = InputMessageSource::default(); ++ let received = unsafe { GetCurrentInputMessageSource(&mut source) != 0 }; ++ received && Self::trusted_pointer_source(source.device_type, source.origin_id) ++ } ++ ++ fn trusted_pointer_source(device_type: i32, origin_id: i32) -> bool { ++ matches!(device_type, 2 | 4 | 8 | 16) // mouse, touch, pen, touchpad ++ && origin_id == 1 // IMO_HARDWARE; injected/post messages fail closed ++ } ++ + pub fn send_mouse_down(&mut self, button: MouseButton, modifiers: KeyModifiers) { + if self.mouse_buttons_down == 0 { + unsafe { +@@ -1878,7 +1898,7 @@ pub fn send_mouse_down(&mut self, button: MouseButton, modifiers: KeyModifiers) + abs: self.last_mouse_pos, + time: self.time_now(), + handled: Cell::new(Area::Empty), +- })); ++ }, Self::hardware_mouse_message())); + } + + pub fn send_mouse_up(&mut self, button: MouseButton, modifiers: KeyModifiers) { +@@ -1896,7 +1916,7 @@ pub fn send_mouse_up(&mut self, button: MouseButton, modifiers: KeyModifiers) { + window_id: self.window_id, + abs: self.last_mouse_pos, + time: self.time_now(), +- })); ++ }, Self::hardware_mouse_message())); + } + + pub fn send_mouse_move(&mut self, pos: Vec2d, modifiers: KeyModifiers) { +@@ -2085,3 +2105,19 @@ pub fn virtual_key_to_key_code(wparam: WPARAM) -> KeyCode { + } + } + } ++ ++#[cfg(test)] ++mod input_source_tests { ++ use super::Win32Window; ++ ++ #[test] ++ fn only_physical_pointer_messages_are_trusted() { ++ for device in [2, 4, 8, 16] { ++ assert!(Win32Window::trusted_pointer_source(device, 1)); ++ assert!(!Win32Window::trusted_pointer_source(device, 2)); ++ assert!(!Win32Window::trusted_pointer_source(device, 4)); ++ } ++ assert!(!Win32Window::trusted_pointer_source(0, 1)); ++ assert!(!Win32Window::trusted_pointer_source(1, 1)); ++ } ++} +diff --git a/platform/src/os/windows/windows.rs b/platform/src/os/windows/windows.rs +index fb1dd91..4d1a501 100644 +--- a/platform/src/os/windows/windows.rs ++++ b/platform/src/os/windows/windows.rs +@@ -293,11 +293,11 @@ fn win32_event_callback( + let time_now = with_win32_app(|app| app.time_now()); + self.paint_tick(time_now, true, d3d11_cx, d3d11_windows); + } +- Win32Event::MouseDown(mut e) => { ++ Win32Event::MouseDown(mut e, trusted) => { + self.dpi_override_scale(&mut e.abs, e.window_id); + self.fingers.process_tap_count(e.abs, e.time); + self.fingers.mouse_down(e.button, e.window_id); +- self.call_event_handler(&Event::MouseDown(e.into())) ++ self.call_event_handler_with_input_trust(&Event::MouseDown(e.into()), trusted) + } + Win32Event::MouseMove(mut e) => { + self.dpi_override_scale(&mut e.abs, e.window_id); +@@ -305,10 +305,10 @@ fn win32_event_callback( + self.fingers.cycle_hover_area(live_id!(mouse).into()); + self.fingers.switch_captures(); + } +- Win32Event::MouseUp(mut e) => { ++ Win32Event::MouseUp(mut e, trusted) => { + self.dpi_override_scale(&mut e.abs, e.window_id); + let button = e.button; +- self.call_event_handler(&Event::MouseUp(e.into())); ++ self.call_event_handler_with_input_trust(&Event::MouseUp(e.into()), trusted); + self.fingers.mouse_up(button); + self.fingers.cycle_hover_area(live_id!(mouse).into()); + } From 666029e4668ea2a2e4132f9159f19b2a55918cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?power=E8=83=96=E4=B8=B8?= <77043631+power246@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:06:27 +0800 Subject: [PATCH 3/3] Clarify Windows hardware-class input provenance Document the UIAccess injection caveat in both language variants, rename the source test, and restamp the locked Makepad patch. --- crates/oauth-service/README.md | 2 +- crates/oauth-service/README.zh-CN.md | 2 +- docs/architecture.md | 8 +++--- docs/architecture.zh-CN.md | 8 +++--- docs/mail-composable-cards.md | 26 +++++++++++-------- docs/mail-composable-cards.zh-CN.md | 8 +++--- runtime-patches.lock.json | 6 ++--- .../makepad-windows-trusted-input.patch | 10 +++---- 8 files changed, 37 insertions(+), 33 deletions(-) diff --git a/crates/oauth-service/README.md b/crates/oauth-service/README.md index cec7b5719..6dad3cee5 100644 --- a/crates/oauth-service/README.md +++ b/crates/oauth-service/README.md @@ -42,7 +42,7 @@ Inbox Assistant and Google Calendar have not run there. | Platform | Provider authorization | Credential storage | Gmail send approval | | --- | --- | --- | --- | | macOS | GitHub device flow; Google browser/PKCE loopback | Mail's platform Keychain adapter, separate OAuth namespace | Native pointer provenance; remote clicks refused; physical acceptance unverified | -| Windows | Same desktop flows; platform execution unverified | Windows Credential Manager; unverified on Windows | Physical pointer provenance is available to native review; real Gmail send remains unverified | +| Windows | Same desktop flows; platform execution unverified | Windows Credential Manager; unverified on Windows | Native review accepts OS hardware-class pointer messages, including UIAccess injection; real Gmail send remains unverified | | Linux | Protocol tests and host compilation passed on Linux; browser login and GUI unverified | Secret Service; unlocked service required, no plaintext fallback; native vault test refused the locked/unavailable build-host store | Unsupported: fails closed | | Android | GitHub flow present but unverified; **Google connection refused until its native adapter is implemented** | Mail's Android platform vault, separate namespace | Existing physical-touch provenance; this sample unverified | diff --git a/crates/oauth-service/README.zh-CN.md b/crates/oauth-service/README.zh-CN.md index fdb8317ec..894979e91 100644 --- a/crates/oauth-service/README.zh-CN.md +++ b/crates/oauth-service/README.zh-CN.md @@ -11,7 +11,7 @@ Rust 授权协议、连接器、原生审阅、账户生命周期和示例界面 | 平台 | 提供商授权 | 凭据保存 | Gmail 发信审批 | | --- | --- | --- | --- | | macOS | GitHub 设备授权;Google 浏览器/PKCE/回环回调 | 复用 Mail 的 Keychain 适配器,独立 OAuth 命名空间 | 原生鼠标来源校验;宿主拒绝远程点击,亲手点按未验证 | -| Windows | 同样的桌面流程,平台运行未验证 | Windows Credential Manager;未在 Windows 验证 | 原生审阅可识别物理指针输入;真实 Gmail 发信仍未验证 | +| Windows | 同样的桌面流程,平台运行未验证 | Windows Credential Manager;未在 Windows 验证 | 原生审阅接受系统归类为硬件来源的指针消息,包括 UIAccess 注入;真实 Gmail 发信仍未验证 | | Linux | 已在 Linux 通过协议测试和主机编译;浏览器登录和 GUI 未验证 | 需要解锁 Secret Service,不回退到明文;构建主机的凭据库未解锁或不可用,原生测试遭到拒绝 | 不支持,明确拒绝 | | Android | GitHub 流程存在但未验证;**Google 原生适配器完成前拒绝连接** | Mail 的 Android 凭据库,独立命名空间 | 现有的亲手点按来源校验;本示例未验证 | diff --git a/docs/architecture.md b/docs/architecture.md index 9a9eacd09..412e2aed2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -362,9 +362,9 @@ flowchart TB 4. **Standing rules** on (owning app, tool), whoever calls. Runs started by incoming content or an unknown trigger skip them unless a rule opts in. 5. **A shell-drawn sheet** with the owning app, tool, exact arguments and any calling app. The system agent's approvals for one request can share one sheet. -**Mail sending** never goes through the router. Mail's own composer (`mail.review_send`) and its reply cards end in one host-owned review of the exact message, drawn by the shell inside the card (`crates/shell/src/mail_review.rs`); Mail's agent can only propose a send (`mail.propose_send`). Only a trusted press and release of the review's Approve & Send control sends, and only a physical press is trusted: a tap on Android or a click on macOS or Windows. Windows still has no Mail send adapter. Developer mode and standing rules cannot authorize a send, and `mail.send` answers only `approval_required`. [Composed Mail cards](mail-composable-cards.md) describes the flow. +**Mail sending** never goes through the router. Mail's own composer (`mail.review_send`) and its reply cards end in one host-owned review of the exact message, drawn by the shell inside the card (`crates/shell/src/mail_review.rs`); Mail's agent can only propose a send (`mail.propose_send`). Only a trusted press and release of the review's Approve & Send control sends: a tap on Android, a click on macOS, or an OS hardware-class pointer message on Windows. Windows still has no Mail send adapter. Developer mode and standing rules cannot authorize a send, and `mail.send` answers only `approval_required`. [Composed Mail cards](mail-composable-cards.md) describes the flow. -Three reviewed Makepad patches establish that trust (`tools/runtime-patches/`): `makepad-trusted-user-input.patch` for Android touchscreens, `makepad-desktop-trusted-input.patch` for macOS pointer events from the HID source without another posting process, and `makepad-windows-trusted-input.patch` for physical Windows mouse, touch, pen or touchpad messages. This Windows provenance applies to every host review using `trusted_user_input()`, including connected-account saves and sends and device consent. Injected, posted and remote input, nested callbacks inside their dispatch, accessibility input and Linux input remain untrusted. The macOS Mail path is **unverified**: no real message has been sent from a Mac. Windows provenance does not supply a Mail send adapter or prove a real connected-account write. +Three reviewed Makepad patches establish that trust (`tools/runtime-patches/`): `makepad-trusted-user-input.patch` for Android touchscreens, `makepad-desktop-trusted-input.patch` for macOS pointer events from the HID source without another posting process, and `makepad-windows-trusted-input.patch` for Windows mouse, touch, pen or touchpad messages classified by user32 as `IMO_HARDWARE`. This Windows provenance applies to every host review using `trusted_user_input()`, including connected-account saves and sends and device consent. Windows rejects `IMO_INJECTED`, `IMO_SYSTEM` and posted messages, but Microsoft also classifies input injected by UIAccess applications as `IMO_HARDWARE`; remote-desktop input has not been characterized. Nested callbacks inside untrusted dispatch, accessibility input without this classification, and Linux input remain untrusted. The macOS Mail path is **unverified**: no real message has been sent from a Mac. Windows provenance does not supply a Mail send adapter or prove a real connected-account write. **Connected-account writes and sends** do not go through the router either. A GitHub commit (`github.review_save`) or a Google Calendar write (`gcalendar.review_save`) opens a host review of the exact change, and only its Approve & Save control saves it. A Gmail send (`gmail.draft.review`) opens the host's review of the exact reply, and only its Approve & Send control sends. On `main`, all three reviews are native (`crates/shell/src/connected_review.rs`) and accept only a trusted physical press and release, and each approval works once (`crates/oauth-service/src/host_api.rs`, `host_inbox.rs`). `desktop-v0.1.0-beta.2` checks for a physical press only on the Gmail send; its GitHub and Calendar saves use a host sheet that does not check how Approve & Save was pressed. An agent cannot open any of these screens: its tool calls reach the services with `may_prompt: false` (`host_tools/script_apps.rs`). The platform table in the [OAuth service README](../crates/oauth-service/README.md#current-delivery-boundary) shows where the physical press is supported and what is verified. @@ -437,8 +437,8 @@ The kernel's core dir, `~/.octosense/octos-home/.octos` on the desktop and ` @@ -131,7 +131,7 @@ Lab **0468** 还验证了拒绝首次助手同意:Enable assistant 按钮消 | 剩余手机 UI 矩阵 | 完整输入法组合输入、光标/选择、Unicode、主题、触摸目标尺寸、冲突、账户切换、到期与重试覆盖仍不完整。相关本地测试不能替代这些检查。 | | 通知浮层触摸路由 | 四项新增回归测试通过。在 0415 上,可见通知经注入的 Android 点击打开准确绑定的卡片,保留修订版 44。这证明导航有效,不是发送授权。 | | 亲手点按批准与真实线程回复 | 没有执行真实 SMTP 发送。用户亲手点按批准、提供商接受,以及卡片与审阅界面的回执是否一致,仍未验证。 | -| 桌面/无障碍审批 | 在该历史检查点暂缓。后来的构建接受 macOS(**未验证**)和 Windows 的物理指针输入;Windows Mail 仍没有发信适配器。Linux 和无障碍输入不能批准。见[运行时契约与输入来源](#运行时契约与输入来源)。 | +| 桌面/无障碍审批 | 在该历史检查点暂缓。后来的构建接受 macOS(**未验证**)的物理指针输入,以及 Windows 系统归类为硬件来源的输入(包括 UIAccess 注入);Windows Mail 仍没有发信适配器。Linux 输入不能批准。见[运行时契约与输入来源](#运行时契约与输入来源)。 | ### 按提供商记录的工程审查 diff --git a/runtime-patches.lock.json b/runtime-patches.lock.json index 52e427dfe..380936984 100644 --- a/runtime-patches.lock.json +++ b/runtime-patches.lock.json @@ -4,7 +4,7 @@ "base_revision": "d653bc4493619ab979ccff165cf48c0c63b77d19", "patch": "tools/runtime-patches/makepad-settings.patch", "sha256": "3a79cddddc89dc4d064a9f0bdd17dc8111dcab9c796a334d95f61ecb9d5011aa", - "tree": "7d6eccdbc1626449a87d86d5bd5533a2251ae4ad", + "tree": "286598c629540456ea1ba52727ac3299f85f39f0", "product_fixes": [ "Settings integration: Android IME synchronization, accessibility and reduced motion, GLES program-cache validation, obscured-gesture cancellation and scroll-position observation.", "Contained-app isolation and secret-input restrictions come from the pinned upstream runtime; the Settings overlay preserves those gates.", @@ -48,7 +48,7 @@ "A native host may stage up to 64 MiB into an app's jail (MAX_IMPORT_BYTES; prepare_import and commit_import), so files.import can take documents above the 1 MiB script write limit. Staged bytes never enter the script heap; the jail's byte quota and entry cap still apply, and script writes, photo picks and exports keep 1 MiB.", "An OS resume restarts only the Video playback its pause stopped, including a start it cancelled; a player paused by the person, or paused or stopped by its app during the pause, stays paused (#406).", "Camera capture and recording use explicit per-call boolean audio/library options, defaulting to silent app-local output. Permission availability never opts into recording audio or exporting captures; async microphone checks retain capture intent and background provenance. Unsupported library export fails before capture.", - "Windows physical pointer messages carry scoped input provenance to host review sheets. Injected, posted, remote and nested synthetic input remains untrusted; approval eligibility changes across host services, not only camera consent." + "Windows user32 hardware-class pointer messages carry scoped input provenance to host review sheets. Posted and ordinary injected messages remain untrusted; UIAccess injection can be classified as hardware, and remote-desktop behavior is unverified. Approval eligibility changes across host services, not only camera consent." ], "stacked": [ { @@ -189,7 +189,7 @@ }, { "patch": "tools/runtime-patches/makepad-windows-trusted-input.patch", - "sha256": "a5786f0121051d074813b7ad908da5388dcad3c46f804d4def57c1e1ef057ed3" + "sha256": "a72d973f7d7a2d629a87bb83df85e5b91b2acd6d8a322796a2637d5b8006a102" } ] } diff --git a/tools/runtime-patches/makepad-windows-trusted-input.patch b/tools/runtime-patches/makepad-windows-trusted-input.patch index 9fb7bde8e..0797ba82f 100644 --- a/tools/runtime-patches/makepad-windows-trusted-input.patch +++ b/tools/runtime-patches/makepad-windows-trusted-input.patch @@ -21,7 +21,7 @@ index 01a356d..3c79a78 100644 self.do_callback(Win32Event::WindowLostFocus(self.window_id)); } -+ fn hardware_mouse_message() -> bool { ++ fn os_reported_pointer_message() -> bool { + // This pinned windows-rs shim lacks Win32::UI::Input's message-origin + // bindings. Query user32 directly while processing the native message. + #[repr(C)] @@ -38,7 +38,7 @@ index 01a356d..3c79a78 100644 + + fn trusted_pointer_source(device_type: i32, origin_id: i32) -> bool { + matches!(device_type, 2 | 4 | 8 | 16) // mouse, touch, pen, touchpad -+ && origin_id == 1 // IMO_HARDWARE; injected/post messages fail closed ++ && origin_id == 1 // IMO_HARDWARE also includes UIAccess-injected input + } + pub fn send_mouse_down(&mut self, button: MouseButton, modifiers: KeyModifiers) { @@ -49,7 +49,7 @@ index 01a356d..3c79a78 100644 time: self.time_now(), handled: Cell::new(Area::Empty), - })); -+ }, Self::hardware_mouse_message())); ++ }, Self::os_reported_pointer_message())); } pub fn send_mouse_up(&mut self, button: MouseButton, modifiers: KeyModifiers) { @@ -58,7 +58,7 @@ index 01a356d..3c79a78 100644 abs: self.last_mouse_pos, time: self.time_now(), - })); -+ }, Self::hardware_mouse_message())); ++ }, Self::os_reported_pointer_message())); } pub fn send_mouse_move(&mut self, pos: Vec2d, modifiers: KeyModifiers) { @@ -72,7 +72,7 @@ index 01a356d..3c79a78 100644 + use super::Win32Window; + + #[test] -+ fn only_physical_pointer_messages_are_trusted() { ++ fn only_os_hardware_class_pointer_messages_are_trusted() { + for device in [2, 4, 8, 16] { + assert!(Win32Window::trusted_pointer_source(device, 1)); + assert!(!Win32Window::trusted_pointer_source(device, 2));