diff --git a/apps/ai-providers/host-service/Cargo.toml b/apps/ai-providers/host-service/Cargo.toml index 19df95f90..0f4194f65 100644 --- a/apps/ai-providers/host-service/Cargo.toml +++ b/apps/ai-providers/host-service/Cargo.toml @@ -28,7 +28,7 @@ ureq = { version = "2", default-features = false, features = ["tls"] } # A provider QR read out of a picked or dropped image (src/image_qr.rs): PNG # and JPEG only, decoded to grey and searched with rqrr, the decoder the # phone's scanner is tested against. -image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] } rqrr = { version = "0.10", default-features = false } # The shell's octos kernel (crates/octos-core), with the `octos-core` diff --git a/apps/ai-providers/host-service/README.md b/apps/ai-providers/host-service/README.md index 3098ee7ed..b81f79891 100644 --- a/apps/ai-providers/host-service/README.md +++ b/apps/ai-providers/host-service/README.md @@ -247,6 +247,20 @@ charges a per-app daily budget (defaults: 6 calls a minute, 100 calls and never sees the provider, the model id or the key. The method table and the error codes are in [src/complete/mod.rs](src/complete/mod.rs). +On Windows, `model.complete@2` also accepts one optional +`image: {app_path}` pointing into the authenticated caller's own storage. +The host reads at most 10 MiB of JPEG, PNG or WebP, then sends it through a +configured OpenAI-compatible chat route in the person's order. A configured +route may still reject images; discovery reports host support, not provider +readiness. Other platforms continue to advertise `model.complete@1` without +image input. Apps can declare version 2 as optional and inspect +`runtime.describe({method:"model.complete"})`; they should offer a manual +fallback when it is absent. The image path is separate from the 32 KiB JSON +`input` budget because the host reads the file only after caller admission. +`files.import` can stage up to 64 MiB on desktop, while `files.pick_photo` +stops at 1 MiB; callers must check the 10 MiB image limit before requesting +the model. Provider credentials and model names remain host-owned. + The shell registers it through [`crates/ai-host`](../../../crates/ai-host). Capability declarations describe usage: an app can call `model.complete` without listing `model`. The host must supply `complete::Options::grants` diff --git a/apps/ai-providers/host-service/README.zh-CN.md b/apps/ai-providers/host-service/README.zh-CN.md index cb3330fc3..f2dfaa3f0 100644 --- a/apps/ai-providers/host-service/README.zh-CN.md +++ b/apps/ai-providers/host-service/README.zh-CN.md @@ -59,6 +59,8 @@ Android 内核从 profile 的 `env_vars` 读取密钥,因此不能直接使用 `model.complete` 接受任务、输入、JSON Schema,以及可选的 `fast`/`strong` 模型类别和 URL 输出选项。宿主选择供应商,校验返回 JSON,不合格时重试一次。应用不指定供应商或密钥,不建立工具调用、浏览、记忆或持久对话。 +Windows 宿主的 `model.complete@2` 还可接收一个 `image: {app_path}`,路径必须位于已准入应用自己的存储目录内。宿主最多读取 10 MiB 的 JPEG、PNG 或 WebP,并按用户配置顺序调用 OpenAI 兼容的聊天接口。接口可能拒绝图片;能力发现只表示宿主支持,不表示提供方已经可用。其他平台仍公布不带图片输入的 `model.complete@1`。应用可将版本 2 声明为可选,通过 `runtime.describe({method:"model.complete"})` 检查,不支持时提供手动流程。图片由宿主在准入后读取,不计入 JSON `input` 的 32 KiB 上限。桌面 `files.import` 最多可导入 64 MiB,`files.pick_photo` 上限为 1 MiB;调用模型前还须检查图片不超过 10 MiB。密钥与模型名称仍由宿主管理。 + 主要限制包括:任务 4 KiB、应用输入 32 KiB、结构化输出 16 KiB;默认每个应用每分钟 6 次、每天 100 次和 100,000 tokens。账本保存在 `/model/ledger.json`。`model.budget` 返回调用应用的预算。工具箱的模型调用也共用这条预算路径。 `capabilities` 是使用说明,省略 `model` 不阻止调用。授权仍由宿主完成: diff --git a/apps/ai-providers/host-service/src/complete/media/mod.rs b/apps/ai-providers/host-service/src/complete/media/mod.rs index 262a1dceb..5285b3ab8 100644 --- a/apps/ai-providers/host-service/src/complete/media/mod.rs +++ b/apps/ai-providers/host-service/src/complete/media/mod.rs @@ -438,10 +438,13 @@ fn run_call(context: &Context) -> Result { context.host.attach(&context.call.host_dir); match context.call.method() { "complete" => { - let request = super::Request::from_args(&context.call.args)?; + let mut args = context.call.args.clone(); + let image = args.as_object_mut().and_then(|o| o.remove("image")); + let request = super::Request::from_args(&args)?; + let image = image.as_ref().map(|value| super::vision::load(&context.call.app_id, &context.call.host_dir, value)).transpose()?; context .host - .complete_while(&context.call.app_id, request, || context.check().is_ok()) + .complete_while(&context.call.app_id, request, image.as_deref(), || context.check().is_ok()) .map(|completion| completion.to_reply()) } "budget" => { @@ -755,12 +758,27 @@ fn public_url(raw: &str) -> Result { } pub(super) fn catalog() -> Vec { - wire::schemas() + let mut methods: Vec<_> = wire::schemas() .into_iter() .map(|(name, summary, args, result)| { HostApiMethod::new(name, 1, "model", summary, args, result) .with_platforms(&["macos", "android", "windows", "linux", "ios", "openharmony"]) .with_agent_access(AgentAccess::Allowed) }) - .collect() + .collect(); + let mut properties = json!({ + "task":{"type":"string"}, "input":{}, "schema":{"type":"object"}, + "class":{"enum":["fast","strong"]}, "allow_urls":{"type":"boolean"} + }); + if cfg!(target_os = "windows") { + properties["image"] = json!({"type":"object","required":["app_path"], + "properties":{"app_path":{"type":"string"}},"additionalProperties":false}); + } + methods.push(HostApiMethod::new("model.complete", if cfg!(target_os = "windows") { 2 } else { 1 }, "model", + "Make one schema-checked model call; version 2 accepts one bounded app-local image on Windows", + json!({"type":"object","required":["task","schema"],"properties":properties,"additionalProperties":false}), + json!({"type":"object"})) + .with_platforms(&["macos", "android", "windows", "linux", "ios", "openharmony"]) + .with_agent_access(AgentAccess::Allowed)); + methods } diff --git a/apps/ai-providers/host-service/src/complete/mod.rs b/apps/ai-providers/host-service/src/complete/mod.rs index 992209f77..5e9203801 100644 --- a/apps/ai-providers/host-service/src/complete/mod.rs +++ b/apps/ai-providers/host-service/src/complete/mod.rs @@ -8,7 +8,8 @@ //! //! | method | args | answer | //! |---|---|---| -//! | `model.complete` | `{task, input, schema, class?, allow_urls?}` | `{output, meta: {class, requested, attempts, usage: {input_tokens, output_tokens, estimated}, budget}}` | +//! | `model.complete@2` on Windows | `{task, input, schema, class?, allow_urls?, image?: {app_path}}` | `{output, meta: {class, requested, attempts, usage: {input_tokens, output_tokens, estimated}, budget}}` | +//! | `model.complete@1` elsewhere | `{task, input, schema, class?, allow_urls?}` | the same reply, without image input | //! | `model.budget` | – | the caller's `budget` alone | //! //! - `task` (1 to [`TASK_MAX`] bytes): what to do, in words. @@ -60,7 +61,7 @@ use crate::model::{effective_model, ProfileStore}; use crate::probe; use crate::vault::Vault; use octosense_appstore::services::{HostService, Replier, ServiceCall, ServiceHost}; -use octosense_llm_config::{catalog, registry, Provider}; +use octosense_llm_config::{catalog, registry, ApiType, Provider}; use serde_json::{json, Value}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; @@ -70,6 +71,7 @@ pub mod ledger; pub mod media; pub mod schema; pub mod wire; +mod vision; use ledger::{Budget, Ledger, Limits, Refused}; use schema::Schema; @@ -124,6 +126,8 @@ pub enum Code { Capability, /// The person has no usable AI provider. NoProvider, + /// This host does not implement the requested input mode. + Unavailable, /// Too many calls in the last minute. Rate, /// Today's calls or tokens are used up. @@ -143,6 +147,7 @@ impl Code { match self { Code::Capability => "capability", Code::NoProvider => "no_provider", + Code::Unavailable => "unavailable", Code::Rate => "rate", Code::Budget => "budget", Code::BadRequest => "bad_request", @@ -507,10 +512,10 @@ impl ModelHost { /// One call for `app`, blocking (run it off the UI thread). The caller /// has checked that `app` may call at all. pub fn complete(&self, app: &str, request: Request) -> Result { - self.complete_while(app, request, || true) + self.complete_while(app, request, None, || true) } - fn complete_while(&self, app: &str, request: Request, active: impl Fn() -> bool) -> Result { + fn complete_while(&self, app: &str, request: Request, image: Option<&str>, active: impl Fn() -> bool) -> Result { let check = || if active() { Ok(()) } else { Err(Refusal::new(Code::Capability, "The request ended or this app no longer has model access.")) }; check()?; let schema = request.check()?; @@ -521,7 +526,13 @@ impl ModelHost { if candidates.is_empty() { return Err(Refusal::new(Code::NoProvider, "No AI provider is set up. Add one in AI providers.")); } - let ordered = order(candidates, request.class); + let mut ordered = order(candidates, request.class); + if image.is_some() { + ordered.retain(|candidate| crate::model::effective_api_type(&candidate.provider) == ApiType::OpenAi); + if ordered.is_empty() { + return Err(Refusal::new(Code::NoProvider, "No configured OpenAI-compatible AI route is available for image input.")); + } + } let now = (self.clock)(); let admitted = self.ledger.lock().unwrap().admit(app, now, request.estimate()); if let Err(refused) = admitted { @@ -554,7 +565,7 @@ impl ModelHost { "{user}\n\nYour previous answer was refused: {why}. Answer again with only one JSON value that validates against the schema." ), }; - let text = match self.send(candidate, &system, &user) { + let text = match self.send(candidate, &system, &user, image) { Ok((text, used)) => { usage.add(used); self.ledger.lock().unwrap().charge(app, (self.clock)(), used.total()); @@ -592,9 +603,9 @@ impl ModelHost { /// One request to one provider: the reply text and its usage, or why the /// provider failed (with the key removed). - fn send(&self, c: &Candidate, system: &str, user: &str) -> Result<(String, Usage), String> { + fn send(&self, c: &Candidate, system: &str, user: &str, image: Option<&str>) -> Result<(String, Usage), String> { let key = c.key.as_deref(); - let prepared = wire::prepare(&c.provider, key, system, user)?; + let prepared = wire::prepare_with_image(&c.provider, key, system, user, image)?; let (status, body) = self.transport.post(&prepared.url, &prepared.headers, &prepared.body).map_err(|e| { format!("{}: {}", probe::transport_reason(&e), probe::describe_error(&e, key)) })?; diff --git a/apps/ai-providers/host-service/src/complete/vision.rs b/apps/ai-providers/host-service/src/complete/vision.rs new file mode 100644 index 000000000..f41f105ef --- /dev/null +++ b/apps/ai-providers/host-service/src/complete/vision.rs @@ -0,0 +1,59 @@ +//! One bounded image from the authenticated calling app's own storage. +use super::{Code, Refusal}; +use base64::{engine::general_purpose::STANDARD, Engine as _}; +use image::{ImageFormat, ImageReader}; +use serde_json::Value; +use std::{fs::File, io::{Cursor, Read}, path::Path}; + +const MAX_BYTES: u64 = 10 * 1024 * 1024; + +fn invalid() -> Refusal { + Refusal::new(Code::BadRequest, "The app image is missing, invalid, or over 10 MiB.") +} + +pub(super) fn load(app: &str, host_dir: &Path, value: &Value) -> Result { + if !cfg!(target_os = "windows") { + return Err(Refusal::new(Code::Unavailable, "Image input is unavailable on this host.")); + } + if app.is_empty() || app.len() > 128 || !app.bytes().next().is_some_and(|b| b.is_ascii_alphanumeric()) + || !app.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) + || app.contains("..") + { + return Err(invalid()); + } + let object = value.as_object().ok_or_else(invalid)?; + if object.len() != 1 { + return Err(invalid()); + } + let path = object.get("app_path").and_then(Value::as_str).ok_or_else(invalid)?; + let path = path.strip_prefix('/').unwrap_or(path); + if path.is_empty() || path.contains('\\') || path.contains(':') + || path.split('/').any(|part| part.is_empty() || part == "." || part == "..") + { + return Err(invalid()); + } + let root = host_dir.parent().ok_or_else(invalid)?.join(app).canonicalize().map_err(|_| invalid())?; + // This follows the existing canonical-path jail boundary. There is no + // handle-relative open here, so a concurrent local filesystem mutation + // between canonicalization and open is not excluded by this check. + let file = root.join(path).canonicalize().map_err(|_| invalid())?; + if !file.starts_with(&root) || !file.is_file() { + return Err(invalid()); + } + let opened = File::open(file).map_err(|_| invalid())?; + if opened.metadata().map_err(|_| invalid())?.len() > MAX_BYTES { return Err(invalid()); } + let mut bytes = Vec::new(); + opened.take(MAX_BYTES + 1).read_to_end(&mut bytes).map_err(|_| invalid())?; + if bytes.is_empty() || bytes.len() as u64 > MAX_BYTES { + return Err(invalid()); + } + let format = image::guess_format(&bytes).map_err(|_| invalid())?; + let mime = match format { + ImageFormat::Jpeg => "image/jpeg", + ImageFormat::Png => "image/png", + ImageFormat::WebP => "image/webp", + _ => return Err(invalid()), + }; + ImageReader::with_format(Cursor::new(&bytes), format).into_dimensions().map_err(|_| invalid())?; + Ok(format!("data:{mime};base64,{}", STANDARD.encode(bytes))) +} diff --git a/apps/ai-providers/host-service/src/complete/wire.rs b/apps/ai-providers/host-service/src/complete/wire.rs index e20a778d4..2c9b178d7 100644 --- a/apps/ai-providers/host-service/src/complete/wire.rs +++ b/apps/ai-providers/host-service/src/complete/wire.rs @@ -24,6 +24,10 @@ pub struct Prepared { /// The request for `provider`: `system` and one `user` message. pub fn prepare(provider: &Provider, key: Option<&str>, system: &str, user: &str) -> Result { + prepare_with_image(provider, key, system, user, None) +} + +pub fn prepare_with_image(provider: &Provider, key: Option<&str>, system: &str, user: &str, image: Option<&str>) -> Result { let family = registry::lookup(&provider.family).map(|f| f.id).unwrap_or(provider.family.as_str()); if family == "vertex" { return Err("Vertex AI is not supported for direct calls".into()); @@ -36,6 +40,9 @@ pub fn prepare(provider: &Provider, key: Option<&str>, system: &str, user: &str) let model = effective_model(provider).ok_or("no model")?; let mut headers = vec![("Content-Type".to_string(), "application/json".to_string())]; let api = effective_api_type(provider); + if image.is_some() && api != ApiType::OpenAi { + return Err("the selected image model requires the OpenAI chat protocol".into()); + } let (url, body) = match api { ApiType::Anthropic => { if let Some(key) = key { @@ -55,10 +62,14 @@ pub fn prepare(provider: &Provider, key: Option<&str>, system: &str, user: &str) } else { format!("{base}/chat/completions") }; + let content = match image { + Some(data) => json!([{"type":"text","text":user},{"type":"image_url","image_url":{"url":data}}]), + None => json!(user), + }; ( url, json!({"model": model, "stream": false, "messages": [ - {"role": "system", "content": system}, {"role": "user", "content": user} + {"role": "system", "content": system}, {"role": "user", "content": content} ]}), ) } diff --git a/apps/ai-providers/host-service/tests/complete.rs b/apps/ai-providers/host-service/tests/complete.rs index 938588cba..a49e6b0b9 100644 --- a/apps/ai-providers/host-service/tests/complete.rs +++ b/apps/ai-providers/host-service/tests/complete.rs @@ -17,6 +17,7 @@ static SERIAL: Mutex<()> = Mutex::new(()); static NEXT: AtomicUsize = AtomicUsize::new(41_000); const APP: &str = "com.example.notes"; +const OTHER_APP: &str = "com.example.photos"; const KEY: &str = "sk-test-deepseek-0000aaaa1234"; struct NoSheets; @@ -102,7 +103,7 @@ impl Rig { let options = complete::Options::default() .providers(Arc::new(Fixed(providers))) .transport(fake.clone()) - .grants(|app: &str, _: &Path| app == APP) + .grants(|app: &str, _: &Path| app == APP || app == OTHER_APP) .clock(move || clock.load(Ordering::SeqCst)); let host = complete::register_with(extra(options)); Rig { dir, fake, now, host, _serial: serial } @@ -463,3 +464,93 @@ fn a_host_caller_may_send_a_larger_input_than_an_app() { let huge = Request { input: json!("x".repeat(complete::HOST_INPUT_MAX + 1)), ..request(Some("s")) }; assert_eq!(rig.host.complete(APP, huge).unwrap_err().code, Code::BadRequest); } + +#[test] +fn image_contract_is_versioned_and_has_no_app_specific_probe() { + let rig = Rig::new("image-contract", vec![deepseek("configured-model")]); + let versions = octosense_appstore::host_api::available_versions(); + assert_eq!(versions.get("model.complete"), Some(&if cfg!(target_os = "windows") { 2 } else { 1 })); + assert!(!versions.contains_key("model.vision_capability")); + assert!(rig.call("unadmitted.app", "complete", args()).unwrap_err().starts_with("capability:")); + assert!(rig.fake.seen().is_empty()); +} + +#[cfg(target_os = "windows")] +#[test] +fn image_uses_the_authenticated_apps_jail_and_configured_openai_route() { + for (family, model, tag) in [ + ("minimax", "configurable-vision-model", "image-minimax"), + ("minimax-cn", "another-vision-model", "image-minimax-cn"), + ("moonshot", "selected-by-person", "image-moonshot"), + ] { + let rig = Rig::new(tag, vec![Candidate { provider: Provider::new(family, Some(model.into())), key: Some("vision-test-key".into()) }]); + let dir = rig.dir.join(OTHER_APP); + std::fs::create_dir_all(&dir).unwrap(); + image::DynamicImage::new_rgba8(1, 1) + .save_with_format(dir.join("photo.png"), image::ImageFormat::Png).unwrap(); + rig.fake.says(GOOD); + let mut request = args(); + request["image"] = json!({"app_path":"/photo.png"}); + assert_eq!(rig.call(OTHER_APP, "complete", request).unwrap()["output"]["title"], "Errands"); + let seen = rig.fake.seen(); + assert_eq!(seen.len(), 1); + let expected_host = match family { + "minimax" => "api.minimax.io", + "minimax-cn" => "api.minimaxi.com", + _ => "api.moonshot.ai", + }; + assert!(seen[0].0.starts_with(&format!("https://{expected_host}/v1/chat/completions"))); + let body = &seen[0].2; + assert_eq!(body["model"], model); + assert_eq!(body["messages"][1]["content"][0]["type"], "text"); + assert_eq!(body["messages"][1]["content"][1]["type"], "image_url"); + assert!(body["messages"][1]["content"][1]["image_url"]["url"] + .as_str().unwrap().starts_with("data:image/png;base64,")); + assert!(!body.to_string().contains("photo.png")); + } +} + +#[cfg(target_os = "windows")] +#[test] +fn image_validation_precedes_any_provider_call() { + let rig = Rig::new("image-refusal", vec![anthropic("configured-text-model")]); + let dir = rig.dir.join(OTHER_APP); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("fake.png"), b"not an image").unwrap(); + std::fs::write(dir.join("large.png"), vec![0u8; 10 * 1024 * 1024 + 1]).unwrap(); + for path in ["../.host/secret", "C:/Windows/win.ini", "missing.png", "fake.png", "large.png"] { + let mut request = args(); + request["image"] = json!({"app_path":path}); + assert!(rig.call(OTHER_APP, "complete", request).unwrap_err().starts_with("bad_request:"), "{path}"); + } + let mut request = args(); + request["image"] = json!({"app_path":"fake.png"}); + assert!(rig.call(APP, "complete", request).unwrap_err().starts_with("bad_request:")); + image::DynamicImage::new_rgba8(1, 1) + .save_with_format(dir.join("valid.png"), image::ImageFormat::Png).unwrap(); + let mut request = args(); + request["image"] = json!({"app_path":"valid.png"}); + let error = rig.call(OTHER_APP, "complete", request).unwrap_err(); + assert!(error.starts_with("no_provider:")); + assert!(!error.contains("configured-text-model")); + assert!(rig.fake.seen().is_empty()); + std::fs::remove_file(dir.join("large.png")).unwrap(); +} + +#[cfg(target_os = "windows")] +#[test] +fn image_keeps_the_structured_output_validator() { + let rig = Rig::new("image-schema", vec![Candidate { + provider: Provider::new("minimax", Some("configurable-model".into())), + key: Some("vision-test-key".into()), + }]); + let dir = rig.dir.join(APP); + std::fs::create_dir_all(&dir).unwrap(); + image::DynamicImage::new_rgba8(1, 1) + .save_with_format(dir.join("valid.png"), image::ImageFormat::Png).unwrap(); + rig.fake.says(r#"{"title":5,"tags":[]}"#).says(r#"{"title":5,"tags":[]}"#); + let mut request = args(); + request["image"] = json!({"app_path":"valid.png"}); + assert!(rig.call(APP, "complete", request).unwrap_err().starts_with("invalid_output:")); + assert_eq!(rig.fake.seen().len(), 2); +} diff --git a/crates/oauth-service/README.md b/crates/oauth-service/README.md index 0d8f11318..2db152bfe 100644 --- a/crates/oauth-service/README.md +++ b/crates/oauth-service/README.md @@ -42,7 +42,7 @@ Inbox Assistant and Google Calendar have not run there. | Platform | Provider authorization | Credential storage | Gmail send approval | | --- | --- | --- | --- | | macOS | GitHub device flow; Google browser/PKCE loopback | Mail's platform Keychain adapter, separate OAuth namespace | Native pointer provenance; remote clicks refused; physical acceptance unverified | -| Windows | Same desktop flows; platform execution unverified | Windows Credential Manager; unverified on Windows | Unsupported: fails closed | +| Windows | Same desktop flows; platform execution unverified | Windows Credential Manager; unverified on Windows | Native review accepts OS hardware-class pointer messages, including UIAccess injection; real Gmail send remains unverified | | Linux | Protocol tests and host compilation passed on Linux; browser login and GUI unverified | Secret Service; unlocked service required, no plaintext fallback; native vault test refused the locked/unavailable build-host store | Unsupported: fails closed | | Android | GitHub flow present but unverified; **Google connection refused until its native adapter is implemented** | Mail's Android platform vault, separate namespace | Existing physical-touch provenance; this sample unverified | diff --git a/crates/oauth-service/README.zh-CN.md b/crates/oauth-service/README.zh-CN.md index 8980a4cef..a80bf88f3 100644 --- a/crates/oauth-service/README.zh-CN.md +++ b/crates/oauth-service/README.zh-CN.md @@ -11,7 +11,7 @@ Rust 授权协议、连接器、原生审阅、账户生命周期和示例界面 | 平台 | 提供商授权 | 凭据保存 | Gmail 发信审批 | | --- | --- | --- | --- | | macOS | GitHub 设备授权;Google 浏览器/PKCE/回环回调 | 复用 Mail 的 Keychain 适配器,独立 OAuth 命名空间 | 原生鼠标来源校验;宿主拒绝远程点击,亲手点按未验证 | -| Windows | 同样的桌面流程,平台运行未验证 | Windows Credential Manager;未在 Windows 验证 | 不支持,明确拒绝 | +| Windows | 同样的桌面流程,平台运行未验证 | Windows Credential Manager;未在 Windows 验证 | 原生审阅接受系统归类为硬件来源的指针消息,包括 UIAccess 注入;真实 Gmail 发信仍未验证 | | Linux | 已在 Linux 通过协议测试和主机编译;浏览器登录和 GUI 未验证 | 需要解锁 Secret Service,不回退到明文;构建主机的凭据库未解锁或不可用,原生测试遭到拒绝 | 不支持,明确拒绝 | | Android | GitHub 流程存在但未验证;**Google 原生适配器完成前拒绝连接** | Mail 的 Android 凭据库,独立命名空间 | 现有的亲手点按来源校验;本示例未验证 | diff --git a/crates/shell/src/platform_services/mod.rs b/crates/shell/src/platform_services/mod.rs index 07309e6b3..3b2e22417 100644 --- a/crates/shell/src/platform_services/mod.rs +++ b/crates/shell/src/platform_services/mod.rs @@ -116,11 +116,12 @@ fn permission(family: &str) -> Option { _ => None, } } -fn permission_supported() -> bool { +fn permission_supported(family: &str) -> bool { cfg!(any(target_os = "android", target_os = "macos")) + || (cfg!(target_os = "windows") && family == "camera") } -fn prompt_supported() -> bool { - cfg!(any(target_os = "android", target_os = "macos")) +fn prompt_supported(family: &str) -> bool { + permission_supported(family) } fn status_name(status: PermissionStatus) -> &'static str { match status { @@ -131,10 +132,10 @@ fn status_name(status: PermissionStatus) -> &'static str { } } fn status(work: &Work, os: &str) -> Value { - json!({"capability": work.family, "supported": permission_supported(), "app_policy_granted": work.policy_allows(), + json!({"capability": work.family, "supported": permission_supported(work.family), "app_policy_granted": work.policy_allows(), "app_consent": work.current_grant().map(|g| g.allowed).unwrap_or(false), "os_permission": os, "location_read_supported": work.family == "location" && cfg!(target_os = "android"), - "location_sample_supported": work.family == "location" && permission_supported(), + "location_sample_supported": work.family == "location" && permission_supported("location"), "scope": "host_api_v1_device_access", "background_permission": false}) } fn queue(work: Work) { @@ -180,7 +181,7 @@ impl HostService for DeviceService { json!({"type":"object","required":["abi","defaults","options","library_export_supported","library_export_requires_foreground","microphone_consent_required"]})) .with_agent_access(AgentAccess::Allowed)] } else { vec![] }; - if !permission_supported() { + if !permission_supported(self.family) { return capture_intent; } let input = json!({"type":"object","additionalProperties":false}); @@ -197,7 +198,7 @@ impl HostService for DeviceService { input.clone(), output.clone(), ) - .with_platforms(&["android", "macos"]) + .with_platforms(if self.family == "camera" { &["android", "macos", "windows"] } else { &["android", "macos"] }) .with_agent_access(AgentAccess::Allowed), HostApiMethod::new( format!("{}.permission.request", self.family), @@ -207,7 +208,7 @@ impl HostService for DeviceService { input.clone(), output.clone(), ) - .with_platforms(&["android", "macos"]) + .with_platforms(if self.family == "camera" { &["android", "macos", "windows"] } else { &["android", "macos"] }) .with_agent_access(AgentAccess::ForegroundOnly), HostApiMethod::new( format!("{}.permission.revoke", self.family), @@ -218,7 +219,7 @@ impl HostService for DeviceService { json!({"type":"object","required":["app_consent","os_permission_unchanged","scope"],"properties":{ "app_consent":{"const":false},"os_permission_unchanged":{"const":true},"scope":{"const":"host_api_v1_device_access"}}}), ) - .with_platforms(&["android", "macos"]) + .with_platforms(if self.family == "camera" { &["android", "macos", "windows"] } else { &["android", "macos"] }) .with_agent_access(AgentAccess::Allowed), ]; methods.extend(capture_intent); @@ -311,7 +312,7 @@ impl HostService for DeviceService { return; } if self.family == "location" && call.method() == "sample.cancel" { - if !permission_supported() { + if !permission_supported(self.family) { reply.send(Err( "unsupported_platform: Location sampling is unavailable".into(), )); @@ -363,7 +364,7 @@ impl HostService for DeviceService { requesting: false, sample: sample_options.map(|options| location::Sample { options, fix: None }), }; - if !permission_supported() { + if !permission_supported(self.family) { if operation == Operation::Status { work.reply.clone().send(Ok(status(&work, "unsupported"))); } else { @@ -405,7 +406,7 @@ impl HostService for DeviceService { ); return; } - if !prompt_supported() { + if !prompt_supported(self.family) { work.fail( "unsupported_platform", "Trusted device consent is unavailable on this platform", diff --git a/crates/shell/src/platform_services/tests.rs b/crates/shell/src/platform_services/tests.rs index 2c35b29c5..7106a5392 100644 --- a/crates/shell/src/platform_services/tests.rs +++ b/crates/shell/src/platform_services/tests.rs @@ -141,7 +141,7 @@ fn glance_devices_use_host_consent_for_declared_and_legacy_apps() { let replies = services::take_replies_for(&[heap]); assert_eq!(replies.len(), 1); let error = replies[0].2.as_ref().unwrap_err(); - if permission_supported() { + if permission_supported("camera") { assert_eq!( error, "camera.permission.request is unavailable to agents/background surfaces" @@ -257,7 +257,7 @@ fn descriptors_do_not_claim_unimplemented_location_or_background_authorization() for family in ["camera", "microphone", "location"] { let service = DeviceService { family }; let methods = service.api_methods(); - if !permission_supported() { + if !permission_supported(family) { assert_eq!(methods.len(), usize::from(family == "camera")); continue; } @@ -266,7 +266,7 @@ fn descriptors_do_not_claim_unimplemented_location_or_background_authorization() .find(|m| m.name.ends_with("permission.request")) .unwrap(); assert_eq!(request.agent_access, services::AgentAccess::ForegroundOnly); - assert!(!request.platforms.contains(&"windows".into())); + assert_eq!(request.platforms.contains(&"windows".into()), family == "camera"); assert!(request.input_schema["additionalProperties"] == false); assert_eq!( methods.iter().any(|m| m.name == "location.get"), @@ -361,7 +361,7 @@ fn location_samples_validate_bounds_age_accuracy_and_platform_data() { } let methods = DeviceService { family: "location" }.api_methods(); let sample = methods.iter().find(|m| m.name == "location.sample"); - assert_eq!(sample.is_some(), permission_supported()); + assert_eq!(sample.is_some(), permission_supported("location")); if let Some(sample) = sample { assert_eq!(sample.agent_access, services::AgentAccess::ForegroundOnly); assert_eq!(sample.platforms, ["android", "macos"]); @@ -374,7 +374,7 @@ fn location_samples_validate_bounds_age_accuracy_and_platform_data() { /// events. A child process owns global registries; no OS/device is opened. #[test] fn location_sampling_broker_lifecycle() { - if !permission_supported() { + if !permission_supported("location") { return; } const CHILD: &str = "OCTOSENSE_LOCATION_TEST_CHILD"; diff --git a/docs/architecture.md b/docs/architecture.md index 23bda75e1..4e1bc665c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -362,9 +362,9 @@ flowchart TB 4. **Standing rules** on (owning app, tool), whoever calls. Runs started by incoming content or an unknown trigger skip them unless a rule opts in. 5. **A shell-drawn sheet** with the owning app, tool, exact arguments and any calling app. The system agent's approvals for one request can share one sheet. -**Mail sending** never goes through the router. Mail's own composer (`mail.review_send`) and its reply cards end in one host-owned review of the exact message, drawn by the shell inside the card (`crates/shell/src/mail_review.rs`); Mail's agent can only propose a send (`mail.propose_send`). Only a trusted press and release of the review's Approve & Send control sends, and only a physical press is trusted: a tap on Android or a click on macOS. Developer mode and standing rules cannot authorize a send, and `mail.send` answers only `approval_required`. [Composed Mail cards](mail-composable-cards.md) describes the flow. +**Mail sending** never goes through the router. Mail's own composer (`mail.review_send`) and its reply cards end in one host-owned review of the exact message, drawn by the shell inside the card (`crates/shell/src/mail_review.rs`); Mail's agent can only propose a send (`mail.propose_send`). Only a trusted press and release of the review's Approve & Send control sends: a tap on Android, a click on macOS, or an OS hardware-class pointer message on Windows. Windows still has no Mail send adapter. Developer mode and standing rules cannot authorize a send, and `mail.send` answers only `approval_required`. [Composed Mail cards](mail-composable-cards.md) describes the flow. -Two reviewed Makepad patches establish that trust (`tools/runtime-patches/`): `makepad-trusted-user-input.patch` for Android touchscreens, and `makepad-desktop-trusted-input.patch` for macOS pointer events that come from the HID source and that no other process posted. Synthetic and remote input are refused, and so is a native callback that runs inside their dispatch. Input on Windows or Linux, and accessibility input, cannot approve a send. The macOS path is **unverified**: no real message has been sent from a Mac. +Three reviewed Makepad patches establish that trust (`tools/runtime-patches/`): `makepad-trusted-user-input.patch` for Android touchscreens, `makepad-desktop-trusted-input.patch` for macOS pointer events from the HID source without another posting process, and `makepad-windows-trusted-input.patch` for Windows mouse, touch, pen or touchpad messages classified by user32 as `IMO_HARDWARE`. This Windows provenance applies to every host review using `trusted_user_input()`, including connected-account saves and sends and device consent. Windows rejects `IMO_INJECTED`, `IMO_SYSTEM` and posted messages, but Microsoft also classifies input injected by UIAccess applications as `IMO_HARDWARE`; remote-desktop input has not been characterized. Nested callbacks inside untrusted dispatch, accessibility input without this classification, and Linux input remain untrusted. The macOS Mail path is **unverified**: no real message has been sent from a Mac. Windows provenance does not supply a Mail send adapter or prove a real connected-account write. **Connected-account writes and sends** do not go through the router either. A GitHub commit (`github.review_save`) or a Google Calendar write (`gcalendar.review_save`) opens a host review of the exact change, and only its Approve & Save control saves it. A Gmail send (`gmail.draft.review`) opens the host's review of the exact reply, and only its Approve & Send control sends. On `main`, all three reviews are native (`crates/shell/src/connected_review.rs`) and accept only a trusted physical press and release, and each approval works once (`crates/oauth-service/src/host_api.rs`, `host_inbox.rs`). `desktop-v0.1.0-beta.2` checks for a physical press only on the Gmail send; its GitHub and Calendar saves use a host sheet that does not check how Approve & Save was pressed. An agent cannot open any of these screens: its tool calls reach the services with `may_prompt: false` (`host_tools/script_apps.rs`). The platform table in the [OAuth service README](../crates/oauth-service/README.md#current-delivery-boundary) shows where the physical press is supported and what is verified. @@ -437,8 +437,8 @@ The kernel's core dir, `~/.octosense/octos-home/.octos` on the desktop and ` @@ -131,7 +131,7 @@ Lab **0468** 还验证了拒绝首次助手同意:Enable assistant 按钮消 | 剩余手机 UI 矩阵 | 完整输入法组合输入、光标/选择、Unicode、主题、触摸目标尺寸、冲突、账户切换、到期与重试覆盖仍不完整。相关本地测试不能替代这些检查。 | | 通知浮层触摸路由 | 四项新增回归测试通过。在 0415 上,可见通知经注入的 Android 点击打开准确绑定的卡片,保留修订版 44。这证明导航有效,不是发送授权。 | | 亲手点按批准与真实线程回复 | 没有执行真实 SMTP 发送。用户亲手点按批准、提供商接受,以及卡片与审阅界面的回执是否一致,仍未验证。 | -| 桌面/无障碍审批 | 在该检查点暂缓;当时的输入来源边界拒绝这些路径。后来的构建也把 macOS 上用鼠标或触控板的点击视为亲手点按(**未验证**);Windows 和 Linux 上的任何输入以及无障碍输入仍不能批准。见[运行时契约与输入来源](#运行时契约与输入来源)。 | +| 桌面/无障碍审批 | 在该历史检查点暂缓。后来的构建接受 macOS(**未验证**)的物理指针输入,以及 Windows 系统归类为硬件来源的输入(包括 UIAccess 注入);Windows Mail 仍没有发信适配器。Linux 输入不能批准。见[运行时契约与输入来源](#运行时契约与输入来源)。 | ### 按提供商记录的工程审查 diff --git a/runtime-patches.lock.json b/runtime-patches.lock.json index 35933d1d4..4b4d87dfe 100644 --- a/runtime-patches.lock.json +++ b/runtime-patches.lock.json @@ -4,7 +4,7 @@ "base_revision": "d653bc4493619ab979ccff165cf48c0c63b77d19", "patch": "tools/runtime-patches/makepad-settings.patch", "sha256": "3a79cddddc89dc4d064a9f0bdd17dc8111dcab9c796a334d95f61ecb9d5011aa", - "tree": "d047e32fdfb91aa26b95a62464645020bfa6a4f7", + "tree": "d80d143b230f705c4c122fefb025d4d10598a0ba", "product_fixes": [ "Settings integration: Android IME synchronization, accessibility and reduced motion, GLES program-cache validation, obscured-gesture cancellation and scroll-position observation.", "Contained-app isolation and secret-input restrictions come from the pinned upstream runtime; the Settings overlay preserves those gates.", @@ -49,7 +49,9 @@ "An OS resume restarts only the Video playback its pause stopped, including a start it cancelled; a player paused by the person, or paused or stopped by its app during the pause, stays paused (#406).", "Camera capture and recording use explicit per-call boolean audio/library options, defaulting to silent app-local output. Permission availability never opts into recording audio or exporting captures; async microphone checks retain capture intent and background provenance. Unsupported library export fails before capture.", "A hidden Android native composer remembers animated IME insets without requesting a root layout every frame; showing it applies the latest inset before its first visible frame.", - "Launch-input hardening: the Studio connection and the OCTOS_PROXY app HTTP proxy, which Makepad itself acts on from an Android Intent extra, are honoured only when the build opts into launch-time development input (debuggable, or the manifest meta-data dev.makepad.LAUNCH_STUDIO). A shipped build ignores them and drops any Studio host a development build saved. The exported activity would otherwise let another installed app point it at a host or a proxy." + "Launch-input hardening: the Studio connection and the OCTOS_PROXY app HTTP proxy, which Makepad itself acts on from an Android Intent extra, are honoured only when the build opts into launch-time development input (debuggable, or the manifest meta-data dev.makepad.LAUNCH_STUDIO). A shipped build ignores them and drops any Studio host a development build saved. The exported activity would otherwise let another installed app point it at a host or a proxy.", + "Windows user32 hardware-class pointer messages carry scoped input provenance to host review sheets. Posted and ordinary injected messages remain untrusted; UIAccess injection can be classified as hardware, and remote-desktop behavior is unverified. Approval eligibility changes across host services, not only camera consent.", + "A Media Foundation camera source feeds the existing YUV video preview and still-capture path, with latest-frame buffering and cleanup; macOS retains its native AVFoundation preview." ], "stacked": [ { @@ -195,6 +197,14 @@ { "patch": "tools/runtime-patches/makepad-launch-input-hardening.patch", "sha256": "8eaafef3ef0591bec2a25c05d5187564d5dfb87a32e6b937b6d9c4e9f6fa5b84" + }, + { + "patch": "tools/runtime-patches/makepad-windows-trusted-input.patch", + "sha256": "a72d973f7d7a2d629a87bb83df85e5b91b2acd6d8a322796a2637d5b8006a102" + }, + { + "patch": "tools/runtime-patches/makepad-windows-camera-preview.patch", + "sha256": "4fd133379eb6d8973d146656198df33ed20320a3711986a5cf271ab8e730de3e" } ] } diff --git a/tools/runtime-patches/makepad-windows-camera-preview.patch b/tools/runtime-patches/makepad-windows-camera-preview.patch new file mode 100644 index 000000000..d69d1c178 --- /dev/null +++ b/tools/runtime-patches/makepad-windows-camera-preview.patch @@ -0,0 +1,352 @@ +diff --git a/platform/src/os/windows/mod.rs b/platform/src/os/windows/mod.rs +index 4856a92..0132132 100644 +--- a/platform/src/os/windows/mod.rs ++++ b/platform/src/os/windows/mod.rs +@@ -20,6 +20,7 @@ + pub mod windows_media_engine_notify; + pub mod windows_mf_source_reader; + pub mod windows_video_playback; ++pub mod windows_camera_player; + pub mod windows_video_player; + pub mod winrt_midi; + +diff --git a/platform/src/os/windows/windows.rs b/platform/src/os/windows/windows.rs +index fb1dd91..5f099d8 100644 +--- a/platform/src/os/windows/windows.rs ++++ b/platform/src/os/windows/windows.rs +@@ -1219,6 +1219,9 @@ fn handle_platform_ops( + let tex_y_id = tex_y.texture_id(); + let tex_u_id = tex_u.texture_id(); + let tex_v_id = tex_v.texture_id(); ++ let camera_access = if matches!(&source, VideoSource::Camera(..)) { ++ Some(self.os.media.media_foundation()) ++ } else { None }; + let player = WindowsUnifiedVideoPlayer::new( + &device, + video_id, +@@ -1229,6 +1232,7 @@ fn handle_platform_ops( + source, + autoplay, + should_loop, ++ camera_access, + ); + self.os.video_players.insert(video_id, player); + // Notify widget so it can bind textures to shader slots +diff --git a/platform/src/os/windows/windows_camera_player.rs b/platform/src/os/windows/windows_camera_player.rs +new file mode 100644 +index 0000000..060c7bb +--- /dev/null ++++ b/platform/src/os/windows/windows_camera_player.rs +@@ -0,0 +1,84 @@ ++//! Media Foundation camera frames as a bounded latest-frame video source. ++use { ++ super::media_foundation::MediaFoundationAccess, ++ crate::{ ++ media_plugin::PlaybackPrepared, ++ video::{CameraFrameInputFn, CameraFrameOwned, CameraFramePool, CameraFrameRef, VideoFormatId, VideoInputId}, ++ }, ++ std::{sync::{Arc, Mutex}, time::{Duration, Instant}}, ++}; ++ ++pub struct WindowsCameraPlayer { ++ frame_pool: Arc>, ++ access: Option>>, ++ started: Instant, ++ prepared: bool, ++ playing: bool, ++} ++ ++impl WindowsCameraPlayer { ++ pub fn new(input_id: VideoInputId, format_id: VideoFormatId, ++ access: Arc>, autoplay: bool) -> Self { ++ let frame_pool = Arc::new(Mutex::new(CameraFramePool::new(4))); ++ let pending = frame_pool.clone(); ++ let callback: CameraFrameInputFn = Box::new(move |source: CameraFrameRef<'_>| { ++ let mut pool = pending.lock().unwrap(); ++ let mut frame = pool.checkout(); ++ if frame.convert_to_i420(source) { ++ pool.publish_latest(frame); ++ } else { ++ pool.recycle(frame); ++ } ++ }); ++ { ++ let mut camera = access.lock().unwrap(); ++ *camera.camera_frame_input_cb[0].lock().unwrap() = Some(callback); ++ *camera.video_input_cb[0].lock().unwrap() = None; ++ camera.use_video_input(&[(input_id, format_id)]); ++ } ++ Self { frame_pool, access: Some(access), started: Instant::now(), ++ prepared: false, playing: autoplay } ++ } ++ ++ pub fn check_prepared(&mut self) -> Option> { ++ if self.prepared { return None; } ++ let mut pool = self.frame_pool.lock().unwrap(); ++ if let Some(frame) = pool.take_latest() { ++ let (width, height) = (frame.width as u32, frame.height as u32); ++ pool.publish_latest(frame); ++ self.prepared = true; ++ return Some(Ok(PlaybackPrepared::new(width, height, 0, false, ++ vec!["camera".to_string()], vec![]))); ++ } ++ if self.started.elapsed() > Duration::from_secs(6) { ++ self.prepared = true; ++ return Some(Err("Camera did not deliver a video frame".to_string())); ++ } ++ None ++ } ++ ++ pub fn take_frame(&self) -> Option { ++ if self.playing { self.frame_pool.lock().unwrap().take_latest() } else { None } ++ } ++ ++ pub fn recycle_frame(&self, frame: CameraFrameOwned) { ++ self.frame_pool.lock().unwrap().recycle(frame); ++ } ++ ++ pub fn is_playing(&self) -> bool { self.playing } ++ pub fn keep_polling(&self) -> bool { !self.prepared || self.playing } ++ pub fn play(&mut self) { self.playing = true; } ++ pub fn pause(&mut self) { self.playing = false; } ++ pub fn cleanup(&mut self) { ++ if let Some(access) = self.access.take() { ++ let mut camera = access.lock().unwrap(); ++ camera.use_video_input(&[]); ++ *camera.camera_frame_input_cb[0].lock().unwrap() = None; ++ } ++ self.playing = false; ++ } ++} ++ ++impl Drop for WindowsCameraPlayer { ++ fn drop(&mut self) { self.cleanup(); } ++} +diff --git a/platform/src/os/windows/windows_video_player.rs b/platform/src/os/windows/windows_video_player.rs +index 9a7ac30..7604ccb 100644 +--- a/platform/src/os/windows/windows_video_player.rs ++++ b/platform/src/os/windows/windows_video_player.rs +@@ -1,5 +1,7 @@ + use { + super::{ ++ media_foundation::MediaFoundationAccess, ++ windows_camera_player::WindowsCameraPlayer, + windows_mf_source_reader::WindowsMfSourceReaderPlayer, + windows_video_playback::WindowsVideoPlayer, + }, +@@ -12,7 +14,8 @@ + CxTexturePool, TextureAlloc, TextureCategory, TextureFormat, TextureId, TexturePixel, + }, + video_decode::software_video::PlaybackSessionHandle, +- video_decode::yuv::YuvPlaneData, ++ video_decode::yuv::{YuvColorMatrix, YuvLayout, YuvPlaneData}, ++ video::CameraColorMatrix, + windows::{ + core::Interface, + Win32::Graphics::{ +@@ -25,6 +28,7 @@ + }, + }, + }, ++ std::sync::{Arc, Mutex}, + }; + + struct D3d11R8GpuPlane { +@@ -250,6 +254,7 @@ pub struct WindowsUnifiedVideoPlayer { + } + + enum WindowsPlayerMode { ++ Camera(WindowsCameraPlayer), + /// Progressive files: Source Reader + DXGI NV12 (true zero-copy when possible). + SourceReader(WindowsMfSourceReaderPlayer), + /// Adaptive / fallback: MediaEngine Transfer present. +@@ -268,9 +273,13 @@ pub fn new( + source: VideoSource, + autoplay: bool, + is_looping: bool, ++ camera_access: Option>>, + ) -> Self { + let force_software = std::env::var_os("MAKEPAD_FORCE_SOFTWARE_VIDEO").is_some(); +- let mode = if force_software { ++ let mode = if let VideoSource::Camera(input_id, format_id) = &source { ++ WindowsPlayerMode::Camera(WindowsCameraPlayer::new(*input_id, *format_id, ++ camera_access.expect("camera access required for camera source"), autoplay)) ++ } else if force_software { + crate::log!("VIDEO: MAKEPAD_FORCE_SOFTWARE_VIDEO set, using software video decoder"); + WindowsPlayerMode::Software(PlaybackSessionHandle::new( + video_id, +@@ -396,6 +405,7 @@ fn switch_to_software(&mut self, reason: &str) { + /// Pump the active native worker once per UI paint (coalesces Tick). + pub fn sync_worker(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.sync_worker(), + WindowsPlayerMode::Native(player) => player.sync_worker(), + WindowsPlayerMode::Software(_) => {} +@@ -414,11 +424,13 @@ pub fn sync_worker(&mut self) { + + pub fn check_prepared(&mut self) -> Option> { + let out = match &mut self.mode { ++ WindowsPlayerMode::Camera(player) => player.check_prepared(), + WindowsPlayerMode::SourceReader(player) => match player.check_prepared() { + Some(Err(err)) => { + self.switch_to_media_engine(&err); + self.prepare_resolved = false; + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => Some(Err(err.clone())), + WindowsPlayerMode::Native(native) => native.check_prepared(), + WindowsPlayerMode::Software(software) => software.check_prepared(), + WindowsPlayerMode::SourceReader(_) => Some(Err(err)), +@@ -455,7 +467,39 @@ pub fn check_prepared(&mut self) -> Option> { + } + + pub fn poll_frame(&mut self, textures: &mut CxTexturePool) -> bool { ++ if let WindowsPlayerMode::Camera(player) = &mut self.mode { ++ let Some(mut frame) = player.take_frame() else { return false; }; ++ if frame.width == 0 || frame.height == 0 || frame.plane_count < 3 { ++ player.recycle_frame(frame); ++ return false; ++ } ++ let matrix = match frame.matrix { ++ CameraColorMatrix::BT601 => YuvColorMatrix::BT601, ++ CameraColorMatrix::BT2020 => YuvColorMatrix::BT2020, ++ _ => YuvColorMatrix::BT709, ++ }; ++ let planes = YuvPlaneData { ++ y: std::mem::take(&mut frame.planes[0].bytes), ++ u: std::mem::take(&mut frame.planes[1].bytes), ++ v: std::mem::take(&mut frame.planes[2].bytes), ++ width: frame.width as u32, height: frame.height as u32, ++ layout: YuvLayout::I420, matrix, ++ y_stride: frame.planes[0].row_stride as u32, ++ u_stride: frame.planes[1].row_stride as u32, ++ v_stride: frame.planes[2].row_stride as u32, ++ }; ++ self.yuv_matrix = matrix.as_f32(); ++ self.yuv_biplanar = false; ++ self.yuv_array = false; ++ self.upload_yuv_to_d3d11(textures, &planes); ++ frame.planes[0].bytes = planes.y; ++ frame.planes[1].bytes = planes.u; ++ frame.planes[2].bytes = planes.v; ++ if let WindowsPlayerMode::Camera(player) = &self.mode { player.recycle_frame(frame); } ++ return true; ++ } + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => unreachable!(), + WindowsPlayerMode::SourceReader(player) => { + let ok = player.poll_frame(textures); + self.yuv_biplanar = player.presents_nv12(); +@@ -568,7 +612,7 @@ pub fn is_software_mode(&self) -> bool { + + /// YUV shader path: software decode, SourceReader DXGI NV12, or MF Transfer NV12. + pub fn uses_yuv(&self) -> bool { +- self.is_software_mode() ++ matches!(self.mode, WindowsPlayerMode::Camera(_)) || self.is_software_mode() + || matches!(self.mode, WindowsPlayerMode::SourceReader(_)) + || self.yuv_biplanar + } +@@ -591,6 +635,7 @@ pub fn yuv_full_range(&self) -> bool { + + pub fn check_eos(&mut self) -> bool { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => false, + WindowsPlayerMode::SourceReader(player) => player.check_eos(), + WindowsPlayerMode::Native(player) => player.check_eos(), + WindowsPlayerMode::Software(player) => player.check_eos(), +@@ -599,6 +644,7 @@ pub fn check_eos(&mut self) -> bool { + + pub fn is_playing(&self) -> bool { + match &self.mode { ++ WindowsPlayerMode::Camera(player) => player.is_playing(), + WindowsPlayerMode::SourceReader(player) => player.is_playing(), + WindowsPlayerMode::Native(player) => player.is_playing(), + WindowsPlayerMode::Software(player) => player.is_playing(), +@@ -608,6 +654,7 @@ pub fn is_playing(&self) -> bool { + /// Keep Poll while Media Foundation is still buffering, or while playing. + pub fn keep_polling(&self) -> bool { + match &self.mode { ++ WindowsPlayerMode::Camera(player) => player.keep_polling(), + WindowsPlayerMode::SourceReader(player) => player.keep_polling(), + WindowsPlayerMode::Native(player) => player.keep_polling(), + WindowsPlayerMode::Software(player) => !self.prepare_resolved || player.is_playing(), +@@ -616,6 +663,7 @@ pub fn keep_polling(&self) -> bool { + + pub fn play(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(player) => player.play(), + WindowsPlayerMode::SourceReader(player) => player.play(), + WindowsPlayerMode::Native(player) => player.play(), + WindowsPlayerMode::Software(player) => player.play(), +@@ -624,6 +672,7 @@ pub fn play(&mut self) { + + pub fn pause(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(player) => player.pause(), + WindowsPlayerMode::SourceReader(player) => player.pause(), + WindowsPlayerMode::Native(player) => player.pause(), + WindowsPlayerMode::Software(player) => player.pause(), +@@ -632,6 +681,7 @@ pub fn pause(&mut self) { + + pub fn resume(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(player) => player.play(), + WindowsPlayerMode::SourceReader(player) => player.resume(), + WindowsPlayerMode::Native(player) => player.resume(), + WindowsPlayerMode::Software(player) => player.resume(), +@@ -640,6 +690,7 @@ pub fn resume(&mut self) { + + pub fn mute(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.mute(), + WindowsPlayerMode::Native(player) => player.mute(), + WindowsPlayerMode::Software(player) => player.mute(), +@@ -648,6 +699,7 @@ pub fn mute(&mut self) { + + pub fn unmute(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.unmute(), + WindowsPlayerMode::Native(player) => player.unmute(), + WindowsPlayerMode::Software(player) => player.unmute(), +@@ -656,6 +708,7 @@ pub fn unmute(&mut self) { + + pub fn seek_to(&mut self, position_ms: u64) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.seek_to(position_ms), + WindowsPlayerMode::Native(player) => player.seek_to(position_ms), + WindowsPlayerMode::Software(player) => player.seek_to(position_ms), +@@ -664,6 +717,7 @@ pub fn seek_to(&mut self, position_ms: u64) { + + pub fn set_volume(&mut self, volume: f64) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.set_volume(volume), + WindowsPlayerMode::Native(player) => player.set_volume(volume), + WindowsPlayerMode::Software(player) => player.set_volume(volume), +@@ -672,6 +726,7 @@ pub fn set_volume(&mut self, volume: f64) { + + pub fn set_playback_rate(&mut self, rate: f64) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(_) => {}, + WindowsPlayerMode::SourceReader(player) => player.set_playback_rate(rate), + WindowsPlayerMode::Native(player) => player.set_playback_rate(rate), + WindowsPlayerMode::Software(player) => player.set_playback_rate(rate), +@@ -680,6 +735,7 @@ pub fn set_playback_rate(&mut self, rate: f64) { + + pub fn current_position_ms(&self) -> u128 { + match &self.mode { ++ WindowsPlayerMode::Camera(_) => 0, + WindowsPlayerMode::SourceReader(player) => player.current_position_ms(), + WindowsPlayerMode::Native(player) => player.current_position_ms(), + WindowsPlayerMode::Software(player) => player.current_position_ms(), +@@ -688,6 +744,7 @@ pub fn current_position_ms(&self) -> u128 { + + pub fn cleanup(&mut self) { + match &mut self.mode { ++ WindowsPlayerMode::Camera(player) => player.cleanup(), + WindowsPlayerMode::SourceReader(player) => player.cleanup(), + WindowsPlayerMode::Native(player) => player.cleanup(), + WindowsPlayerMode::Software(player) => player.cleanup(), diff --git a/tools/runtime-patches/makepad-windows-trusted-input.patch b/tools/runtime-patches/makepad-windows-trusted-input.patch new file mode 100644 index 000000000..0797ba82f --- /dev/null +++ b/tools/runtime-patches/makepad-windows-trusted-input.patch @@ -0,0 +1,115 @@ +diff --git a/platform/src/os/windows/win32_event.rs b/platform/src/os/windows/win32_event.rs +index 8a02d73..3299138 100644 +--- a/platform/src/os/windows/win32_event.rs ++++ b/platform/src/os/windows/win32_event.rs +@@ -30,8 +30,8 @@ pub enum Win32Event { + primary: bool, + }, + +- MouseDown(MouseDownEvent), +- MouseUp(MouseUpEvent), ++ MouseDown(MouseDownEvent, bool), ++ MouseUp(MouseUpEvent, bool), + MouseMove(MouseMoveEvent), + MouseLeave(MouseLeaveEvent), + Scroll(ScrollEvent), +diff --git a/platform/src/os/windows/win32_window.rs b/platform/src/os/windows/win32_window.rs +index 01a356d..3c79a78 100644 +--- a/platform/src/os/windows/win32_window.rs ++++ b/platform/src/os/windows/win32_window.rs +@@ -1864,6 +1864,26 @@ pub fn send_focus_lost_event(&mut self) { + self.do_callback(Win32Event::WindowLostFocus(self.window_id)); + } + ++ fn os_reported_pointer_message() -> bool { ++ // This pinned windows-rs shim lacks Win32::UI::Input's message-origin ++ // bindings. Query user32 directly while processing the native message. ++ #[repr(C)] ++ #[derive(Default)] ++ struct InputMessageSource { device_type: i32, origin_id: i32 } ++ #[link(name = "user32")] ++ unsafe extern "system" { ++ fn GetCurrentInputMessageSource(source: *mut InputMessageSource) -> i32; ++ } ++ let mut source = InputMessageSource::default(); ++ let received = unsafe { GetCurrentInputMessageSource(&mut source) != 0 }; ++ received && Self::trusted_pointer_source(source.device_type, source.origin_id) ++ } ++ ++ fn trusted_pointer_source(device_type: i32, origin_id: i32) -> bool { ++ matches!(device_type, 2 | 4 | 8 | 16) // mouse, touch, pen, touchpad ++ && origin_id == 1 // IMO_HARDWARE also includes UIAccess-injected input ++ } ++ + pub fn send_mouse_down(&mut self, button: MouseButton, modifiers: KeyModifiers) { + if self.mouse_buttons_down == 0 { + unsafe { +@@ -1878,7 +1898,7 @@ pub fn send_mouse_down(&mut self, button: MouseButton, modifiers: KeyModifiers) + abs: self.last_mouse_pos, + time: self.time_now(), + handled: Cell::new(Area::Empty), +- })); ++ }, Self::os_reported_pointer_message())); + } + + pub fn send_mouse_up(&mut self, button: MouseButton, modifiers: KeyModifiers) { +@@ -1896,7 +1916,7 @@ pub fn send_mouse_up(&mut self, button: MouseButton, modifiers: KeyModifiers) { + window_id: self.window_id, + abs: self.last_mouse_pos, + time: self.time_now(), +- })); ++ }, Self::os_reported_pointer_message())); + } + + pub fn send_mouse_move(&mut self, pos: Vec2d, modifiers: KeyModifiers) { +@@ -2085,3 +2105,19 @@ pub fn virtual_key_to_key_code(wparam: WPARAM) -> KeyCode { + } + } + } ++ ++#[cfg(test)] ++mod input_source_tests { ++ use super::Win32Window; ++ ++ #[test] ++ fn only_os_hardware_class_pointer_messages_are_trusted() { ++ for device in [2, 4, 8, 16] { ++ assert!(Win32Window::trusted_pointer_source(device, 1)); ++ assert!(!Win32Window::trusted_pointer_source(device, 2)); ++ assert!(!Win32Window::trusted_pointer_source(device, 4)); ++ } ++ assert!(!Win32Window::trusted_pointer_source(0, 1)); ++ assert!(!Win32Window::trusted_pointer_source(1, 1)); ++ } ++} +diff --git a/platform/src/os/windows/windows.rs b/platform/src/os/windows/windows.rs +index fb1dd91..4d1a501 100644 +--- a/platform/src/os/windows/windows.rs ++++ b/platform/src/os/windows/windows.rs +@@ -293,11 +293,11 @@ fn win32_event_callback( + let time_now = with_win32_app(|app| app.time_now()); + self.paint_tick(time_now, true, d3d11_cx, d3d11_windows); + } +- Win32Event::MouseDown(mut e) => { ++ Win32Event::MouseDown(mut e, trusted) => { + self.dpi_override_scale(&mut e.abs, e.window_id); + self.fingers.process_tap_count(e.abs, e.time); + self.fingers.mouse_down(e.button, e.window_id); +- self.call_event_handler(&Event::MouseDown(e.into())) ++ self.call_event_handler_with_input_trust(&Event::MouseDown(e.into()), trusted) + } + Win32Event::MouseMove(mut e) => { + self.dpi_override_scale(&mut e.abs, e.window_id); +@@ -305,10 +305,10 @@ fn win32_event_callback( + self.fingers.cycle_hover_area(live_id!(mouse).into()); + self.fingers.switch_captures(); + } +- Win32Event::MouseUp(mut e) => { ++ Win32Event::MouseUp(mut e, trusted) => { + self.dpi_override_scale(&mut e.abs, e.window_id); + let button = e.button; +- self.call_event_handler(&Event::MouseUp(e.into())); ++ self.call_event_handler_with_input_trust(&Event::MouseUp(e.into()), trusted); + self.fingers.mouse_up(button); + self.fingers.cycle_hover_area(live_id!(mouse).into()); + }