Feat/windows camera vision host input

apps/ai-providers/host-service/Cargo.toml

@@ -28,7 +28,7 @@ ureq = { version = "2", default-features = false, features = ["tls"] }
 # A provider QR read out of a picked or dropped image (src/image_qr.rs): PNG
 # and JPEG only, decoded to grey and searched with rqrr, the decoder the
 # phone's scanner is tested against.
-image = { version = "0.25", default-features = false, features = ["png", "jpeg"] }
+image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] }
 rqrr = { version = "0.10", default-features = false }
 
 # The shell's octos kernel (crates/octos-core), with the `octos-core`

apps/ai-providers/host-service/README.md

@@ -247,6 +247,20 @@ charges a per-app daily budget (defaults: 6 calls a minute, 100 calls and
 never sees the provider, the model id or the key. The method table and the
 error codes are in [src/complete/mod.rs](src/complete/mod.rs).
 
+On Windows, `model.complete@2` also accepts one optional
+`image: {app_path}` pointing into the authenticated caller's own storage.
+The host reads at most 10 MiB of JPEG, PNG or WebP, then sends it through a
+configured OpenAI-compatible chat route in the person's order. A configured
+route may still reject images; discovery reports host support, not provider
+readiness. Other platforms continue to advertise `model.complete@1` without
+image input. Apps can declare version 2 as optional and inspect
+`runtime.describe({method:"model.complete"})`; they should offer a manual
+fallback when it is absent. The image path is separate from the 32 KiB JSON
+`input` budget because the host reads the file only after caller admission.
+`files.import` can stage up to 64 MiB on desktop, while `files.pick_photo`
+stops at 1 MiB; callers must check the 10 MiB image limit before requesting
+the model. Provider credentials and model names remain host-owned.
+
 The shell registers it through [`crates/ai-host`](../../../crates/ai-host).
 Capability declarations describe usage: an app can call `model.complete`
 without listing `model`. The host must supply `complete::Options::grants`

apps/ai-providers/host-service/README.zh-CN.md

@@ -59,6 +59,8 @@ Android 内核从 profile 的 `env_vars` 读取密钥，因此不能直接使用
 
 `model.complete` 接受任务、输入、JSON Schema，以及可选的 `fast`/`strong` 模型类别和 URL 输出选项。宿主选择供应商，校验返回 JSON，不合格时重试一次。应用不指定供应商或密钥，不建立工具调用、浏览、记忆或持久对话。
 
+Windows 宿主的 `model.complete@2` 还可接收一个 `image: {app_path}`，路径必须位于已准入应用自己的存储目录内。宿主最多读取 10 MiB 的 JPEG、PNG 或 WebP，并按用户配置顺序调用 OpenAI 兼容的聊天接口。接口可能拒绝图片；能力发现只表示宿主支持，不表示提供方已经可用。其他平台仍公布不带图片输入的 `model.complete@1`。应用可将版本 2 声明为可选，通过 `runtime.describe({method:"model.complete"})` 检查，不支持时提供手动流程。图片由宿主在准入后读取，不计入 JSON `input` 的 32 KiB 上限。桌面 `files.import` 最多可导入 64 MiB，`files.pick_photo` 上限为 1 MiB；调用模型前还须检查图片不超过 10 MiB。密钥与模型名称仍由宿主管理。
+
 主要限制包括：任务 4 KiB、应用输入 32 KiB、结构化输出 16 KiB；默认每个应用每分钟 6 次、每天 100 次和 100,000 tokens。账本保存在 `<host dir>/model/ledger.json`。`model.budget` 返回调用应用的预算。工具箱的模型调用也共用这条预算路径。
 
 `capabilities` 是使用说明，省略 `model` 不阻止调用。授权仍由宿主完成：

apps/ai-providers/host-service/src/complete/media/mod.rs

@@ -438,10 +438,13 @@ fn run_call(context: &Context) -> Result<Value, Refusal> {
     context.host.attach(&context.call.host_dir);
     match context.call.method() {
         "complete" => {
-            let request = super::Request::from_args(&context.call.args)?;
+            let mut args = context.call.args.clone();
+            let image = args.as_object_mut().and_then(|o| o.remove("image"));
+            let request = super::Request::from_args(&args)?;
+            let image = image.as_ref().map(|value| super::vision::load(&context.call.app_id, &context.call.host_dir, value)).transpose()?;
             context
                 .host
-                .complete_while(&context.call.app_id, request, || context.check().is_ok())
+                .complete_while(&context.call.app_id, request, image.as_deref(), || context.check().is_ok())
                 .map(|completion| completion.to_reply())
         }
         "budget" => {
@@ -755,12 +758,27 @@ fn public_url(raw: &str) -> Result<String, Refusal> {
 }
 
 pub(super) fn catalog() -> Vec<HostApiMethod> {
-    wire::schemas()
+    let mut methods: Vec<_> = wire::schemas()
         .into_iter()
         .map(|(name, summary, args, result)| {
             HostApiMethod::new(name, 1, "model", summary, args, result)
                 .with_platforms(&["macos", "android", "windows", "linux", "ios", "openharmony"])
                 .with_agent_access(AgentAccess::Allowed)
         })
-        .collect()
+        .collect();
+    let mut properties = json!({
+        "task":{"type":"string"}, "input":{}, "schema":{"type":"object"},
+        "class":{"enum":["fast","strong"]}, "allow_urls":{"type":"boolean"}
+    });
+    if cfg!(target_os = "windows") {
+        properties["image"] = json!({"type":"object","required":["app_path"],
+            "properties":{"app_path":{"type":"string"}},"additionalProperties":false});
+    }
+    methods.push(HostApiMethod::new("model.complete", if cfg!(target_os = "windows") { 2 } else { 1 }, "model",
+        "Make one schema-checked model call; version 2 accepts one bounded app-local image on Windows",
+        json!({"type":"object","required":["task","schema"],"properties":properties,"additionalProperties":false}),
+        json!({"type":"object"}))
+        .with_platforms(&["macos", "android", "windows", "linux", "ios", "openharmony"])
+        .with_agent_access(AgentAccess::Allowed));
+    methods
 }

apps/ai-providers/host-service/src/complete/mod.rs

@@ -8,7 +8,8 @@
 //!
 //! | method | args | answer |
 //! |---|---|---|
-//! | `model.complete` | `{task, input, schema, class?, allow_urls?}` | `{output, meta: {class, requested, attempts, usage: {input_tokens, output_tokens, estimated}, budget}}` |
+//! | `model.complete@2` on Windows | `{task, input, schema, class?, allow_urls?, image?: {app_path}}` | `{output, meta: {class, requested, attempts, usage: {input_tokens, output_tokens, estimated}, budget}}` |
+//! | `model.complete@1` elsewhere | `{task, input, schema, class?, allow_urls?}` | the same reply, without image input |
 //! | `model.budget` | – | the caller's `budget` alone |
 //!
 //! - `task` (1 to [`TASK_MAX`] bytes): what to do, in words.
@@ -60,7 +61,7 @@ use crate::model::{effective_model, ProfileStore};
 use crate::probe;
 use crate::vault::Vault;
 use octosense_appstore::services::{HostService, Replier, ServiceCall, ServiceHost};
-use octosense_llm_config::{catalog, registry, Provider};
+use octosense_llm_config::{catalog, registry, ApiType, Provider};
 use serde_json::{json, Value};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
@@ -70,6 +71,7 @@ pub mod ledger;
 pub mod media;
 pub mod schema;
 pub mod wire;
+mod vision;
 
 use ledger::{Budget, Ledger, Limits, Refused};
 use schema::Schema;
@@ -124,6 +126,8 @@ pub enum Code {
     Capability,
     /// The person has no usable AI provider.
     NoProvider,
+    /// This host does not implement the requested input mode.
+    Unavailable,
     /// Too many calls in the last minute.
     Rate,
     /// Today's calls or tokens are used up.
@@ -143,6 +147,7 @@ impl Code {
         match self {
             Code::Capability => "capability",
             Code::NoProvider => "no_provider",
+            Code::Unavailable => "unavailable",
             Code::Rate => "rate",
             Code::Budget => "budget",
             Code::BadRequest => "bad_request",
@@ -507,10 +512,10 @@ impl ModelHost {
     /// One call for `app`, blocking (run it off the UI thread). The caller
     /// has checked that `app` may call at all.
     pub fn complete(&self, app: &str, request: Request) -> Result<Completion, Refusal> {
-        self.complete_while(app, request, || true)
+        self.complete_while(app, request, None, || true)
     }
 
-    fn complete_while(&self, app: &str, request: Request, active: impl Fn() -> bool) -> Result<Completion, Refusal> {
+    fn complete_while(&self, app: &str, request: Request, image: Option<&str>, active: impl Fn() -> bool) -> Result<Completion, Refusal> {
         let check = || if active() { Ok(()) } else { Err(Refusal::new(Code::Capability, "The request ended or this app no longer has model access.")) };
         check()?;
         let schema = request.check()?;
@@ -521,7 +526,13 @@ impl ModelHost {
         if candidates.is_empty() {
             return Err(Refusal::new(Code::NoProvider, "No AI provider is set up. Add one in AI providers."));
         }
-        let ordered = order(candidates, request.class);
+        let mut ordered = order(candidates, request.class);
+        if image.is_some() {
+            ordered.retain(|candidate| crate::model::effective_api_type(&candidate.provider) == ApiType::OpenAi);
+            if ordered.is_empty() {
+                return Err(Refusal::new(Code::NoProvider, "No configured OpenAI-compatible AI route is available for image input."));
+            }
+        }
         let now = (self.clock)();
         let admitted = self.ledger.lock().unwrap().admit(app, now, request.estimate());
         if let Err(refused) = admitted {
@@ -554,7 +565,7 @@ impl ModelHost {
                         "{user}\n\nYour previous answer was refused: {why}. Answer again with only one JSON value that validates against the schema."
                     ),
                 };
-                let text = match self.send(candidate, &system, &user) {
+                let text = match self.send(candidate, &system, &user, image) {
                     Ok((text, used)) => {
                         usage.add(used);
                         self.ledger.lock().unwrap().charge(app, (self.clock)(), used.total());
@@ -592,9 +603,9 @@ impl ModelHost {
 
     /// One request to one provider: the reply text and its usage, or why the
     /// provider failed (with the key removed).
-    fn send(&self, c: &Candidate, system: &str, user: &str) -> Result<(String, Usage), String> {
+    fn send(&self, c: &Candidate, system: &str, user: &str, image: Option<&str>) -> Result<(String, Usage), String> {
         let key = c.key.as_deref();
-        let prepared = wire::prepare(&c.provider, key, system, user)?;
+        let prepared = wire::prepare_with_image(&c.provider, key, system, user, image)?;
         let (status, body) = self.transport.post(&prepared.url, &prepared.headers, &prepared.body).map_err(|e| {
             format!("{}: {}", probe::transport_reason(&e), probe::describe_error(&e, key))
         })?;

apps/ai-providers/host-service/src/complete/vision.rs

@@ -0,0 +1,59 @@
+//! One bounded image from the authenticated calling app's own storage.
+use super::{Code, Refusal};
+use base64::{engine::general_purpose::STANDARD, Engine as _};
+use image::{ImageFormat, ImageReader};
+use serde_json::Value;
+use std::{fs::File, io::{Cursor, Read}, path::Path};
+
+const MAX_BYTES: u64 = 10 * 1024 * 1024;
+
+fn invalid() -> Refusal {
+    Refusal::new(Code::BadRequest, "The app image is missing, invalid, or over 10 MiB.")
+}
+
+pub(super) fn load(app: &str, host_dir: &Path, value: &Value) -> Result<String, Refusal> {
+    if !cfg!(target_os = "windows") {
+        return Err(Refusal::new(Code::Unavailable, "Image input is unavailable on this host."));
+    }
+    if app.is_empty() || app.len() > 128 || !app.bytes().next().is_some_and(|b| b.is_ascii_alphanumeric())
+        || !app.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-'))
+        || app.contains("..")
+    {
+        return Err(invalid());
+    }
+    let object = value.as_object().ok_or_else(invalid)?;
+    if object.len() != 1 {
+        return Err(invalid());
+    }
+    let path = object.get("app_path").and_then(Value::as_str).ok_or_else(invalid)?;
+    let path = path.strip_prefix('/').unwrap_or(path);
+    if path.is_empty() || path.contains('\\') || path.contains(':')
+        || path.split('/').any(|part| part.is_empty() || part == "." || part == "..")
+    {
+        return Err(invalid());
+    }
+    let root = host_dir.parent().ok_or_else(invalid)?.join(app).canonicalize().map_err(|_| invalid())?;
+    // This follows the existing canonical-path jail boundary. There is no
+    // handle-relative open here, so a concurrent local filesystem mutation
+    // between canonicalization and open is not excluded by this check.
+    let file = root.join(path).canonicalize().map_err(|_| invalid())?;
+    if !file.starts_with(&root) || !file.is_file() {
+        return Err(invalid());
+    }
+    let opened = File::open(file).map_err(|_| invalid())?;
+    if opened.metadata().map_err(|_| invalid())?.len() > MAX_BYTES { return Err(invalid()); }
+    let mut bytes = Vec::new();
+    opened.take(MAX_BYTES + 1).read_to_end(&mut bytes).map_err(|_| invalid())?;
+    if bytes.is_empty() || bytes.len() as u64 > MAX_BYTES {
+        return Err(invalid());
+    }
+    let format = image::guess_format(&bytes).map_err(|_| invalid())?;
+    let mime = match format {
+        ImageFormat::Jpeg => "image/jpeg",
+        ImageFormat::Png => "image/png",
+        ImageFormat::WebP => "image/webp",
+        _ => return Err(invalid()),
+    };
+    ImageReader::with_format(Cursor::new(&bytes), format).into_dimensions().map_err(|_| invalid())?;
+    Ok(format!("data:{mime};base64,{}", STANDARD.encode(bytes)))
+}

apps/ai-providers/host-service/src/complete/wire.rs

@@ -24,6 +24,10 @@ pub struct Prepared {
 
 /// The request for `provider`: `system` and one `user` message.
 pub fn prepare(provider: &Provider, key: Option<&str>, system: &str, user: &str) -> Result<Prepared, String> {
+    prepare_with_image(provider, key, system, user, None)
+}
+
+pub fn prepare_with_image(provider: &Provider, key: Option<&str>, system: &str, user: &str, image: Option<&str>) -> Result<Prepared, String> {
     let family = registry::lookup(&provider.family).map(|f| f.id).unwrap_or(provider.family.as_str());
     if family == "vertex" {
         return Err("Vertex AI is not supported for direct calls".into());
@@ -36,6 +40,9 @@ pub fn prepare(provider: &Provider, key: Option<&str>, system: &str, user: &str)
     let model = effective_model(provider).ok_or("no model")?;
     let mut headers = vec![("Content-Type".to_string(), "application/json".to_string())];
     let api = effective_api_type(provider);
+    if image.is_some() && api != ApiType::OpenAi {
+        return Err("the selected image model requires the OpenAI chat protocol".into());
+    }
     let (url, body) = match api {
         ApiType::Anthropic => {
             if let Some(key) = key {
@@ -55,10 +62,14 @@ pub fn prepare(provider: &Provider, key: Option<&str>, system: &str, user: &str)
             } else {
                 format!("{base}/chat/completions")
             };
+            let content = match image {
+                Some(data) => json!([{"type":"text","text":user},{"type":"image_url","image_url":{"url":data}}]),
+                None => json!(user),
+            };
             (
                 url,
                 json!({"model": model, "stream": false, "messages": [
-                    {"role": "system", "content": system}, {"role": "user", "content": user}
+                    {"role": "system", "content": system}, {"role": "user", "content": content}
                 ]}),
             )
         }