diff --git a/Cargo.lock b/Cargo.lock index 36b7d6be..30cba497 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2818,8 +2818,8 @@ dependencies = [ "libc", "log", "rustversion", - "windows-link 0.1.3", - "windows-result 0.3.4", + "windows-link 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "windows-result 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4049,7 +4049,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", + "webpki-roots 1.0.9", ] [[package]] @@ -4116,7 +4116,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.2", + "windows-core 0.62.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4751,7 +4751,7 @@ dependencies = [ "tokio", "tokio-rustls", "url", - "webpki-roots", + "webpki-roots 1.0.9", ] [[package]] @@ -7164,6 +7164,8 @@ dependencies = [ "makepad-wm-theme", "octosense-app-hub-app", "octosense-appcard", + "octosense-llm-config", + "octosense-llm-service", "octosense-mail-service", "octosense-reference", "rinx", @@ -7182,7 +7184,7 @@ dependencies = [ "rand_core 0.9.5", "serde", "serde_json", - "ureq", + "ureq 3.4.2", ] [[package]] @@ -7249,6 +7251,18 @@ dependencies = [ "url", ] +[[package]] +name = "octosense-llm-service" +version = "0.1.0" +dependencies = [ + "image", + "octosense-appstore", + "octosense-llm-config", + "rqrr", + "serde_json", + "ureq 2.12.1", +] + [[package]] name = "octosense-mail-service" version = "0.1.0" @@ -7265,7 +7279,7 @@ dependencies = [ "scraper", "serde_json", "sha2 0.10.9", - "webpki-roots", + "webpki-roots 1.0.9", ] [[package]] @@ -8286,7 +8300,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams 0.4.2", "web-sys", - "webpki-roots", + "webpki-roots 1.0.9", ] [[package]] @@ -10954,6 +10968,21 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "log", + "once_cell 1.21.4", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "ureq" version = "3.4.2" @@ -10968,7 +10997,7 @@ dependencies = [ "rustls-pki-types", "ureq-proto", "utf8-zero", - "webpki-roots", + "webpki-roots 1.0.9", ] [[package]] @@ -11361,6 +11390,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + [[package]] name = "webpki-roots" version = "1.0.9" diff --git a/Cargo.toml b/Cargo.toml index c4c9d059..4a06e61c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,8 +65,8 @@ octosense-appcard = { path = "apps/appcard", optional = true } # module (`card`) that runs contained script apps, the system apps packed at # build time from the selection `OCTOSENSE_SYSTEM_APPS` names # (system-apps.json, .cargo/config.toml), installed apps and their icons. -# Same rev as the Mail host service's octosense-appstore (OctoSense-System-Apps): -# one App Hub source, so one host-service registry. +# Same rev as the Mail and llm host services' octosense-appstore +# (OctoSense-System-Apps): one App Hub source, so one host-service registry. octosense-app-hub-app = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "3e993d4c596560c42f011b2fd00bd40dbcac7cc5", optional = true } # The `mail` host service the Mail system app calls through `host.request`: # accounts, message lists and send run here, with the passwords the host @@ -74,6 +74,11 @@ octosense-app-hub-app = { git = "https://github.com/OctoSense-org/OctoSense-App- # OctoSense-System-Apps checkout beside this repository # (native-apps.lock.json, tools/setup-native.py). octosense-mail-service = { path = "../OctoSense-System-Apps/apps/mail/host-service", optional = true } +# The `llm` host service the AI providers system app calls: the assistant's +# LLM providers, their keys (macOS keychain) and the phone QR, on host-owned +# sheets. The config crate names the AppCard kernel's octos home it writes. +octosense-llm-service = { path = "../OctoSense-System-Apps/apps/ai-providers/host-service", optional = true } +octosense-llm-config = { path = "../OctoSense-System-Apps/apps/ai-providers/config", optional = true } # Rinx (the Matrix client with Moments and articles) as an in-process module. # Built without its standalone window entry point. rinx = { git = "https://github.com/upstreamlabs/Rinx", rev = "5ad2c1739e5b4776588518ab76440a5367c9968e", default-features = false, features = ["octosense-module"], optional = true } @@ -90,7 +95,7 @@ makepad-sheets = { git = "https://github.com/OctoSense-org/makepad.git", rev = " [features] # App Hub is part of the standard launcher: the store, the Card runner and -# the system apps (News, Photos, Maps, Camera, Mail) it runs. +# the system apps (News, Photos, Maps, Camera, Mail, AI providers) it runs. default = ["app-hub"] # Also usable on desktop for testing the same embedded apps with --module. # Photos is the system app here; `app-photos` links Makepad's native Photos @@ -98,7 +103,7 @@ default = ["app-hub"] # AppCard is not among them: it ships only with `--features app-appcard`. mobile-apps = ["app-reference", "app-sheets", "app-hub"] # App Hub and the host services its system apps call. -app-hub = ["dep:octosense-app-hub-app", "dep:octosense-mail-service"] +app-hub = ["dep:octosense-app-hub-app", "dep:octosense-mail-service", "dep:octosense-llm-service", "dep:octosense-llm-config"] app-reference = ["dep:octosense-reference"] # Sheets linked in as a module (hosting is still Process unless switched). app-sheets = ["dep:makepad-sheets"] diff --git a/README.md b/README.md index 18ece22b..07a74040 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ OctoSense-Desktop is the desktop shell of [OctoSense](https://github.com/OctoSen | Repository | Role for this repo | | --- | --- | | [OctoSense-ROM](https://github.com/OctoSense-org/OctoSense-ROM) | The phone shell (`home/`). Same app model, same runtime, same system apps. | -| [OctoSense-System-Apps](https://github.com/OctoSense-org/OctoSense-System-Apps) | News, Photos, Maps, Camera and Mail bundles, the Mail host service and the AppCard assistant (`octos-app`). Pinned sibling checkout. | +| [OctoSense-System-Apps](https://github.com/OctoSense-org/OctoSense-System-Apps) | News, Photos, Maps, Camera, Mail and AI providers bundles, the Mail and `llm` host services and the AppCard assistant (`octos-app`). Pinned sibling checkout. | | [OctoSense-App-Hub](https://github.com/OctoSense-org/OctoSense-App-Hub) | The signed catalog, the store and the Card runner. Linked as the Git crate `octosense-app-hub-app`. | | [OctoScript-App-Design-Flow](https://github.com/OctoSense-org/OctoScript-App-Design-Flow) | Where apps are designed, built and published to the App Hub. | | [OctoScript-Makepad](https://github.com/OctoSense-org/OctoScript-Makepad) | The runtime release that pins Makepad and OctoScript. Pinned sibling checkout. | @@ -33,7 +33,7 @@ OctoSense-Desktop is the desktop shell of [OctoSense](https://github.com/OctoSen | `native-apps.lock.json` | The OctoSense-System-Apps revision. | | `runtime-patches.lock.json` | Reviewed patches on top of the pinned Makepad, with their expected trees. Empty today: the pinned Makepad carries everything the shell needs. | | `tools/setup-native.py` | Prepares and checks the pinned sibling checkouts. | -| `scripts/` | `upstream.py` (WM provenance and catalog regeneration), `smoke.py` (native smoke test), their Python tests, and `provision-appcard-llm.sh` (Android). | +| `scripts/` | `upstream.py` (WM provenance and catalog regeneration), `smoke.py` (native smoke test), their Python tests, `system_apps_remote.sh` and `ai_providers_remote.sh` (hidden `--remote` end-to-end runs of the system apps and of AI providers), and `provision-appcard-llm.sh` (Android). | | `upstream/makepad.json` | Provenance of every file imported from Makepad's `apps/wm`. | | `resources/` | Themes, wallpapers, icons, Android manifest template, startup script. | | `docs/` | [Validation record](docs/validation.md), [upstream sync](docs/upstream.md), [local AI](docs/local-ai.md), [Android AppCard build](docs/android-appcard-build.md), dated plans. | @@ -105,7 +105,7 @@ A relocatable `.app`, installers and a Linux session compositor are not provided | Feature | Default | Effect | | --- | --- | --- | -| `app-hub` | on | Links `octosense-app-hub-app` (store `apphub`, Card runner `card`, system apps) and the Mail host service `octosense-mail-service`. Without it the build has no App Hub and no system apps. | +| `app-hub` | on | Links `octosense-app-hub-app` (store `apphub`, Card runner `card`, system apps) and the host services `octosense-mail-service` (Mail) and `octosense-llm-service` (AI providers). Without it the build has no App Hub and no system apps. | | `app-reference` | off | Links Reference as a module. | | `app-sheets` | off | Links Makepad's Sheets as a module. | | `app-photos` | off | Links Makepad's native Photos module; it replaces the Photos system app of the same id (for comparison). | @@ -138,6 +138,7 @@ App Hub's modules are the exception: they have no process form and always open i | `OCTOSENSE_SYSTEM_APPS` | The system-app selection file; `.cargo/config.toml` sets it to `system-apps.json`. | | `MAKEPAD_APP_CONFIG='{"mail_demo":true}'` | Serve Mail's demo mailbox (see [Demos](#demos)). | | `OCTOSENSE_MAIL_VAULT=file` | Keep Mail passwords in a 0600 file instead of the macOS keychain. | +| `OCTOSENSE_LLM_VAULT=file` | Keep AI providers' keys in the owner-only octos profile instead of the macOS keychain. | | `OCTOS_APP_CORE_BIN`, `OCTOS_APP_CORE_DIR` | Point the AppCard assistant at a local octos kernel. | | `MAKEPAD_REMOTE`, `MAKEPAD_HIDE_WINDOWS` | Remote-control bridge; hidden windows (see [Demos](#demos)). | @@ -147,7 +148,7 @@ The launcher lists four kinds of app together: | Kind | Comes from | Runs as | Launcher id | | --- | --- | --- | --- | -| **System apps**: News, Photos, Maps, Camera, Mail | OctoSense-System-Apps `apps//bundle`, selected by `system-apps.json`, packed into the build | Contained Splash programs in App Hub's Card runner, each in its own isolate under the capabilities its manifest asks for | `` (manifest id `os.`) | +| **System apps**: News, Photos, Maps, Camera, Mail, AI providers | OctoSense-System-Apps `apps//bundle`, selected by `system-apps.json`, packed into the build | Contained Splash programs in App Hub's Card runner, each in its own isolate under the capabilities its manifest asks for | `` (manifest id `os.`) | | **Store apps** | The signed App Hub catalog, installed from the store (`apphub`) | The same Card runner. Every open is checked against the catalog; an update closes old instances. | `hub:` | | **Native modules** | Rust crates linked into this binary | In-process `AppModule`s. Trusted code only: App Hub, AppCard, Reference and the `app-*` features. | module id | | **Developer programs** | `config/apps.json` | Separate processes in tiles, over Makepad's `--stdin-loop` hosting protocol, built on first launch | catalog `id` | @@ -168,6 +169,8 @@ Mail is the worked example (`octosense-mail-service`, from `../OctoSense-System- - The service tests the account, stores the password in the platform secret store (macOS keychain), and grants the account only to the app that added it. - Mail state lives under the host's own directory, outside every app's jail. +AI providers (`os.ai-providers`) edits the LLM providers the AppCard assistant runs on through the `llm` service (`octosense-llm-service`, from `../OctoSense-System-Apps/apps/ai-providers/host-service`). Keys are typed only on host sheets and go to the macOS keychain entry octos reads; the providers are written to the AppCard kernel's octos profile (`OCTOS_APP_CORE_DIR`, else `~/octos-home/.octos`). A phone's provider QR is imported from a picture of it: **Choose image** opens the open panel, or drop a screenshot on the import sheet. **Start → Settings → AI providers** opens it. With `--features app-appcard`, a change restarts a running AppCard so its kernel reads the new providers. + New app features that need a password, PIN or token belong in a host service and a host sheet, never in the app's own UI. ### Store apps (App Hub) @@ -194,7 +197,7 @@ Open **App Hub**, choose the app, **Get**, scroll to **Install**, then **Open**: { "schema": 1, "source": "../OctoSense-System-Apps/apps", - "apps": ["news", "photos", "maps", "camera", "mail"], + "apps": ["news", "photos", "maps", "camera", "mail", "ai-providers"], "assets": {} } ``` @@ -313,7 +316,7 @@ Phone builds always link Reference and Sheets, and App Hub with the system apps | --- | --- | --- | | Makepad / OctoScript | `native-runtime.lock.json` (a new OctoScript-Makepad release), and the `rev` of the Makepad Git dependencies in `Cargo.toml` and `apps/*/Cargo.toml` | If a patch is needed on top, record it in `runtime-patches.lock.json` (base, sha256, tree) | | System apps, Mail service, AppCard | `revision` in `native-apps.lock.json` | — | -| App Hub | `rev` of `octosense-app-hub-app` in `Cargo.toml` | Keep it at the App Hub rev the pinned System-Apps Mail service names (`octosense-appstore`), so the graph has one App Hub | +| App Hub | `rev` of `octosense-app-hub-app` in `Cargo.toml` | Keep it at the App Hub rev the pinned System-Apps Mail and `llm` services name (`octosense-appstore`), so the graph has one App Hub | After any change: `python3 tools/setup-native.py --update`, `cargo update` as needed, then `python3 tools/setup-native.py --check --cargo-manifest Cargo.toml` and the tests below. diff --git a/README.zh-CN.md b/README.zh-CN.md index e9929d32..df259d6d 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -11,7 +11,7 @@ OctoSense-Desktop 是 [OctoSense](https://github.com/OctoSense-org)（运行在 | 仓库 | 与本仓库的关系 | | --- | --- | | [OctoSense-ROM](https://github.com/OctoSense-org/OctoSense-ROM/blob/main/README.zh-CN.md) | 手机 Shell（`home/`）。相同的应用模型、相同的运行时、相同的系统应用。 | -| [OctoSense-System-Apps](https://github.com/OctoSense-org/OctoSense-System-Apps) | 新闻、相册、地图、相机、邮件的应用包，邮件宿主服务，以及 AppCard 助手（`octos-app`）。以固定版本的同级目录检出。 | +| [OctoSense-System-Apps](https://github.com/OctoSense-org/OctoSense-System-Apps) | 新闻、相册、地图、相机、邮件、AI 提供商的应用包，邮件与 `llm` 宿主服务，以及 AppCard 助手（`octos-app`）。以固定版本的同级目录检出。 | | [OctoSense-App-Hub](https://github.com/OctoSense-org/OctoSense-App-Hub) | 签名目录、商店和 Card 运行器。以 Git crate `octosense-app-hub-app` 链接。 | | [OctoScript-App-Design-Flow](https://github.com/OctoSense-org/OctoScript-App-Design-Flow) | 设计、构建应用并发布到 App Hub 的地方。 | | [OctoScript-Makepad](https://github.com/OctoSense-org/OctoScript-Makepad) | 固定 Makepad 与 OctoScript 版本的运行时发行版。以固定版本的同级目录检出。 | @@ -33,7 +33,7 @@ OctoSense-Desktop 是 [OctoSense](https://github.com/OctoSense-org)（运行在 | `native-apps.lock.json` | OctoSense-System-Apps 的版本。 | | `runtime-patches.lock.json` | 固定 Makepad 之上经过评审的补丁及其期望的源码树。目前为空：固定的 Makepad 已包含 Shell 所需的一切。 | | `tools/setup-native.py` | 准备并检查固定版本的同级目录。 | -| `scripts/` | `upstream.py`（WM 来源记录与目录重新生成）、`smoke.py`（原生冒烟测试）、它们的 Python 测试，以及 `provision-appcard-llm.sh`（Android）。 | +| `scripts/` | `upstream.py`（WM 来源记录与目录重新生成）、`smoke.py`（原生冒烟测试）、它们的 Python 测试、`system_apps_remote.sh` 和 `ai_providers_remote.sh`（以隐藏窗口 `--remote` 运行的系统应用与 AI 提供商端到端测试），以及 `provision-appcard-llm.sh`（Android）。 | | `upstream/makepad.json` | 每个从 Makepad `apps/wm` 引入的文件的来源记录。 | | `resources/` | 主题、壁纸、图标、Android manifest 模板、启动脚本。 | | `docs/` | [验证记录](docs/validation.md)、[上游同步](docs/upstream.md)、[本地 AI](docs/local-ai.md)、[Android AppCard 构建](docs/android-appcard-build.md)、按日期的计划文档。 | @@ -105,7 +105,7 @@ cargo build --release --workspace | Feature | 默认 | 作用 | | --- | --- | --- | -| `app-hub` | 开 | 链接 `octosense-app-hub-app`（商店 `apphub`、Card 运行器 `card`、系统应用）以及邮件宿主服务 `octosense-mail-service`。关闭后构建中没有 App Hub，也没有系统应用。 | +| `app-hub` | 开 | 链接 `octosense-app-hub-app`（商店 `apphub`、Card 运行器 `card`、系统应用）以及宿主服务 `octosense-mail-service`（邮件）和 `octosense-llm-service`（AI 提供商）。关闭后构建中没有 App Hub，也没有系统应用。 | | `app-reference` | 关 | 把 Reference 作为模块链接。 | | `app-sheets` | 关 | 把 Makepad 的 Sheets 作为模块链接。 | | `app-photos` | 关 | 链接 Makepad 的原生 Photos 模块；它会替换同 id 的相册系统应用（用于对比）。 | @@ -138,6 +138,7 @@ App Hub 的模块是例外：它们没有进程形态，总是在进程内打开 | `OCTOSENSE_SYSTEM_APPS` | 系统应用选择文件；`.cargo/config.toml` 将其设为 `system-apps.json`。 | | `MAKEPAD_APP_CONFIG='{"mail_demo":true}'` | 提供邮件的演示邮箱（见[演示](#演示)）。 | | `OCTOSENSE_MAIL_VAULT=file` | 把邮件密码保存在权限为 0600 的文件中，而不是 macOS 钥匙串。 | +| `OCTOSENSE_LLM_VAULT=file` | 把 AI 提供商的密钥保存在仅所有者可读的 octos 配置文件中，而不是 macOS 钥匙串。 | | `OCTOS_APP_CORE_BIN`、`OCTOS_APP_CORE_DIR` | 让 AppCard 助手使用本地的 octos 内核。 | | `MAKEPAD_REMOTE`、`MAKEPAD_HIDE_WINDOWS` | 远程控制桥；隐藏窗口（见[演示](#演示)）。 | @@ -147,7 +148,7 @@ launcher 把四类应用列在一起： | 类别 | 来源 | 运行方式 | Launcher id | | --- | --- | --- | --- | -| **系统应用**：新闻、相册、地图、相机、邮件 | OctoSense-System-Apps 的 `apps//bundle`，由 `system-apps.json` 选择，打包进构建 | App Hub 的 Card 运行器中隔离运行的 Splash 程序，每个应用一个 isolate，只拥有其清单申请的能力 | ``（清单 id `os.`） | +| **系统应用**：新闻、相册、地图、相机、邮件、AI 提供商 | OctoSense-System-Apps 的 `apps//bundle`，由 `system-apps.json` 选择，打包进构建 | App Hub 的 Card 运行器中隔离运行的 Splash 程序，每个应用一个 isolate，只拥有其清单申请的能力 | ``（清单 id `os.`） | | **商店应用** | 签名的 App Hub 目录，从商店（`apphub`）安装 | 同一个 Card 运行器。每次打开都会对照目录检查；更新会关闭旧实例。 | `hub:` | | **原生模块** | 链接进本二进制的 Rust crate | 进程内的 `AppModule`。只允许受信任的代码：App Hub、AppCard、Reference 以及各 `app-*` feature。 | 模块 id | | **开发者程序** | `config/apps.json` | tile 中的独立进程，通过 Makepad 的 `--stdin-loop` 托管协议运行，首次启动时构建 | 目录 `id` | @@ -168,6 +169,8 @@ launcher 把四类应用列在一起： - 服务先测试账户，再把密码存入平台的密钥存储（macOS 钥匙串），并且只把账户授予添加它的应用。 - 邮件状态保存在宿主自己的目录中，位于所有应用的沙箱之外。 +AI 提供商（`os.ai-providers`）通过 `llm` 服务（`octosense-llm-service`，来自 `../OctoSense-System-Apps/apps/ai-providers/host-service`）编辑 AppCard 助手使用的 LLM 提供商。密钥只在宿主面板上输入，保存到 octos 读取的 macOS 钥匙串条目；提供商写入 AppCard 内核的 octos 配置（`OCTOS_APP_CORE_DIR`，否则为 `~/octos-home/.octos`）。手机上的提供商二维码可从图片导入：**Choose image** 打开文件面板，或把截图拖到导入面板上。**开始 → 设置 → AI providers** 可打开它。使用 `--features app-appcard` 时，更改会重启正在运行的 AppCard，使其内核读取新的提供商。 + 需要密码、PIN 或令牌的新功能，应放在宿主服务和宿主自有面板中，绝不放在应用自己的界面里。 ### 商店应用（App Hub） @@ -194,7 +197,7 @@ OCTOSENSE_HUB= OCTOSENSE_HUB_ANCHOR= \ { "schema": 1, "source": "../OctoSense-System-Apps/apps", - "apps": ["news", "photos", "maps", "camera", "mail"], + "apps": ["news", "photos", "maps", "camera", "mail", "ai-providers"], "assets": {} } ``` diff --git a/scripts/ai_providers_remote.sh b/scripts/ai_providers_remote.sh new file mode 100755 index 00000000..778e51e2 --- /dev/null +++ b/scripts/ai_providers_remote.sh @@ -0,0 +1,667 @@ +#!/usr/bin/env bash +# AI providers' QR import from an image, and the phone QR it exports, on the +# desktop, driven through Makepad's remote (instrument) mode: no OS +# screenshots, only the app's own frames. +# +# cargo build --release [--config …] --features mobile-apps +# cargo build --release [--config …] -p octosense-llm-service --example read_qr_image +# scripts/ai_providers_remote.sh [artifacts-dir] +# +# Runs the release binary hidden with its own OCTOSENSE_HOME, octos core dir +# and file vaults under a temp directory (neither ~/.octosense nor the login +# keychain is touched). Every key in it is fake, and the binary runs under +# sandbox-exec with outbound HTTPS denied: no fake key reaches a real +# provider, and a provider test fails as a network failure. A local fake +# endpoint (fake.py, 127.0.0.1) stands in for a provider that answers. The +# flow: +# 1. AI providers opens; the host reports an image picker ("Import code +# from image"). +# W1. Add DeepSeek with the add wizard, step by step: 1 search the families +# and choose DeepSeek, 2 the model pull-down (DeepSeek V4 Flash, +# Recommended), 3 the route pull-down (Official API), 4 a fake key, 5 +# Test connection: the provider cannot be reached, Save stays disabled +# and "Save without testing" saves it as the primary. +# W2. Change its model with the card's model pull-down (DeepSeek V4 Pro). +# W3. Add Z.ai as a fallback on another model (GLM-4.7) and route (a +# custom OpenAI-compatible endpoint: the fake endpoint); Test connection +# passes ("Connected"), and "Add as fallback" saves it. +# W4. The saved-models list and _main.json say so; Test connection on a +# row; then both are removed with the cards' Remove (tap twice). +# 2. Import into the empty list by PASTE: an old single-provider code +# (DeepSeek deepseek-v4-flash, fake key ending fcb0) typed into the +# import sheet's code field is saved as the primary. +# 3. Add OpenAI with the wizard (its default gpt-4o on the official route) +# with the fake key sk-test-0000000000001234, saved without testing. +# 4. Import by DROP: the fixture QR-A (OctoSense-System-Apps +# config/tests/fixtures/qr-a.png, PIN 7K3M-9QX2: DeepSeek deepseek-chat +# and Z.ai glm-4.6) dropped on the import sheet through the native drop +# path (/drop), PIN typed. An import only adds, so there is no confirm +# step: the primary and OpenAI stay first, QR-A's providers are appended +# as fallbacks, and its DeepSeek key goes to a slot of its own +# (DEEPSEEK_2_API_KEY) so the primary's key is not overwritten. +# 5. Import QR-A again by PICKER: "Choose image" (OCTOSENSE_LLM_TEST_IMAGE +# answers with the fixture instead of the open panel, which a hidden run +# cannot click through); a wrong PIN is refused and changes nothing, the +# right one changes nothing either ("Already saved"). +# 6. Show QR for phone, grab the sheet, read the QR out of the grab (the +# service's own image search, rqrr) and open it with the PIN on the +# sheet: it equals the saved set and keys, both DeepSeek slots included. +# The countdown +# ticks, and at 0 the sheet closes by itself (OCTOSENSE_LLM_QR_SECONDS +# shortens its five minutes to $QR_SECONDS s), with no Splash error in +# the log. +# 7. No key text in /snap, /d, /log or the host log; no panic, no Splash +# error; /gq exits. +# Artifacts: grabs/, export.png (the export sheet at full scale) and +# export-pin.txt, for scanning on a phone. The script-drawn controls (the +# wizard's fields, rows and pull-downs, the app's cards) are not in /snap: +# they are found in grabs by their colours (blocks.py). +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +BIN=${OCTOSENSE_BIN:-$ROOT/target/release/octosense} +READ_QR=${READ_QR_IMAGE:-$ROOT/target/release/examples/read_qr_image} +SRC=$(cd "$ROOT" && python3 -c 'import json; print(json.load(open("system-apps.json"))["source"])') +FIXTURE=$(cd "$ROOT/$SRC/ai-providers/config/tests/fixtures" && pwd)/qr-a.png +FIXTURE_PIN=7K3M-9QX2 +FAKE_KEY=sk-test-0000000000001234 +# The old single-provider code the empty list is filled from (its key is fake). +LEGACY_CODE='{"llm_family":"deepseek","llm_model":"deepseek-v4-flash","llm_key":"sk-test-legacy-00000000fcb0"}' +# The phone QR's lifetime in this run: long enough to grab and read the code. +QR_SECONDS=${QR_SECONDS:-20} +# What a Splash script error logs (the export sheet's countdown used to log +# these every second). +SPLASH_ERRORS='pop_stack_resolved|mes empty|splash host callback error|\[E\] splash' +# Every key the run handles: none may show up in the UI tree or the logs. +SECRETS='sk-test-|zai-test-|0000000000001234|0000000000005678' +ZAI_KEY=sk-test-0000000000005678 +WORK=${1:-$(mktemp -d -t octosense-ai-providers)} +mkdir -p "$WORK/home" "$WORK/octos" "$WORK/grabs" +LOG=$WORK/host.log +PROFILE=$WORK/octos/profiles/_main.json +[ -x "$BIN" ] || { echo "no $BIN: build it first"; exit 2; } +[ -x "$READ_QR" ] || { echo "no $READ_QR: cargo build --release -p octosense-llm-service --example read_qr_image"; exit 2; } +echo "artifacts: $WORK" + +cd "$WORK" +: >"$LOG" +env -u MAKEPAD_HOME -u MAKEPAD_WM_ROOT -u MAKEPAD_WM_THEME \ + OCTOSENSE_HOME="$WORK/home" OCTOS_APP_CORE_DIR="$WORK/octos" \ + OCTOSENSE_MAIL_VAULT=file OCTOSENSE_LLM_VAULT=file \ + OCTOSENSE_LLM_TEST_IMAGE="$FIXTURE" OCTOSENSE_LLM_QR_SECONDS="$QR_SECONDS" \ + MAKEPAD_HIDE_WINDOWS=1 sandbox-exec -p '(version 1)(allow default)(deny network-outbound (remote tcp "*:443"))' \ + "$BIN" --remote >"$LOG" 2>&1 & +PID=$! +PORT= +# A provider that answers: OpenAI-compatible chat completions for the Z.ai +# fake key, on 127.0.0.1 (the sandbox lets local traffic through). +cat >"$WORK/fake.py" <<'EOF' +import http.server, json, sys +KEY = sys.argv[1] +class H(http.server.BaseHTTPRequestHandler): + def answer(self, ok, body): + data = json.dumps(body if ok else {"error": {"message": "Incorrect API key"}}).encode() + self.send_response(200 if ok else 401) + self.send_header("content-type", "application/json"); self.send_header("content-length", str(len(data))) + self.end_headers(); self.wfile.write(data) + def do_POST(self): + self.rfile.read(int(self.headers.get("content-length", 0))) + self.answer(self.headers.get("authorization") == "Bearer " + KEY, {"choices": []}) + def do_GET(self): + self.answer(self.headers.get("authorization") == "Bearer " + KEY, {"data": [{"id": "glm-4.7"}]}) + def log_message(self, *args): pass +server = http.server.HTTPServer(("127.0.0.1", 0), H) +print(server.server_address[1], flush=True) +server.serve_forever() +EOF +python3 "$WORK/fake.py" "$ZAI_KEY" >"$WORK/fake.port" 2>/dev/null & +FAKE=$! +disown "$FAKE" +cleanup() { + kill "$FAKE" 2>/dev/null || true + if kill -0 "$PID" 2>/dev/null; then + [ -n "$PORT" ] && curl -s "127.0.0.1:$PORT/quit" >/dev/null || true + sleep 2 + kill "$PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +fail() { echo "FAIL: $*"; [ -z "${HOLD_ON_FAIL:-}" ] || sleep "$HOLD_ON_FAIL"; exit 1; } +pass() { echo "PASS: $*"; } +for _ in $(seq 1 120); do + PORT=$(sed -n 's/.*\[makepad-remote\] listening on 127\.0\.0\.1:\([0-9]*\) pid=.*/\1/p' "$LOG" | head -1) + [ -n "$PORT" ] && break + sleep 0.5 +done +[ -n "$PORT" ] || fail "no remote port in $LOG" +get() { curl -fsS "127.0.0.1:$PORT/$1"; } +q() { python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$1"; } +key() { + local out + out=$(curl -s "127.0.0.1:$PORT/k?k=press&c=$1&wait=1${2:-}") + case $out in *'"err"'*) echo "note: key $1: $out" ;; esac +} +type_keys() { + local text=$1 i c + for ((i = 0; i < ${#text}; i++)); do + c=${text:i:1} + case $c in + " ") key Space ;; + *) key "Key$(printf %s "$c" | tr a-z A-Z)" ;; + esac + done +} +# Input waits for the next frame; a hidden window sometimes misses it and +# the remote answers "retry" although the input was delivered. What input +# did is checked through the profile, the log and snapshots, so a missed +# frame is only noted (retrying could click twice). +input() { + local out + out=$(curl -s "127.0.0.1:$PORT/$1&wait=1") + case $out in *'"err"'*) echo "note: ${1%%\?*}: $out" ;; esac +} +text() { input "t?t=$(q "$1")"; } +click() { input "click?x=$1&y=$2"; } +scroll() { input "m?k=scroll&x=$1&y=$2&dy=$3"; } +# A grab at layout scale (one pixel a point); prints its path. +grab() { + local png + for _ in 1 2 3 4 5; do + png=$(curl -s "127.0.0.1:$PORT/g?scale=${2:-0.5}" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("png",""))' || true) + [ -n "$png" ] && { cp "$png" "$WORK/grabs/$1.png"; echo "$WORK/grabs/$1.png"; return; } + sleep 1 + done + fail "no frame for $1" +} +# The centre of the card widget whose text is exactly $1: "x y". +widget() { + get "snap?q=$(q "$1")" | python3 -c ' +import json,sys +hits=[e for e in json.load(sys.stdin)["s"] if e.get("t")==sys.argv[1] and e["r"][2]>0] +if hits: x,y,w,h=hits[-1]["r"]; print(x+w/2, y+h/2)' "$1" +} +wait_widget() { + for _ in $(seq 1 ${2:-40}); do [ -n "$(widget "$1")" ] && return 0; sleep 0.5; done + fail "never showed: $1" +} +# The host's sheet (its rect and its program) once it runs `$1`. +sheet_rect() { + get "snap?q=$(q "$1")" | python3 -c ' +import json,sys +hits=[e for e in json.load(sys.stdin)["s"] if e["i"]=="sheet" and sys.argv[1] in (e.get("t") or "")] +if hits: print(*hits[-1]["r"])' "$1" +} +wait_sheet() { + local r + for _ in $(seq 1 40); do r=$(sheet_rect "$1"); [ -n "$r" ] && { echo "$r"; return 0; }; sleep 0.5; done + fail "no sheet running $1" +} +wait_log() { + for _ in $(seq 1 ${2:-40}); do grep -q "$1" "$LOG" && return 0; sleep 0.5; done + fail "log never said: $1" +} +# Sheets are Splash isolates whose widgets /snap does not list: find their +# controls in a grab instead. Prints "pill X Y" (the blue action at the top +# right: Import, Save, Close) and "full Y" / "half X Y" for each grey field +# or button, top to bottom. +cat >"$WORK/sheet.py" <<'EOF' +import sys, zlib, struct +def read_png(path): + data = open(path, "rb").read() + pos, chunks = 8, {} + idat = b"" + while pos < len(data): + n, kind = struct.unpack(">I4s", data[pos:pos + 8]) + body = data[pos + 8:pos + 8 + n] + if kind == b"IHDR": w, h, depth, color = struct.unpack(">IIBB", body[:10]) + if kind == b"IDAT": idat += body + pos += 12 + n + assert depth == 8 and color in (2, 6), (depth, color) + bpp = 4 if color == 6 else 3 + raw, stride, rows, prev = zlib.decompress(idat), w * bpp, [], bytearray(w * bpp) + i = 0 + for _ in range(h): + f, line = raw[i], bytearray(raw[i + 1:i + 1 + stride]); i += 1 + stride + for x in range(stride): + a = line[x - bpp] if x >= bpp else 0 + b = prev[x]; c = prev[x - bpp] if x >= bpp else 0 + if f == 1: line[x] = (line[x] + a) & 255 + elif f == 2: line[x] = (line[x] + b) & 255 + elif f == 3: line[x] = (line[x] + (a + b) // 2) & 255 + elif f == 4: + p = a + b - c; pa, pb, pc = abs(p - a), abs(p - b), abs(p - c) + line[x] = (line[x] + (a if pa <= pb and pa <= pc else b if pb <= pc else c)) & 255 + rows.append(bytes(line)); prev = line + return w, h, bpp, rows +path, sx, sy, sw, sh = sys.argv[1], *map(float, sys.argv[2:6]) +w, h, bpp, rows = read_png(path) +px = lambda x, y: rows[y][x * bpp:x * bpp + 3] +x0, x1 = int(sx + 20), int(sx + sw - 20) +inner = x1 - x0 +grey = lambda p: 200 <= p[0] <= 248 and abs(p[0] - p[1]) <= 2 and 3 <= p[2] - p[0] <= 7 +blue = lambda p: p[0] < 60 and 100 <= p[1] <= 160 and p[2] > 200 +# The pill: the blue columns tall enough to be a filled button. +top = [y for y in range(int(sy), int(sy + 110))] +cols = [x for x in range(x0, x1) if sum(blue(px(x, y)) for y in top) >= 20] +if cols: + ys = [y for y in top if sum(blue(px(x, y)) for x in cols[::3]) >= len(cols[::3]) // 2] + print("pill", (cols[0] + cols[-1]) // 2, (ys[0] + ys[-1]) // 2 if ys else int(sy + 60)) +out, y = [], int(sy + 60) +count = lambda y: sum(grey(px(x, y)) for x in range(x0, x1, 2)) * 2 +while y < int(min(sy + sh, h)): + if count(y) >= 0.4 * inner: + start = y + while y < h and count(y) >= 0.4 * inner: y += 1 + if y - start >= 20: + row, runs, run = start + 3, [], None + for x in range(x0, x1): + if grey(px(x, row)): + run = [x, x] if run is None else [run[0], x] + elif run: runs.append(run); run = None + if run: runs.append(run) + runs = [r for r in runs if r[1] - r[0] >= 0.3 * inner] + mid = (start + y) // 2 + if runs and runs[0][1] - runs[0][0] >= 0.85 * inner: + if not any(l.startswith("full") for l in out): + # The note line just above the first full-width control: + # how far right its text reaches (0: no text). + ink = [x for x in range(x0, x1) for yy in range(start - 40, start - 6) if max(px(x, yy)) < 120] + out.append("note %d" % (max(ink) - sx if ink else 0)) + out.append("full %d" % mid) + else: + for r in runs: out.append("half %d %d" % ((r[0] + r[1]) // 2, mid)) + y += 1 +print("\n".join(out)) +EOF +# `controls NAME RECT…` grabs, then prints the sheet's controls. +controls() { local png; png=$(grab "$1"); shift; python3 "$WORK/sheet.py" "$png" "$@"; } +# The import sheet moved on to the PIN once its note reads "Code read from +# the image. Type the PIN shown beside it." (wider than "Choose a screenshot +# or photo of the code…" or none), with no script error on the way. +pin_prompt() { + local C n + for _ in $(seq 1 10); do + sleep 1 + C=$(controls "$@") + n=$(echo "$C" | awk '$1=="note"{print $2; exit}') + [ "${n:-0}" -ge 420 ] && break + done + if grep -q "not found in prototype chain" "$LOG"; then + grep "not found in prototype chain" "$LOG" | head -3 + fail "the import sheet's script failed" + fi + [ "${n:-0}" -ge 420 ] || return 1 + echo "$C" +} +nth_full() { awk -v n="$1" '$1=="full"{i++; if(i==n){print $2; exit}}'; } +pill() { awk '$1=="pill"{print $2, $3; exit}'; } +# Blocks of one colour in a grab (the wizard's fields, rows, pull-downs and +# buttons, the cards' controls): "cx cy w h" per block, top to bottom. +# `blocks.py SHEET_PY PNG RRGGBB [MINCOUNT] [MINW]`: rows with at least +# MINCOUNT pixels of the colour (+-3; +-30 for text, MINCOUNT 1) make bands +# (a text band may pause for 3 rows); a band splits where the colour pauses +# for more than 12 px across (24 for text); narrower blocks than MINW drop. +cat >"$WORK/blocks.py" <<'EOF' +import sys +exec(open(sys.argv[1]).read().split("path, sx, sy")[0]) +w, h, bpp, rows = read_png(sys.argv[2]) +want = tuple(int(sys.argv[3][i:i + 2], 16) for i in (0, 2, 4)) +mincount = int(sys.argv[4]) if len(sys.argv) > 4 else 40 +minw = int(sys.argv[5]) if len(sys.argv) > 5 else 40 +text = mincount <= 1 +gap, tol = (24, 30) if text else (12, 3) +def hits(y): + r = rows[y] + return [x for x in range(w) if abs(r[x * bpp] - want[0]) <= tol and abs(r[x * bpp + 1] - want[1]) <= tol and abs(r[x * bpp + 2] - want[2]) <= tol] +bands, cur, quiet = [], None, 0 +for y in range(h): + xs = hits(y) + if len(xs) >= mincount: + if cur is None: cur = [y, y, set()] + cur[1] = y; cur[2].update(xs); quiet = 0 + elif cur is not None: + quiet += 1 + if quiet > (3 if text else 0): bands.append(cur); cur = None; quiet = 0 +if cur: bands.append(cur) +out = [] +for y0, y1, xs in bands: + xs = sorted(xs); start = prev = xs[0] + for x in xs[1:] + [10 ** 9]: + if x - prev > gap: + if prev - start + 1 >= minw and y1 - y0 + 1 >= (6 if text else 14): + out.append(((start + prev) // 2, (y0 + y1) // 2, prev - start + 1, y1 - y0 + 1)) + start = x + prev = x +for b in sorted(out, key=lambda b: (b[1], b[0])): print(*b) +EOF +# The wizard's and the cards' colours (bundle/main.splash and +# host-service/src/sheets.rs in OctoSense-System-Apps): pull-down, option +# row, chosen row, field, Test connection, Save without testing, accent, +# danger. +PULLDOWN=edf2fc; OPTION=f6f8fc; CHOSEN=e8f0fe; FIELD=f2f2f7; TEST=e3edff; BYPASS=fff1e0; BLUE=007aff; RED=ff3b30 +in_png() { python3 "$WORK/blocks.py" "$WORK/sheet.py" "$1" "$2" "${3:-40}" "${4:-40}"; } +# `blocks NAME COLOUR [MINCOUNT] [MINW]`: grab, then the colour's blocks. +blocks() { local png; png=$(grab "$1"); in_png "$png" "$2" "${3:-40}" "${4:-40}"; } +nth() { awk -v n="$1" 'NR==n{print $1, $2; exit}'; } +# The app scrolls as one list: its top (the cards) or its end (the QR +# buttons, under the cards). +top_app() { scroll 548 400 -3000; sleep 1; } +bottom_app() { scroll 548 400 3000; sleep 1; } + +profile_families() { + python3 -c ' +import json,sys +llm=json.load(open(sys.argv[1]))["config"]["llm"] +print(" ".join(s["family_id"] for s in [llm.get("primary")]+llm.get("fallbacks",[]) if s))' "$PROFILE" +} + +for _ in $(seq 1 60); do curl -s "127.0.0.1:$PORT/snap?q=main_window" | grep -q '"s":\[{' && break; sleep 0.5; done +pass "desktop up on port $PORT (pid $PID)" + +# 1. AI providers opens and offers the image import. +key Space "&cmd=1"; type_keys "ai providers"; key Return +wait_log "card: os.ai-providers running under" +wait_widget "Import code from image" +pass "AI providers opened; the host has an image picker (Import code from image)" + +# The rows are drawn by the app's script (on_render), which /snap does not +# list: the masked keys are in the grab; the keys themselves in the profile. +keys_end_with() { + python3 -c ' +import json,sys +env=json.load(open(sys.argv[1]))["config"]["env_vars"] +want=dict(a.split("=") for a in sys.argv[2:]) +sys.exit(0 if sorted(env)==sorted(want) and all(env[k].endswith(v) for k,v in want.items()) else 1)' "$PROFILE" "$@" +} +# Each saved route's model and the key env var octos reads it from. +profile_routes() { + python3 -c ' +import json,sys +llm=json.load(open(sys.argv[1]))["config"]["llm"] +print(" ".join("%s:%s" % (s.get("model_id","-"), (s.get("route") or {}).get("api_key_env","-")) for s in [llm.get("primary")]+llm.get("fallbacks",[]) if s))' "$PROFILE" +} +# The import sheet closes by itself once the code is imported. +sheet_closes() { + for _ in $(seq 1 40); do [ -z "$(sheet_rect "$1")" ] && return 0; sleep 0.5; done + return 1 +} + +# The add wizard, driven like a person. `next_step NAME` grabs (NAME) and +# presses the footer's Next (or the save on step 5): the lowest blue button +# narrower than a field; greyed out, it is not blue, and the step fails. +next_step() { + local X Y + read -r X Y _ < <(blocks "$1" $BLUE 40 60 | awk '$3<400' | tail -1) || true + [ -n "$Y" ] || fail "$1: Next is not enabled" + click "$X" "$Y"; sleep 1.5 +} +# `pick NAME N`: open the step's pull-down and take its option N. +pick() { + local X Y W H X2 Y2 + read -r X Y W H _ < <(blocks "$1-closed" $PULLDOWN 40 400 | head -1) || true + [ -n "$Y" ] || fail "$1: no pull-down on this step" + click "$X" "$Y"; sleep 1.5 + read -r X2 Y2 _ < <(blocks "$1" $OPTION 40 400 | awk -v y=$((Y + H / 2)) '$2>y' | nth "$2") || true + [ -n "$Y2" ] || fail "$1: the pull-down shows no option $2" + click "$X2" "$Y2"; sleep 1 +} +# `add_model NAME QUERY MODEL_N ROUTE_N KEY [BASE]`: the five steps. Step 1 +# searches the families for QUERY and chooses the first; steps 2 and 3 take +# option MODEL_N / ROUTE_N of their pull-downs (0: keep the preselected +# one), and BASE goes into the custom route's base URL; step 4 takes KEY; +# step 5 tests, then saves (Save once the test passes, else Save without +# testing after a network failure); SAVED says which: tested or untested. +add_model() { + local name=$1 query=$2 model_n=$3 route_n=$4 key=$5 base=${6:-} X Y before + SAVED= + before=$(cat "$PROFILE" 2>/dev/null || true) + read -r X Y _ < <(widget "Add model") || true; click "$X" "$Y" + read -r SX SY SW SH < <(wait_sheet "llm.sheet.fetch_models") + sleep 1.5 + read -r X Y _ < <(blocks "$name-1-start" $FIELD 40 400 | nth 1) || true + [ -n "$Y" ] || fail "$name: no search field on step 1" + click "$X" "$Y"; text "$query"; sleep 1.5 + read -r X Y _ < <(blocks "$name-1-search" $OPTION 40 400 | nth 1) || true + [ -n "$Y" ] || fail "$name: no family matches $query" + click "$X" "$Y"; sleep 1.5 + [ -n "$(blocks "$name-1-family" $CHOSEN 40 400)" ] || fail "$name: the family row is not marked chosen" + next_step "$name-1-next" + [ "$model_n" -gt 0 ] && pick "$name-2-model" "$model_n" + next_step "$name-2-next" + [ "$route_n" -gt 0 ] && pick "$name-3-route" "$route_n" + if [ -n "$base" ]; then + read -r X Y _ < <(blocks "$name-3-custom" $FIELD 40 400 | nth 1) || true + [ -n "$Y" ] || fail "$name: no base URL field" + click "$X" "$Y"; text "$base"; sleep 1 + fi + next_step "$name-3-next" + read -r X Y _ < <(blocks "$name-4-key" $FIELD 40 400 | nth 1) || true + [ -n "$Y" ] || fail "$name: no key field on step 4" + click "$X" "$Y"; text "$key"; sleep 1 + next_step "$name-4-next" + read -r X Y _ < <(blocks "$name-5-review" $TEST 40 400 | head -1) || true + [ -n "$Y" ] || fail "$name: no Test connection on step 5" + [ -z "$(blocks "$name-5-review" $BLUE 40 60 | awk '$3<400')" ] || fail "$name: Save is enabled before a test" + click "$X" "$Y" + for _ in $(seq 1 15); do + sleep 2 + read -r X Y _ < <(blocks "$name-5-tested" $BLUE 40 60 | awk '$3<400' | tail -1) || true + if [ -n "$Y" ]; then click "$X" "$Y"; SAVED=tested; break; fi + read -r X Y _ < <(in_png "$WORK/grabs/$name-5-tested.png" $BYPASS 40 100 | head -1) || true + if [ -n "$Y" ]; then click "$X" "$Y"; SAVED=untested; break; fi + done + [ -n "$Y" ] || fail "$name: the test neither passed nor offered Save without testing" + # The result line: green "Connected · N ms", or red "Couldn't connect: …". + if [ "$SAVED" = tested ]; then + [ -n "$(in_png "$WORK/grabs/$name-5-tested.png" 248a3d 1 100)" ] || fail "$name: no green Connected line (grab $name-5-tested)" + else + [ -n "$(in_png "$WORK/grabs/$name-5-tested.png" c4281c 1 100)" ] || fail "$name: no red Couldn't connect line (grab $name-5-tested)" + fi + for _ in $(seq 1 20); do [ "$(cat "$PROFILE" 2>/dev/null || true)" != "$before" ] && return 0; sleep 0.5; done + fail "$name: the save did not change the profile" +} +profile_json() { python3 -c 'import json,sys; print(json.dumps(json.load(open(sys.argv[1]))["config"].get(sys.argv[2]), sort_keys=True))' "$PROFILE" "$1"; } + +# W1. DeepSeek with the wizard; the provider is unreachable (sandbox). +add_model w1 deepseek 1 1 "$FAKE_KEY" +[ "$SAVED" = untested ] || fail "DeepSeek was not saved without testing" +wait_widget "Added DeepSeek · deepseek-v4-flash" +sheet_closes "llm.sheet.fetch_models" || fail "the wizard stayed up after the save" +[ "$(profile_json llm)" = '{"fallbacks": [], "primary": {"family_id": "deepseek", "model_id": "deepseek-v4-flash"}}' ] \ + || fail "profile after the wizard: $(profile_json llm)" +keys_end_with DEEPSEEK_API_KEY=1234 || fail "the DeepSeek key is not in the profile" +pass "DeepSeek added with the wizard: family search, model pull-down (DeepSeek V4 Flash, Recommended), route pull-down (Official API), key, Test connection failed (unreachable), Save without testing (grabs w1-1-family, w1-2-model, w1-3-route, w1-4-next, w1-5-review, w1-5-tested)" + +# W2. The card's model pull-down. +read -r X Y _ < <(blocks w2-list $PULLDOWN 40 400 | nth 1) || true +[ -n "$Y" ] || fail "no model pull-down on the DeepSeek card" +click "$X" "$Y"; sleep 2 +read -r X Y _ < <(blocks w2-pulldown $OPTION 40 400 | nth 2) || true +[ -n "$Y" ] || fail "the card's model pull-down has no second model" +click "$X" "$Y" +wait_widget "DeepSeek now uses DeepSeek V4 Pro." +[ "$(profile_json llm)" = '{"fallbacks": [], "primary": {"family_id": "deepseek", "model_id": "deepseek-v4-pro"}}' ] \ + || fail "profile after the model change: $(profile_json llm)" +pass "the card's model pull-down listed DeepSeek's catalog models and switched to DeepSeek V4 Pro (grab w2-pulldown)" + +# W3. Z.ai as a fallback: GLM-4.7 (the second model) on a custom +# OpenAI-compatible route (the route pull-down's option after Official API), +# the local fake endpoint: the test passes. +FAKE_PORT=$(head -1 "$WORK/fake.port") +[ -n "$FAKE_PORT" ] || fail "the fake endpoint did not start" +ZAI_BASE="http://127.0.0.1:$FAKE_PORT/v1" +add_model w3 "z.ai" 2 2 "$ZAI_KEY" "$ZAI_BASE" +[ "$SAVED" = tested ] || fail "Z.ai was not saved after a passing test" +wait_widget "Added Z.ai · glm-4.7" +sheet_closes "llm.sheet.fetch_models" || fail "the wizard stayed up after the Z.ai save" +[ "$(profile_json llm)" = '{"fallbacks": [{"family_id": "zai", "model_id": "glm-4.7", "route": {"api_type": "openai", "base_url": "'"$ZAI_BASE"'"}}], "primary": {"family_id": "deepseek", "model_id": "deepseek-v4-pro"}}' ] \ + || fail "profile after the Z.ai add: $(profile_json llm)" +keys_end_with DEEPSEEK_API_KEY=1234 ZAI_API_KEY=5678 || fail "the keys after the Z.ai add are wrong" +pass "Z.ai added as a fallback on GLM-4.7 and a custom OpenAI-compatible route; Test connection passed, Add as fallback saved it (grabs w3-2-model, w3-3-custom, w3-5-tested)" + +# W4. The list shows both; Test connection on the fallback's row passes; +# remove both with the cards' Remove (tap twice). +# Both cards in view: the list scrolled past the header. +top_app; scroll 548 400 220; sleep 1 +[ "$(blocks w4-saved-models $PULLDOWN 40 400 | wc -l | tr -d ' ')" = 2 ] || fail "the list does not show two model pull-downs (grab w4-saved-models)" +# Green text: the rows' key statuses, and "Connected" once the test passes. +GREEN=248a3d +read -r X Y _ < <(blocks w4-rows $TEST 40 60 | tail -1) || true +[ -n "$Y" ] || fail "no Test connection on the second row" +BEFORE=$(in_png "$WORK/grabs/w4-rows.png" $GREEN 1 60 | wc -l) +click "$X" "$Y" +for _ in $(seq 1 20); do sleep 1; [ "$(blocks w4-row-tested $GREEN 1 60 | wc -l)" -gt "$BEFORE" ] && break; done +[ "$(in_png "$WORK/grabs/w4-row-tested.png" $GREEN 1 60 | wc -l)" -gt "$BEFORE" ] || fail "the row's Test connection did not say Connected (grab w4-row-tested)" +for n in 1 2; do + top_app + read -r X Y _ < <(blocks "w4-remove-$n" $RED 1 30 | head -1) || true + [ -n "$Y" ] || fail "no Remove on the first card (grab w4-remove-$n)" + click "$X" "$Y"; sleep 1 + click "$X" "$Y"; sleep 2 +done +python3 -c 'import json,sys; sys.exit(0 if "llm" not in json.load(open(sys.argv[1]))["config"] else 1)' "$PROFILE" || fail "the list is not empty after removing both: $(profile_json llm)" +top_app; grab w4-empty >/dev/null +pass "the list showed DeepSeek V4 Pro (primary) and Z.ai GLM-4.7 (fallback 1); the fallback's Test connection said Connected; both removed with Remove, tapped twice (grabs w4-saved-models, w4-row-tested, w4-empty)" + +# 2. Import into the empty list by pasting an old single-provider code. +bottom_app; read -r X Y < <(widget "Import code from image"); click "$X" "$Y" +read -r SX SY SW SH < <(wait_sheet "llm.sheet.import") +sleep 1 +C=$(controls paste-sheet "$SX" "$SY" "$SW" "$SH") +CODE_Y=$(echo "$C" | nth_full 2); read -r PX PY < <(echo "$C" | pill) +[ -n "$CODE_Y" ] && [ -n "$PX" ] || { echo "$C"; fail "import sheet controls not found"; } +click $((SX + SW / 2)) "$CODE_Y"; text "$LEGACY_CODE"; click "$PX" "$PY" +wait_widget "Saved DeepSeek · deepseek-v4-flash as primary." +sheet_closes "llm.sheet.import" || fail "the import sheet stayed up after the paste" +[ "$(profile_routes)" = "deepseek-v4-flash:-" ] || fail "profile after the paste: $(profile_routes)" +keys_end_with DEEPSEEK_API_KEY=fcb0 || fail "the pasted key is not in the profile" +grab imported-by-paste >/dev/null +pass "an old single-provider code pasted into the empty list: DeepSeek deepseek-v4-flash is the primary (grab imported-by-paste)" + +# 3. Add OpenAI with a fake key on the host's sheet. +add_openai() { + add_model "$1" openai 0 0 "$FAKE_KEY" + [ "$SAVED" = untested ] || fail "OpenAI was not saved without testing" + wait_widget "Added OpenAI · gpt-4o" +} +add_openai add1 +[ "$(profile_families)" = "deepseek openai" ] || fail "profile after the add: $(profile_families)" +keys_end_with DEEPSEEK_API_KEY=fcb0 OPENAI_API_KEY=1234 || fail "OpenAI's key is not in the profile" +grab added-openai >/dev/null +pass "OpenAI added with the wizard (gpt-4o, official route), saved without testing (grab added-openai)" + +# 4. Import QR-A by drop onto the sheet: appended, no confirm step. +bottom_app; read -r X Y < <(widget "Import code from image"); click "$X" "$Y" +read -r SX SY SW SH < <(wait_sheet "llm.sheet.image") +sleep 1 +DROP=$(curl -s --get --data-urlencode "path=$FIXTURE" --data-urlencode "x=$((SX + SW / 2))" \ + --data-urlencode "y=$((SY + SH / 2))" --data-urlencode wait=1 "127.0.0.1:$PORT/drop") +case $DROP in *'"drop_handled":true'*'"drag_response":"copy"'*) ;; *) fail "the sheet did not take the drop: $DROP" ;; esac +wait_log "llm: image dropped on the import sheet" +C=$(pin_prompt drop-read "$SX" "$SY" "$SW" "$SH") || fail "the sheet did not ask for the PIN after the drop" +PIN_Y=$(echo "$C" | nth_full 3); read -r PX PY < <(echo "$C" | pill) +[ -n "$PIN_Y" ] && [ -n "$PX" ] || { echo "$C"; fail "import sheet controls not found"; } +click $((SX + SW / 2)) "$PIN_Y"; text "$FIXTURE_PIN"; click "$PX" "$PY" +wait_widget "Added DeepSeek · deepseek-chat, Z.ai · glm-4.6 as fallbacks." +sheet_closes "llm.sheet.image" || fail "the import sheet stayed up (a confirm step?)" +[ "$(profile_families)" = "deepseek openai deepseek zai" ] || fail "profile after the drop: $(profile_families)" +[ "$(profile_routes)" = "deepseek-v4-flash:- gpt-4o:- deepseek-chat:DEEPSEEK_2_API_KEY glm-4.6:-" ] || fail "routes after the drop: $(profile_routes)" +keys_end_with DEEPSEEK_API_KEY=fcb0 OPENAI_API_KEY=1234 DEEPSEEK_2_API_KEY=0000 ZAI_API_KEY=0000 \ + || fail "the keys after the drop are wrong (the primary's key must be untouched)" +grab imported-by-drop >/dev/null +pass "dropped QR-A appended as fallbacks without a confirm step: primary + OpenAI first, QR-A's DeepSeek in DEEPSEEK_2_API_KEY, the primary's key untouched (grab imported-by-drop)" + +# 5. Import QR-A again by the picker (test image), a wrong PIN first: the same +# code again changes nothing. +pick_and_unlock() { + local X Y C + bottom_app; read -r X Y < <(widget "Import code from image"); click "$X" "$Y" + read -r SX SY SW SH < <(wait_sheet "llm.sheet.pick") + sleep 1 + C=$(controls "$1-sheet" "$SX" "$SY" "$SW" "$SH") + CHOOSE_Y=$(echo "$C" | nth_full 1); PIN_Y=$(echo "$C" | nth_full 3); read -r PX PY < <(echo "$C" | pill) + click $((SX + SW / 2)) "$CHOOSE_Y" + pin_prompt "$1-picked" "$SX" "$SY" "$SW" "$SH" >/dev/null || fail "the sheet did not ask for the PIN after the pick" +} +pick_and_unlock pick1 +BEFORE=$(cat "$PROFILE") +click $((SX + SW / 2)) "$PIN_Y"; text "0000-0000"; click "$PX" "$PY"; sleep 4 +[ -n "$(sheet_rect "llm.sheet.pick")" ] || fail "a wrong PIN closed the sheet" +[ "$(cat "$PROFILE")" = "$BEFORE" ] || fail "a wrong PIN changed the profile" +grab wrong-pin >/dev/null +click $((SX + SW / 2)) "$PIN_Y" +for _ in 1 2 3 4 5 6 7 8 9; do key Backspace; done +text "$FIXTURE_PIN"; click "$PX" "$PY" +wait_widget "Already saved: DeepSeek · deepseek-chat, Z.ai · glm-4.6." +sheet_closes "llm.sheet.pick" || fail "the import sheet stayed up after the second import" +[ "$(cat "$PROFILE")" = "$BEFORE" ] || fail "importing QR-A again changed the profile" +grab reimported >/dev/null +pass "QR-A picked and imported again after a refused wrong PIN: nothing changed, no duplicates (grab reimported)" + +# 6. The phone QR: show the code, read it from the grab. +LOG_MARK=$(( $(wc -l <"$LOG") + 1 )) +bottom_app; read -r X Y < <(widget "Show QR for phone"); click "$X" "$Y" +read -r SX SY SW SH < <(wait_sheet "countdown :=") +SHOWN=$(date +%s) +sleep 1 +PIN=$(get "snap?q=$(q "pin := Label")" | python3 -c ' +import json,re,sys +for e in json.load(sys.stdin)["s"]: + m=re.search(r"pin := Label\{[^\n]*text: \"([A-Z0-9-]+)\"", e.get("t") or "") + if m: print(m.group(1))' | tail -1) +[ -n "$PIN" ] || fail "no PIN on the export sheet" +EXPORT=$(grab export 1) +cp "$EXPORT" "$WORK/export.png"; echo "$PIN" >"$WORK/export-pin.txt" +READ=$("$READ_QR" "$EXPORT" "$PIN" "$WORK/octos") || fail "the exported QR does not match the profile: $READ" +echo "$READ" | grep -q '"same_as_profile":true' || fail "$READ" +echo "$READ" | grep -q '"family":"openai"' || fail "OpenAI missing from the code: $READ" +echo "$READ" | grep -q '"DEEPSEEK_2_API_KEY":"••••0000"' || fail "the second DeepSeek slot is missing from the code: $READ" +echo "$READ" | grep -q '"DEEPSEEK_API_KEY":"••••fcb0"' || fail "the primary's key is missing from the code: $READ" +pass "export QR read back from the grab (rqrr) with PIN $PIN: same set and keys as _main.json, both DeepSeek slots included" +# The countdown ticks: two grabs of the sheet two seconds apart differ (only +# the "Expires in" line, below the PIN, changes on it: scroll down to it). +same_sheet() { + python3 - "$1" "$2" "$SX" "$SY" "$SW" "$SH" "$WORK/sheet.py" <<'PY' +import sys +exec(open(sys.argv[7]).read().split("path, sx, sy")[0]) +a, b = read_png(sys.argv[1]), read_png(sys.argv[2]) +sx, sy, sw, sh = map(int, map(float, sys.argv[3:7])) +rows = range(sy, min(sy + sh, a[1], b[1])) +diff = sum(a[3][y][sx * a[2]:(sx + sw) * a[2]] != b[3][y][sx * b[2]:(sx + sw) * b[2]] for y in rows) +sys.exit(0 if diff == 0 else 1) +PY +} +scroll $((SX + SW / 2)) $((SY + SH / 2)) 600; sleep 1 +T1=$(grab countdown-1); sleep 2; T2=$(grab countdown-2) +same_sheet "$T1" "$T2" && fail "the countdown does not tick (grabs countdown-1 and countdown-2 are the same)" +# At 0 the sheet closes itself: the service's own close comes 5 s after the +# lifetime (counted from before the code was sealed), so a sheet gone within +# the lifetime + 2 s closed itself. +for _ in $(seq 1 $((QR_SECONDS * 2 + 20))); do [ -z "$(sheet_rect "countdown :=")" ] && break; sleep 0.5; done +CLOSED=$(( $(date +%s) - SHOWN )) +[ -z "$(sheet_rect "countdown :=")" ] || fail "the export sheet did not close after $QR_SECONDS s" +[ "$CLOSED" -le $((QR_SECONDS + 2)) ] || fail "the export sheet closed after ${CLOSED} s, not by its own countdown ($QR_SECONDS s)" +grab export-closed >/dev/null +if tail -n +"$LOG_MARK" "$LOG" | grep -Eq "$SPLASH_ERRORS"; then + tail -n +"$LOG_MARK" "$LOG" | grep -E "$SPLASH_ERRORS" | head -5 + fail "Splash errors while the export sheet was up" +fi +pass "the countdown ticked and the sheet closed itself after ${CLOSED} s ($QR_SECONDS s lifetime); no Splash error during the export" + +# 7. No key text anywhere the UI or logs show; no panic. +get "snap?all=1" >"$WORK/snap.json"; get "d" >"$WORK/dump.txt"; get "log?n=5000" >"$WORK/remote-log.json" +if grep -Eq "$SECRETS" "$WORK/snap.json" "$WORK/dump.txt" "$WORK/remote-log.json" "$LOG"; then + grep -Eo ".{40}($SECRETS).{10}" "$WORK/snap.json" "$WORK/dump.txt" "$WORK/remote-log.json" "$LOG" | head -5 + fail "key text in the UI tree or a log" +fi +grep -q "panicked" "$LOG" "$WORK/remote-log.json" && fail "panic in the log" +if grep -Eq "$SPLASH_ERRORS" "$LOG"; then grep -E "$SPLASH_ERRORS" "$LOG" | head -5; fail "Splash errors in the host log"; fi +pass "no key text in /snap, /d, /log or the host log; no panic, no Splash error" + +get "gq?scale=0.5" >/dev/null +for _ in $(seq 1 30); do kill -0 "$PID" 2>/dev/null || break; sleep 0.5; done +kill -0 "$PID" 2>/dev/null && fail "the shell did not exit after /gq" +kill "$FAKE" 2>/dev/null || true +trap - EXIT +pass "shell exited after /gq; grabs in $WORK/grabs, export.png + export-pin.txt in $WORK" diff --git a/scripts/system_apps_remote.sh b/scripts/system_apps_remote.sh new file mode 100755 index 00000000..b4e4f198 --- /dev/null +++ b/scripts/system_apps_remote.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# The system apps on the desktop, driven through Makepad's remote +# (instrument) mode: no OS screenshots, only the app's own frames. +# +# cargo build --release [--config …] && scripts/system_apps_remote.sh [artifacts-dir] +# +# Runs the release binary hidden, with its own OCTOSENSE_HOME, octos core dir +# and file vaults under a temp directory, so neither ~/.octosense nor the login +# keychain is touched. Checks: the launcher lists every selected system app; +# Photos opens and draws (no network needed); Mail opens and its host-owned +# sign-in sheet appears (nobody signs in); each app runs in an isolate under +# its manifest's policy; bundles unpack under /apps/.system//; +# AI providers (when selected) opens on its empty provider list; the log has +# no panic and no UI hang outside the known ones. Grabs are kept as evidence. +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +BIN=${OCTOSENSE_BIN:-$ROOT/target/release/octosense} +WORK=${1:-$(mktemp -d -t octosense-system-apps)} +mkdir -p "$WORK/home" "$WORK/octos" "$WORK/grabs" +LOG=$WORK/host.log +APPS=$(python3 -c 'import json,sys; print(" ".join(json.load(open(sys.argv[1]))["apps"]))' "$ROOT/system-apps.json") +echo "artifacts: $WORK" +echo "system apps: $APPS" + +# From the temp directory: Maps' tile cache is written relative to the +# working directory (./local/tilecache_v4), not under the app's jail. +cd "$WORK" +env -u MAKEPAD_HOME -u MAKEPAD_WM_ROOT -u MAKEPAD_WM_THEME \ + OCTOSENSE_HOME="$WORK/home" OCTOS_APP_CORE_DIR="$WORK/octos" \ + OCTOSENSE_MAIL_VAULT=file OCTOSENSE_LLM_VAULT=file \ + MAKEPAD_HIDE_WINDOWS=1 "$BIN" --remote >"$LOG" 2>&1 & +PID=$! +PORT= +cleanup() { + if kill -0 "$PID" 2>/dev/null; then + [ -n "$PORT" ] && curl -s "127.0.0.1:$PORT/quit" >/dev/null || true + sleep 2 + kill "$PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +fail() { echo "FAIL: $*"; exit 1; } +pass() { echo "PASS: $*"; } +for _ in $(seq 1 120); do + PORT=$(sed -n 's/.*\[makepad-remote\] listening on 127\.0\.0\.1:\([0-9]*\) pid=.*/\1/p' "$LOG" | head -1) + [ -n "$PORT" ] && break + sleep 0.5 +done +[ -n "$PORT" ] || fail "no remote port in $LOG" +get() { curl -fsS "127.0.0.1:$PORT/$1"; } +# Input waits for the next frame; while a bundle unpacks on the UI thread the +# wait can time out after the key was delivered. What a key did is checked +# through the log and snapshots, so a timed-out wait is only noted. +key() { + local out + out=$(curl -s "127.0.0.1:$PORT/k?k=press&c=$1&wait=1${2:-}") + case $out in *'"err"'*) echo "note: key $1: $out" ;; esac +} +grab() { + local png + for _ in 1 2 3 4 5; do + # A hidden window's grab can miss its frame ("retry"): ask again. + png=$(curl -s "127.0.0.1:$PORT/g?scale=0.5" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("png",""))' || true) + [ -n "$png" ] && { cp "$png" "$WORK/grabs/$1.png"; return; } + sleep 1 + done + fail "no frame for $1" +} +# `/snap` texts, one per line, for a query. +snap() { get "snap?q=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$1")" \ + | python3 -c 'import json,sys; [print(e["i"], e["ty"]) for e in json.load(sys.stdin)["s"]]'; } +wait_log() { + for _ in $(seq 1 ${2:-40}); do grep -q "$1" "$LOG" && return 0; sleep 0.5; done + fail "log never said: $1" +} +# Type into the Start menu's search (KeyDown characters, as the smoke test does). +type_keys() { + local text=$1 i c + for ((i = 0; i < ${#text}; i++)); do + c=${text:i:1} + case $c in + " ") key Space ;; + -) key Minus ;; + *) key "Key$(printf %s "$c" | tr a-z A-Z)" ;; + esac + done +} +launch() { key Space "&cmd=1"; type_keys "$1"; key Return; } +label_of() { # the manifest name of a selected app + python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["name"])' \ + "$ROOT/$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["source"])' "$ROOT/system-apps.json")/$1/bundle/manifest.json" +} + +for _ in $(seq 1 60); do curl -s "127.0.0.1:$PORT/snap?q=main_window" | grep -q '"s":\[{' && break; sleep 0.5; done +pass "desktop up on port $PORT (pid $PID)" + +# 1. The launcher lists each system app (its short id, its manifest name). +for app in $APPS; do + key Space "&cmd=1" + type_keys "$app" + grab "launcher-$app" + key Escape +done +python3 - "$WORK/grabs" $APPS <<'EOF' || fail "launcher grabs missing" +import sys, pathlib +d = pathlib.Path(sys.argv[1]) +assert all((d / f"launcher-{a}.png").stat().st_size > 10000 for a in sys.argv[2:]) +EOF +grep -q 'wm: modules linked: .*"card"' "$LOG" || fail "the Card runner is not linked" +pass "launcher searched for: $APPS (grabs launcher-*.png); Card runner linked" + +# 2. Photos: opens in the Card runner and draws its library. +launch photos +wait_log "card: os.photos running under" +for _ in $(seq 1 20); do snap "Collections" | grep -q "^card Splash" && break; sleep 0.5; done +snap "Collections" | grep -q "^card Splash" || fail "Photos drew nothing" +grab photos +pass "Photos opened: $(grep -o 'card: os.photos running under.*' "$LOG" | head -1)" + +# 3. Mail: opens, and its sign-in is the host's sheet, not the app's. +launch mail +wait_log "card: os.mail running under" +sleep 2 +grab mail-welcome +snap "Add account" | grep -q "^card Splash" || fail "Mail shows no Add account" +# The button is the Splash's; click its centre as drawn in the welcome pane. +read -r X Y < <(get "snap?q=card" | python3 -c ' +import json,sys +cards=[e for e in json.load(sys.stdin)["s"] if e["i"]=="card" and "account = nil" in (e.get("t") or "")] +x,y,w,h=cards[-1]["r"]; print(x+w/2, y+h*0.567)') +get "click?x=$X&y=$Y&wait=1" >/dev/null +for _ in $(seq 1 20); do snap "Add a mail account" | grep -q "^sheet Splash" && break; sleep 0.5; done +snap "Add a mail account" | grep -q "^sheet Splash" || fail "Mail's sign-in sheet did not open" +grab mail-sheet +pass "Mail's host-owned sign-in sheet is up (not signed in)" + +# 4. AI providers, when this build selects it: opens on an empty list. +if [[ " $APPS " == *" ai-providers "* ]]; then + key Escape + launch "ai providers" + sleep 3 + if grep -q 'os.ai-providers requests unknown capability' "$LOG"; then + fail "App Hub's policy at this pin does not know the \`llm\` capability; AI providers is refused (its notice is drawn)" + fi + wait_log "card: os.ai-providers running under" + sleep 2 + grab ai-providers + snap "No models yet" | grep -q "^card Splash" || fail "AI providers shows no empty list" + NAME=$(label_of ai-providers) + [ -z "$(ls -A "$WORK/octos" 2>/dev/null)" ] || echo "note: octos core dir is not empty" + pass "$NAME opened: $(grep -o 'card: os.ai-providers running under.*' "$LOG" | head -1)" +fi + +# 5. Containment: one isolate per app under its policy, `os.` ids, bundles +# unpacked outside every jail under the temp home. +for app in photos mail; do + [ -f "$WORK/home/apps/.system/os.$app/"*/manifest.json ] || fail "os.$app not unpacked under $WORK/home/apps/.system" + [ -d "$WORK/home/apps/os.$app" ] || fail "os.$app has no jail" +done +pass "bundles unpacked under /apps/.system///, jails at /apps/" + +# 6. The log: no panic; UI hangs only where known (see KNOWN below). +get "log?n=2000" >"$WORK/remote-log.json" +grep -q "panicked" "$LOG" "$WORK/remote-log.json" && fail "panic in the log" +# Known: the catalog's startup Makepad-source probe, and App Hub unpacking and +# hashing a system bundle, or building an app's Splash VM, on the UI thread +# the first time it opens. +KNOWN='makepad_source::makepad_root|octosense_appstore::system::prepare|CxSplashVmExt>::alloc_splash_vm' +UNKNOWN=$(grep "\[ui-hang\]" "$LOG" | grep -Ev "$KNOWN" || true) +[ -z "$UNKNOWN" ] || { echo "$UNKNOWN" | cut -c1-400; fail "unexpected UI hang"; } +echo "note: $(grep -c "\[ui-hang\]" "$LOG" || true) known UI hang(s) (startup catalog probe, first-open bundle unpack or VM)" +pass "no panic, no unexpected [ui-hang]" + +# Finish: grab and quit, then confirm the process is gone. +get "gq?scale=0.5" >/dev/null +for _ in $(seq 1 30); do kill -0 "$PID" 2>/dev/null || break; sleep 0.5; done +kill -0 "$PID" 2>/dev/null && fail "the shell did not exit after /gq" +trap - EXIT +pass "shell exited after /gq; grabs in $WORK/grabs" diff --git a/src/apps.rs b/src/apps.rs index 385f924a..30ac4244 100644 --- a/src/apps.rs +++ b/src/apps.rs @@ -127,10 +127,21 @@ pub fn system_card_apps() -> Vec { Vec::new() } -/// The services contained apps call through `host.request` (ADR 0004): `mail` -/// keeps accounts and passwords for the Mail app. `mail_demo` in -/// MAKEPAD_APP_CONFIG serves a demo mailbox from a file vault instead (no -/// keychain, no network): `MAKEPAD_APP_CONFIG='{"mail_demo":true}'`. +/// The services contained apps call through `host.request` (ADR 0004), +/// registered once, before the first system app can open: +/// +/// - `mail` keeps accounts and passwords for the Mail app. `mail_demo` in +/// MAKEPAD_APP_CONFIG serves a demo mailbox from a file vault instead (no +/// keychain, no network): `MAKEPAD_APP_CONFIG='{"mail_demo":true}'`. +/// - `llm` keeps the assistant's LLM providers for AI providers +/// (`os.ai-providers`), written to the AppCard kernel's octos profile +/// (`OCTOS_APP_CORE_DIR`, else `~/octos-home/.octos`), keys in the keychain +/// entry octos reads (`OCTOSENSE_LLM_VAULT=file` keeps them in the +/// owner-only profile). No camera scanner: a phone's profile QR is imported +/// from a picture of it (the open panel, or an image dropped on the import +/// sheet: `llm_image`), or by pasting its text. With AppCard linked +/// (`app-appcard`), a change restarts its core so the kernel reads the new +/// providers; without it there is nothing to restart. #[cfg(feature = "app-hub")] pub fn register_host_services() { static ONCE: std::sync::Once = std::sync::Once::new(); @@ -145,9 +156,36 @@ pub fn register_host_services() { } else { octosense_mail_service::register() } + let mut llm = octosense_llm_service::Options::default(); + if let Some(dir) = octosense_llm_config::profile::default_core_dir() { + llm = llm.core_dir(dir); + } + #[cfg(not(any(target_os = "android", target_os = "ios")))] + { + llm = llm.image_picker(crate::llm_image::picker()).image_drops(true); + } + #[cfg(feature = "app-appcard")] + { + llm = llm.on_changed(|| { + RESTART_APPCARD.store(true, std::sync::atomic::Ordering::Release); + makepad_widgets::makepad_platform::thread::SignalToUI::set_ui_signal(); + }); + } + octosense_llm_service::register_with(llm); }); } +/// Set (any thread) when AI providers changed the saved provider set. +#[cfg(all(feature = "app-hub", feature = "app-appcard"))] +static RESTART_APPCARD: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// UI thread: whether the AppCard core should restart to read a changed +/// provider set (once per change). +#[cfg(all(feature = "app-hub", feature = "app-appcard"))] +pub fn take_appcard_restart() -> bool { + RESTART_APPCARD.swap(false, std::sync::atomic::Ordering::AcqRel) +} + /// Apps App Hub installed: each is an app of its own in the launcher, hosted /// by the linked `card` module under its `hub:` identity. Read /// fresh each time, so an install shows up without a restart. @@ -344,16 +382,16 @@ mod tests { let catalog = bundled_catalog(); // AppCard is opt-in (`app-appcard`), not part of `mobile-apps`. let expected: &[&str] = if cfg!(feature = "app-appcard") { - &["reference", "sheets", "appcard", "apphub", "news", "photos", "maps", "camera", "mail"] + &["reference", "sheets", "appcard", "apphub", "news", "photos", "maps", "camera", "mail", "ai-providers"] } else { - &["reference", "sheets", "apphub", "news", "photos", "maps", "camera", "mail"] + &["reference", "sheets", "apphub", "news", "photos", "maps", "camera", "mail", "ai-providers"] }; assert_eq!(catalog.iter().map(|app| app.id.as_str()).collect::>(), expected); assert!(catalog.iter().all(|app| app.manifest.is_none())); // The system apps have no native module: the Card runner hosts them, // launched by their manifest id (ADR 0004). let registry = AppRegistry::default(); - for id in ["news", "photos", "maps", "camera", "mail"] { + for id in ["news", "photos", "maps", "camera", "mail", "ai-providers"] { let app = catalog.iter().find(|app| app.id == id).unwrap(); assert_eq!(card_manifest_id(app), Some(format!("os.{id}").as_str())); assert_eq!(registry.module(id).map(|m| m.id()), Some("card")); @@ -405,7 +443,7 @@ mod tests { #[test] fn the_system_apps_ship_as_card_apps() { let ids: Vec = system_card_apps().into_iter().map(|app| app.id).collect(); - assert_eq!(ids, ["news", "photos", "maps", "camera", "mail"]); + assert_eq!(ids, ["news", "photos", "maps", "camera", "mail", "ai-providers"]); let registry = AppRegistry::default(); for id in &ids { assert_eq!(registry.hosting(id), Hosting::Module); diff --git a/src/llm_image.rs b/src/llm_image.rs new file mode 100644 index 00000000..70da8e21 --- /dev/null +++ b/src/llm_image.rs @@ -0,0 +1,155 @@ +//! The desktop's image source for AI providers' QR import (the `llm` host +//! service's `QrImagePicker`, OctoSense-System-Apps +//! `apps/ai-providers/host-service`): Makepad's native open panel filtered to +//! PNG and JPEG, and image files dropped on the AI providers window while +//! its import sheet is up. Either way the service gets the file's bytes and +//! reads the code out of them on a worker; the sheet then asks for the PIN. +//! +//! Test hook: `OCTOSENSE_LLM_TEST_IMAGE=` makes "Choose image" answer +//! with that file at once, without a panel (the remote e2e run is hidden and +//! cannot click through a modal). Development only. +use makepad_widgets::makepad_platform::file_dialogs::{FileDialog, FileDialogAction}; +use makepad_widgets::makepad_platform::thread::SignalToUI; +use makepad_widgets::*; +use octosense_llm_service::{ImageDone, PickError, QrImagePicker}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +/// The system app whose window takes dropped images (its short id). +const APP: &str = "ai-providers"; +/// Larger files are not read at all; the service refuses over 20 MB anyway. +const MAX_FILE_BYTES: u64 = 20 * 1024 * 1024; +const TEST_IMAGE_ENV: &str = "OCTOSENSE_LLM_TEST_IMAGE"; + +fn dialog_id() -> LiveId { + live_id!(octosense_llm_qr_image) +} + +#[derive(Default)] +pub struct DesktopPicker { + /// The pick waiting on the open panel. + pending: Mutex>, + /// The panel should open on the next UI event. + open: AtomicBool, +} + +impl QrImagePicker for DesktopPicker { + fn pick(&self, done: ImageDone) { + if let Some(path) = std::env::var_os(TEST_IMAGE_ENV).filter(|p| !p.is_empty()) { + let path = PathBuf::from(path); + std::thread::spawn(move || done(read_image(&path))); + return; + } + if let Some(earlier) = self.pending.lock().unwrap().replace(done) { + earlier(Err(PickError::Cancelled)); + } + self.open.store(true, Ordering::SeqCst); + SignalToUI::set_ui_signal(); + } +} + +/// The one picker the service is registered with. +pub fn picker() -> Arc { + static PICKER: OnceLock> = OnceLock::new(); + PICKER.get_or_init(Arc::default).clone() +} + +fn is_image(path: &Path) -> bool { + matches!( + path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref(), + Some("png" | "jpg" | "jpeg") + ) +} + +/// A chosen or dropped file's bytes, off the UI thread. +fn read_image(path: &Path) -> Result, PickError> { + let size = std::fs::metadata(path).map_err(|e| PickError::Failed(e.kind().to_string()))?.len(); + if size > MAX_FILE_BYTES { + return Err(PickError::Failed("larger than 20 MB".into())); + } + std::fs::read(path).map_err(|e| PickError::Failed(e.kind().to_string())) +} + +/// On every UI event: open the panel a pick asked for. +pub fn open_requested(cx: &mut Cx) { + let picker = picker(); + if picker.open.swap(false, Ordering::SeqCst) { + cx.open_select_file_dialog( + FileDialog::new() + .set_id(dialog_id()) + .set_title("Choose a picture of the provider code".into()) + .add_filter("Images".into(), vec!["png".into(), "jpg".into(), "jpeg".into()]), + ); + } +} + +/// The panel's answer, from the actions pass. +pub fn handle_actions(actions: &Actions) { + for action in actions { + let Some(answer) = action.downcast_ref::() else { + continue; + }; + let result = match answer { + FileDialogAction::FileSelected { id, paths } if *id == dialog_id() => paths.first().cloned().ok_or(PickError::Cancelled), + FileDialogAction::FileCancelled { id } if *id == dialog_id() => Err(PickError::Cancelled), + _ => continue, + }; + let Some(done) = picker().pending.lock().unwrap().take() else { + continue; + }; + match result { + Ok(path) if is_image(&path) => { + std::thread::spawn(move || done(read_image(&path))); + } + Ok(_) => done(Err(PickError::Failed("not a PNG or JPEG file".into()))), + Err(e) => done(Err(e)), + } + } +} + +/// The one image file a drag or drop carries. +fn dropped_image(items: &[DragItem]) -> Option { + match items { + [DragItem::FilePath { path, .. }] if is_image(Path::new(path)) => Some(PathBuf::from(path)), + _ => None, + } +} + +/// A drag or drop of one image file over the AI providers window while its +/// import sheet waits for one: answered "copy", and on the drop the file goes +/// to the service. `app_at` names the app whose window is at a point. `true` +/// when the event was the service's. +pub fn handle_drop(event: &Event, app_at: impl Fn(Vec2d) -> Option) -> bool { + let over_import = |abs: Vec2d, items: &[DragItem]| { + let path = dropped_image(items)?; + (octosense_llm_service::wants_image() && app_at(abs).as_deref() == Some(APP)).then_some(path) + }; + match event { + Event::Drag(e) => { + if over_import(e.abs, &e.items).is_some() { + *e.response.lock().unwrap() = DragResponse::Copy; + *e.handled.lock().unwrap() = true; + return true; + } + false + } + Event::Drop(e) => { + let Some(path) = over_import(e.abs, &e.items) else { + return false; + }; + *e.handled.lock().unwrap() = true; + std::thread::spawn(move || { + // A file that cannot be read is offered as nothing: the sheet + // says it is not an image. + let bytes = read_image(&path).unwrap_or_default(); + if !octosense_llm_service::offer_image(bytes) { + log!("llm: the import sheet closed before the dropped image was read"); + } + }); + log!("llm: image dropped on the import sheet"); + true + } + _ => false, + } +} diff --git a/src/main.rs b/src/main.rs index ae5e617d..0c17f042 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,6 +30,8 @@ mod dock_warp; mod host; mod hub; mod layout; +#[cfg(all(feature = "app-hub", not(any(target_os = "android", target_os = "ios"))))] +mod llm_image; mod octosense; mod module_host; mod module_view; @@ -1396,6 +1398,28 @@ impl App { self.redraw_all(cx); } + /// The provider set changed (AI providers): end every AppCard module + /// instance, so its agent and its kernel child stop, and open AppCard + /// again when one was running: its new kernel reads the new profile. + #[cfg(all(feature = "app-hub", feature = "app-appcard"))] + fn restart_appcard_core(&mut self, cx: &mut Cx) { + let mut ended = 0; + while let Some(client) = self.module_host.client_of_module("appcard") { + log!("llm: providers changed: restarting the AppCard core (client {client})"); + self.request_close(cx, client); + ended += 1; + if ended >= 8 || self.module_host.client_of_module("appcard") == Some(client) { + break; + } + } + if ended == 0 { + log!("llm: providers changed; no AppCard core is running, the next one reads them"); + } else { + self.launch_app(cx, "appcard"); + } + self.redraw_all(cx); + } + fn request_close(&mut self, cx: &mut Cx, client: ClientId) { // A module instance has no process to ask politely and nothing to // reap later: it ends now, through the same removal as a death. @@ -4147,6 +4171,8 @@ impl MatchEvent for App { } fn handle_actions(&mut self, cx: &mut Cx, actions: &Actions) { + #[cfg(all(feature = "app-hub", not(any(target_os = "android", target_os = "ios"))))] + llm_image::handle_actions(actions); if self.gallery { let gallery = self.ui.widget(cx, ids!(shell_gallery)); { @@ -4461,6 +4487,29 @@ impl AppMain for App { { return; } + // AI providers' QR import: the open panel a pick asked for, and an + // image dropped on its window while the import sheet waits. + #[cfg(all(feature = "app-hub", not(any(target_os = "android", target_os = "ios"))))] + if self.state.is_some() { + llm_image::open_requested(cx); + if matches!(event, Event::Drag(_) | Event::Drop(_)) { + let desk = self.desk(cx); + let desk = desk.borrow::(); + let state = self.state.as_ref(); + let app_at = |p: Vec2d| { + let client = desk.as_ref()?.window_at(p)?; + state?.clients.get(&client).map(|slot| slot.app.clone()) + }; + if llm_image::handle_drop(event, app_at) { + return; + } + } + } + // AI providers changed the saved providers: restart the AppCard core. + #[cfg(all(feature = "app-hub", feature = "app-appcard"))] + if apps::take_appcard_restart() { + self.restart_appcard_core(cx); + } if self.phone_search_event(cx,event) {return;} if self.state.is_some() && self.phone_pointer(cx,event) {return;} if self.state.is_some() && self.snap_event(cx,event) {return;} diff --git a/src/shell/menu.rs b/src/shell/menu.rs index df6bcbf6..2b293852 100644 --- a/src/shell/menu.rs +++ b/src/shell/menu.rs @@ -482,6 +482,18 @@ impl MenuModel { ] {items.push(MenuItem::new(id,label,kind).icon(icon));} } let available = launcher::apps(); + // Settings gains a submenu once the build ships the AI providers + // system app (`os.ai-providers`): Appearance, then AI providers. + if path.starts_with("start") && available.iter().any(|app| app.id == AI_PROVIDERS) { + items.push( + MenuItem::new("start.settings.style", "Appearance", MenuKind::Menu) + .icon(Ico::Monitor), + ); + items.push( + MenuItem::new("start.settings.ai-providers", "AI providers", MenuKind::App) + .icon(Ico::Cpu), + ); + } items.retain(|item| { let app = match item.id.as_str() { "workspace.files" | "start.documents" => "apps.files", @@ -727,7 +739,8 @@ impl MenuModel { MenuKind::Menu => { let target=match row.target.as_str() { "start.programs"|"start.search"|"start.run"|"workspace.apps"=>"apps", - "start.settings"=>"style", + "start.settings" if !self.items.iter().any(|item| item.id=="start.settings.ai-providers")=>"style", + "start.settings.style"=>"style", "workspace.desktop"=>"desktop", other=>other, }; @@ -739,12 +752,16 @@ impl MenuModel { "workspace.files" => "apps.files".into(), "workspace.tools.terminal" => "apps.terminal".into(), "workspace.tools.task" => "apps.task".into(), + "start.settings.ai-providers" => AI_PROVIDERS.into(), _ => row.target, }), } } } +/// The launcher row of the AI providers system app (`os.ai-providers`). +const AI_PROVIDERS: &str = "apps.ai-providers"; + /// `MenuModel.js` `matchesQuery` + `searchScore`, lower is better. /// /// Every whitespace term has to match the label, the leaf id or an alias by @@ -1588,6 +1605,29 @@ mod tests { assert_eq!(centered_card_top(smaller, 280.0, 8.0, Some(300.0)), 138.0); } + /// Start > Settings is Appearance alone until the build ships the AI + /// providers system app; then it is a submenu that also opens it. + #[test] + fn start_settings_offers_ai_providers_once_it_ships() { + let mut model = MenuModel::default(); + model.open_at("", MenuSkin::Menu); + model.descend("start"); + let ships = launcher::apps().iter().any(|app| app.id == AI_PROVIDERS); + model.sel = model.rows.iter().position(|r| r.target == "start.settings").unwrap(); + assert_eq!(model.activate(), None); + if ships { + assert_eq!(model.path, "start.settings"); + model.sel = model + .rows + .iter() + .position(|r| r.target == "start.settings.ai-providers") + .unwrap(); + assert_eq!(model.activate().as_deref(), Some(AI_PROVIDERS)); + } else { + assert_eq!(model.path, "style"); + } + } + #[test] fn navigating_between_menus_recenters_after_a_search() { let mut model = MenuModel::default(); diff --git a/system-apps.json b/system-apps.json index 390574e0..933675f8 100644 --- a/system-apps.json +++ b/system-apps.json @@ -1,6 +1,6 @@ { "schema": 1, "source": "../OctoSense-System-Apps/apps", - "apps": ["news", "photos", "maps", "camera", "mail"], + "apps": ["news", "photos", "maps", "camera", "mail", "ai-providers"], "assets": {} }