From fd41ca4867ccb991ed15cc34d7cc54463d1ece7b Mon Sep 17 00:00:00 2001 From: ymote <151983+ymote@users.noreply.github.com> Date: Sat, 26 Sep 2026 17:04:22 -0700 Subject: [PATCH 1/2] AI providers system app and the llm host service Link the AI providers system app from the pinned OctoSense-System-Apps (c0f2199c, which carries #1: the AI providers app, the octosense-llm-service `llm` host service and octosense-llm-config) on App Hub 3e993d4c (which has the `llm` capability), the rev both the Mail and llm services pin: one App Hub in the graph, no alias. - system-apps.json selects ai-providers. - The `app-hub` feature links octosense-llm-service and -config; register_host_services registers `llm` next to Mail with the AppCard kernel's octos home as its core dir, the desktop image picker (the open panel, filtered to PNG and JPEG) and image drops on the import sheet (src/llm_image.rs; OCTOSENSE_LLM_TEST_IMAGE answers a pick for hidden test runs). - With app-appcard linked, a change to the provider set restarts the running AppCard module so its kernel reads the new profile; without it nothing is registered as a change hook. - Start > Settings becomes Appearance plus AI providers once the build ships the app. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 56 ++++++++++++++--- Cargo.toml | 13 ++-- README.md | 15 +++-- README.zh-CN.md | 13 ++-- src/apps.rs | 54 +++++++++++++--- src/llm_image.rs | 155 ++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 49 +++++++++++++++ src/shell/menu.rs | 42 ++++++++++++- system-apps.json | 2 +- 9 files changed, 365 insertions(+), 34 deletions(-) create mode 100644 src/llm_image.rs diff --git a/Cargo.lock b/Cargo.lock index 36b7d6be..30cba497 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2818,8 +2818,8 @@ dependencies = [ "libc", "log", "rustversion", - "windows-link 0.1.3", - "windows-result 0.3.4", + "windows-link 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "windows-result 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4049,7 +4049,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", + "webpki-roots 1.0.9", ] [[package]] @@ -4116,7 +4116,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.2", + "windows-core 0.62.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4751,7 +4751,7 @@ dependencies = [ "tokio", "tokio-rustls", "url", - "webpki-roots", + "webpki-roots 1.0.9", ] [[package]] @@ -7164,6 +7164,8 @@ dependencies = [ "makepad-wm-theme", "octosense-app-hub-app", "octosense-appcard", + "octosense-llm-config", + "octosense-llm-service", "octosense-mail-service", "octosense-reference", "rinx", @@ -7182,7 +7184,7 @@ dependencies = [ "rand_core 0.9.5", "serde", "serde_json", - "ureq", + "ureq 3.4.2", ] [[package]] @@ -7249,6 +7251,18 @@ dependencies = [ "url", ] +[[package]] +name = "octosense-llm-service" +version = "0.1.0" +dependencies = [ + "image", + "octosense-appstore", + "octosense-llm-config", + "rqrr", + "serde_json", + "ureq 2.12.1", +] + [[package]] name = "octosense-mail-service" version = "0.1.0" @@ -7265,7 +7279,7 @@ dependencies = [ "scraper", "serde_json", "sha2 0.10.9", - "webpki-roots", + "webpki-roots 1.0.9", ] [[package]] @@ -8286,7 +8300,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams 0.4.2", "web-sys", - "webpki-roots", + "webpki-roots 1.0.9", ] [[package]] @@ -10954,6 +10968,21 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "log", + "once_cell 1.21.4", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "ureq" version = "3.4.2" @@ -10968,7 +10997,7 @@ dependencies = [ "rustls-pki-types", "ureq-proto", "utf8-zero", - "webpki-roots", + "webpki-roots 1.0.9", ] [[package]] @@ -11361,6 +11390,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + [[package]] name = "webpki-roots" version = "1.0.9" diff --git a/Cargo.toml b/Cargo.toml index c4c9d059..4a06e61c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,8 +65,8 @@ octosense-appcard = { path = "apps/appcard", optional = true } # module (`card`) that runs contained script apps, the system apps packed at # build time from the selection `OCTOSENSE_SYSTEM_APPS` names # (system-apps.json, .cargo/config.toml), installed apps and their icons. -# Same rev as the Mail host service's octosense-appstore (OctoSense-System-Apps): -# one App Hub source, so one host-service registry. +# Same rev as the Mail and llm host services' octosense-appstore +# (OctoSense-System-Apps): one App Hub source, so one host-service registry. octosense-app-hub-app = { git = "https://github.com/OctoSense-org/OctoSense-App-Hub", rev = "3e993d4c596560c42f011b2fd00bd40dbcac7cc5", optional = true } # The `mail` host service the Mail system app calls through `host.request`: # accounts, message lists and send run here, with the passwords the host @@ -74,6 +74,11 @@ octosense-app-hub-app = { git = "https://github.com/OctoSense-org/OctoSense-App- # OctoSense-System-Apps checkout beside this repository # (native-apps.lock.json, tools/setup-native.py). octosense-mail-service = { path = "../OctoSense-System-Apps/apps/mail/host-service", optional = true } +# The `llm` host service the AI providers system app calls: the assistant's +# LLM providers, their keys (macOS keychain) and the phone QR, on host-owned +# sheets. The config crate names the AppCard kernel's octos home it writes. +octosense-llm-service = { path = "../OctoSense-System-Apps/apps/ai-providers/host-service", optional = true } +octosense-llm-config = { path = "../OctoSense-System-Apps/apps/ai-providers/config", optional = true } # Rinx (the Matrix client with Moments and articles) as an in-process module. # Built without its standalone window entry point. rinx = { git = "https://github.com/upstreamlabs/Rinx", rev = "5ad2c1739e5b4776588518ab76440a5367c9968e", default-features = false, features = ["octosense-module"], optional = true } @@ -90,7 +95,7 @@ makepad-sheets = { git = "https://github.com/OctoSense-org/makepad.git", rev = " [features] # App Hub is part of the standard launcher: the store, the Card runner and -# the system apps (News, Photos, Maps, Camera, Mail) it runs. +# the system apps (News, Photos, Maps, Camera, Mail, AI providers) it runs. default = ["app-hub"] # Also usable on desktop for testing the same embedded apps with --module. # Photos is the system app here; `app-photos` links Makepad's native Photos @@ -98,7 +103,7 @@ default = ["app-hub"] # AppCard is not among them: it ships only with `--features app-appcard`. mobile-apps = ["app-reference", "app-sheets", "app-hub"] # App Hub and the host services its system apps call. -app-hub = ["dep:octosense-app-hub-app", "dep:octosense-mail-service"] +app-hub = ["dep:octosense-app-hub-app", "dep:octosense-mail-service", "dep:octosense-llm-service", "dep:octosense-llm-config"] app-reference = ["dep:octosense-reference"] # Sheets linked in as a module (hosting is still Process unless switched). app-sheets = ["dep:makepad-sheets"] diff --git a/README.md b/README.md index 18ece22b..07a74040 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ OctoSense-Desktop is the desktop shell of [OctoSense](https://github.com/OctoSen | Repository | Role for this repo | | --- | --- | | [OctoSense-ROM](https://github.com/OctoSense-org/OctoSense-ROM) | The phone shell (`home/`). Same app model, same runtime, same system apps. | -| [OctoSense-System-Apps](https://github.com/OctoSense-org/OctoSense-System-Apps) | News, Photos, Maps, Camera and Mail bundles, the Mail host service and the AppCard assistant (`octos-app`). Pinned sibling checkout. | +| [OctoSense-System-Apps](https://github.com/OctoSense-org/OctoSense-System-Apps) | News, Photos, Maps, Camera, Mail and AI providers bundles, the Mail and `llm` host services and the AppCard assistant (`octos-app`). Pinned sibling checkout. | | [OctoSense-App-Hub](https://github.com/OctoSense-org/OctoSense-App-Hub) | The signed catalog, the store and the Card runner. Linked as the Git crate `octosense-app-hub-app`. | | [OctoScript-App-Design-Flow](https://github.com/OctoSense-org/OctoScript-App-Design-Flow) | Where apps are designed, built and published to the App Hub. | | [OctoScript-Makepad](https://github.com/OctoSense-org/OctoScript-Makepad) | The runtime release that pins Makepad and OctoScript. Pinned sibling checkout. | @@ -33,7 +33,7 @@ OctoSense-Desktop is the desktop shell of [OctoSense](https://github.com/OctoSen | `native-apps.lock.json` | The OctoSense-System-Apps revision. | | `runtime-patches.lock.json` | Reviewed patches on top of the pinned Makepad, with their expected trees. Empty today: the pinned Makepad carries everything the shell needs. | | `tools/setup-native.py` | Prepares and checks the pinned sibling checkouts. | -| `scripts/` | `upstream.py` (WM provenance and catalog regeneration), `smoke.py` (native smoke test), their Python tests, and `provision-appcard-llm.sh` (Android). | +| `scripts/` | `upstream.py` (WM provenance and catalog regeneration), `smoke.py` (native smoke test), their Python tests, `system_apps_remote.sh` and `ai_providers_remote.sh` (hidden `--remote` end-to-end runs of the system apps and of AI providers), and `provision-appcard-llm.sh` (Android). | | `upstream/makepad.json` | Provenance of every file imported from Makepad's `apps/wm`. | | `resources/` | Themes, wallpapers, icons, Android manifest template, startup script. | | `docs/` | [Validation record](docs/validation.md), [upstream sync](docs/upstream.md), [local AI](docs/local-ai.md), [Android AppCard build](docs/android-appcard-build.md), dated plans. | @@ -105,7 +105,7 @@ A relocatable `.app`, installers and a Linux session compositor are not provided | Feature | Default | Effect | | --- | --- | --- | -| `app-hub` | on | Links `octosense-app-hub-app` (store `apphub`, Card runner `card`, system apps) and the Mail host service `octosense-mail-service`. Without it the build has no App Hub and no system apps. | +| `app-hub` | on | Links `octosense-app-hub-app` (store `apphub`, Card runner `card`, system apps) and the host services `octosense-mail-service` (Mail) and `octosense-llm-service` (AI providers). Without it the build has no App Hub and no system apps. | | `app-reference` | off | Links Reference as a module. | | `app-sheets` | off | Links Makepad's Sheets as a module. | | `app-photos` | off | Links Makepad's native Photos module; it replaces the Photos system app of the same id (for comparison). | @@ -138,6 +138,7 @@ App Hub's modules are the exception: they have no process form and always open i | `OCTOSENSE_SYSTEM_APPS` | The system-app selection file; `.cargo/config.toml` sets it to `system-apps.json`. | | `MAKEPAD_APP_CONFIG='{"mail_demo":true}'` | Serve Mail's demo mailbox (see [Demos](#demos)). | | `OCTOSENSE_MAIL_VAULT=file` | Keep Mail passwords in a 0600 file instead of the macOS keychain. | +| `OCTOSENSE_LLM_VAULT=file` | Keep AI providers' keys in the owner-only octos profile instead of the macOS keychain. | | `OCTOS_APP_CORE_BIN`, `OCTOS_APP_CORE_DIR` | Point the AppCard assistant at a local octos kernel. | | `MAKEPAD_REMOTE`, `MAKEPAD_HIDE_WINDOWS` | Remote-control bridge; hidden windows (see [Demos](#demos)). | @@ -147,7 +148,7 @@ The launcher lists four kinds of app together: | Kind | Comes from | Runs as | Launcher id | | --- | --- | --- | --- | -| **System apps**: News, Photos, Maps, Camera, Mail | OctoSense-System-Apps `apps//bundle`, selected by `system-apps.json`, packed into the build | Contained Splash programs in App Hub's Card runner, each in its own isolate under the capabilities its manifest asks for | `` (manifest id `os.`) | +| **System apps**: News, Photos, Maps, Camera, Mail, AI providers | OctoSense-System-Apps `apps//bundle`, selected by `system-apps.json`, packed into the build | Contained Splash programs in App Hub's Card runner, each in its own isolate under the capabilities its manifest asks for | `` (manifest id `os.`) | | **Store apps** | The signed App Hub catalog, installed from the store (`apphub`) | The same Card runner. Every open is checked against the catalog; an update closes old instances. | `hub:` | | **Native modules** | Rust crates linked into this binary | In-process `AppModule`s. Trusted code only: App Hub, AppCard, Reference and the `app-*` features. | module id | | **Developer programs** | `config/apps.json` | Separate processes in tiles, over Makepad's `--stdin-loop` hosting protocol, built on first launch | catalog `id` | @@ -168,6 +169,8 @@ Mail is the worked example (`octosense-mail-service`, from `../OctoSense-System- - The service tests the account, stores the password in the platform secret store (macOS keychain), and grants the account only to the app that added it. - Mail state lives under the host's own directory, outside every app's jail. +AI providers (`os.ai-providers`) edits the LLM providers the AppCard assistant runs on through the `llm` service (`octosense-llm-service`, from `../OctoSense-System-Apps/apps/ai-providers/host-service`). Keys are typed only on host sheets and go to the macOS keychain entry octos reads; the providers are written to the AppCard kernel's octos profile (`OCTOS_APP_CORE_DIR`, else `~/octos-home/.octos`). A phone's provider QR is imported from a picture of it: **Choose image** opens the open panel, or drop a screenshot on the import sheet. **Start → Settings → AI providers** opens it. With `--features app-appcard`, a change restarts a running AppCard so its kernel reads the new providers. + New app features that need a password, PIN or token belong in a host service and a host sheet, never in the app's own UI. ### Store apps (App Hub) @@ -194,7 +197,7 @@ Open **App Hub**, choose the app, **Get**, scroll to **Install**, then **Open**: { "schema": 1, "source": "../OctoSense-System-Apps/apps", - "apps": ["news", "photos", "maps", "camera", "mail"], + "apps": ["news", "photos", "maps", "camera", "mail", "ai-providers"], "assets": {} } ``` @@ -313,7 +316,7 @@ Phone builds always link Reference and Sheets, and App Hub with the system apps | --- | --- | --- | | Makepad / OctoScript | `native-runtime.lock.json` (a new OctoScript-Makepad release), and the `rev` of the Makepad Git dependencies in `Cargo.toml` and `apps/*/Cargo.toml` | If a patch is needed on top, record it in `runtime-patches.lock.json` (base, sha256, tree) | | System apps, Mail service, AppCard | `revision` in `native-apps.lock.json` | — | -| App Hub | `rev` of `octosense-app-hub-app` in `Cargo.toml` | Keep it at the App Hub rev the pinned System-Apps Mail service names (`octosense-appstore`), so the graph has one App Hub | +| App Hub | `rev` of `octosense-app-hub-app` in `Cargo.toml` | Keep it at the App Hub rev the pinned System-Apps Mail and `llm` services name (`octosense-appstore`), so the graph has one App Hub | After any change: `python3 tools/setup-native.py --update`, `cargo update` as needed, then `python3 tools/setup-native.py --check --cargo-manifest Cargo.toml` and the tests below. diff --git a/README.zh-CN.md b/README.zh-CN.md index e9929d32..df259d6d 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -11,7 +11,7 @@ OctoSense-Desktop 是 [OctoSense](https://github.com/OctoSense-org)（运行在 | 仓库 | 与本仓库的关系 | | --- | --- | | [OctoSense-ROM](https://github.com/OctoSense-org/OctoSense-ROM/blob/main/README.zh-CN.md) | 手机 Shell（`home/`）。相同的应用模型、相同的运行时、相同的系统应用。 | -| [OctoSense-System-Apps](https://github.com/OctoSense-org/OctoSense-System-Apps) | 新闻、相册、地图、相机、邮件的应用包，邮件宿主服务，以及 AppCard 助手（`octos-app`）。以固定版本的同级目录检出。 | +| [OctoSense-System-Apps](https://github.com/OctoSense-org/OctoSense-System-Apps) | 新闻、相册、地图、相机、邮件、AI 提供商的应用包，邮件与 `llm` 宿主服务，以及 AppCard 助手（`octos-app`）。以固定版本的同级目录检出。 | | [OctoSense-App-Hub](https://github.com/OctoSense-org/OctoSense-App-Hub) | 签名目录、商店和 Card 运行器。以 Git crate `octosense-app-hub-app` 链接。 | | [OctoScript-App-Design-Flow](https://github.com/OctoSense-org/OctoScript-App-Design-Flow) | 设计、构建应用并发布到 App Hub 的地方。 | | [OctoScript-Makepad](https://github.com/OctoSense-org/OctoScript-Makepad) | 固定 Makepad 与 OctoScript 版本的运行时发行版。以固定版本的同级目录检出。 | @@ -33,7 +33,7 @@ OctoSense-Desktop 是 [OctoSense](https://github.com/OctoSense-org)（运行在 | `native-apps.lock.json` | OctoSense-System-Apps 的版本。 | | `runtime-patches.lock.json` | 固定 Makepad 之上经过评审的补丁及其期望的源码树。目前为空：固定的 Makepad 已包含 Shell 所需的一切。 | | `tools/setup-native.py` | 准备并检查固定版本的同级目录。 | -| `scripts/` | `upstream.py`（WM 来源记录与目录重新生成）、`smoke.py`（原生冒烟测试）、它们的 Python 测试，以及 `provision-appcard-llm.sh`（Android）。 | +| `scripts/` | `upstream.py`（WM 来源记录与目录重新生成）、`smoke.py`（原生冒烟测试）、它们的 Python 测试、`system_apps_remote.sh` 和 `ai_providers_remote.sh`（以隐藏窗口 `--remote` 运行的系统应用与 AI 提供商端到端测试），以及 `provision-appcard-llm.sh`（Android）。 | | `upstream/makepad.json` | 每个从 Makepad `apps/wm` 引入的文件的来源记录。 | | `resources/` | 主题、壁纸、图标、Android manifest 模板、启动脚本。 | | `docs/` | [验证记录](docs/validation.md)、[上游同步](docs/upstream.md)、[本地 AI](docs/local-ai.md)、[Android AppCard 构建](docs/android-appcard-build.md)、按日期的计划文档。 | @@ -105,7 +105,7 @@ cargo build --release --workspace | Feature | 默认 | 作用 | | --- | --- | --- | -| `app-hub` | 开 | 链接 `octosense-app-hub-app`（商店 `apphub`、Card 运行器 `card`、系统应用）以及邮件宿主服务 `octosense-mail-service`。关闭后构建中没有 App Hub，也没有系统应用。 | +| `app-hub` | 开 | 链接 `octosense-app-hub-app`（商店 `apphub`、Card 运行器 `card`、系统应用）以及宿主服务 `octosense-mail-service`（邮件）和 `octosense-llm-service`（AI 提供商）。关闭后构建中没有 App Hub，也没有系统应用。 | | `app-reference` | 关 | 把 Reference 作为模块链接。 | | `app-sheets` | 关 | 把 Makepad 的 Sheets 作为模块链接。 | | `app-photos` | 关 | 链接 Makepad 的原生 Photos 模块；它会替换同 id 的相册系统应用（用于对比）。 | @@ -138,6 +138,7 @@ App Hub 的模块是例外：它们没有进程形态，总是在进程内打开 | `OCTOSENSE_SYSTEM_APPS` | 系统应用选择文件；`.cargo/config.toml` 将其设为 `system-apps.json`。 | | `MAKEPAD_APP_CONFIG='{"mail_demo":true}'` | 提供邮件的演示邮箱（见[演示](#演示)）。 | | `OCTOSENSE_MAIL_VAULT=file` | 把邮件密码保存在权限为 0600 的文件中，而不是 macOS 钥匙串。 | +| `OCTOSENSE_LLM_VAULT=file` | 把 AI 提供商的密钥保存在仅所有者可读的 octos 配置文件中，而不是 macOS 钥匙串。 | | `OCTOS_APP_CORE_BIN`、`OCTOS_APP_CORE_DIR` | 让 AppCard 助手使用本地的 octos 内核。 | | `MAKEPAD_REMOTE`、`MAKEPAD_HIDE_WINDOWS` | 远程控制桥；隐藏窗口（见[演示](#演示)）。 | @@ -147,7 +148,7 @@ launcher 把四类应用列在一起： | 类别 | 来源 | 运行方式 | Launcher id | | --- | --- | --- | --- | -| **系统应用**：新闻、相册、地图、相机、邮件 | OctoSense-System-Apps 的 `apps//bundle`，由 `system-apps.json` 选择，打包进构建 | App Hub 的 Card 运行器中隔离运行的 Splash 程序，每个应用一个 isolate，只拥有其清单申请的能力 | ``（清单 id `os.`） | +| **系统应用**：新闻、相册、地图、相机、邮件、AI 提供商 | OctoSense-System-Apps 的 `apps//bundle`，由 `system-apps.json` 选择，打包进构建 | App Hub 的 Card 运行器中隔离运行的 Splash 程序，每个应用一个 isolate，只拥有其清单申请的能力 | ``（清单 id `os.`） | | **商店应用** | 签名的 App Hub 目录，从商店（`apphub`）安装 | 同一个 Card 运行器。每次打开都会对照目录检查；更新会关闭旧实例。 | `hub:` | | **原生模块** | 链接进本二进制的 Rust crate | 进程内的 `AppModule`。只允许受信任的代码：App Hub、AppCard、Reference 以及各 `app-*` feature。 | 模块 id | | **开发者程序** | `config/apps.json` | tile 中的独立进程，通过 Makepad 的 `--stdin-loop` 托管协议运行，首次启动时构建 | 目录 `id` | @@ -168,6 +169,8 @@ launcher 把四类应用列在一起： - 服务先测试账户，再把密码存入平台的密钥存储（macOS 钥匙串），并且只把账户授予添加它的应用。 - 邮件状态保存在宿主自己的目录中，位于所有应用的沙箱之外。 +AI 提供商（`os.ai-providers`）通过 `llm` 服务（`octosense-llm-service`，来自 `../OctoSense-System-Apps/apps/ai-providers/host-service`）编辑 AppCard 助手使用的 LLM 提供商。密钥只在宿主面板上输入，保存到 octos 读取的 macOS 钥匙串条目；提供商写入 AppCard 内核的 octos 配置（`OCTOS_APP_CORE_DIR`，否则为 `~/octos-home/.octos`）。手机上的提供商二维码可从图片导入：**Choose image** 打开文件面板，或把截图拖到导入面板上。**开始 → 设置 → AI providers** 可打开它。使用 `--features app-appcard` 时，更改会重启正在运行的 AppCard，使其内核读取新的提供商。 + 需要密码、PIN 或令牌的新功能，应放在宿主服务和宿主自有面板中，绝不放在应用自己的界面里。 ### 商店应用（App Hub） @@ -194,7 +197,7 @@ OCTOSENSE_HUB= OCTOSENSE_HUB_ANCHOR= \ { "schema": 1, "source": "../OctoSense-System-Apps/apps", - "apps": ["news", "photos", "maps", "camera", "mail"], + "apps": ["news", "photos", "maps", "camera", "mail", "ai-providers"], "assets": {} } ``` diff --git a/src/apps.rs b/src/apps.rs index 385f924a..30ac4244 100644 --- a/src/apps.rs +++ b/src/apps.rs @@ -127,10 +127,21 @@ pub fn system_card_apps() -> Vec { Vec::new() } -/// The services contained apps call through `host.request` (ADR 0004): `mail` -/// keeps accounts and passwords for the Mail app. `mail_demo` in -/// MAKEPAD_APP_CONFIG serves a demo mailbox from a file vault instead (no -/// keychain, no network): `MAKEPAD_APP_CONFIG='{"mail_demo":true}'`. +/// The services contained apps call through `host.request` (ADR 0004), +/// registered once, before the first system app can open: +/// +/// - `mail` keeps accounts and passwords for the Mail app. `mail_demo` in +/// MAKEPAD_APP_CONFIG serves a demo mailbox from a file vault instead (no +/// keychain, no network): `MAKEPAD_APP_CONFIG='{"mail_demo":true}'`. +/// - `llm` keeps the assistant's LLM providers for AI providers +/// (`os.ai-providers`), written to the AppCard kernel's octos profile +/// (`OCTOS_APP_CORE_DIR`, else `~/octos-home/.octos`), keys in the keychain +/// entry octos reads (`OCTOSENSE_LLM_VAULT=file` keeps them in the +/// owner-only profile). No camera scanner: a phone's profile QR is imported +/// from a picture of it (the open panel, or an image dropped on the import +/// sheet: `llm_image`), or by pasting its text. With AppCard linked +/// (`app-appcard`), a change restarts its core so the kernel reads the new +/// providers; without it there is nothing to restart. #[cfg(feature = "app-hub")] pub fn register_host_services() { static ONCE: std::sync::Once = std::sync::Once::new(); @@ -145,9 +156,36 @@ pub fn register_host_services() { } else { octosense_mail_service::register() } + let mut llm = octosense_llm_service::Options::default(); + if let Some(dir) = octosense_llm_config::profile::default_core_dir() { + llm = llm.core_dir(dir); + } + #[cfg(not(any(target_os = "android", target_os = "ios")))] + { + llm = llm.image_picker(crate::llm_image::picker()).image_drops(true); + } + #[cfg(feature = "app-appcard")] + { + llm = llm.on_changed(|| { + RESTART_APPCARD.store(true, std::sync::atomic::Ordering::Release); + makepad_widgets::makepad_platform::thread::SignalToUI::set_ui_signal(); + }); + } + octosense_llm_service::register_with(llm); }); } +/// Set (any thread) when AI providers changed the saved provider set. +#[cfg(all(feature = "app-hub", feature = "app-appcard"))] +static RESTART_APPCARD: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// UI thread: whether the AppCard core should restart to read a changed +/// provider set (once per change). +#[cfg(all(feature = "app-hub", feature = "app-appcard"))] +pub fn take_appcard_restart() -> bool { + RESTART_APPCARD.swap(false, std::sync::atomic::Ordering::AcqRel) +} + /// Apps App Hub installed: each is an app of its own in the launcher, hosted /// by the linked `card` module under its `hub:` identity. Read /// fresh each time, so an install shows up without a restart. @@ -344,16 +382,16 @@ mod tests { let catalog = bundled_catalog(); // AppCard is opt-in (`app-appcard`), not part of `mobile-apps`. let expected: &[&str] = if cfg!(feature = "app-appcard") { - &["reference", "sheets", "appcard", "apphub", "news", "photos", "maps", "camera", "mail"] + &["reference", "sheets", "appcard", "apphub", "news", "photos", "maps", "camera", "mail", "ai-providers"] } else { - &["reference", "sheets", "apphub", "news", "photos", "maps", "camera", "mail"] + &["reference", "sheets", "apphub", "news", "photos", "maps", "camera", "mail", "ai-providers"] }; assert_eq!(catalog.iter().map(|app| app.id.as_str()).collect::>(), expected); assert!(catalog.iter().all(|app| app.manifest.is_none())); // The system apps have no native module: the Card runner hosts them, // launched by their manifest id (ADR 0004). let registry = AppRegistry::default(); - for id in ["news", "photos", "maps", "camera", "mail"] { + for id in ["news", "photos", "maps", "camera", "mail", "ai-providers"] { let app = catalog.iter().find(|app| app.id == id).unwrap(); assert_eq!(card_manifest_id(app), Some(format!("os.{id}").as_str())); assert_eq!(registry.module(id).map(|m| m.id()), Some("card")); @@ -405,7 +443,7 @@ mod tests { #[test] fn the_system_apps_ship_as_card_apps() { let ids: Vec = system_card_apps().into_iter().map(|app| app.id).collect(); - assert_eq!(ids, ["news", "photos", "maps", "camera", "mail"]); + assert_eq!(ids, ["news", "photos", "maps", "camera", "mail", "ai-providers"]); let registry = AppRegistry::default(); for id in &ids { assert_eq!(registry.hosting(id), Hosting::Module); diff --git a/src/llm_image.rs b/src/llm_image.rs new file mode 100644 index 00000000..70da8e21 --- /dev/null +++ b/src/llm_image.rs @@ -0,0 +1,155 @@ +//! The desktop's image source for AI providers' QR import (the `llm` host +//! service's `QrImagePicker`, OctoSense-System-Apps +//! `apps/ai-providers/host-service`): Makepad's native open panel filtered to +//! PNG and JPEG, and image files dropped on the AI providers window while +//! its import sheet is up. Either way the service gets the file's bytes and +//! reads the code out of them on a worker; the sheet then asks for the PIN. +//! +//! Test hook: `OCTOSENSE_LLM_TEST_IMAGE=` makes "Choose image" answer +//! with that file at once, without a panel (the remote e2e run is hidden and +//! cannot click through a modal). Development only. +use makepad_widgets::makepad_platform::file_dialogs::{FileDialog, FileDialogAction}; +use makepad_widgets::makepad_platform::thread::SignalToUI; +use makepad_widgets::*; +use octosense_llm_service::{ImageDone, PickError, QrImagePicker}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +/// The system app whose window takes dropped images (its short id). +const APP: &str = "ai-providers"; +/// Larger files are not read at all; the service refuses over 20 MB anyway. +const MAX_FILE_BYTES: u64 = 20 * 1024 * 1024; +const TEST_IMAGE_ENV: &str = "OCTOSENSE_LLM_TEST_IMAGE"; + +fn dialog_id() -> LiveId { + live_id!(octosense_llm_qr_image) +} + +#[derive(Default)] +pub struct DesktopPicker { + /// The pick waiting on the open panel. + pending: Mutex>, + /// The panel should open on the next UI event. + open: AtomicBool, +} + +impl QrImagePicker for DesktopPicker { + fn pick(&self, done: ImageDone) { + if let Some(path) = std::env::var_os(TEST_IMAGE_ENV).filter(|p| !p.is_empty()) { + let path = PathBuf::from(path); + std::thread::spawn(move || done(read_image(&path))); + return; + } + if let Some(earlier) = self.pending.lock().unwrap().replace(done) { + earlier(Err(PickError::Cancelled)); + } + self.open.store(true, Ordering::SeqCst); + SignalToUI::set_ui_signal(); + } +} + +/// The one picker the service is registered with. +pub fn picker() -> Arc { + static PICKER: OnceLock> = OnceLock::new(); + PICKER.get_or_init(Arc::default).clone() +} + +fn is_image(path: &Path) -> bool { + matches!( + path.extension().and_then(|e| e.to_str()).map(str::to_ascii_lowercase).as_deref(), + Some("png" | "jpg" | "jpeg") + ) +} + +/// A chosen or dropped file's bytes, off the UI thread. +fn read_image(path: &Path) -> Result, PickError> { + let size = std::fs::metadata(path).map_err(|e| PickError::Failed(e.kind().to_string()))?.len(); + if size > MAX_FILE_BYTES { + return Err(PickError::Failed("larger than 20 MB".into())); + } + std::fs::read(path).map_err(|e| PickError::Failed(e.kind().to_string())) +} + +/// On every UI event: open the panel a pick asked for. +pub fn open_requested(cx: &mut Cx) { + let picker = picker(); + if picker.open.swap(false, Ordering::SeqCst) { + cx.open_select_file_dialog( + FileDialog::new() + .set_id(dialog_id()) + .set_title("Choose a picture of the provider code".into()) + .add_filter("Images".into(), vec!["png".into(), "jpg".into(), "jpeg".into()]), + ); + } +} + +/// The panel's answer, from the actions pass. +pub fn handle_actions(actions: &Actions) { + for action in actions { + let Some(answer) = action.downcast_ref::() else { + continue; + }; + let result = match answer { + FileDialogAction::FileSelected { id, paths } if *id == dialog_id() => paths.first().cloned().ok_or(PickError::Cancelled), + FileDialogAction::FileCancelled { id } if *id == dialog_id() => Err(PickError::Cancelled), + _ => continue, + }; + let Some(done) = picker().pending.lock().unwrap().take() else { + continue; + }; + match result { + Ok(path) if is_image(&path) => { + std::thread::spawn(move || done(read_image(&path))); + } + Ok(_) => done(Err(PickError::Failed("not a PNG or JPEG file".into()))), + Err(e) => done(Err(e)), + } + } +} + +/// The one image file a drag or drop carries. +fn dropped_image(items: &[DragItem]) -> Option { + match items { + [DragItem::FilePath { path, .. }] if is_image(Path::new(path)) => Some(PathBuf::from(path)), + _ => None, + } +} + +/// A drag or drop of one image file over the AI providers window while its +/// import sheet waits for one: answered "copy", and on the drop the file goes +/// to the service. `app_at` names the app whose window is at a point. `true` +/// when the event was the service's. +pub fn handle_drop(event: &Event, app_at: impl Fn(Vec2d) -> Option) -> bool { + let over_import = |abs: Vec2d, items: &[DragItem]| { + let path = dropped_image(items)?; + (octosense_llm_service::wants_image() && app_at(abs).as_deref() == Some(APP)).then_some(path) + }; + match event { + Event::Drag(e) => { + if over_import(e.abs, &e.items).is_some() { + *e.response.lock().unwrap() = DragResponse::Copy; + *e.handled.lock().unwrap() = true; + return true; + } + false + } + Event::Drop(e) => { + let Some(path) = over_import(e.abs, &e.items) else { + return false; + }; + *e.handled.lock().unwrap() = true; + std::thread::spawn(move || { + // A file that cannot be read is offered as nothing: the sheet + // says it is not an image. + let bytes = read_image(&path).unwrap_or_default(); + if !octosense_llm_service::offer_image(bytes) { + log!("llm: the import sheet closed before the dropped image was read"); + } + }); + log!("llm: image dropped on the import sheet"); + true + } + _ => false, + } +} diff --git a/src/main.rs b/src/main.rs index ae5e617d..0c17f042 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,6 +30,8 @@ mod dock_warp; mod host; mod hub; mod layout; +#[cfg(all(feature = "app-hub", not(any(target_os = "android", target_os = "ios"))))] +mod llm_image; mod octosense; mod module_host; mod module_view; @@ -1396,6 +1398,28 @@ impl App { self.redraw_all(cx); } + /// The provider set changed (AI providers): end every AppCard module + /// instance, so its agent and its kernel child stop, and open AppCard + /// again when one was running: its new kernel reads the new profile. + #[cfg(all(feature = "app-hub", feature = "app-appcard"))] + fn restart_appcard_core(&mut self, cx: &mut Cx) { + let mut ended = 0; + while let Some(client) = self.module_host.client_of_module("appcard") { + log!("llm: providers changed: restarting the AppCard core (client {client})"); + self.request_close(cx, client); + ended += 1; + if ended >= 8 || self.module_host.client_of_module("appcard") == Some(client) { + break; + } + } + if ended == 0 { + log!("llm: providers changed; no AppCard core is running, the next one reads them"); + } else { + self.launch_app(cx, "appcard"); + } + self.redraw_all(cx); + } + fn request_close(&mut self, cx: &mut Cx, client: ClientId) { // A module instance has no process to ask politely and nothing to // reap later: it ends now, through the same removal as a death. @@ -4147,6 +4171,8 @@ impl MatchEvent for App { } fn handle_actions(&mut self, cx: &mut Cx, actions: &Actions) { + #[cfg(all(feature = "app-hub", not(any(target_os = "android", target_os = "ios"))))] + llm_image::handle_actions(actions); if self.gallery { let gallery = self.ui.widget(cx, ids!(shell_gallery)); { @@ -4461,6 +4487,29 @@ impl AppMain for App { { return; } + // AI providers' QR import: the open panel a pick asked for, and an + // image dropped on its window while the import sheet waits. + #[cfg(all(feature = "app-hub", not(any(target_os = "android", target_os = "ios"))))] + if self.state.is_some() { + llm_image::open_requested(cx); + if matches!(event, Event::Drag(_) | Event::Drop(_)) { + let desk = self.desk(cx); + let desk = desk.borrow::(); + let state = self.state.as_ref(); + let app_at = |p: Vec2d| { + let client = desk.as_ref()?.window_at(p)?; + state?.clients.get(&client).map(|slot| slot.app.clone()) + }; + if llm_image::handle_drop(event, app_at) { + return; + } + } + } + // AI providers changed the saved providers: restart the AppCard core. + #[cfg(all(feature = "app-hub", feature = "app-appcard"))] + if apps::take_appcard_restart() { + self.restart_appcard_core(cx); + } if self.phone_search_event(cx,event) {return;} if self.state.is_some() && self.phone_pointer(cx,event) {return;} if self.state.is_some() && self.snap_event(cx,event) {return;} diff --git a/src/shell/menu.rs b/src/shell/menu.rs index df6bcbf6..2b293852 100644 --- a/src/shell/menu.rs +++ b/src/shell/menu.rs @@ -482,6 +482,18 @@ impl MenuModel { ] {items.push(MenuItem::new(id,label,kind).icon(icon));} } let available = launcher::apps(); + // Settings gains a submenu once the build ships the AI providers + // system app (`os.ai-providers`): Appearance, then AI providers. + if path.starts_with("start") && available.iter().any(|app| app.id == AI_PROVIDERS) { + items.push( + MenuItem::new("start.settings.style", "Appearance", MenuKind::Menu) + .icon(Ico::Monitor), + ); + items.push( + MenuItem::new("start.settings.ai-providers", "AI providers", MenuKind::App) + .icon(Ico::Cpu), + ); + } items.retain(|item| { let app = match item.id.as_str() { "workspace.files" | "start.documents" => "apps.files", @@ -727,7 +739,8 @@ impl MenuModel { MenuKind::Menu => { let target=match row.target.as_str() { "start.programs"|"start.search"|"start.run"|"workspace.apps"=>"apps", - "start.settings"=>"style", + "start.settings" if !self.items.iter().any(|item| item.id=="start.settings.ai-providers")=>"style", + "start.settings.style"=>"style", "workspace.desktop"=>"desktop", other=>other, }; @@ -739,12 +752,16 @@ impl MenuModel { "workspace.files" => "apps.files".into(), "workspace.tools.terminal" => "apps.terminal".into(), "workspace.tools.task" => "apps.task".into(), + "start.settings.ai-providers" => AI_PROVIDERS.into(), _ => row.target, }), } } } +/// The launcher row of the AI providers system app (`os.ai-providers`). +const AI_PROVIDERS: &str = "apps.ai-providers"; + /// `MenuModel.js` `matchesQuery` + `searchScore`, lower is better. /// /// Every whitespace term has to match the label, the leaf id or an alias by @@ -1588,6 +1605,29 @@ mod tests { assert_eq!(centered_card_top(smaller, 280.0, 8.0, Some(300.0)), 138.0); } + /// Start > Settings is Appearance alone until the build ships the AI + /// providers system app; then it is a submenu that also opens it. + #[test] + fn start_settings_offers_ai_providers_once_it_ships() { + let mut model = MenuModel::default(); + model.open_at("", MenuSkin::Menu); + model.descend("start"); + let ships = launcher::apps().iter().any(|app| app.id == AI_PROVIDERS); + model.sel = model.rows.iter().position(|r| r.target == "start.settings").unwrap(); + assert_eq!(model.activate(), None); + if ships { + assert_eq!(model.path, "start.settings"); + model.sel = model + .rows + .iter() + .position(|r| r.target == "start.settings.ai-providers") + .unwrap(); + assert_eq!(model.activate().as_deref(), Some(AI_PROVIDERS)); + } else { + assert_eq!(model.path, "style"); + } + } + #[test] fn navigating_between_menus_recenters_after_a_search() { let mut model = MenuModel::default(); diff --git a/system-apps.json b/system-apps.json index 390574e0..933675f8 100644 --- a/system-apps.json +++ b/system-apps.json @@ -1,6 +1,6 @@ { "schema": 1, "source": "../OctoSense-System-Apps/apps", - "apps": ["news", "photos", "maps", "camera", "mail"], + "apps": ["news", "photos", "maps", "camera", "mail", "ai-providers"], "assets": {} } From 9bb85e5b0ca8d00cf0079b9d243dfd70f4fa4b38 Mon Sep 17 00:00:00 2001 From: ymote <151983+ymote@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:23:53 -0700 Subject: [PATCH 2/2] Remote e2e runs for the system apps and AI providers scripts/system_apps_remote.sh and scripts/ai_providers_remote.sh drive a hidden release build (MAKEPAD_HIDE_WINDOWS=1, --remote) with a temp OCTOSENSE_HOME, octos core dir and file vaults. The AI providers run is sandboxed with outbound HTTPS denied and uses fake keys only: the add wizard, paste, drop and picker imports of the QR-A fixture, and the phone QR export read back and compared with the saved profile. The system-apps run also treats the Card runner building an app's first Splash VM on the UI thread as a known first-open hang, as it already did for the bundle unpack. Co-Authored-By: Claude Opus 5.5 --- scripts/ai_providers_remote.sh | 667 +++++++++++++++++++++++++++++++++ scripts/system_apps_remote.sh | 181 +++++++++ 2 files changed, 848 insertions(+) create mode 100755 scripts/ai_providers_remote.sh create mode 100755 scripts/system_apps_remote.sh diff --git a/scripts/ai_providers_remote.sh b/scripts/ai_providers_remote.sh new file mode 100755 index 00000000..778e51e2 --- /dev/null +++ b/scripts/ai_providers_remote.sh @@ -0,0 +1,667 @@ +#!/usr/bin/env bash +# AI providers' QR import from an image, and the phone QR it exports, on the +# desktop, driven through Makepad's remote (instrument) mode: no OS +# screenshots, only the app's own frames. +# +# cargo build --release [--config …] --features mobile-apps +# cargo build --release [--config …] -p octosense-llm-service --example read_qr_image +# scripts/ai_providers_remote.sh [artifacts-dir] +# +# Runs the release binary hidden with its own OCTOSENSE_HOME, octos core dir +# and file vaults under a temp directory (neither ~/.octosense nor the login +# keychain is touched). Every key in it is fake, and the binary runs under +# sandbox-exec with outbound HTTPS denied: no fake key reaches a real +# provider, and a provider test fails as a network failure. A local fake +# endpoint (fake.py, 127.0.0.1) stands in for a provider that answers. The +# flow: +# 1. AI providers opens; the host reports an image picker ("Import code +# from image"). +# W1. Add DeepSeek with the add wizard, step by step: 1 search the families +# and choose DeepSeek, 2 the model pull-down (DeepSeek V4 Flash, +# Recommended), 3 the route pull-down (Official API), 4 a fake key, 5 +# Test connection: the provider cannot be reached, Save stays disabled +# and "Save without testing" saves it as the primary. +# W2. Change its model with the card's model pull-down (DeepSeek V4 Pro). +# W3. Add Z.ai as a fallback on another model (GLM-4.7) and route (a +# custom OpenAI-compatible endpoint: the fake endpoint); Test connection +# passes ("Connected"), and "Add as fallback" saves it. +# W4. The saved-models list and _main.json say so; Test connection on a +# row; then both are removed with the cards' Remove (tap twice). +# 2. Import into the empty list by PASTE: an old single-provider code +# (DeepSeek deepseek-v4-flash, fake key ending fcb0) typed into the +# import sheet's code field is saved as the primary. +# 3. Add OpenAI with the wizard (its default gpt-4o on the official route) +# with the fake key sk-test-0000000000001234, saved without testing. +# 4. Import by DROP: the fixture QR-A (OctoSense-System-Apps +# config/tests/fixtures/qr-a.png, PIN 7K3M-9QX2: DeepSeek deepseek-chat +# and Z.ai glm-4.6) dropped on the import sheet through the native drop +# path (/drop), PIN typed. An import only adds, so there is no confirm +# step: the primary and OpenAI stay first, QR-A's providers are appended +# as fallbacks, and its DeepSeek key goes to a slot of its own +# (DEEPSEEK_2_API_KEY) so the primary's key is not overwritten. +# 5. Import QR-A again by PICKER: "Choose image" (OCTOSENSE_LLM_TEST_IMAGE +# answers with the fixture instead of the open panel, which a hidden run +# cannot click through); a wrong PIN is refused and changes nothing, the +# right one changes nothing either ("Already saved"). +# 6. Show QR for phone, grab the sheet, read the QR out of the grab (the +# service's own image search, rqrr) and open it with the PIN on the +# sheet: it equals the saved set and keys, both DeepSeek slots included. +# The countdown +# ticks, and at 0 the sheet closes by itself (OCTOSENSE_LLM_QR_SECONDS +# shortens its five minutes to $QR_SECONDS s), with no Splash error in +# the log. +# 7. No key text in /snap, /d, /log or the host log; no panic, no Splash +# error; /gq exits. +# Artifacts: grabs/, export.png (the export sheet at full scale) and +# export-pin.txt, for scanning on a phone. The script-drawn controls (the +# wizard's fields, rows and pull-downs, the app's cards) are not in /snap: +# they are found in grabs by their colours (blocks.py). +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +BIN=${OCTOSENSE_BIN:-$ROOT/target/release/octosense} +READ_QR=${READ_QR_IMAGE:-$ROOT/target/release/examples/read_qr_image} +SRC=$(cd "$ROOT" && python3 -c 'import json; print(json.load(open("system-apps.json"))["source"])') +FIXTURE=$(cd "$ROOT/$SRC/ai-providers/config/tests/fixtures" && pwd)/qr-a.png +FIXTURE_PIN=7K3M-9QX2 +FAKE_KEY=sk-test-0000000000001234 +# The old single-provider code the empty list is filled from (its key is fake). +LEGACY_CODE='{"llm_family":"deepseek","llm_model":"deepseek-v4-flash","llm_key":"sk-test-legacy-00000000fcb0"}' +# The phone QR's lifetime in this run: long enough to grab and read the code. +QR_SECONDS=${QR_SECONDS:-20} +# What a Splash script error logs (the export sheet's countdown used to log +# these every second). +SPLASH_ERRORS='pop_stack_resolved|mes empty|splash host callback error|\[E\] splash' +# Every key the run handles: none may show up in the UI tree or the logs. +SECRETS='sk-test-|zai-test-|0000000000001234|0000000000005678' +ZAI_KEY=sk-test-0000000000005678 +WORK=${1:-$(mktemp -d -t octosense-ai-providers)} +mkdir -p "$WORK/home" "$WORK/octos" "$WORK/grabs" +LOG=$WORK/host.log +PROFILE=$WORK/octos/profiles/_main.json +[ -x "$BIN" ] || { echo "no $BIN: build it first"; exit 2; } +[ -x "$READ_QR" ] || { echo "no $READ_QR: cargo build --release -p octosense-llm-service --example read_qr_image"; exit 2; } +echo "artifacts: $WORK" + +cd "$WORK" +: >"$LOG" +env -u MAKEPAD_HOME -u MAKEPAD_WM_ROOT -u MAKEPAD_WM_THEME \ + OCTOSENSE_HOME="$WORK/home" OCTOS_APP_CORE_DIR="$WORK/octos" \ + OCTOSENSE_MAIL_VAULT=file OCTOSENSE_LLM_VAULT=file \ + OCTOSENSE_LLM_TEST_IMAGE="$FIXTURE" OCTOSENSE_LLM_QR_SECONDS="$QR_SECONDS" \ + MAKEPAD_HIDE_WINDOWS=1 sandbox-exec -p '(version 1)(allow default)(deny network-outbound (remote tcp "*:443"))' \ + "$BIN" --remote >"$LOG" 2>&1 & +PID=$! +PORT= +# A provider that answers: OpenAI-compatible chat completions for the Z.ai +# fake key, on 127.0.0.1 (the sandbox lets local traffic through). +cat >"$WORK/fake.py" <<'EOF' +import http.server, json, sys +KEY = sys.argv[1] +class H(http.server.BaseHTTPRequestHandler): + def answer(self, ok, body): + data = json.dumps(body if ok else {"error": {"message": "Incorrect API key"}}).encode() + self.send_response(200 if ok else 401) + self.send_header("content-type", "application/json"); self.send_header("content-length", str(len(data))) + self.end_headers(); self.wfile.write(data) + def do_POST(self): + self.rfile.read(int(self.headers.get("content-length", 0))) + self.answer(self.headers.get("authorization") == "Bearer " + KEY, {"choices": []}) + def do_GET(self): + self.answer(self.headers.get("authorization") == "Bearer " + KEY, {"data": [{"id": "glm-4.7"}]}) + def log_message(self, *args): pass +server = http.server.HTTPServer(("127.0.0.1", 0), H) +print(server.server_address[1], flush=True) +server.serve_forever() +EOF +python3 "$WORK/fake.py" "$ZAI_KEY" >"$WORK/fake.port" 2>/dev/null & +FAKE=$! +disown "$FAKE" +cleanup() { + kill "$FAKE" 2>/dev/null || true + if kill -0 "$PID" 2>/dev/null; then + [ -n "$PORT" ] && curl -s "127.0.0.1:$PORT/quit" >/dev/null || true + sleep 2 + kill "$PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +fail() { echo "FAIL: $*"; [ -z "${HOLD_ON_FAIL:-}" ] || sleep "$HOLD_ON_FAIL"; exit 1; } +pass() { echo "PASS: $*"; } +for _ in $(seq 1 120); do + PORT=$(sed -n 's/.*\[makepad-remote\] listening on 127\.0\.0\.1:\([0-9]*\) pid=.*/\1/p' "$LOG" | head -1) + [ -n "$PORT" ] && break + sleep 0.5 +done +[ -n "$PORT" ] || fail "no remote port in $LOG" +get() { curl -fsS "127.0.0.1:$PORT/$1"; } +q() { python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$1"; } +key() { + local out + out=$(curl -s "127.0.0.1:$PORT/k?k=press&c=$1&wait=1${2:-}") + case $out in *'"err"'*) echo "note: key $1: $out" ;; esac +} +type_keys() { + local text=$1 i c + for ((i = 0; i < ${#text}; i++)); do + c=${text:i:1} + case $c in + " ") key Space ;; + *) key "Key$(printf %s "$c" | tr a-z A-Z)" ;; + esac + done +} +# Input waits for the next frame; a hidden window sometimes misses it and +# the remote answers "retry" although the input was delivered. What input +# did is checked through the profile, the log and snapshots, so a missed +# frame is only noted (retrying could click twice). +input() { + local out + out=$(curl -s "127.0.0.1:$PORT/$1&wait=1") + case $out in *'"err"'*) echo "note: ${1%%\?*}: $out" ;; esac +} +text() { input "t?t=$(q "$1")"; } +click() { input "click?x=$1&y=$2"; } +scroll() { input "m?k=scroll&x=$1&y=$2&dy=$3"; } +# A grab at layout scale (one pixel a point); prints its path. +grab() { + local png + for _ in 1 2 3 4 5; do + png=$(curl -s "127.0.0.1:$PORT/g?scale=${2:-0.5}" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("png",""))' || true) + [ -n "$png" ] && { cp "$png" "$WORK/grabs/$1.png"; echo "$WORK/grabs/$1.png"; return; } + sleep 1 + done + fail "no frame for $1" +} +# The centre of the card widget whose text is exactly $1: "x y". +widget() { + get "snap?q=$(q "$1")" | python3 -c ' +import json,sys +hits=[e for e in json.load(sys.stdin)["s"] if e.get("t")==sys.argv[1] and e["r"][2]>0] +if hits: x,y,w,h=hits[-1]["r"]; print(x+w/2, y+h/2)' "$1" +} +wait_widget() { + for _ in $(seq 1 ${2:-40}); do [ -n "$(widget "$1")" ] && return 0; sleep 0.5; done + fail "never showed: $1" +} +# The host's sheet (its rect and its program) once it runs `$1`. +sheet_rect() { + get "snap?q=$(q "$1")" | python3 -c ' +import json,sys +hits=[e for e in json.load(sys.stdin)["s"] if e["i"]=="sheet" and sys.argv[1] in (e.get("t") or "")] +if hits: print(*hits[-1]["r"])' "$1" +} +wait_sheet() { + local r + for _ in $(seq 1 40); do r=$(sheet_rect "$1"); [ -n "$r" ] && { echo "$r"; return 0; }; sleep 0.5; done + fail "no sheet running $1" +} +wait_log() { + for _ in $(seq 1 ${2:-40}); do grep -q "$1" "$LOG" && return 0; sleep 0.5; done + fail "log never said: $1" +} +# Sheets are Splash isolates whose widgets /snap does not list: find their +# controls in a grab instead. Prints "pill X Y" (the blue action at the top +# right: Import, Save, Close) and "full Y" / "half X Y" for each grey field +# or button, top to bottom. +cat >"$WORK/sheet.py" <<'EOF' +import sys, zlib, struct +def read_png(path): + data = open(path, "rb").read() + pos, chunks = 8, {} + idat = b"" + while pos < len(data): + n, kind = struct.unpack(">I4s", data[pos:pos + 8]) + body = data[pos + 8:pos + 8 + n] + if kind == b"IHDR": w, h, depth, color = struct.unpack(">IIBB", body[:10]) + if kind == b"IDAT": idat += body + pos += 12 + n + assert depth == 8 and color in (2, 6), (depth, color) + bpp = 4 if color == 6 else 3 + raw, stride, rows, prev = zlib.decompress(idat), w * bpp, [], bytearray(w * bpp) + i = 0 + for _ in range(h): + f, line = raw[i], bytearray(raw[i + 1:i + 1 + stride]); i += 1 + stride + for x in range(stride): + a = line[x - bpp] if x >= bpp else 0 + b = prev[x]; c = prev[x - bpp] if x >= bpp else 0 + if f == 1: line[x] = (line[x] + a) & 255 + elif f == 2: line[x] = (line[x] + b) & 255 + elif f == 3: line[x] = (line[x] + (a + b) // 2) & 255 + elif f == 4: + p = a + b - c; pa, pb, pc = abs(p - a), abs(p - b), abs(p - c) + line[x] = (line[x] + (a if pa <= pb and pa <= pc else b if pb <= pc else c)) & 255 + rows.append(bytes(line)); prev = line + return w, h, bpp, rows +path, sx, sy, sw, sh = sys.argv[1], *map(float, sys.argv[2:6]) +w, h, bpp, rows = read_png(path) +px = lambda x, y: rows[y][x * bpp:x * bpp + 3] +x0, x1 = int(sx + 20), int(sx + sw - 20) +inner = x1 - x0 +grey = lambda p: 200 <= p[0] <= 248 and abs(p[0] - p[1]) <= 2 and 3 <= p[2] - p[0] <= 7 +blue = lambda p: p[0] < 60 and 100 <= p[1] <= 160 and p[2] > 200 +# The pill: the blue columns tall enough to be a filled button. +top = [y for y in range(int(sy), int(sy + 110))] +cols = [x for x in range(x0, x1) if sum(blue(px(x, y)) for y in top) >= 20] +if cols: + ys = [y for y in top if sum(blue(px(x, y)) for x in cols[::3]) >= len(cols[::3]) // 2] + print("pill", (cols[0] + cols[-1]) // 2, (ys[0] + ys[-1]) // 2 if ys else int(sy + 60)) +out, y = [], int(sy + 60) +count = lambda y: sum(grey(px(x, y)) for x in range(x0, x1, 2)) * 2 +while y < int(min(sy + sh, h)): + if count(y) >= 0.4 * inner: + start = y + while y < h and count(y) >= 0.4 * inner: y += 1 + if y - start >= 20: + row, runs, run = start + 3, [], None + for x in range(x0, x1): + if grey(px(x, row)): + run = [x, x] if run is None else [run[0], x] + elif run: runs.append(run); run = None + if run: runs.append(run) + runs = [r for r in runs if r[1] - r[0] >= 0.3 * inner] + mid = (start + y) // 2 + if runs and runs[0][1] - runs[0][0] >= 0.85 * inner: + if not any(l.startswith("full") for l in out): + # The note line just above the first full-width control: + # how far right its text reaches (0: no text). + ink = [x for x in range(x0, x1) for yy in range(start - 40, start - 6) if max(px(x, yy)) < 120] + out.append("note %d" % (max(ink) - sx if ink else 0)) + out.append("full %d" % mid) + else: + for r in runs: out.append("half %d %d" % ((r[0] + r[1]) // 2, mid)) + y += 1 +print("\n".join(out)) +EOF +# `controls NAME RECT…` grabs, then prints the sheet's controls. +controls() { local png; png=$(grab "$1"); shift; python3 "$WORK/sheet.py" "$png" "$@"; } +# The import sheet moved on to the PIN once its note reads "Code read from +# the image. Type the PIN shown beside it." (wider than "Choose a screenshot +# or photo of the code…" or none), with no script error on the way. +pin_prompt() { + local C n + for _ in $(seq 1 10); do + sleep 1 + C=$(controls "$@") + n=$(echo "$C" | awk '$1=="note"{print $2; exit}') + [ "${n:-0}" -ge 420 ] && break + done + if grep -q "not found in prototype chain" "$LOG"; then + grep "not found in prototype chain" "$LOG" | head -3 + fail "the import sheet's script failed" + fi + [ "${n:-0}" -ge 420 ] || return 1 + echo "$C" +} +nth_full() { awk -v n="$1" '$1=="full"{i++; if(i==n){print $2; exit}}'; } +pill() { awk '$1=="pill"{print $2, $3; exit}'; } +# Blocks of one colour in a grab (the wizard's fields, rows, pull-downs and +# buttons, the cards' controls): "cx cy w h" per block, top to bottom. +# `blocks.py SHEET_PY PNG RRGGBB [MINCOUNT] [MINW]`: rows with at least +# MINCOUNT pixels of the colour (+-3; +-30 for text, MINCOUNT 1) make bands +# (a text band may pause for 3 rows); a band splits where the colour pauses +# for more than 12 px across (24 for text); narrower blocks than MINW drop. +cat >"$WORK/blocks.py" <<'EOF' +import sys +exec(open(sys.argv[1]).read().split("path, sx, sy")[0]) +w, h, bpp, rows = read_png(sys.argv[2]) +want = tuple(int(sys.argv[3][i:i + 2], 16) for i in (0, 2, 4)) +mincount = int(sys.argv[4]) if len(sys.argv) > 4 else 40 +minw = int(sys.argv[5]) if len(sys.argv) > 5 else 40 +text = mincount <= 1 +gap, tol = (24, 30) if text else (12, 3) +def hits(y): + r = rows[y] + return [x for x in range(w) if abs(r[x * bpp] - want[0]) <= tol and abs(r[x * bpp + 1] - want[1]) <= tol and abs(r[x * bpp + 2] - want[2]) <= tol] +bands, cur, quiet = [], None, 0 +for y in range(h): + xs = hits(y) + if len(xs) >= mincount: + if cur is None: cur = [y, y, set()] + cur[1] = y; cur[2].update(xs); quiet = 0 + elif cur is not None: + quiet += 1 + if quiet > (3 if text else 0): bands.append(cur); cur = None; quiet = 0 +if cur: bands.append(cur) +out = [] +for y0, y1, xs in bands: + xs = sorted(xs); start = prev = xs[0] + for x in xs[1:] + [10 ** 9]: + if x - prev > gap: + if prev - start + 1 >= minw and y1 - y0 + 1 >= (6 if text else 14): + out.append(((start + prev) // 2, (y0 + y1) // 2, prev - start + 1, y1 - y0 + 1)) + start = x + prev = x +for b in sorted(out, key=lambda b: (b[1], b[0])): print(*b) +EOF +# The wizard's and the cards' colours (bundle/main.splash and +# host-service/src/sheets.rs in OctoSense-System-Apps): pull-down, option +# row, chosen row, field, Test connection, Save without testing, accent, +# danger. +PULLDOWN=edf2fc; OPTION=f6f8fc; CHOSEN=e8f0fe; FIELD=f2f2f7; TEST=e3edff; BYPASS=fff1e0; BLUE=007aff; RED=ff3b30 +in_png() { python3 "$WORK/blocks.py" "$WORK/sheet.py" "$1" "$2" "${3:-40}" "${4:-40}"; } +# `blocks NAME COLOUR [MINCOUNT] [MINW]`: grab, then the colour's blocks. +blocks() { local png; png=$(grab "$1"); in_png "$png" "$2" "${3:-40}" "${4:-40}"; } +nth() { awk -v n="$1" 'NR==n{print $1, $2; exit}'; } +# The app scrolls as one list: its top (the cards) or its end (the QR +# buttons, under the cards). +top_app() { scroll 548 400 -3000; sleep 1; } +bottom_app() { scroll 548 400 3000; sleep 1; } + +profile_families() { + python3 -c ' +import json,sys +llm=json.load(open(sys.argv[1]))["config"]["llm"] +print(" ".join(s["family_id"] for s in [llm.get("primary")]+llm.get("fallbacks",[]) if s))' "$PROFILE" +} + +for _ in $(seq 1 60); do curl -s "127.0.0.1:$PORT/snap?q=main_window" | grep -q '"s":\[{' && break; sleep 0.5; done +pass "desktop up on port $PORT (pid $PID)" + +# 1. AI providers opens and offers the image import. +key Space "&cmd=1"; type_keys "ai providers"; key Return +wait_log "card: os.ai-providers running under" +wait_widget "Import code from image" +pass "AI providers opened; the host has an image picker (Import code from image)" + +# The rows are drawn by the app's script (on_render), which /snap does not +# list: the masked keys are in the grab; the keys themselves in the profile. +keys_end_with() { + python3 -c ' +import json,sys +env=json.load(open(sys.argv[1]))["config"]["env_vars"] +want=dict(a.split("=") for a in sys.argv[2:]) +sys.exit(0 if sorted(env)==sorted(want) and all(env[k].endswith(v) for k,v in want.items()) else 1)' "$PROFILE" "$@" +} +# Each saved route's model and the key env var octos reads it from. +profile_routes() { + python3 -c ' +import json,sys +llm=json.load(open(sys.argv[1]))["config"]["llm"] +print(" ".join("%s:%s" % (s.get("model_id","-"), (s.get("route") or {}).get("api_key_env","-")) for s in [llm.get("primary")]+llm.get("fallbacks",[]) if s))' "$PROFILE" +} +# The import sheet closes by itself once the code is imported. +sheet_closes() { + for _ in $(seq 1 40); do [ -z "$(sheet_rect "$1")" ] && return 0; sleep 0.5; done + return 1 +} + +# The add wizard, driven like a person. `next_step NAME` grabs (NAME) and +# presses the footer's Next (or the save on step 5): the lowest blue button +# narrower than a field; greyed out, it is not blue, and the step fails. +next_step() { + local X Y + read -r X Y _ < <(blocks "$1" $BLUE 40 60 | awk '$3<400' | tail -1) || true + [ -n "$Y" ] || fail "$1: Next is not enabled" + click "$X" "$Y"; sleep 1.5 +} +# `pick NAME N`: open the step's pull-down and take its option N. +pick() { + local X Y W H X2 Y2 + read -r X Y W H _ < <(blocks "$1-closed" $PULLDOWN 40 400 | head -1) || true + [ -n "$Y" ] || fail "$1: no pull-down on this step" + click "$X" "$Y"; sleep 1.5 + read -r X2 Y2 _ < <(blocks "$1" $OPTION 40 400 | awk -v y=$((Y + H / 2)) '$2>y' | nth "$2") || true + [ -n "$Y2" ] || fail "$1: the pull-down shows no option $2" + click "$X2" "$Y2"; sleep 1 +} +# `add_model NAME QUERY MODEL_N ROUTE_N KEY [BASE]`: the five steps. Step 1 +# searches the families for QUERY and chooses the first; steps 2 and 3 take +# option MODEL_N / ROUTE_N of their pull-downs (0: keep the preselected +# one), and BASE goes into the custom route's base URL; step 4 takes KEY; +# step 5 tests, then saves (Save once the test passes, else Save without +# testing after a network failure); SAVED says which: tested or untested. +add_model() { + local name=$1 query=$2 model_n=$3 route_n=$4 key=$5 base=${6:-} X Y before + SAVED= + before=$(cat "$PROFILE" 2>/dev/null || true) + read -r X Y _ < <(widget "Add model") || true; click "$X" "$Y" + read -r SX SY SW SH < <(wait_sheet "llm.sheet.fetch_models") + sleep 1.5 + read -r X Y _ < <(blocks "$name-1-start" $FIELD 40 400 | nth 1) || true + [ -n "$Y" ] || fail "$name: no search field on step 1" + click "$X" "$Y"; text "$query"; sleep 1.5 + read -r X Y _ < <(blocks "$name-1-search" $OPTION 40 400 | nth 1) || true + [ -n "$Y" ] || fail "$name: no family matches $query" + click "$X" "$Y"; sleep 1.5 + [ -n "$(blocks "$name-1-family" $CHOSEN 40 400)" ] || fail "$name: the family row is not marked chosen" + next_step "$name-1-next" + [ "$model_n" -gt 0 ] && pick "$name-2-model" "$model_n" + next_step "$name-2-next" + [ "$route_n" -gt 0 ] && pick "$name-3-route" "$route_n" + if [ -n "$base" ]; then + read -r X Y _ < <(blocks "$name-3-custom" $FIELD 40 400 | nth 1) || true + [ -n "$Y" ] || fail "$name: no base URL field" + click "$X" "$Y"; text "$base"; sleep 1 + fi + next_step "$name-3-next" + read -r X Y _ < <(blocks "$name-4-key" $FIELD 40 400 | nth 1) || true + [ -n "$Y" ] || fail "$name: no key field on step 4" + click "$X" "$Y"; text "$key"; sleep 1 + next_step "$name-4-next" + read -r X Y _ < <(blocks "$name-5-review" $TEST 40 400 | head -1) || true + [ -n "$Y" ] || fail "$name: no Test connection on step 5" + [ -z "$(blocks "$name-5-review" $BLUE 40 60 | awk '$3<400')" ] || fail "$name: Save is enabled before a test" + click "$X" "$Y" + for _ in $(seq 1 15); do + sleep 2 + read -r X Y _ < <(blocks "$name-5-tested" $BLUE 40 60 | awk '$3<400' | tail -1) || true + if [ -n "$Y" ]; then click "$X" "$Y"; SAVED=tested; break; fi + read -r X Y _ < <(in_png "$WORK/grabs/$name-5-tested.png" $BYPASS 40 100 | head -1) || true + if [ -n "$Y" ]; then click "$X" "$Y"; SAVED=untested; break; fi + done + [ -n "$Y" ] || fail "$name: the test neither passed nor offered Save without testing" + # The result line: green "Connected · N ms", or red "Couldn't connect: …". + if [ "$SAVED" = tested ]; then + [ -n "$(in_png "$WORK/grabs/$name-5-tested.png" 248a3d 1 100)" ] || fail "$name: no green Connected line (grab $name-5-tested)" + else + [ -n "$(in_png "$WORK/grabs/$name-5-tested.png" c4281c 1 100)" ] || fail "$name: no red Couldn't connect line (grab $name-5-tested)" + fi + for _ in $(seq 1 20); do [ "$(cat "$PROFILE" 2>/dev/null || true)" != "$before" ] && return 0; sleep 0.5; done + fail "$name: the save did not change the profile" +} +profile_json() { python3 -c 'import json,sys; print(json.dumps(json.load(open(sys.argv[1]))["config"].get(sys.argv[2]), sort_keys=True))' "$PROFILE" "$1"; } + +# W1. DeepSeek with the wizard; the provider is unreachable (sandbox). +add_model w1 deepseek 1 1 "$FAKE_KEY" +[ "$SAVED" = untested ] || fail "DeepSeek was not saved without testing" +wait_widget "Added DeepSeek · deepseek-v4-flash" +sheet_closes "llm.sheet.fetch_models" || fail "the wizard stayed up after the save" +[ "$(profile_json llm)" = '{"fallbacks": [], "primary": {"family_id": "deepseek", "model_id": "deepseek-v4-flash"}}' ] \ + || fail "profile after the wizard: $(profile_json llm)" +keys_end_with DEEPSEEK_API_KEY=1234 || fail "the DeepSeek key is not in the profile" +pass "DeepSeek added with the wizard: family search, model pull-down (DeepSeek V4 Flash, Recommended), route pull-down (Official API), key, Test connection failed (unreachable), Save without testing (grabs w1-1-family, w1-2-model, w1-3-route, w1-4-next, w1-5-review, w1-5-tested)" + +# W2. The card's model pull-down. +read -r X Y _ < <(blocks w2-list $PULLDOWN 40 400 | nth 1) || true +[ -n "$Y" ] || fail "no model pull-down on the DeepSeek card" +click "$X" "$Y"; sleep 2 +read -r X Y _ < <(blocks w2-pulldown $OPTION 40 400 | nth 2) || true +[ -n "$Y" ] || fail "the card's model pull-down has no second model" +click "$X" "$Y" +wait_widget "DeepSeek now uses DeepSeek V4 Pro." +[ "$(profile_json llm)" = '{"fallbacks": [], "primary": {"family_id": "deepseek", "model_id": "deepseek-v4-pro"}}' ] \ + || fail "profile after the model change: $(profile_json llm)" +pass "the card's model pull-down listed DeepSeek's catalog models and switched to DeepSeek V4 Pro (grab w2-pulldown)" + +# W3. Z.ai as a fallback: GLM-4.7 (the second model) on a custom +# OpenAI-compatible route (the route pull-down's option after Official API), +# the local fake endpoint: the test passes. +FAKE_PORT=$(head -1 "$WORK/fake.port") +[ -n "$FAKE_PORT" ] || fail "the fake endpoint did not start" +ZAI_BASE="http://127.0.0.1:$FAKE_PORT/v1" +add_model w3 "z.ai" 2 2 "$ZAI_KEY" "$ZAI_BASE" +[ "$SAVED" = tested ] || fail "Z.ai was not saved after a passing test" +wait_widget "Added Z.ai · glm-4.7" +sheet_closes "llm.sheet.fetch_models" || fail "the wizard stayed up after the Z.ai save" +[ "$(profile_json llm)" = '{"fallbacks": [{"family_id": "zai", "model_id": "glm-4.7", "route": {"api_type": "openai", "base_url": "'"$ZAI_BASE"'"}}], "primary": {"family_id": "deepseek", "model_id": "deepseek-v4-pro"}}' ] \ + || fail "profile after the Z.ai add: $(profile_json llm)" +keys_end_with DEEPSEEK_API_KEY=1234 ZAI_API_KEY=5678 || fail "the keys after the Z.ai add are wrong" +pass "Z.ai added as a fallback on GLM-4.7 and a custom OpenAI-compatible route; Test connection passed, Add as fallback saved it (grabs w3-2-model, w3-3-custom, w3-5-tested)" + +# W4. The list shows both; Test connection on the fallback's row passes; +# remove both with the cards' Remove (tap twice). +# Both cards in view: the list scrolled past the header. +top_app; scroll 548 400 220; sleep 1 +[ "$(blocks w4-saved-models $PULLDOWN 40 400 | wc -l | tr -d ' ')" = 2 ] || fail "the list does not show two model pull-downs (grab w4-saved-models)" +# Green text: the rows' key statuses, and "Connected" once the test passes. +GREEN=248a3d +read -r X Y _ < <(blocks w4-rows $TEST 40 60 | tail -1) || true +[ -n "$Y" ] || fail "no Test connection on the second row" +BEFORE=$(in_png "$WORK/grabs/w4-rows.png" $GREEN 1 60 | wc -l) +click "$X" "$Y" +for _ in $(seq 1 20); do sleep 1; [ "$(blocks w4-row-tested $GREEN 1 60 | wc -l)" -gt "$BEFORE" ] && break; done +[ "$(in_png "$WORK/grabs/w4-row-tested.png" $GREEN 1 60 | wc -l)" -gt "$BEFORE" ] || fail "the row's Test connection did not say Connected (grab w4-row-tested)" +for n in 1 2; do + top_app + read -r X Y _ < <(blocks "w4-remove-$n" $RED 1 30 | head -1) || true + [ -n "$Y" ] || fail "no Remove on the first card (grab w4-remove-$n)" + click "$X" "$Y"; sleep 1 + click "$X" "$Y"; sleep 2 +done +python3 -c 'import json,sys; sys.exit(0 if "llm" not in json.load(open(sys.argv[1]))["config"] else 1)' "$PROFILE" || fail "the list is not empty after removing both: $(profile_json llm)" +top_app; grab w4-empty >/dev/null +pass "the list showed DeepSeek V4 Pro (primary) and Z.ai GLM-4.7 (fallback 1); the fallback's Test connection said Connected; both removed with Remove, tapped twice (grabs w4-saved-models, w4-row-tested, w4-empty)" + +# 2. Import into the empty list by pasting an old single-provider code. +bottom_app; read -r X Y < <(widget "Import code from image"); click "$X" "$Y" +read -r SX SY SW SH < <(wait_sheet "llm.sheet.import") +sleep 1 +C=$(controls paste-sheet "$SX" "$SY" "$SW" "$SH") +CODE_Y=$(echo "$C" | nth_full 2); read -r PX PY < <(echo "$C" | pill) +[ -n "$CODE_Y" ] && [ -n "$PX" ] || { echo "$C"; fail "import sheet controls not found"; } +click $((SX + SW / 2)) "$CODE_Y"; text "$LEGACY_CODE"; click "$PX" "$PY" +wait_widget "Saved DeepSeek · deepseek-v4-flash as primary." +sheet_closes "llm.sheet.import" || fail "the import sheet stayed up after the paste" +[ "$(profile_routes)" = "deepseek-v4-flash:-" ] || fail "profile after the paste: $(profile_routes)" +keys_end_with DEEPSEEK_API_KEY=fcb0 || fail "the pasted key is not in the profile" +grab imported-by-paste >/dev/null +pass "an old single-provider code pasted into the empty list: DeepSeek deepseek-v4-flash is the primary (grab imported-by-paste)" + +# 3. Add OpenAI with a fake key on the host's sheet. +add_openai() { + add_model "$1" openai 0 0 "$FAKE_KEY" + [ "$SAVED" = untested ] || fail "OpenAI was not saved without testing" + wait_widget "Added OpenAI · gpt-4o" +} +add_openai add1 +[ "$(profile_families)" = "deepseek openai" ] || fail "profile after the add: $(profile_families)" +keys_end_with DEEPSEEK_API_KEY=fcb0 OPENAI_API_KEY=1234 || fail "OpenAI's key is not in the profile" +grab added-openai >/dev/null +pass "OpenAI added with the wizard (gpt-4o, official route), saved without testing (grab added-openai)" + +# 4. Import QR-A by drop onto the sheet: appended, no confirm step. +bottom_app; read -r X Y < <(widget "Import code from image"); click "$X" "$Y" +read -r SX SY SW SH < <(wait_sheet "llm.sheet.image") +sleep 1 +DROP=$(curl -s --get --data-urlencode "path=$FIXTURE" --data-urlencode "x=$((SX + SW / 2))" \ + --data-urlencode "y=$((SY + SH / 2))" --data-urlencode wait=1 "127.0.0.1:$PORT/drop") +case $DROP in *'"drop_handled":true'*'"drag_response":"copy"'*) ;; *) fail "the sheet did not take the drop: $DROP" ;; esac +wait_log "llm: image dropped on the import sheet" +C=$(pin_prompt drop-read "$SX" "$SY" "$SW" "$SH") || fail "the sheet did not ask for the PIN after the drop" +PIN_Y=$(echo "$C" | nth_full 3); read -r PX PY < <(echo "$C" | pill) +[ -n "$PIN_Y" ] && [ -n "$PX" ] || { echo "$C"; fail "import sheet controls not found"; } +click $((SX + SW / 2)) "$PIN_Y"; text "$FIXTURE_PIN"; click "$PX" "$PY" +wait_widget "Added DeepSeek · deepseek-chat, Z.ai · glm-4.6 as fallbacks." +sheet_closes "llm.sheet.image" || fail "the import sheet stayed up (a confirm step?)" +[ "$(profile_families)" = "deepseek openai deepseek zai" ] || fail "profile after the drop: $(profile_families)" +[ "$(profile_routes)" = "deepseek-v4-flash:- gpt-4o:- deepseek-chat:DEEPSEEK_2_API_KEY glm-4.6:-" ] || fail "routes after the drop: $(profile_routes)" +keys_end_with DEEPSEEK_API_KEY=fcb0 OPENAI_API_KEY=1234 DEEPSEEK_2_API_KEY=0000 ZAI_API_KEY=0000 \ + || fail "the keys after the drop are wrong (the primary's key must be untouched)" +grab imported-by-drop >/dev/null +pass "dropped QR-A appended as fallbacks without a confirm step: primary + OpenAI first, QR-A's DeepSeek in DEEPSEEK_2_API_KEY, the primary's key untouched (grab imported-by-drop)" + +# 5. Import QR-A again by the picker (test image), a wrong PIN first: the same +# code again changes nothing. +pick_and_unlock() { + local X Y C + bottom_app; read -r X Y < <(widget "Import code from image"); click "$X" "$Y" + read -r SX SY SW SH < <(wait_sheet "llm.sheet.pick") + sleep 1 + C=$(controls "$1-sheet" "$SX" "$SY" "$SW" "$SH") + CHOOSE_Y=$(echo "$C" | nth_full 1); PIN_Y=$(echo "$C" | nth_full 3); read -r PX PY < <(echo "$C" | pill) + click $((SX + SW / 2)) "$CHOOSE_Y" + pin_prompt "$1-picked" "$SX" "$SY" "$SW" "$SH" >/dev/null || fail "the sheet did not ask for the PIN after the pick" +} +pick_and_unlock pick1 +BEFORE=$(cat "$PROFILE") +click $((SX + SW / 2)) "$PIN_Y"; text "0000-0000"; click "$PX" "$PY"; sleep 4 +[ -n "$(sheet_rect "llm.sheet.pick")" ] || fail "a wrong PIN closed the sheet" +[ "$(cat "$PROFILE")" = "$BEFORE" ] || fail "a wrong PIN changed the profile" +grab wrong-pin >/dev/null +click $((SX + SW / 2)) "$PIN_Y" +for _ in 1 2 3 4 5 6 7 8 9; do key Backspace; done +text "$FIXTURE_PIN"; click "$PX" "$PY" +wait_widget "Already saved: DeepSeek · deepseek-chat, Z.ai · glm-4.6." +sheet_closes "llm.sheet.pick" || fail "the import sheet stayed up after the second import" +[ "$(cat "$PROFILE")" = "$BEFORE" ] || fail "importing QR-A again changed the profile" +grab reimported >/dev/null +pass "QR-A picked and imported again after a refused wrong PIN: nothing changed, no duplicates (grab reimported)" + +# 6. The phone QR: show the code, read it from the grab. +LOG_MARK=$(( $(wc -l <"$LOG") + 1 )) +bottom_app; read -r X Y < <(widget "Show QR for phone"); click "$X" "$Y" +read -r SX SY SW SH < <(wait_sheet "countdown :=") +SHOWN=$(date +%s) +sleep 1 +PIN=$(get "snap?q=$(q "pin := Label")" | python3 -c ' +import json,re,sys +for e in json.load(sys.stdin)["s"]: + m=re.search(r"pin := Label\{[^\n]*text: \"([A-Z0-9-]+)\"", e.get("t") or "") + if m: print(m.group(1))' | tail -1) +[ -n "$PIN" ] || fail "no PIN on the export sheet" +EXPORT=$(grab export 1) +cp "$EXPORT" "$WORK/export.png"; echo "$PIN" >"$WORK/export-pin.txt" +READ=$("$READ_QR" "$EXPORT" "$PIN" "$WORK/octos") || fail "the exported QR does not match the profile: $READ" +echo "$READ" | grep -q '"same_as_profile":true' || fail "$READ" +echo "$READ" | grep -q '"family":"openai"' || fail "OpenAI missing from the code: $READ" +echo "$READ" | grep -q '"DEEPSEEK_2_API_KEY":"••••0000"' || fail "the second DeepSeek slot is missing from the code: $READ" +echo "$READ" | grep -q '"DEEPSEEK_API_KEY":"••••fcb0"' || fail "the primary's key is missing from the code: $READ" +pass "export QR read back from the grab (rqrr) with PIN $PIN: same set and keys as _main.json, both DeepSeek slots included" +# The countdown ticks: two grabs of the sheet two seconds apart differ (only +# the "Expires in" line, below the PIN, changes on it: scroll down to it). +same_sheet() { + python3 - "$1" "$2" "$SX" "$SY" "$SW" "$SH" "$WORK/sheet.py" <<'PY' +import sys +exec(open(sys.argv[7]).read().split("path, sx, sy")[0]) +a, b = read_png(sys.argv[1]), read_png(sys.argv[2]) +sx, sy, sw, sh = map(int, map(float, sys.argv[3:7])) +rows = range(sy, min(sy + sh, a[1], b[1])) +diff = sum(a[3][y][sx * a[2]:(sx + sw) * a[2]] != b[3][y][sx * b[2]:(sx + sw) * b[2]] for y in rows) +sys.exit(0 if diff == 0 else 1) +PY +} +scroll $((SX + SW / 2)) $((SY + SH / 2)) 600; sleep 1 +T1=$(grab countdown-1); sleep 2; T2=$(grab countdown-2) +same_sheet "$T1" "$T2" && fail "the countdown does not tick (grabs countdown-1 and countdown-2 are the same)" +# At 0 the sheet closes itself: the service's own close comes 5 s after the +# lifetime (counted from before the code was sealed), so a sheet gone within +# the lifetime + 2 s closed itself. +for _ in $(seq 1 $((QR_SECONDS * 2 + 20))); do [ -z "$(sheet_rect "countdown :=")" ] && break; sleep 0.5; done +CLOSED=$(( $(date +%s) - SHOWN )) +[ -z "$(sheet_rect "countdown :=")" ] || fail "the export sheet did not close after $QR_SECONDS s" +[ "$CLOSED" -le $((QR_SECONDS + 2)) ] || fail "the export sheet closed after ${CLOSED} s, not by its own countdown ($QR_SECONDS s)" +grab export-closed >/dev/null +if tail -n +"$LOG_MARK" "$LOG" | grep -Eq "$SPLASH_ERRORS"; then + tail -n +"$LOG_MARK" "$LOG" | grep -E "$SPLASH_ERRORS" | head -5 + fail "Splash errors while the export sheet was up" +fi +pass "the countdown ticked and the sheet closed itself after ${CLOSED} s ($QR_SECONDS s lifetime); no Splash error during the export" + +# 7. No key text anywhere the UI or logs show; no panic. +get "snap?all=1" >"$WORK/snap.json"; get "d" >"$WORK/dump.txt"; get "log?n=5000" >"$WORK/remote-log.json" +if grep -Eq "$SECRETS" "$WORK/snap.json" "$WORK/dump.txt" "$WORK/remote-log.json" "$LOG"; then + grep -Eo ".{40}($SECRETS).{10}" "$WORK/snap.json" "$WORK/dump.txt" "$WORK/remote-log.json" "$LOG" | head -5 + fail "key text in the UI tree or a log" +fi +grep -q "panicked" "$LOG" "$WORK/remote-log.json" && fail "panic in the log" +if grep -Eq "$SPLASH_ERRORS" "$LOG"; then grep -E "$SPLASH_ERRORS" "$LOG" | head -5; fail "Splash errors in the host log"; fi +pass "no key text in /snap, /d, /log or the host log; no panic, no Splash error" + +get "gq?scale=0.5" >/dev/null +for _ in $(seq 1 30); do kill -0 "$PID" 2>/dev/null || break; sleep 0.5; done +kill -0 "$PID" 2>/dev/null && fail "the shell did not exit after /gq" +kill "$FAKE" 2>/dev/null || true +trap - EXIT +pass "shell exited after /gq; grabs in $WORK/grabs, export.png + export-pin.txt in $WORK" diff --git a/scripts/system_apps_remote.sh b/scripts/system_apps_remote.sh new file mode 100755 index 00000000..b4e4f198 --- /dev/null +++ b/scripts/system_apps_remote.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# The system apps on the desktop, driven through Makepad's remote +# (instrument) mode: no OS screenshots, only the app's own frames. +# +# cargo build --release [--config …] && scripts/system_apps_remote.sh [artifacts-dir] +# +# Runs the release binary hidden, with its own OCTOSENSE_HOME, octos core dir +# and file vaults under a temp directory, so neither ~/.octosense nor the login +# keychain is touched. Checks: the launcher lists every selected system app; +# Photos opens and draws (no network needed); Mail opens and its host-owned +# sign-in sheet appears (nobody signs in); each app runs in an isolate under +# its manifest's policy; bundles unpack under /apps/.system//; +# AI providers (when selected) opens on its empty provider list; the log has +# no panic and no UI hang outside the known ones. Grabs are kept as evidence. +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +BIN=${OCTOSENSE_BIN:-$ROOT/target/release/octosense} +WORK=${1:-$(mktemp -d -t octosense-system-apps)} +mkdir -p "$WORK/home" "$WORK/octos" "$WORK/grabs" +LOG=$WORK/host.log +APPS=$(python3 -c 'import json,sys; print(" ".join(json.load(open(sys.argv[1]))["apps"]))' "$ROOT/system-apps.json") +echo "artifacts: $WORK" +echo "system apps: $APPS" + +# From the temp directory: Maps' tile cache is written relative to the +# working directory (./local/tilecache_v4), not under the app's jail. +cd "$WORK" +env -u MAKEPAD_HOME -u MAKEPAD_WM_ROOT -u MAKEPAD_WM_THEME \ + OCTOSENSE_HOME="$WORK/home" OCTOS_APP_CORE_DIR="$WORK/octos" \ + OCTOSENSE_MAIL_VAULT=file OCTOSENSE_LLM_VAULT=file \ + MAKEPAD_HIDE_WINDOWS=1 "$BIN" --remote >"$LOG" 2>&1 & +PID=$! +PORT= +cleanup() { + if kill -0 "$PID" 2>/dev/null; then + [ -n "$PORT" ] && curl -s "127.0.0.1:$PORT/quit" >/dev/null || true + sleep 2 + kill "$PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +fail() { echo "FAIL: $*"; exit 1; } +pass() { echo "PASS: $*"; } +for _ in $(seq 1 120); do + PORT=$(sed -n 's/.*\[makepad-remote\] listening on 127\.0\.0\.1:\([0-9]*\) pid=.*/\1/p' "$LOG" | head -1) + [ -n "$PORT" ] && break + sleep 0.5 +done +[ -n "$PORT" ] || fail "no remote port in $LOG" +get() { curl -fsS "127.0.0.1:$PORT/$1"; } +# Input waits for the next frame; while a bundle unpacks on the UI thread the +# wait can time out after the key was delivered. What a key did is checked +# through the log and snapshots, so a timed-out wait is only noted. +key() { + local out + out=$(curl -s "127.0.0.1:$PORT/k?k=press&c=$1&wait=1${2:-}") + case $out in *'"err"'*) echo "note: key $1: $out" ;; esac +} +grab() { + local png + for _ in 1 2 3 4 5; do + # A hidden window's grab can miss its frame ("retry"): ask again. + png=$(curl -s "127.0.0.1:$PORT/g?scale=0.5" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("png",""))' || true) + [ -n "$png" ] && { cp "$png" "$WORK/grabs/$1.png"; return; } + sleep 1 + done + fail "no frame for $1" +} +# `/snap` texts, one per line, for a query. +snap() { get "snap?q=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$1")" \ + | python3 -c 'import json,sys; [print(e["i"], e["ty"]) for e in json.load(sys.stdin)["s"]]'; } +wait_log() { + for _ in $(seq 1 ${2:-40}); do grep -q "$1" "$LOG" && return 0; sleep 0.5; done + fail "log never said: $1" +} +# Type into the Start menu's search (KeyDown characters, as the smoke test does). +type_keys() { + local text=$1 i c + for ((i = 0; i < ${#text}; i++)); do + c=${text:i:1} + case $c in + " ") key Space ;; + -) key Minus ;; + *) key "Key$(printf %s "$c" | tr a-z A-Z)" ;; + esac + done +} +launch() { key Space "&cmd=1"; type_keys "$1"; key Return; } +label_of() { # the manifest name of a selected app + python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["name"])' \ + "$ROOT/$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["source"])' "$ROOT/system-apps.json")/$1/bundle/manifest.json" +} + +for _ in $(seq 1 60); do curl -s "127.0.0.1:$PORT/snap?q=main_window" | grep -q '"s":\[{' && break; sleep 0.5; done +pass "desktop up on port $PORT (pid $PID)" + +# 1. The launcher lists each system app (its short id, its manifest name). +for app in $APPS; do + key Space "&cmd=1" + type_keys "$app" + grab "launcher-$app" + key Escape +done +python3 - "$WORK/grabs" $APPS <<'EOF' || fail "launcher grabs missing" +import sys, pathlib +d = pathlib.Path(sys.argv[1]) +assert all((d / f"launcher-{a}.png").stat().st_size > 10000 for a in sys.argv[2:]) +EOF +grep -q 'wm: modules linked: .*"card"' "$LOG" || fail "the Card runner is not linked" +pass "launcher searched for: $APPS (grabs launcher-*.png); Card runner linked" + +# 2. Photos: opens in the Card runner and draws its library. +launch photos +wait_log "card: os.photos running under" +for _ in $(seq 1 20); do snap "Collections" | grep -q "^card Splash" && break; sleep 0.5; done +snap "Collections" | grep -q "^card Splash" || fail "Photos drew nothing" +grab photos +pass "Photos opened: $(grep -o 'card: os.photos running under.*' "$LOG" | head -1)" + +# 3. Mail: opens, and its sign-in is the host's sheet, not the app's. +launch mail +wait_log "card: os.mail running under" +sleep 2 +grab mail-welcome +snap "Add account" | grep -q "^card Splash" || fail "Mail shows no Add account" +# The button is the Splash's; click its centre as drawn in the welcome pane. +read -r X Y < <(get "snap?q=card" | python3 -c ' +import json,sys +cards=[e for e in json.load(sys.stdin)["s"] if e["i"]=="card" and "account = nil" in (e.get("t") or "")] +x,y,w,h=cards[-1]["r"]; print(x+w/2, y+h*0.567)') +get "click?x=$X&y=$Y&wait=1" >/dev/null +for _ in $(seq 1 20); do snap "Add a mail account" | grep -q "^sheet Splash" && break; sleep 0.5; done +snap "Add a mail account" | grep -q "^sheet Splash" || fail "Mail's sign-in sheet did not open" +grab mail-sheet +pass "Mail's host-owned sign-in sheet is up (not signed in)" + +# 4. AI providers, when this build selects it: opens on an empty list. +if [[ " $APPS " == *" ai-providers "* ]]; then + key Escape + launch "ai providers" + sleep 3 + if grep -q 'os.ai-providers requests unknown capability' "$LOG"; then + fail "App Hub's policy at this pin does not know the \`llm\` capability; AI providers is refused (its notice is drawn)" + fi + wait_log "card: os.ai-providers running under" + sleep 2 + grab ai-providers + snap "No models yet" | grep -q "^card Splash" || fail "AI providers shows no empty list" + NAME=$(label_of ai-providers) + [ -z "$(ls -A "$WORK/octos" 2>/dev/null)" ] || echo "note: octos core dir is not empty" + pass "$NAME opened: $(grep -o 'card: os.ai-providers running under.*' "$LOG" | head -1)" +fi + +# 5. Containment: one isolate per app under its policy, `os.` ids, bundles +# unpacked outside every jail under the temp home. +for app in photos mail; do + [ -f "$WORK/home/apps/.system/os.$app/"*/manifest.json ] || fail "os.$app not unpacked under $WORK/home/apps/.system" + [ -d "$WORK/home/apps/os.$app" ] || fail "os.$app has no jail" +done +pass "bundles unpacked under /apps/.system///, jails at /apps/" + +# 6. The log: no panic; UI hangs only where known (see KNOWN below). +get "log?n=2000" >"$WORK/remote-log.json" +grep -q "panicked" "$LOG" "$WORK/remote-log.json" && fail "panic in the log" +# Known: the catalog's startup Makepad-source probe, and App Hub unpacking and +# hashing a system bundle, or building an app's Splash VM, on the UI thread +# the first time it opens. +KNOWN='makepad_source::makepad_root|octosense_appstore::system::prepare|CxSplashVmExt>::alloc_splash_vm' +UNKNOWN=$(grep "\[ui-hang\]" "$LOG" | grep -Ev "$KNOWN" || true) +[ -z "$UNKNOWN" ] || { echo "$UNKNOWN" | cut -c1-400; fail "unexpected UI hang"; } +echo "note: $(grep -c "\[ui-hang\]" "$LOG" || true) known UI hang(s) (startup catalog probe, first-open bundle unpack or VM)" +pass "no panic, no unexpected [ui-hang]" + +# Finish: grab and quit, then confirm the process is gone. +get "gq?scale=0.5" >/dev/null +for _ in $(seq 1 30); do kill -0 "$PID" 2>/dev/null || break; sleep 0.5; done +kill -0 "$PID" 2>/dev/null && fail "the shell did not exit after /gq" +trap - EXIT +pass "shell exited after /gq; grabs in $WORK/grabs"