From 2b735d66cf860688d8f8706a527bc68c581e161b Mon Sep 17 00:00:00 2001 From: shuyuzhang Date: Thu, 12 Nov 2026 11:31:09 +0800 Subject: [PATCH 1/5] docs: drop the start.sh and model-bench.json note from the 145B and 120B pages --- edge0-infer/docs/edge0-120b.md | 2 +- edge0-infer/docs/edge0-145b.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edge0-infer/docs/edge0-120b.md b/edge0-infer/docs/edge0-120b.md index 75501de3..d2db844c 100644 --- a/edge0-infer/docs/edge0-120b.md +++ b/edge0-infer/docs/edge0-120b.md @@ -13,7 +13,7 @@ This page is the reference for running Edge0 120B with `edge0-infer`; [Getting s ## Model directory -Use each released model folder as provided: one model and one expert format per folder, with files laid out flat. `edge0-infer` reads only the `model_name` field of the folder's `model.json` and ignores its other fields (`architecture`, `args`, `env` and the rest); a `model.json` that contains `format_version` is an asset manifest of the native engine instead. The folder also holds a `start.sh` and a `model-bench.json`, which `edge0-infer` does not use. +Use each released model folder as provided: one model and one expert format per folder, with files laid out flat. `edge0-infer` reads only the `model_name` field of the folder's `model.json` and ignores its other fields (`architecture`, `args`, `env` and the rest); a `model.json` that contains `format_version` is an asset manifest of the native engine instead. | model_name | Registration (default alias) | Expert format | | --- | --- | --- | diff --git a/edge0-infer/docs/edge0-145b.md b/edge0-infer/docs/edge0-145b.md index 3e70a430..d373190b 100644 --- a/edge0-infer/docs/edge0-145b.md +++ b/edge0-infer/docs/edge0-145b.md @@ -13,7 +13,7 @@ This page is the reference for running Edge0 145B with `edge0-infer`; [Getting s ## Model directory -Use each released model folder as provided: one model and one expert format per folder, with files laid out flat. `edge0-infer` reads only the `model_name` field of the folder's `model.json` and ignores its other fields (`architecture`, `args`, `env` and the rest); a `model.json` that contains `format_version` is an asset manifest of the native engine instead. The folder also holds a `start.sh` and a `model-bench.json`, which `edge0-infer` does not use. +Use each released model folder as provided: one model and one expert format per folder, with files laid out flat. `edge0-infer` reads only the `model_name` field of the folder's `model.json` and ignores its other fields (`architecture`, `args`, `env` and the rest); a `model.json` that contains `format_version` is an asset manifest of the native engine instead. | model_name | Registration (default alias) | Expert format | | --- | --- | --- | From cfe48bdc5179d859c57165bdca2d5e3769a56b2f Mon Sep 17 00:00:00 2001 From: shuyuzhang Date: Thu, 12 Nov 2026 11:31:09 +0800 Subject: [PATCH 2/5] desktop: use the edge0-infer beside the app when no executable is set The Windows release package puts Edge0 Chat.exe and edge0-infer.exe in the same folder, yet a first start stopped at "No edge0-infer executable is configured" until the path was entered by hand. Reading the settings now fills an empty executable with the edge0-infer next to the app's own executable when it is there; a path the user set is kept, and without that file nothing changes. --- .../examples/desktop/src-tauri/src/service.rs | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/edge0-infer/examples/desktop/src-tauri/src/service.rs b/edge0-infer/examples/desktop/src-tauri/src/service.rs index 2455581e..76685475 100644 --- a/edge0-infer/examples/desktop/src-tauri/src/service.rs +++ b/edge0-infer/examples/desktop/src-tauri/src/service.rs @@ -236,11 +236,24 @@ impl Manager { } pub fn settings(&self) -> Result { let path = self.directory.join("settings.json"); - if !path.exists() { - return Ok(Settings::default()); + let mut settings: Settings = if path.exists() { + serde_json::from_slice(&fs::read(path).map_err(|e| e.to_string())?) + .map_err(|e| format!("Cannot read settings: {e}"))? + } else { + Settings::default() + }; + // No executable chosen yet: use the edge0-infer that the release package puts beside this app. + if settings.executable.trim().is_empty() { + if let Some(found) = std::env::current_exe() + .ok() + .as_deref() + .and_then(Path::parent) + .and_then(bundled_executable) + { + settings.executable = found; + } } - serde_json::from_slice(&fs::read(path).map_err(|e| e.to_string())?) - .map_err(|e| format!("Cannot read settings: {e}")) + Ok(settings) } pub fn save_settings(&self, settings: &Settings) -> Result<(), String> { settings.validate_options()?; @@ -677,6 +690,13 @@ impl Drop for Manager { let _ = self.shutdown(); } } +/// The edge0-infer in `app_dir` (the directory of this app's own executable), as the release package lays +/// them out side by side; None when it is not there (a development build, an app bundle). +pub(crate) fn bundled_executable(app_dir: &Path) -> Option { + let path = app_dir.join(format!("edge0-infer{}", std::env::consts::EXE_SUFFIX)); + path.is_file().then(|| path.to_string_lossy().into_owned()) +} + #[cfg(test)] mod tests { use super::*; @@ -689,6 +709,22 @@ mod tests { )) } #[test] + fn finds_the_bundled_executable_beside_the_app() { + let dir = directory(); + fs::create_dir_all(&dir).unwrap(); + assert_eq!(bundled_executable(&dir), None); + let exe = dir.join(format!("edge0-infer{}", std::env::consts::EXE_SUFFIX)); + fs::create_dir(&exe).unwrap(); // a directory of that name is not an executable + assert_eq!(bundled_executable(&dir), None); + fs::remove_dir(&exe).unwrap(); + fs::write(&exe, b"").unwrap(); + assert_eq!( + bundled_executable(&dir), + Some(exe.to_string_lossy().into_owned()) + ); + fs::remove_dir_all(dir).unwrap(); + } + #[test] fn parses_only_local_ready_lines() { assert_eq!( ready_url("Listening: http://127.0.0.1:41325 model=local"), From 11d9abdcb7e30bb6813894f8affe0dacfd5bf461 Mon Sep 17 00:00:00 2001 From: shuyuzhang Date: Thu, 12 Nov 2026 11:41:29 +0800 Subject: [PATCH 3/5] desktop: read the engine status once a model is loaded The sidebar names the device from /status, but the app polled /status only on the Status and Service pages, so after loading a model it showed "device unknown" until one of those pages was opened. --- edge0-infer/examples/desktop/src/main.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/edge0-infer/examples/desktop/src/main.tsx b/edge0-infer/examples/desktop/src/main.tsx index 806f0d32..01920e8b 100644 --- a/edge0-infer/examples/desktop/src/main.tsx +++ b/edge0-infer/examples/desktop/src/main.tsx @@ -95,6 +95,8 @@ export function App() { const response = await fetch(started.url + '/v1/models', { signal: abort.signal }); if (!response.ok) throw new Error(`Model capabilities failed (${response.status})`); const selected: Model | undefined = (await response.json()).data?.[0]; if (!selected?.capabilities) throw new Error('The service does not provide model capabilities. Rebuild edge0-infer.'); runtimeSettings.current = nextSettings; setActivePath(path); setModel(selected); setService(started); setStatus(undefined); + // The sidebar shows the device from /status, which is otherwise polled only on the Status and Service pages. + const loadedEpoch = serviceEpoch.current; void fetch(started.url + '/status').then(r => r.ok ? r.json() as Promise : undefined).then(next => { if (next && loadedEpoch === serviceEpoch.current) setStatus(next); }).catch(() => undefined); changeConversation(restore ?? newConversation(nextSettings.system_prompt, selected.capabilities.thinking && selected.capabilities.thinking_default, path)); setDraft(''); setPage('chat'); } catch (failure) { if (!abort.signal.aborted) report(failure); await invoke('stop_service').catch(report); } finally { controller.current = null; setLoading(false); } From 57d6840bb48a8baae09955577449e8960bc46036 Mon Sep 17 00:00:00 2001 From: shuyuzhang Date: Thu, 12 Nov 2026 14:10:06 +0800 Subject: [PATCH 4/5] docs: link the Edge0 145B and 120B Hugging Face repositories Replace the coming-soon placeholders with the four Hugging Face repositories, add Edge0-145B and Edge0-120B badges after the ModelScope badges, and use the README.md Edge0-120B scores (51.3 / 75.3) in the translations and the model page. --- README.md | 8 +++++--- README_es.md | 10 ++++++---- README_fr.md | 10 ++++++---- README_ja.md | 10 ++++++---- README_zh.md | 10 ++++++---- docs/models/edge0-145b.md | 4 ++-- 6 files changed, 31 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index cb489cbc..3f87cb45 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--8B--A1B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) [![ModelScope](https://img.shields.io/badge/ModelScope-Edge0--35B--A3B--preview-624AFF?style=for-the-badge)](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) [![ModelScope](https://img.shields.io/badge/ModelScope-Edge0--8B--A1B--preview-624AFF?style=for-the-badge)](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--145B-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-145B-q3k) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--120B-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-120B-q3k) [![arXiv](https://img.shields.io/badge/arXiv-2609.18063-B31B1B?style=for-the-badge&logo=arxiv&logoColor=white)](https://arxiv.org/abs/2609.18063) [![GitHub](https://img.shields.io/badge/GitHub-Edge0--AI%2FEdge0-black?style=for-the-badge&logo=github)](https://github.com/Edge0-AI/Edge0) [![Discord](https://img.shields.io/badge/Discord-Join%20us-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://edge0.ai/discord) @@ -23,7 +25,7 @@ English | [中文](README_zh.md) | [日本語](README_ja.md) | [Español](README ## News -- **[2026-11-13]** We released **`edge0-145b`** and **`edge0-120b`**, latent mixture-of-experts models derived from Qwen3.8-Flash-Next for a single NVIDIA GPU on Linux or Windows. They run on [edge0-strata](edge0-strata/EDGE0.md), which `edge0-infer serve` and `edge0-infer chat` start for them; see the [model page](docs/models/edge0-145b.md). +- **[2026-11-13]** We released **`edge0-145b`** and **`edge0-120b`**, latent mixture-of-experts models derived from Qwen3.8-Flash-Next for a single NVIDIA GPU on Linux or Windows. They run on [edge0-strata](edge0-strata/EDGE0.md), which `edge0-infer serve` and `edge0-infer chat` start for them; see the [model page](docs/models/edge0-145b.md). Weights on Hugging Face: [`Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [`Edge0-145B-NVFP4`](https://huggingface.co/Edge0/Edge0-145B-NVFP4) · [`Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [`Edge0-120B-NVFP4`](https://huggingface.co/Edge0/Edge0-120B-NVFP4). - **[2026-11-04]** We released the **unified inference engine [`edge0-infer`](edge0-infer/README.md)**: built on [GGML](https://github.com/ggml-org/ggml), one engine and one native C++ API run the same model on macOS, iOS, Android and Windows, with a CLI, an OpenAI-compatible local server, and chat app examples for desktop, Android and iOS. The first release supports `edge0-35b`; `edge0-8b` follows in a later release. It replaces the Python framework and the per-platform engines; the previous code is kept at tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer). - **[2026-09-30]** We released the **edge0 inference engines for four platforms — iOS, macOS, Android and Windows** (source at tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer)). - **[2026-09-16]** Our technical report is on arXiv: [The Other Half of the Memory Wall: Serving 35B MoEs from SSD with Trained Routing Prediction](https://arxiv.org/abs/2609.18063). @@ -75,8 +77,8 @@ trained prerouter heads work together as one unit. |---|---|---|---| | `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4-bit, 40 layers, 256 experts, prerouter K=4 | Native engine ✅ | | `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4-bit, 24 layers, 128 experts, prerouter K=8 | Native engine (planned) | -| [`edge0-145b`](docs/models/edge0-145b.md) | coming soon | Q3_K or NVFP4 latent experts (1792), 48 layers, 512 experts, K=10 | edge0-strata ✅ (Linux or Windows, NVIDIA GPU) | -| [`edge0-120b`](docs/models/edge0-145b.md) | coming soon | Q3_K or NVFP4 latent experts (1280), 48 layers, 512 experts, K=10 | edge0-strata ✅ (Linux or Windows, NVIDIA GPU) | +| [`edge0-145b`](docs/models/edge0-145b.md) | [`Edge0/Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [NVFP4](https://huggingface.co/Edge0/Edge0-145B-NVFP4) | Q3_K or NVFP4 latent experts (1792), 48 layers, 512 experts, K=10 | edge0-strata ✅ (Linux or Windows, NVIDIA GPU) | +| [`edge0-120b`](docs/models/edge0-145b.md) | [`Edge0/Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [NVFP4](https://huggingface.co/Edge0/Edge0-120B-NVFP4) | Q3_K or NVFP4 latent experts (1280), 48 layers, 512 experts, K=10 | edge0-strata ✅ (Linux or Windows, NVIDIA GPU) | The two released checkpoints are built on open sparse-MoE base models (Qwen3.6-35B-A3B and the Ling 3.0 bailing hybrid respectively) and ship with the diff --git a/README_es.md b/README_es.md index c1b3782b..2cdc11ed 100644 --- a/README_es.md +++ b/README_es.md @@ -12,6 +12,8 @@ [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--8B--A1B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) [![ModelScope](https://img.shields.io/badge/ModelScope-Edge0--35B--A3B--preview-624AFF?style=for-the-badge)](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) [![ModelScope](https://img.shields.io/badge/ModelScope-Edge0--8B--A1B--preview-624AFF?style=for-the-badge)](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--145B-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-145B-q3k) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--120B-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-120B-q3k) [![arXiv](https://img.shields.io/badge/arXiv-2609.18063-B31B1B?style=for-the-badge&logo=arxiv&logoColor=white)](https://arxiv.org/abs/2609.18063) [![GitHub](https://img.shields.io/badge/GitHub-Edge0--AI%2FEdge0-black?style=for-the-badge&logo=github)](https://github.com/Edge0-AI/Edge0) [![Discord](https://img.shields.io/badge/Discord-Join%20us-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://edge0.ai/discord) @@ -23,7 +25,7 @@ ## Novedades -- **[2026-11-13]** Lanzamos **`edge0-145b`** y **`edge0-120b`**, modelos latent mixture-of-experts derivados de Qwen3.8-Flash-Next para una sola GPU NVIDIA en Linux o Windows. Se ejecutan en [edge0-strata](edge0-strata/EDGE0.md), que `edge0-infer serve` y `edge0-infer chat` inician para ellos; consulta la [página del modelo](docs/models/edge0-145b.md). +- **[2026-11-13]** Lanzamos **`edge0-145b`** y **`edge0-120b`**, modelos latent mixture-of-experts derivados de Qwen3.8-Flash-Next para una sola GPU NVIDIA en Linux o Windows. Se ejecutan en [edge0-strata](edge0-strata/EDGE0.md), que `edge0-infer serve` y `edge0-infer chat` inician para ellos; consulta la [página del modelo](docs/models/edge0-145b.md). Pesos en Hugging Face: [`Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [`Edge0-145B-NVFP4`](https://huggingface.co/Edge0/Edge0-145B-NVFP4) · [`Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [`Edge0-120B-NVFP4`](https://huggingface.co/Edge0/Edge0-120B-NVFP4). - **[2026-11-04]** Lanzamos el **motor de inferencia unificado [`edge0-infer`](edge0-infer/README_es.md)**: construido sobre [GGML](https://github.com/ggml-org/ggml), un solo motor y una sola API nativa en C++ ejecutan el mismo modelo en macOS, iOS, Android y Windows, con una CLI, un servidor local compatible con OpenAI y ejemplos de aplicaciones de chat para escritorio, Android e iOS. La primera versión admite `edge0-35b`; `edge0-8b` llegará en una versión posterior. Sustituye al framework de Python y a los motores de cada plataforma; el código anterior se conserva en la etiqueta [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer). - **[2026-09-30]** Lanzamos los **motores de inferencia de edge0 para cuatro plataformas — iOS, macOS, Android y Windows** (código fuente en la etiqueta [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer)). - **[2026-09-16]** Nuestro informe técnico está en arXiv: [The Other Half of the Memory Wall: Serving 35B MoEs from SSD with Trained Routing Prediction](https://arxiv.org/abs/2609.18063). @@ -78,8 +80,8 @@ las cabezas prerouter entrenadas funcionan en conjunto como una sola unidad. |---|---|---|---| | `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4 bits, 40 capas, 256 expertos, prerouter K=4 | Motor nativo ✅ | | `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4 bits, 24 capas, 128 expertos, prerouter K=8 | Motor nativo (previsto) | -| [`edge0-145b`](docs/models/edge0-145b.md) | coming soon | expertos latentes Q3_K o NVFP4 (1792), 48 capas, 512 expertos, K=10 | edge0-strata ✅ (Linux o Windows, GPU NVIDIA) | -| [`edge0-120b`](docs/models/edge0-145b.md) | coming soon | expertos latentes Q3_K o NVFP4 (1280), 48 capas, 512 expertos, K=10 | edge0-strata ✅ (Linux o Windows, GPU NVIDIA) | +| [`edge0-145b`](docs/models/edge0-145b.md) | [`Edge0/Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [NVFP4](https://huggingface.co/Edge0/Edge0-145B-NVFP4) | expertos latentes Q3_K o NVFP4 (1792), 48 capas, 512 expertos, K=10 | edge0-strata ✅ (Linux o Windows, GPU NVIDIA) | +| [`edge0-120b`](docs/models/edge0-145b.md) | [`Edge0/Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [NVFP4](https://huggingface.co/Edge0/Edge0-120B-NVFP4) | expertos latentes Q3_K o NVFP4 (1280), 48 capas, 512 expertos, K=10 | edge0-strata ✅ (Linux o Windows, GPU NVIDIA) | Los dos checkpoints publicados se construyen sobre modelos base sparse-MoE abiertos (Qwen3.6-35B-A3B y el híbrido bailing Ling 3.0, respectivamente) e incluyen @@ -136,7 +138,7 @@ Máximo 100: | Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | | --- | ---: | ---: | ---: | ---: | ---: | ---: | | **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | -| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| **Edge0-120B** | 51.3 | 75.3 | 81.0 | 79.3 | 87.4 | 31.0 | | DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | | Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | | Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | diff --git a/README_fr.md b/README_fr.md index e419cc49..68dfd3f0 100644 --- a/README_fr.md +++ b/README_fr.md @@ -12,6 +12,8 @@ [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--8B--A1B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) [![ModelScope](https://img.shields.io/badge/ModelScope-Edge0--35B--A3B--preview-624AFF?style=for-the-badge)](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) [![ModelScope](https://img.shields.io/badge/ModelScope-Edge0--8B--A1B--preview-624AFF?style=for-the-badge)](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--145B-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-145B-q3k) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--120B-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-120B-q3k) [![arXiv](https://img.shields.io/badge/arXiv-2609.18063-B31B1B?style=for-the-badge&logo=arxiv&logoColor=white)](https://arxiv.org/abs/2609.18063) [![GitHub](https://img.shields.io/badge/GitHub-Edge0--AI%2FEdge0-black?style=for-the-badge&logo=github)](https://github.com/Edge0-AI/Edge0) [![Discord](https://img.shields.io/badge/Discord-Join%20us-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://edge0.ai/discord) @@ -23,7 +25,7 @@ ## Actualités -- **[2026-11-13]** Nous avons publié **`edge0-145b`** et **`edge0-120b`**, des modèles latent mixture-of-experts dérivés de Qwen3.8-Flash-Next pour un seul GPU NVIDIA sous Linux ou Windows. Ils s'exécutent sur [edge0-strata](edge0-strata/EDGE0.md), que `edge0-infer serve` et `edge0-infer chat` démarrent pour eux ; voir la [page du modèle](docs/models/edge0-145b.md). +- **[2026-11-13]** Nous avons publié **`edge0-145b`** et **`edge0-120b`**, des modèles latent mixture-of-experts dérivés de Qwen3.8-Flash-Next pour un seul GPU NVIDIA sous Linux ou Windows. Ils s'exécutent sur [edge0-strata](edge0-strata/EDGE0.md), que `edge0-infer serve` et `edge0-infer chat` démarrent pour eux ; voir la [page du modèle](docs/models/edge0-145b.md). Poids sur Hugging Face : [`Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [`Edge0-145B-NVFP4`](https://huggingface.co/Edge0/Edge0-145B-NVFP4) · [`Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [`Edge0-120B-NVFP4`](https://huggingface.co/Edge0/Edge0-120B-NVFP4). - **[2026-11-04]** Nous avons publié le **moteur d'inférence unifié [`edge0-infer`](edge0-infer/README_fr.md)** : construit sur [GGML](https://github.com/ggml-org/ggml), un seul moteur et une seule API C++ native exécutent le même modèle sur macOS, iOS, Android et Windows, avec une CLI, un serveur local compatible OpenAI et des exemples d'applications de discussion pour le bureau, Android et iOS. La première version prend en charge `edge0-35b` ; `edge0-8b` suivra dans une version ultérieure. Il remplace le framework Python et les moteurs propres à chaque plateforme ; l'ancien code est conservé au tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer). - **[2026-09-30]** Nous avons publié les **moteurs d'inférence edge0 pour quatre plateformes — iOS, macOS, Android et Windows** (code source au tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer)). - **[2026-09-16]** Notre rapport technique est sur arXiv : [The Other Half of the Memory Wall: Serving 35B MoEs from SSD with Trained Routing Prediction](https://arxiv.org/abs/2609.18063). @@ -79,8 +81,8 @@ une seule unité. |---|---|---|---| | `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4-bit, 40 couches, 256 experts, prerouter K=4 | Moteur natif ✅ | | `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4-bit, 24 couches, 128 experts, prerouter K=8 | Moteur natif (prévu) | -| [`edge0-145b`](docs/models/edge0-145b.md) | coming soon | experts latents Q3_K ou NVFP4 (1792), 48 couches, 512 experts, K=10 | edge0-strata ✅ (Linux ou Windows, GPU NVIDIA) | -| [`edge0-120b`](docs/models/edge0-145b.md) | coming soon | experts latents Q3_K ou NVFP4 (1280), 48 couches, 512 experts, K=10 | edge0-strata ✅ (Linux ou Windows, GPU NVIDIA) | +| [`edge0-145b`](docs/models/edge0-145b.md) | [`Edge0/Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [NVFP4](https://huggingface.co/Edge0/Edge0-145B-NVFP4) | experts latents Q3_K ou NVFP4 (1792), 48 couches, 512 experts, K=10 | edge0-strata ✅ (Linux ou Windows, GPU NVIDIA) | +| [`edge0-120b`](docs/models/edge0-145b.md) | [`Edge0/Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [NVFP4](https://huggingface.co/Edge0/Edge0-120B-NVFP4) | experts latents Q3_K ou NVFP4 (1280), 48 couches, 512 experts, K=10 | edge0-strata ✅ (Linux ou Windows, GPU NVIDIA) | Les deux checkpoints publiés reposent sur des modèles de base sparse-MoE ouverts (respectivement Qwen3.6-35B-A3B et l'hybride bailing Ling 3.0) et intègrent @@ -137,7 +139,7 @@ au-dessus de la base). Maximum 100 : | Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | | --- | ---: | ---: | ---: | ---: | ---: | ---: | | **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | -| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| **Edge0-120B** | 51.3 | 75.3 | 81.0 | 79.3 | 87.4 | 31.0 | | DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | | Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | | Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | diff --git a/README_ja.md b/README_ja.md index dc2d46ed..aae52b0f 100644 --- a/README_ja.md +++ b/README_ja.md @@ -12,6 +12,8 @@ [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--8B--A1B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) [![ModelScope](https://img.shields.io/badge/ModelScope-Edge0--35B--A3B--preview-624AFF?style=for-the-badge)](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) [![ModelScope](https://img.shields.io/badge/ModelScope-Edge0--8B--A1B--preview-624AFF?style=for-the-badge)](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--145B-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-145B-q3k) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--120B-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-120B-q3k) [![arXiv](https://img.shields.io/badge/arXiv-2609.18063-B31B1B?style=for-the-badge&logo=arxiv&logoColor=white)](https://arxiv.org/abs/2609.18063) [![GitHub](https://img.shields.io/badge/GitHub-Edge0--AI%2FEdge0-black?style=for-the-badge&logo=github)](https://github.com/Edge0-AI/Edge0) [![Discord](https://img.shields.io/badge/Discord-Join%20us-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://edge0.ai/discord) @@ -23,7 +25,7 @@ ## ニュース -- **[2026-11-13]** **`edge0-145b`** と **`edge0-120b`** をリリースしました。Qwen3.8-Flash-Next から派生した latent MoE モデルで、Linux または Windows 上の NVIDIA GPU 1 枚で動作します。[edge0-strata](edge0-strata/EDGE0.md) で動作し、`edge0-infer serve` と `edge0-infer chat` がこれらのモデル用に起動します。[モデルページ](docs/models/edge0-145b.md)を参照してください。 +- **[2026-11-13]** **`edge0-145b`** と **`edge0-120b`** をリリースしました。Qwen3.8-Flash-Next から派生した latent MoE モデルで、Linux または Windows 上の NVIDIA GPU 1 枚で動作します。[edge0-strata](edge0-strata/EDGE0.md) で動作し、`edge0-infer serve` と `edge0-infer chat` がこれらのモデル用に起動します。[モデルページ](docs/models/edge0-145b.md)を参照してください。Hugging Face の重み：[`Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [`Edge0-145B-NVFP4`](https://huggingface.co/Edge0/Edge0-145B-NVFP4) · [`Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [`Edge0-120B-NVFP4`](https://huggingface.co/Edge0/Edge0-120B-NVFP4)。 - **[2026-11-04]** **統合推論エンジン [`edge0-infer`](edge0-infer/README_ja.md)** をリリースしました。[GGML](https://github.com/ggml-org/ggml) をベースに、1 つのエンジンと 1 つのネイティブ C++ API で、同じモデルを macOS、iOS、Android、Windows で実行します。CLI、OpenAI 互換のローカルサーバー、デスクトップ・Android・iOS 向けのチャットアプリのサンプルも付属します。最初のリリースは `edge0-35b` をサポートし、`edge0-8b` は今後のリリースでサポートします。これまでの Python フレームワークと各プラットフォームのエンジンを置き換えるもので、以前のコードは tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer) に残しています。 - **[2026-09-30]** **4 つのプラットフォーム — iOS、macOS、Android、Windows — 向けの edge0 推論エンジン**をリリースしました(ソースは tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer) を参照)。 - **[2026-09-16]** 技術レポートが arXiv に掲載: [The Other Half of the Memory Wall: Serving 35B MoEs from SSD with Trained Routing Prediction](https://arxiv.org/abs/2609.18063)。 @@ -61,8 +63,8 @@ |---|---|---|---| | `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4-bit、40 層、256 エキスパート、prerouter K=4 | ネイティブエンジン ✅ | | `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4-bit、24 層、128 エキスパート、prerouter K=8 | ネイティブエンジン(予定) | -| [`edge0-145b`](docs/models/edge0-145b.md) | coming soon | Q3_K または NVFP4 latent エキスパート（1792）、48 層、512 エキスパート、K=10 | edge0-strata ✅（Linux または Windows、NVIDIA GPU） | -| [`edge0-120b`](docs/models/edge0-145b.md) | coming soon | Q3_K または NVFP4 latent エキスパート（1280）、48 層、512 エキスパート、K=10 | edge0-strata ✅（Linux または Windows、NVIDIA GPU） | +| [`edge0-145b`](docs/models/edge0-145b.md) | [`Edge0/Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [NVFP4](https://huggingface.co/Edge0/Edge0-145B-NVFP4) | Q3_K または NVFP4 latent エキスパート（1792）、48 層、512 エキスパート、K=10 | edge0-strata ✅（Linux または Windows、NVIDIA GPU） | +| [`edge0-120b`](docs/models/edge0-145b.md) | [`Edge0/Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [NVFP4](https://huggingface.co/Edge0/Edge0-120B-NVFP4) | Q3_K または NVFP4 latent エキスパート（1280）、48 層、512 エキスパート、K=10 | edge0-strata ✅（Linux または Windows、NVIDIA GPU） | 公開済みの 2 つのチェックポイントはいずれもオープンのスパース MoE ベースモデル(それぞれ Qwen3.6-35B-A3B と Ling 3.0 bailing hybrid)上に構築されており、本フレームワークのために実施された LoRA と prerouter の訓練成果物を同梱しています。edge0-infer は付属のアセットツールで公開チェックポイントをダウンロードし、エンジン用のモデルアセットに変換します([はじめに](#はじめに)を参照)。 `edge0-145b` と `edge0-120b` は単一の NVIDIA GPU 向けの latent-MoE です。edge0-strata 上で動作し、`edge0-infer serve` と `edge0-infer chat` がそれを起動します([edge0-145b と edge0-120b](docs/models/edge0-145b.md) を参照)。 @@ -95,7 +97,7 @@ | Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | | --- | ---: | ---: | ---: | ---: | ---: | ---: | | **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | -| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| **Edge0-120B** | 51.3 | 75.3 | 81.0 | 79.3 | 87.4 | 31.0 | | DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | | Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | | Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | diff --git a/README_zh.md b/README_zh.md index 923e9adc..055c2e53 100644 --- a/README_zh.md +++ b/README_zh.md @@ -12,6 +12,8 @@ [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--8B--A1B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) [![ModelScope](https://img.shields.io/badge/ModelScope-Edge0--35B--A3B--preview-624AFF?style=for-the-badge)](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) [![ModelScope](https://img.shields.io/badge/ModelScope-Edge0--8B--A1B--preview-624AFF?style=for-the-badge)](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--145B-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-145B-q3k) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--120B-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-120B-q3k) [![arXiv](https://img.shields.io/badge/arXiv-2609.18063-B31B1B?style=for-the-badge&logo=arxiv&logoColor=white)](https://arxiv.org/abs/2609.18063) [![GitHub](https://img.shields.io/badge/GitHub-Edge0--AI%2FEdge0-black?style=for-the-badge&logo=github)](https://github.com/Edge0-AI/Edge0) [![Discord](https://img.shields.io/badge/Discord-Join%20us-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://edge0.ai/discord) @@ -23,7 +25,7 @@ ## 新闻 -- **【2026-11-13】** 发布 **`edge0-145b`** 与 **`edge0-120b`**：由 Qwen3.8-Flash-Next 派生的 latent MoE 模型，在 Linux 或 Windows 上用单张 NVIDIA GPU 运行。它们运行在 [edge0-strata](edge0-strata/EDGE0.md) 上，由 `edge0-infer serve` 与 `edge0-infer chat` 为它们启动；见[模型页](docs/models/edge0-145b.md)。 +- **【2026-11-13】** 发布 **`edge0-145b`** 与 **`edge0-120b`**：由 Qwen3.8-Flash-Next 派生的 latent MoE 模型，在 Linux 或 Windows 上用单张 NVIDIA GPU 运行。它们运行在 [edge0-strata](edge0-strata/EDGE0.md) 上，由 `edge0-infer serve` 与 `edge0-infer chat` 为它们启动；见[模型页](docs/models/edge0-145b.md)。Hugging Face 权重：[`Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [`Edge0-145B-NVFP4`](https://huggingface.co/Edge0/Edge0-145B-NVFP4) · [`Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [`Edge0-120B-NVFP4`](https://huggingface.co/Edge0/Edge0-120B-NVFP4)。 - **【2026-11-04】** 发布**统一推理引擎 [`edge0-infer`](edge0-infer/README_zh.md)**： 基于 [GGML](https://github.com/ggml-org/ggml)，一个引擎、一套原生 C++ API， 在 macOS、iOS、Android、Windows 上运行同一个模型，并附带 CLI、OpenAI 兼容 @@ -84,8 +86,8 @@ MoE 模型跑在消费级硬件上——峰值内存由**激活**专家集而非 |---|---|---|---| | `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4bit，40 层，256 专家，prerouter K=4 | 原生引擎 ✅ | | `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4bit，24 层，128 专家，prerouter K=8 | 原生引擎（计划中） | -| [`edge0-145b`](docs/models/edge0-145b.md) | coming soon | Q3_K 或 NVFP4 latent 专家（1792），48 层，512 专家，K=10 | edge0-strata ✅（Linux 或 Windows，NVIDIA GPU） | -| [`edge0-120b`](docs/models/edge0-145b.md) | coming soon | Q3_K 或 NVFP4 latent 专家（1280），48 层，512 专家，K=10 | edge0-strata ✅（Linux 或 Windows，NVIDIA GPU） | +| [`edge0-145b`](docs/models/edge0-145b.md) | [`Edge0/Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [NVFP4](https://huggingface.co/Edge0/Edge0-145B-NVFP4) | Q3_K 或 NVFP4 latent 专家（1792），48 层，512 专家，K=10 | edge0-strata ✅（Linux 或 Windows，NVIDIA GPU） | +| [`edge0-120b`](docs/models/edge0-145b.md) | [`Edge0/Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [NVFP4](https://huggingface.co/Edge0/Edge0-120B-NVFP4) | Q3_K 或 NVFP4 latent 专家（1280），48 层，512 专家，K=10 | edge0-strata ✅（Linux 或 Windows，NVIDIA GPU） | 两个已发布的 checkpoint 均基于开源稀疏 MoE 基座（分别为 Qwen3.6-35B-A3B 与 Ling 3.0 混合架构），并携带为本框架训练的 LoRA 与 prerouter 权重。 @@ -130,7 +132,7 @@ edge0-infer 用自带的资产工具下载发布的 checkpoint，并转换成引 | Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | | --- | ---: | ---: | ---: | ---: | ---: | ---: | | **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | -| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| **Edge0-120B** | 51.3 | 75.3 | 81.0 | 79.3 | 87.4 | 31.0 | | DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | | Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | | Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | diff --git a/docs/models/edge0-145b.md b/docs/models/edge0-145b.md index 7086c988..08a84874 100644 --- a/docs/models/edge0-145b.md +++ b/docs/models/edge0-145b.md @@ -2,7 +2,7 @@ Two latent mixture-of-experts models for a single NVIDIA GPU, derived from Qwen3.8-Flash-Next: Edge0 145B and Edge0 120B. Their experts read compressed inputs, and both models run on [edge0-strata](../../edge0-strata/EDGE0.md), a fork of [Strata](https://github.com/Niko1221/Strata) whose scheduling of CPU, GPU and PCIe work they depend on. [edge0-infer](../../edge0-infer/README.md) starts edge0-strata for `serve` and `chat`, so the commands and endpoints are those of the other models. -Model card: coming soon. +Weights on Hugging Face: [`Edge0-145B-q3k`](https://huggingface.co/Edge0/Edge0-145B-q3k) · [`Edge0-145B-NVFP4`](https://huggingface.co/Edge0/Edge0-145B-NVFP4) · [`Edge0-120B-q3k`](https://huggingface.co/Edge0/Edge0-120B-q3k) · [`Edge0-120B-NVFP4`](https://huggingface.co/Edge0/Edge0-120B-NVFP4). ## Model profile @@ -22,7 +22,7 @@ Model card: coming soon. | Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | | --- | ---: | ---: | ---: | ---: | ---: | ---: | | **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | -| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| **Edge0-120B** | 51.3 | 75.3 | 81.0 | 79.3 | 87.4 | 31.0 | | DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | | Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | | Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | From b1fac3412d57b5786dff7943caaca94c3a05502c Mon Sep 17 00:00:00 2001 From: JiahaoWang Date: Thu, 12 Nov 2026 15:10:23 +0800 Subject: [PATCH 5/5] desktop: reload with a VRAM reserve when a model leaves little commit memory On a GPU that shares system memory (Windows on RTX Spark), the engine's automatic expert cache fills the free VRAM, which Windows charges to the commit limit. Edge0 145B then left ~0.5 GB of commit free and the first generation could crash with an out-of-memory error. A load now waits for the committed memory to stay steady for 3 s. When less than 10 GiB is free, the engine is reloaded with --strata-arg --vram-reserve-mib covering the shortfall (at most twice); the reserve is remembered per model in memory-reserve.json. A model whose reload frees no commit memory is not reloaded again. Below 8 GiB the UI shows a warning, and an engine that dies mid-generation reports the likely cause. Co-authored-by: Cursor --- .../examples/desktop/src-tauri/Cargo.toml | 2 +- .../desktop/src-tauri/src/platform/macos.rs | 4 + .../desktop/src-tauri/src/platform/windows.rs | 11 + .../examples/desktop/src-tauri/src/service.rs | 203 +++++++++++++++++- edge0-infer/examples/desktop/src/chat.test.ts | 13 +- edge0-infer/examples/desktop/src/chat.ts | 7 + edge0-infer/examples/desktop/src/main.tsx | 11 +- edge0-infer/examples/desktop/src/style.css | 3 +- 8 files changed, 237 insertions(+), 17 deletions(-) diff --git a/edge0-infer/examples/desktop/src-tauri/Cargo.toml b/edge0-infer/examples/desktop/src-tauri/Cargo.toml index 987ba912..8295aacf 100644 --- a/edge0-infer/examples/desktop/src-tauri/Cargo.toml +++ b/edge0-infer/examples/desktop/src-tauri/Cargo.toml @@ -21,4 +21,4 @@ tauri-build = { version = "=2.7.4", features = [] } libc = "=0.2.191" [target.'cfg(target_os = "windows")'.dependencies] -windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_System_JobObjects", "Win32_System_Threading", "Win32_System_Console", "Win32_Security", "Win32_UI_WindowsAndMessaging"] } +windows-sys = { version = "=0.61.2", features = ["Win32_Foundation", "Win32_System_JobObjects", "Win32_System_Threading", "Win32_System_Console", "Win32_System_SystemInformation", "Win32_Security", "Win32_UI_WindowsAndMessaging"] } diff --git a/edge0-infer/examples/desktop/src-tauri/src/platform/macos.rs b/edge0-infer/examples/desktop/src-tauri/src/platform/macos.rs index 3a6b9f82..456100b4 100644 --- a/edge0-infer/examples/desktop/src-tauri/src/platform/macos.rs +++ b/edge0-infer/examples/desktop/src-tauri/src/platform/macos.rs @@ -35,6 +35,10 @@ pub fn force(pid: u32) { libc::kill(pid as i32, libc::SIGKILL); } } +/// macOS grows its swap on demand and has no fixed commit limit to run into. +pub fn commit_memory() -> Option { + None +} // AppKit panels must run on the application's main thread. The command routes // this call through AppHandle::run_on_main_thread and waits off the UI thread. diff --git a/edge0-infer/examples/desktop/src-tauri/src/platform/windows.rs b/edge0-infer/examples/desktop/src-tauri/src/platform/windows.rs index f5280cbd..da6e9da9 100644 --- a/edge0-infer/examples/desktop/src-tauri/src/platform/windows.rs +++ b/edge0-infer/examples/desktop/src-tauri/src/platform/windows.rs @@ -15,6 +15,7 @@ use windows_sys::Win32::{ SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, }, + SystemInformation::{GlobalMemoryStatusEx, MEMORYSTATUSEX}, Threading::{ OpenProcess, QueryFullProcessImageNameW, TerminateProcess, CREATE_NEW_PROCESS_GROUP, PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_TERMINATE, @@ -113,3 +114,13 @@ pub fn force(pid: u32) { } } } +/// The system commit limit (RAM visible to Windows plus the page file) and how much of it is still free. An +/// allocation past the limit fails even when RAM looks free, and a GPU that shares system memory charges its VRAM to it. +pub fn commit_memory() -> Option { + let mut status: MEMORYSTATUSEX = unsafe { std::mem::zeroed() }; + status.dwLength = std::mem::size_of::() as u32; + (unsafe { GlobalMemoryStatusEx(&mut status) } != 0).then_some(crate::service::CommitMemory { + limit: status.ullTotalPageFile, + available: status.ullAvailPageFile, + }) +} diff --git a/edge0-infer/examples/desktop/src-tauri/src/service.rs b/edge0-infer/examples/desktop/src-tauri/src/service.rs index 76685475..4708882b 100644 --- a/edge0-infer/examples/desktop/src-tauri/src/service.rs +++ b/edge0-infer/examples/desktop/src-tauri/src/service.rs @@ -105,6 +105,60 @@ pub struct Inspection { pub struct Service { url: String, pid: u32, + /// Set when the model left so little commit memory that the first generations may fail to allocate. + memory_warning: Option, +} +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CommitMemory { + pub limit: u64, + pub available: u64, +} +const GIB: u64 = 1 << 30; +const MEMORY_POLL: Duration = Duration::from_millis(250); +/// Samples that must agree before the load counts as finished: 3 s at MEMORY_POLL. +const STEADY_SAMPLES: usize = 13; +const STEADY_TOLERANCE: u64 = 256 << 20; +const STEADY_LIMIT: Duration = Duration::from_secs(30); +/// Edge0 145B's first request on a GPU that shares system memory was measured committing ~6.5 GB more than the loaded +/// model held, so less than this left after the load can end that request with an allocation failure. +const LOW_COMMIT: u64 = 8 * GIB; +/// An engine's automatic expert cache fills the free VRAM, which a GPU sharing system memory charges to the commit +/// limit. A load leaving less than this free is reloaded with that much more VRAM left out of the cache. +const RELOAD_BELOW: u64 = 10 * GIB; +/// edge0-strata's own --vram-reserve-mib when none is given. +const ENGINE_RESERVE_MIB: u32 = 700; +/// A larger reserve leaves the expert cache too small to be worth it; that load keeps its warning instead. +const MAX_RESERVE_MIB: u32 = 24 * 1024; +const MAX_RELOADS: usize = 2; +const RESERVES_FILE: &str = "memory-reserve.json"; +/// The reserve that brings a load with `memory` free up to RELOAD_BELOW, a quarter more since leaving 1 GiB of VRAM +/// free was measured to free ~0.75 GiB of commit memory; None when it is enough already or would pass MAX_RESERVE_MIB. +fn next_reserve(memory: CommitMemory, current: Option) -> Option { + let short_mib = RELOAD_BELOW.checked_sub(memory.available).filter(|short| *short > 0)? >> 20; + let base = current.filter(|mib| *mib > 0).unwrap_or(ENGINE_RESERVE_MIB); + let added = (short_mib * 5 / 4).div_ceil(256) * 256; + (base < MAX_RESERVE_MIB).then(|| (u64::from(base) + added).min(u64::from(MAX_RESERVE_MIB)) as u32) +} +fn reserve_arguments(mib: u32) -> [String; 4] { + ["--strata-arg".into(), "--vram-reserve-mib".into(), "--strata-arg".into(), mib.to_string()] +} +/// The committed bytes of the last STEADY_SAMPLES samples stay within STEADY_TOLERANCE of each other. +fn steady(committed: &VecDeque) -> bool { + committed.len() >= STEADY_SAMPLES + && committed.iter().max().zip(committed.iter().min()).is_some_and(|(max, min)| max - min <= STEADY_TOLERANCE) +} +pub fn commit_warning(memory: CommitMemory) -> Option { + (memory.available < LOW_COMMIT).then(|| { + let gb = |bytes: u64| bytes as f64 / 1e9; + format!( + "Low memory: with the model loaded, only {:.1} GB of the {:.1} GB Windows commit limit is free, so \ + generating may crash with an out-of-memory error. Increase the page file (System Properties > Advanced > \ + Performance Settings > Advanced > Virtual memory; for example a fixed size of 96 GB), restart Windows, \ + and load the model again.", + gb(memory.available), + gb(memory.limit) + ) + }) } #[derive(Serialize)] pub struct Status { @@ -480,6 +534,44 @@ impl Manager { if self.epoch.load(Ordering::SeqCst) != epoch || self.closing.load(Ordering::SeqCst) { return Err("Model loading cancelled".into()); } + let key = canonical(model).to_string_lossy().into_owned(); + let mut reserve = if delegated { self.reserves().get(&key).copied() } else { None }; + let mut reloads = 0; + // the commit memory free and the reserve of the load before a reload + let mut previous: Option<(u64, u32)> = None; + loop { + let mut args = network_arguments(settings, model, &self.directory.join("api-token"), delegated); + if let Some(mib) = reserve.filter(|mib| *mib > 0) { + args.extend(reserve_arguments(mib)); + } + let (url, pid) = self.launch(epoch, settings, &executable, args)?; + let memory = self.settle_memory(epoch)?; + if let (Some(memory), Some((before, before_mib))) = (memory, previous) { + let added = u64::from(reserve.unwrap_or(0).saturating_sub(before_mib)) << 20; + if memory.available.saturating_sub(before) < added / 2 { + // Free VRAM is not commit memory here (a GPU with its own memory): reloading cannot help this model. + trace("the reserve freed no commit memory; this model is not reloaded for it again"); + self.remember_reserve(&key, 0); + reserve = Some(0); + } + } + let next = memory + .filter(|_| delegated && reserve != Some(0) && reloads < MAX_RELOADS) + .and_then(|memory| next_reserve(memory, reserve)); + let (Some(memory), Some(next)) = (memory, next) else { + let memory_warning = memory.and_then(commit_warning); + trace(&format!("ready: {url} memory_warning={memory_warning:?}")); + return Ok(Service { url, pid, memory_warning }); + }; + trace(&format!("reloading with --vram-reserve-mib {next}: {} bytes of commit memory free", memory.available)); + self.remember_reserve(&key, next); + previous = Some((memory.available, reserve.filter(|mib| *mib > 0).unwrap_or(ENGINE_RESERVE_MIB))); + reserve = Some(next); + reloads += 1; + } + } + /// Starts edge0-infer with `args` in place of any running one and waits for its ready line: the URL and pid. + fn launch(&self, epoch: u64, settings: &Settings, executable: &Path, args: Vec) -> Result<(String, u32), String> { self.stop_process(None).map_err(|e| { trace(&format!("stop_process failed: {e}")); e @@ -488,13 +580,7 @@ impl Manager { if self.epoch.load(Ordering::SeqCst) != epoch || self.closing.load(Ordering::SeqCst) { return Err("Model loading cancelled".into()); } - let mut command = Command::new(&executable); - let args = network_arguments( - settings, - model, - &self.directory.join("api-token"), - delegated, - ); + let mut command = Command::new(executable); if settings.lan_access { if crate::storage::read_token(&self.directory)?.is_none() { return Err("Enable network access with an API token first".into()); @@ -528,7 +614,7 @@ impl Manager { } }; let pid = child.id(); - let record = PidRecord { pid, executable }; + let record = PidRecord { pid, executable: executable.to_path_buf() }; if let Err(e) = write_json(&self.record_path(), &record) { trace(&format!( "write_json({}) failed: {e}", @@ -593,7 +679,7 @@ impl Manager { .iter() .find_map(|line| ready_url(line)); if let Some(url) = url { - return Ok(Service { url, pid }); + return Ok((url, pid)); } if Instant::now() >= deadline { let detail = status.detail; @@ -603,6 +689,59 @@ impl Manager { thread::sleep(Duration::from_millis(50)); } } + /// The --vram-reserve-mib each model was last reloaded with on this machine, by its canonical path; 0 marks a model + /// that reloading did not help. + fn reserves(&self) -> BTreeMap { + fs::read(self.directory.join(RESERVES_FILE)) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .unwrap_or_default() + } + fn remember_reserve(&self, model: &str, mib: u32) { + let mut reserves = self.reserves(); + reserves.insert(model.into(), mib); + if let Err(e) = write_json(&self.directory.join(RESERVES_FILE), &reserves) { + trace(&format!("cannot remember the reserve: {e}")); + } + } + /// After the ready line an engine can still be filling its caches; the load is finished once the system's committed + /// memory has stopped moving (or STEADY_LIMIT passed). The last reading, or None where the platform has no limit. + fn settle_memory(&self, epoch: u64) -> Result, String> { + let Some(mut last) = platform::commit_memory() else { + return Ok(None); + }; + let deadline = Instant::now() + STEADY_LIMIT; + let mut committed = VecDeque::from([last.limit.saturating_sub(last.available)]); + while !steady(&committed) && Instant::now() < deadline { + thread::sleep(MEMORY_POLL); + if self.epoch.load(Ordering::SeqCst) != epoch || self.closing.load(Ordering::SeqCst) { + self.stop_process(Some(epoch))?; + return Err("Model loading cancelled".into()); + } + let status = self.status(); + if !status.running { + self.stop_process(Some(epoch))?; + return Err(format!( + "Service exited before becoming ready:\n{}", + status.detail + )); + } + if let Some(next) = platform::commit_memory() { + last = next; + committed.push_back(next.limit.saturating_sub(next.available)); + if committed.len() > STEADY_SAMPLES { + committed.pop_front(); + } + } + } + trace(&format!( + "memory: steady={} available={} limit={}", + steady(&committed), + last.available, + last.limit + )); + Ok(Some(last)) + } pub fn status(&self) -> Status { let Ok(mut process) = self.process.lock() else { return Status { @@ -725,6 +864,52 @@ mod tests { fs::remove_dir_all(dir).unwrap(); } #[test] + fn memory_is_steady_only_after_a_full_window_within_tolerance() { + let gb = |v: f64| (v * GIB as f64) as u64; + let mut committed: VecDeque = (0..STEADY_SAMPLES - 1).map(|_| gb(90.0)).collect(); + assert!(!steady(&committed), "a window not yet full"); + committed.push_back(gb(90.2)); + assert!(steady(&committed)); + // the expert cache filling: 40 GB in a few samples + let filling: VecDeque = (0..STEADY_SAMPLES).map(|i| gb(61.0 + 3.5 * i as f64)).collect(); + assert!(!steady(&filling)); + } + #[test] + fn warns_only_when_little_commit_memory_is_left() { + // measured on a 128 GB RTX Spark N1X with Edge0 145B loaded: 0.6 GB left of 104.5 GB + let low = commit_warning(CommitMemory { limit: 104_500_000_000, available: 600_000_000 }).unwrap(); + assert!(low.starts_with("Low memory:"), "{low}"); + assert!(low.contains("0.6 GB of the 104.5 GB"), "{low}"); + assert!(low.contains("page file"), "{low}"); + // the same PC with the development model: 24 GB left + assert_eq!(commit_warning(CommitMemory { limit: 102_500_000_000, available: 24_000_000_000 }), None); + assert!(commit_warning(CommitMemory { limit: 100 * GIB, available: LOW_COMMIT - 1 }).is_some()); + assert!(commit_warning(CommitMemory { limit: 100 * GIB, available: LOW_COMMIT }).is_none()); + } + #[test] + fn reloads_with_a_reserve_covering_the_commit_memory_short_of_the_target() { + let free = |available| CommitMemory { limit: 100 * GIB, available }; + assert_eq!(next_reserve(free(RELOAD_BELOW), None), None); + // 0.5 GiB free is 9728 MiB short: 12160 MiB at the measured ratio, 12288 in whole 256 MiB, on top of the engine's 700 + assert_eq!(next_reserve(free(GIB / 2), None), Some(700 + 12288)); + assert_eq!(next_reserve(free(9 * GIB), Some(4000)), Some(4000 + 1280)); + assert_eq!(next_reserve(free(0), Some(20000)), Some(MAX_RESERVE_MIB)); + assert_eq!(next_reserve(free(0), Some(MAX_RESERVE_MIB)), None); + assert_eq!( + reserve_arguments(5000), + ["--strata-arg", "--vram-reserve-mib", "--strata-arg", "5000"].map(String::from) + ); + } + #[test] + fn remembers_the_reserve_of_each_model() { + let manager = Manager::new(directory()).unwrap(); + assert!(manager.reserves().is_empty()); + manager.remember_reserve("/models/a", 9000); + manager.remember_reserve("/models/b", 0); + manager.remember_reserve("/models/a", 12000); + assert_eq!(manager.reserves(), BTreeMap::from([("/models/a".into(), 12000), ("/models/b".into(), 0)])); + } + #[test] fn parses_only_local_ready_lines() { assert_eq!( ready_url("Listening: http://127.0.0.1:41325 model=local"), diff --git a/edge0-infer/examples/desktop/src/chat.test.ts b/edge0-infer/examples/desktop/src/chat.test.ts index 2bf767a9..bc55d43a 100644 --- a/edge0-infer/examples/desktop/src/chat.test.ts +++ b/edge0-infer/examples/desktop/src/chat.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it, vi } from 'vitest'; import shared from '../../shared/app.json'; import metricsCases from '../../shared/metrics-cases.json'; import mathCases from '../../shared/math-cases.json'; -import { appData, buildRequest, fetchWithRetry, formatMetrics, mapAsset, selectableAssets, protectMath, SseAccumulator, streamChat } from './chat'; +import { appData, buildRequest, explainFailure, fetchWithRetry, formatMetrics, mapAsset, selectableAssets, protectMath, SseAccumulator, streamChat } from './chat'; describe('shared vectors and bundled app data', () => { it('uses the canonical prompt and ordered suggestions', () => { @@ -112,3 +112,14 @@ for (const thinking of [false, true]) it(`normal HTTP request changes only tempe expect(body.messages).toEqual([{ role: 'system', content: shared.system_prompt }, ...history]); expect(body.chat_template_kwargs).toEqual({ enable_thinking: thinking }); }); +it('adds the low-memory warning to a generation that ended because the engine stopped', async () => { + const warning = 'Low memory: with the model loaded, only 0.6 GB of the 104.5 GB Windows commit limit is free.'; + // edge0-strata ends the stream with this event when its engine process dies mid-answer + const fetcher = async () => new Response('data: {"error": {"type": "server_error", "message": "the engine stopped unexpectedly (exit code 0xc0000409); the next request restarts it"}}\n\ndata: [DONE]\n\n'); + const failure = await streamChat(fetcher, 'http://127.0.0.1:1', buildRequest([], false, 'edge0_145b'), new AbortController().signal, () => {}).catch(e => e); + const shown = explainFailure(failure, warning); + expect(shown).toContain('the engine stopped unexpectedly (exit code 0xc0000409)'); + expect(shown).toContain('ran out of memory. ' + warning); + expect(explainFailure(failure)).toBe(String(failure)); + expect(explainFailure(new Error('Generation failed (400): bad request'), warning)).toBe('Error: Generation failed (400): bad request'); +}); diff --git a/edge0-infer/examples/desktop/src/chat.ts b/edge0-infer/examples/desktop/src/chat.ts index 61f3552b..75d31a94 100644 --- a/edge0-infer/examples/desktop/src/chat.ts +++ b/edge0-infer/examples/desktop/src/chat.ts @@ -164,6 +164,13 @@ export class SseAccumulator { } } } +/** A generation that failed because the engine process ended (edge0-strata: "the engine stopped unexpectedly (exit + * code ...)"), with the load's low-memory warning, if there was one: the likely reason, and what to do about it. */ +export function explainFailure(failure: unknown, memoryWarning = ''): string { + const text = String(failure); + const engineEnded = /engine (stopped|ended|exited|is unavailable)|service connection closed/i.test(text); + return engineEnded && memoryWarning ? `${text}\nThe engine most likely ran out of memory. ${memoryWarning}` : text; +} export type Fetcher = (input: string, init?: RequestInit) => Promise; const pause = (ms: number, signal: AbortSignal) => new Promise((resolve, reject) => { signal.throwIfAborted(); diff --git a/edge0-infer/examples/desktop/src/main.tsx b/edge0-infer/examples/desktop/src/main.tsx index 01920e8b..f07ed655 100644 --- a/edge0-infer/examples/desktop/src/main.tsx +++ b/edge0-infer/examples/desktop/src/main.tsx @@ -3,7 +3,7 @@ import { flushSync } from 'react-dom'; import { createRoot } from 'react-dom/client'; import { invoke } from '@tauri-apps/api/core'; import { fetch } from '@tauri-apps/plugin-http'; -import { buildRequest, streamChat, type Message } from './chat'; +import { buildRequest, explainFailure, streamChat, type Message } from './chat'; import { validateServerPort, type GenerationPhase } from './conversation'; import { Icon, type IconName } from './Icon'; import { Chat } from './pages/Chat'; @@ -24,7 +24,7 @@ import 'streamdown/styles.css'; import './theme.css'; import './style.css'; type Page = 'chat' | 'models' | 'status' | 'service' | 'settings'; -interface RunningService { url: string; pid: number; } +interface RunningService { url: string; pid: number; memory_warning?: string | null; } interface Model { id: string; context_length?: number; capabilities: { thinking: boolean; thinking_default: boolean }; } export function App() { const [settings, setSettings] = useState(defaultSettings), settingsRef = useRef(defaultSettings); @@ -43,6 +43,7 @@ export function App() { const archiveWrites = useRef(new ArchiveWrites(persistence.current, conversation => invoke('save_conversation', { id: conversation.id, conversation }), id => invoke('delete_conversation', { id }))); const [deleting, setDeleting] = useState(false), deletingRef = useRef(false); const busy = phase !== 'idle', disabled = busy || loading || initializing || deleting; + const memoryWarning = service?.memory_warning ?? ''; const report = (failure: unknown) => setError(String(failure)); function changeConversation(next: Conversation) { conversationRef.current = next; setConversation(next); } function remember(next: Conversation) { @@ -79,7 +80,7 @@ export function App() { useEffect(() => { if (!service) return; let cancelled = false; const epoch = serviceEpoch.current; - const timer = setInterval(() => { void invoke<{ running: boolean; detail: string }>('service_status').then(next => { if (!cancelled && epoch === serviceEpoch.current && !next.running) { controller.current?.abort(); serviceEpoch.current++; setService(undefined); setModel(undefined); setError(`The model service exited. ${next.detail}`); } }).catch(report); }, 1500); + const timer = setInterval(() => { void invoke<{ running: boolean; detail: string }>('service_status').then(next => { if (!cancelled && epoch === serviceEpoch.current && !next.running) { controller.current?.abort(); serviceEpoch.current++; setService(undefined); setModel(undefined); setError(explainFailure(`The model service exited. ${next.detail}`, service.memory_warning ?? '')); } }).catch(report); }, 1500); return () => { cancelled = true; clearInterval(timer); }; }, [service]); async function stopService() { if (busy) return; serviceEpoch.current++; setLoading(true); try { await invoke('stop_service'); setService(undefined); setModel(undefined); setStatus(previous => previous ? { ...previous, current: null } : undefined); } catch (failure) { report(failure); } finally { setLoading(false); } } @@ -110,7 +111,7 @@ export function App() { try { await streamChat(fetch, service.url, buildRequest(history, base.thinking, model.id, temperatures[settingsRef.current.generation_style], base.systemPrompt), abort.signal, message => { if (abort.signal.aborted || conversationRef.current.id !== id) return; const previous = conversationRef.current; changeConversation({ ...previous, messages: [...previous.messages.slice(0, -1), { ...message, reasoningExpanded: previous.messages.at(-1)?.reasoningExpanded }] }); - }); } catch (failure) { failed = true; if (!abort.signal.aborted) report(failure); } + }); } catch (failure) { failed = true; if (!abort.signal.aborted) setError(explainFailure(failure, memoryWarning)); } finally { if (conversationRef.current.id === id) { const previous = conversationRef.current; const messages = previous.messages.map((message, index) => index === previous.messages.length - 1 ? { ...message, stopped: abort.signal.aborted, ...((failed || abort.signal.aborted) ? { metrics: undefined } : {}) } : message); const next = { ...previous, updatedAt: Date.now(), messages }; changeConversation(next); remember(next); } controller.current = null; setPhase('idle'); @@ -139,7 +140,7 @@ export function App() { async function network(nextToken = false) { if (disabled) return; try { if (nextToken) { const value = generateToken(); await invoke('write_api_token', { token: value }); tokenRef.current = value; setToken(value); } else if (!settingsRef.current.lan_access) await ensureToken(); const next = { ...settingsRef.current, lan_access: nextToken || !settingsRef.current.lan_access }; await saveSettings(next); settingsRef.current = next; setSettings(next); if (service && activePath) { const current = conversationRef.current; const previousPage = page; await select(activePath, current, next); setPage(previousPage); } } catch (failure) { report(failure); } } const loaded = !!service && !!model; const nav: [Page, string, IconName][] = [['chat', 'Chat', 'chat'], ['models', 'Models', 'box'], ['status', 'Status', 'pulse'], ['service', 'Service', 'server']]; - return
{error &&
{error}
}{loading &&
Loading model… First load can take several minutes.
} + return
{error &&
{error}
}{memoryWarning && !error &&
{memoryWarning}
}{loading &&
Loading model… First load can take several minutes.
} {page === 'chat' && changeConversation({ ...conversation, thinking })} reasoningExpanded={(index, expanded) => { const next = { ...conversationRef.current, messages: conversationRef.current.messages.map((message, i) => i === index ? { ...message, reasoningExpanded: expanded } : message) }; changeConversation(next); if (!busy) remember(next); }} edit={(index, text) => { if (!disabled) void generate(editAndResend(conversationRef.current.messages, index, text)); }} redo={() => { if (!disabled) void generate(regenerate(conversationRef.current.messages)); }} copy={text => void copy(text)} models={() => setPage('models')}/>} {page === 'models' && void select(path)} unload={() => void stopService()} chat={() => setPage('chat')} refresh={() => { void inspect().catch(report); }} add={() => void addLocation()} remove={path => void removeLocation(path)}/>} {page === 'status' && keepTotals(resetTotals(totalsRef.current))}/>} diff --git a/edge0-infer/examples/desktop/src/style.css b/edge0-infer/examples/desktop/src/style.css index 5110b2db..db74f7d3 100644 --- a/edge0-infer/examples/desktop/src/style.css +++ b/edge0-infer/examples/desktop/src/style.css @@ -201,8 +201,9 @@ button, input, textarea, select { color: inherit; } .thermal-ok { color: var(--ok); } .thermal-warn { color: var(--warn); } .thermal-error { color: var(--err); } -.error, .loading { flex-shrink: 0; padding: var(--sp-3) var(--sp-5); color: var(--err); background: var(--err-soft); font-size: var(--fs-md); overflow-wrap: anywhere; max-height: 20dvh; overflow-y: auto; } +.error, .loading, .warning { flex-shrink: 0; padding: var(--sp-3) var(--sp-5); color: var(--err); background: var(--err-soft); font-size: var(--fs-md); overflow-wrap: anywhere; max-height: 20dvh; overflow-y: auto; } .loading { color: var(--text-2); background: var(--surface-2); } +.warning { color: var(--warn); background: var(--warn-soft); } .thinking-control { display: flex; flex-direction: column; align-items: flex-end; gap: var(--sp-1); } .thinking-control.is-locked { opacity: .55; } .thinking-switch { display: inline-flex; align-items: center; gap: var(--sp-2); border: 0; background: transparent; padding: 0; font-size: var(--fs-sm); white-space: nowrap; }