diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index bd5b52bf..debcc102 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -23,6 +23,9 @@ body: - Android (CPU) - Windows (CPU) - Windows (Vulkan) + - Linux (CPU) + - Linux (NVIDIA GPU, edge0-strata) + - Windows (NVIDIA GPU, edge0-strata) - Other (describe below) validations: required: true @@ -33,6 +36,8 @@ body: label: Model tier options: - edge0-35b + - edge0-145b + - edge0-120b - other / custom checkpoint validations: required: true @@ -59,7 +64,7 @@ body: id: os attributes: label: OS version - placeholder: macOS 15.1 / Windows 11 23H2 / Android 15 / iOS 18.1 + placeholder: macOS 15.1 / Windows 11 23H2 / Ubuntu 24.04 / Android 15 / iOS 18.1 validations: required: true @@ -68,7 +73,7 @@ body: attributes: label: Engine build description: "The engine.version and engine.source_commit lines printed by: edge0-infer info" - placeholder: "engine.version: 0.1.0, engine.source_commit: ..." + placeholder: "engine.version: 0.2.0, engine.source_commit: ..." validations: required: false @@ -78,7 +83,7 @@ body: label: How are you running it? options: - edge0-infer CLI (run / chat / bench) - - edge0-infer serve (OpenAI-compatible server) + - edge0-infer serve (local OpenAI- and Anthropic-compatible server) - Native C++ API in my own application - Chat app example (desktop / Android / iOS) - Other (describe below) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 4d0d5cef..a56a48b1 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -15,7 +15,7 @@ body: options: - New backend (GPU / NPU / other accelerator) - New platform support - - Server / OpenAI-compatible API + - Server / OpenAI- and Anthropic-compatible API - CLI / native C++ API - Quantization / adapters (LoRA / prerouter) - Documentation diff --git a/.github/ISSUE_TEMPLATE/performance_report.yml b/.github/ISSUE_TEMPLATE/performance_report.yml index e2214c4d..b5b461fb 100644 --- a/.github/ISSUE_TEMPLATE/performance_report.yml +++ b/.github/ISSUE_TEMPLATE/performance_report.yml @@ -7,7 +7,9 @@ body: value: | Use this for "slower than the README numbers" or "high memory / OOM" reports (e.g. #17, #110). The reference figures and the devices they were measured on are in - [Benchmarking](https://github.com/Edge0-AI/Edge0/blob/main/edge0-infer/docs/benchmarking.md#reference-figures). + [Benchmarking](https://github.com/Edge0-AI/Edge0/blob/main/edge0-infer/docs/benchmarking.md#reference-figures) + (`edge0-35b`) and on the [model page](https://github.com/Edge0-AI/Edge0/blob/main/docs/models/edge0-145b.md) + (`edge0-145b`, `edge0-120b`; include the GPU and its memory). Throughput depends on memory and storage speed — please include your total RAM and storage type. - type: dropdown @@ -16,6 +18,8 @@ body: label: Model tier options: - edge0-35b + - edge0-145b + - edge0-120b validations: required: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 755e32aa..a43292dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,3 +29,5 @@ jobs: run: cmake --build --preset cpu - name: Test (L0, L1) run: ctest --preset cpu --no-tests=error --output-on-failure -L "L0|L1" + - name: Test (L2, delegated engine) + run: ctest --preset cpu --no-tests=error --output-on-failure -L L2 -R strata diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 707c3873..cc899a94 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,8 @@ platform/backend work. | Path | What | |---|---| -| [`edge0-infer/`](edge0-infer/) | The inference engine (C++, GGML), its CLI and OpenAI-compatible server, the model asset tools, and the chat app examples for desktop, Android and iOS | +| [`edge0-infer/`](edge0-infer/) | The program you run: the native inference engine (C++, GGML), its CLI and local OpenAI- and Anthropic-compatible server, the model asset tools, and the chat app examples for desktop, Android and iOS | +| [`edge0-strata/`](edge0-strata/EDGE0.md) | The engine for `edge0-145b` and `edge0-120b` (NVIDIA GPU, Linux or Windows), a fork of Strata that `edge0-infer` starts for these models | | [`docs/`](docs/) | Model design docs: MoE routing, the prerouter, and the model tiers | | [`paper/`](paper/) | The technical report | @@ -32,7 +33,8 @@ is applied there first and arrives here with the next sync. Build the engine and get the model assets as described in [Getting started](edge0-infer/docs/getting-started.md). It covers macOS, iOS, -Android and Windows. +Android, Linux and Windows. For `edge0-145b` and `edge0-120b`, also build +`edge0-strata/` as [`edge0-strata/EDGE0.md`](edge0-strata/EDGE0.md) describes. ## Running the tests @@ -84,4 +86,5 @@ instead. ## License By contributing, you agree that your contributions will be licensed under the -project's [Apache-2.0](LICENSE) license. +project's [Apache-2.0](LICENSE) license. Contributions to `edge0-strata/` are +licensed under that directory's [MIT](edge0-strata/LICENSE) license. diff --git a/NOTICE b/NOTICE index 3744eed0..0cee9448 100644 --- a/NOTICE +++ b/NOTICE @@ -19,15 +19,27 @@ llama.cpp tokenizer code (MIT License) cpp-httplib (MIT License) - `edge0-infer/third_party/cpp-httplib/` (version 0.60.1), used only by the - CLI and the OpenAI-compatible server. + CLI and the local server. License: `edge0-infer/third_party/cpp-httplib/LICENSE`. Copyright (c) 2017 yhirose. nlohmann/json (MIT License) - `edge0-infer/third_party/nlohmann-json/` (version 3.12.0), used only by the - CLI and the OpenAI-compatible server. + CLI and the local server. License: `edge0-infer/third_party/nlohmann-json/LICENSE.MIT`. Copyright (c) 2013-2025 Niels Lohmann. +Strata (MIT License) + - `edge0-strata/` is a fork of Strata (https://github.com/Niko1221/Strata), + imported from the fork at commit f1933b1 plus the source files recorded in + `edge0-strata/edge0/IMPORTED_UNCOMMITTED.txt`, whose merge base with upstream + Strata is commit c62d5e3. The license is kept verbatim in + `edge0-strata/LICENSE`; `edge0-strata/EDGE0.md` records the origin and the + changes. Third-party code in it keeps its own license and notice files + (`edge0-strata/third_party/`): ggml, project-maya, the libstrix notice, and + `ggml-tq`, which is derived from the ggml of the agentionai/llama.cpp fork + (MIT License, the ggml authors). + Copyright (c) 2026 Niko1221 and the Strata contributors. + All other files are Copyright (c) 2025 edge0 contributors, licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/README.md b/README.md index 1e890f4b..6fe697e5 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ **An open-source streaming MoE inference framework — SSD expert offload + Recover-LoRA + prerouter routing prediction.** -**macOS** · **iOS** · **Android** · **Windows** — one engine, one native API. +**macOS** · **iOS** · **Android** · **Linux** · **Windows** — one program for every Edge0 model. [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--35B--A3B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--8B--A1B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) @@ -23,6 +23,7 @@ English | [中文](README_zh.md) | [日本語](README_ja.md) | [Español](README ## News +- **[2026-11-13]** We released **`edge0-145b`** and **`edge0-120b`**, latent mixture-of-experts models derived from Qwen3.8-Flash-Next for a single NVIDIA GPU on Linux or Windows. They run on [edge0-strata](edge0-strata/EDGE0.md), which `edge0-infer serve` and `edge0-infer chat` start for them; see the [model page](docs/models/edge0-145b.md). - **[2026-11-04]** We released the **unified inference engine [`edge0-infer`](edge0-infer/README.md)**: built on [GGML](https://github.com/ggml-org/ggml), one engine and one native C++ API run the same model on macOS, iOS, Android and Windows, with a CLI, an OpenAI-compatible local server, and chat app examples for desktop, Android and iOS. The first release supports `edge0-35b`; `edge0-8b` follows in a later release. It replaces the Python framework and the per-platform engines; the previous code is kept at tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer). - **[2026-09-30]** We released the **edge0 inference engines for four platforms — iOS, macOS, Android and Windows** (source at tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer)). - **[2026-09-16]** Our technical report is on arXiv: [The Other Half of the Memory Wall: Serving 35B MoEs from SSD with Trained Routing Prediction](https://arxiv.org/abs/2609.18063). @@ -52,33 +53,37 @@ memory is bounded by the *active* expert set, not the parameter count. ### Platforms -One engine and one native API, with the same code on every platform — see [`edge0-infer/`](edge0-infer/README.md): +[`edge0-infer`](edge0-infer/README.md) is the program you run (CLI, local OpenAI- and Anthropic-compatible server, chat apps), with the same commands for every model. Each model runs on the engine built for it, and edge0-infer picks the engine from the model folder; there is nothing to choose: `edge0-35b` runs on the **native engine** in `edge0-infer/` (built on GGML, with a native C++ API), and `edge0-145b` and `edge0-120b` run on **edge0-strata** in [`edge0-strata/`](edge0-strata/EDGE0.md), a fork of Strata. -| Platform | Backends | Verified on | +| Platform | `edge0-35b` (native engine) | `edge0-145b`, `edge0-120b` (edge0-strata) | |---|---|---| -| **macOS** (Apple Silicon) | Metal, CPU | Apple M5, 16 GiB | -| **iOS** | Metal | iPhone 17 Pro Max | -| **Android** | CPU | OnePlus PKR110 (Snapdragon 8 Elite) | -| **Windows** (x86-64) | CPU, Vulkan | Intel Core i5-12400, NVIDIA GeForce RTX 4070 SUPER | +| **macOS** (Apple Silicon) | Metal, CPU | — | +| **iOS** | Metal | — | +| **Android** | CPU | — | +| **Linux** | CPU | NVIDIA GPU (CUDA 13) | +| **Windows** | CPU, Vulkan | NVIDIA GPU (CUDA 13) | -Prefill and decode can run on different backends of the same device, for example Metal prefill with CPU decode. Chat app examples for desktop (macOS, Windows), Android and iOS are in [`edge0-infer/examples/`](edge0-infer/examples/README.md#chat-applications). +Measurements: [Benchmarking](edge0-infer/docs/benchmarking.md#reference-figures) for `edge0-35b`, and the [model page](docs/models/edge0-145b.md) for `edge0-145b` and `edge0-120b`. On the native engine, prefill and decode can run on different backends of the same device, for example Metal prefill with CPU decode. Chat app examples for desktop (macOS, Windows), Android and iOS are in [`edge0-infer/examples/`](edge0-infer/examples/README.md#chat-applications). ### Models -Two model tiers ship with the framework. Each tier is an end-to-end +Two model tiers, `edge0-35b` and `edge0-8b`, ship with the framework. Each tier is an end-to-end release: the released checkpoint, the trained LoRA adapters, and the trained prerouter heads work together as one unit. -| Tier | Released checkpoint | Inference profile | edge0-infer | +| Tier | Released checkpoint | Inference profile | Engine | |---|---|---|---| -| `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4-bit, 40 layers, 256 experts, prerouter K=4 | ✅ Supported | -| `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4-bit, 24 layers, 128 experts, prerouter K=8 | Planned | +| `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4-bit, 40 layers, 256 experts, prerouter K=4 | Native engine ✅ | +| `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4-bit, 24 layers, 128 experts, prerouter K=8 | Native engine (planned) | +| [`edge0-145b`](docs/models/edge0-145b.md) | coming soon | Q3_K or NVFP4 latent experts (1792), 48 layers, 512 experts, K=10 | edge0-strata ✅ (Linux or Windows, NVIDIA GPU) | +| [`edge0-120b`](docs/models/edge0-145b.md) | coming soon | Q3_K or NVFP4 latent experts (1280), 48 layers, 512 experts, K=10 | edge0-strata ✅ (Linux or Windows, NVIDIA GPU) | -Both checkpoints are built on open sparse-MoE base models (Qwen3.6-35B-A3B +The two released checkpoints are built on open sparse-MoE base models (Qwen3.6-35B-A3B and the Ling 3.0 bailing hybrid respectively) and ship with the LoRA and prerouter training done for this framework. edge0-infer downloads the released checkpoint with its asset tools and converts it into the engine's model assets (see [Getting Started](#getting-started)). +`edge0-145b` and `edge0-120b` are a separate latent-MoE tier for a single NVIDIA GPU; they run on edge0-strata, which `edge0-infer serve` and `edge0-infer chat` start for them (see [edge0-145b and edge0-120b](docs/models/edge0-145b.md)). ### Design @@ -87,7 +92,7 @@ engine's model assets (see [Getting Started](#getting-started)). downloads nothing; - **One native API**: applications link the CMake target `edge0::edge0` and include the headers in `include/edge0/` to use the engine in - process; the reference CLI and the OpenAI-compatible server use the same + process; the reference CLI and the local server use the same settings and defaults as the native API; - **Experts on demand**: routed experts are read from storage on demand into a fixed number of expert slots per layer, so a model larger than the @@ -118,6 +123,17 @@ edge0-35b, 2.8 for edge0-8b** (MMLU-Pro is even above the base). Max 100: | IFBench | 57.9 | 61.7 | 53.9 | 60.6 | | **Average** | **79.2** | **83.2** | **69.9** | **72.7** | +`edge0-145b` and `edge0-120b` (**bold** = best in column; see the [model page](docs/models/edge0-145b.md#quality)): + +| Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | +| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | +| Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | +| Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | +| Claude-Opus-4.6 (Max) | / | 78.2 | 78.0 | 62.5 | 91.3 | **40.0** | + ### Benchmark `edge0-35b` with `predicted` routing, measured with edge0-infer (each range is the minimum to maximum of three processes): @@ -133,6 +149,33 @@ edge0-35b, 2.8 for edge0-8b** (MMLU-Pro is even above the base). Max 100: Memory metrics differ by platform and cannot be compared directly. The measurement method, the `actual` routing figures and the commands to reproduce them are in [Benchmarking](edge0-infer/docs/benchmarking.md#reference-figures). +`edge0-145b` and `edge0-120b` on edge0-strata, one NVIDIA GPU (tok/s): + +#### edge0-145b + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 2169.9 | 3554.9 | 99.7 | **121.2** | +| RTX 5090 D 24 GB | NVFP4 | 1839.6 | 3165.7 | 79.6 | 98.3 | +| RTX 4090 24 GB | Q3_K | 2042.0 | 3179.4 | 69.9 | 91.7 | +| RTX Spark N1X 60 GiB | Q3_K | 674.0 | 770.3 | 37.7 | 36.7 | +| RTX Spark N1X 60 GiB | NVFP4 | 821.2 | 922.4 | 40.1 | 46.0 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 249 | 335 | 36.8 | 44.9 | + +#### edge0-120b + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 3236.7 | 4576.5 | 130.2 | **144.6** | +| RTX 5090 D 24 GB | NVFP4 | 2668.5 | 4357.1 | 124.9 | 135.5 | +| RTX 4090 24 GB | Q3_K | 2637.5 | 3551.1 | 95.0 | 104.9 | +| RTX Spark N1X 60 GiB | Q3_K | 987.0 | 1081.7 | 47.2 | 68.4 | +| RTX Spark N1X 60 GiB | NVFP4 | 996.4 | 1066.9 | 45.1 | 68.9 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 430 | 778 | 62.4 | 73.3 | +| 16 GiB VRAM + 20 GiB RAM | NVFP4 | 361.1 | 619.7 | 51.3 | 59.9 | + +`16 GiB VRAM + 20 GiB RAM` rows are the same RTX 5090 D 24 GB, limited by cgroup to 16 GiB of GPU memory and 20 GiB of system memory. Details are on the [model page](docs/models/edge0-145b.md#speed-measured-one-card). + ## Getting Started Run the following commands in `edge0-infer/`. You need CMake 3.21+, @@ -151,7 +194,9 @@ build/mac-metal/bin/edge0-infer info # expect a line: model.0: edge0_35b On machines without Metal use `--preset cpu`, `build/cpu/bin/edge0-infer`, and `--device cpu` in the commands below. -### 2) Download and convert the model assets +### 2) Get a model + +**edge0-35b (native engine).** Download and convert the model assets: ```bash export MOE_SOURCE_35B= @@ -168,6 +213,8 @@ assets_py tools/assets/check_assets.py check "$MOE_ASSETS_35B" This downloads `Edge0/Edge0-35B-A3B-preview` from Hugging Face and checks it against the locked revision and digests. +**edge0-145b and edge0-120b (edge0-strata).** Build edge0-strata as [`edge0-strata/EDGE0.md`](edge0-strata/EDGE0.md) describes and use a released model folder (see [edge0-145b and edge0-120b](docs/models/edge0-145b.md)). + ### 3) Run ```bash @@ -175,10 +222,12 @@ it against the locked revision and digests. build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "Introduce yourself in one sentence." # multi-turn chat build/mac-metal/bin/edge0-infer chat --model "$MOE_ASSETS_35B" --device metal -# serve (OpenAI-compatible, default http://127.0.0.1:8000/v1) +# serve (OpenAI- and Anthropic-compatible, default http://127.0.0.1:8000) build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal ``` +For edge0-145b and edge0-120b, `chat` and `serve` take the model folder the same way (`--model `); `run` is for the native engine. + For the Android, Windows and iOS builds, and for using the engine from your own CMake project, see [Getting started](edge0-infer/docs/getting-started.md). @@ -187,17 +236,22 @@ your own CMake project, see [Getting started](edge0-infer/docs/getting-started.m **Models** - [MoE routing and experts](docs/moe.md) / [prerouter](docs/prerouter.md) -- [edge0-35b](docs/models/edge0-35b.md) / [edge0-8b](docs/models/edge0-8b.md) +- [edge0-35b](docs/models/edge0-35b.md) / [edge0-8b](docs/models/edge0-8b.md) / [edge0-145b and edge0-120b](docs/models/edge0-145b.md) - Technical report: [The Other Half of the Memory Wall](https://arxiv.org/abs/2609.18063) ([PDF](paper/main.pdf)) -**Engine (edge0-infer)** +**edge0-infer** - [Getting started](edge0-infer/docs/getting-started.md): build, model assets, run, chat and serve on every platform -- [Configuration](edge0-infer/docs/configuration.md) / [Native API](edge0-infer/docs/native-api.md) / [OpenAI-compatible server](edge0-infer/docs/openai-compatible-server.md) +- [Configuration](edge0-infer/docs/configuration.md) / [Native API](edge0-infer/docs/native-api.md) / [Local server](edge0-infer/docs/local-server.md) - [Architecture](edge0-infer/docs/architecture.md) / [Known limitations](edge0-infer/docs/limitations.md) - [Benchmarking](edge0-infer/docs/benchmarking.md) / [Testing](edge0-infer/docs/testing.md) - [Model asset tools and adding a model](edge0-infer/tools/assets/README.md) +**Engines** + +- Native engine: [Architecture](edge0-infer/docs/architecture.md#native-engine) / [Native API](edge0-infer/docs/native-api.md) +- edge0-strata: [build and launch contract](edge0-strata/EDGE0.md) / [edge0-145b](edge0-infer/docs/edge0-145b.md) and [edge0-120b](edge0-infer/docs/edge0-120b.md) in edge0-infer + ## Roadmap ### Q4 2026 diff --git a/README_es.md b/README_es.md index 17cfee01..c1b3782b 100644 --- a/README_es.md +++ b/README_es.md @@ -6,7 +6,7 @@ **Un framework de inferencia MoE en streaming de código abierto — SSD expert offload + Recover-LoRA + predicción de enrutamiento del prerouter.** -**macOS** · **iOS** · **Android** · **Windows** — un motor, una API nativa. +**macOS** · **iOS** · **Android** · **Linux** · **Windows** — un programa para todos los modelos Edge0. [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--35B--A3B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--8B--A1B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) @@ -23,6 +23,7 @@ ## Novedades +- **[2026-11-13]** Lanzamos **`edge0-145b`** y **`edge0-120b`**, modelos latent mixture-of-experts derivados de Qwen3.8-Flash-Next para una sola GPU NVIDIA en Linux o Windows. Se ejecutan en [edge0-strata](edge0-strata/EDGE0.md), que `edge0-infer serve` y `edge0-infer chat` inician para ellos; consulta la [página del modelo](docs/models/edge0-145b.md). - **[2026-11-04]** Lanzamos el **motor de inferencia unificado [`edge0-infer`](edge0-infer/README_es.md)**: construido sobre [GGML](https://github.com/ggml-org/ggml), un solo motor y una sola API nativa en C++ ejecutan el mismo modelo en macOS, iOS, Android y Windows, con una CLI, un servidor local compatible con OpenAI y ejemplos de aplicaciones de chat para escritorio, Android e iOS. La primera versión admite `edge0-35b`; `edge0-8b` llegará en una versión posterior. Sustituye al framework de Python y a los motores de cada plataforma; el código anterior se conserva en la etiqueta [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer). - **[2026-09-30]** Lanzamos los **motores de inferencia de edge0 para cuatro plataformas — iOS, macOS, Android y Windows** (código fuente en la etiqueta [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer)). - **[2026-09-16]** Nuestro informe técnico está en arXiv: [The Other Half of the Memory Wall: Serving 35B MoEs from SSD with Trained Routing Prediction](https://arxiv.org/abs/2609.18063). @@ -55,34 +56,38 @@ número de parámetros. ### Plataformas -Un solo motor y una sola API nativa, con el mismo código en todas las plataformas; consulta [`edge0-infer/`](edge0-infer/README_es.md): +[`edge0-infer`](edge0-infer/README_es.md) es el programa que ejecutas (CLI, servidor local compatible con OpenAI y Anthropic, aplicaciones de chat), con los mismos comandos para todos los modelos. Cada modelo se ejecuta en el motor preparado para él, y edge0-infer elige el motor a partir de la carpeta del modelo; no hay nada que elegir: `edge0-35b` se ejecuta en el **motor nativo** de `edge0-infer/` (construido sobre GGML, con una API nativa en C++), y `edge0-145b` y `edge0-120b` en **edge0-strata**, en [`edge0-strata/`](edge0-strata/EDGE0.md), un fork de Strata. -| Plataforma | Backends | Verificado en | +| Plataforma | `edge0-35b` (motor nativo) | `edge0-145b`, `edge0-120b` (edge0-strata) | |---|---|---| -| **macOS** (Apple Silicon) | Metal, CPU | Apple M5, 16 GiB | -| **iOS** | Metal | iPhone 17 Pro Max | -| **Android** | CPU | OnePlus PKR110 (Snapdragon 8 Elite) | -| **Windows** (x86-64) | CPU, Vulkan | Intel Core i5-12400, NVIDIA GeForce RTX 4070 SUPER | +| **macOS** (Apple Silicon) | Metal, CPU | — | +| **iOS** | Metal | — | +| **Android** | CPU | — | +| **Linux** | CPU | GPU NVIDIA (CUDA 13) | +| **Windows** | CPU, Vulkan | GPU NVIDIA (CUDA 13) | -El prefill y el decode pueden ejecutarse en backends distintos del mismo dispositivo, por ejemplo prefill en Metal y decode en CPU. Los ejemplos de aplicaciones de chat para escritorio (macOS, Windows), Android e iOS están en [`edge0-infer/examples/`](edge0-infer/examples/README.md#chat-applications). +Mediciones: [Benchmarking](edge0-infer/docs/benchmarking.md#reference-figures) para `edge0-35b` y la [página del modelo](docs/models/edge0-145b.md) para `edge0-145b` y `edge0-120b`. En el motor nativo, el prefill y el decode pueden ejecutarse en backends distintos del mismo dispositivo, por ejemplo prefill en Metal y decode en CPU. Los ejemplos de aplicaciones de chat para escritorio (macOS, Windows), Android e iOS están en [`edge0-infer/examples/`](edge0-infer/examples/README.md#chat-applications). ### Modelos -El framework incluye dos niveles de modelos. Cada nivel es una versión de +El framework incluye dos niveles de modelos, `edge0-35b` y `edge0-8b`. Cada nivel es una versión de extremo a extremo: el checkpoint publicado, los adaptadores LoRA entrenados y las cabezas prerouter entrenadas funcionan en conjunto como una sola unidad. -| Nivel | Checkpoint publicado | Perfil de inferencia | edge0-infer | +| Nivel | Checkpoint publicado | Perfil de inferencia | Motor | |---|---|---|---| -| `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4 bits, 40 capas, 256 expertos, prerouter K=4 | ✅ Compatible | -| `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4 bits, 24 capas, 128 expertos, prerouter K=8 | Previsto | +| `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4 bits, 40 capas, 256 expertos, prerouter K=4 | Motor nativo ✅ | +| `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4 bits, 24 capas, 128 expertos, prerouter K=8 | Motor nativo (previsto) | +| [`edge0-145b`](docs/models/edge0-145b.md) | coming soon | expertos latentes Q3_K o NVFP4 (1792), 48 capas, 512 expertos, K=10 | edge0-strata ✅ (Linux o Windows, GPU NVIDIA) | +| [`edge0-120b`](docs/models/edge0-145b.md) | coming soon | expertos latentes Q3_K o NVFP4 (1280), 48 capas, 512 expertos, K=10 | edge0-strata ✅ (Linux o Windows, GPU NVIDIA) | -Ambos checkpoints se construyen sobre modelos base sparse-MoE abiertos +Los dos checkpoints publicados se construyen sobre modelos base sparse-MoE abiertos (Qwen3.6-35B-A3B y el híbrido bailing Ling 3.0, respectivamente) e incluyen el entrenamiento de LoRA y prerouter realizado para este framework. edge0-infer descarga el checkpoint publicado con sus herramientas de recursos y lo convierte en los recursos de modelo del motor (consulta [Primeros pasos](#primeros-pasos)). +`edge0-145b` y `edge0-120b` forman un nivel latent-MoE aparte para una sola GPU NVIDIA; se ejecutan con edge0-strata, que `edge0-infer serve` y `edge0-infer chat` inician (consulta [edge0-145b y edge0-120b](docs/models/edge0-145b.md)). ### Diseño @@ -91,8 +96,8 @@ y lo convierte en los recursos de modelo del motor (consulta fija; la compilación no descarga nada; - **Una API nativa**: las aplicaciones enlazan el target de CMake `edge0::edge0` e incluyen las cabeceras de `include/edge0/` para usar el - motor dentro del proceso; la CLI de referencia y el servidor compatible con - OpenAI usan los mismos ajustes y valores por defecto que la API nativa; + motor dentro del proceso; la CLI de referencia y el servidor local + usan los mismos ajustes y valores por defecto que la API nativa; - **Expertos bajo demanda**: los expertos enrutados se leen del almacenamiento bajo demanda en un número fijo de ranuras por capa, de modo que un modelo más grande que el presupuesto de memoria del dispositivo se ejecuta sin @@ -126,6 +131,17 @@ Máximo 100: | IFBench | 57.9 | 61.7 | 53.9 | 60.6 | | **Media** | **79.2** | **83.2** | **69.9** | **72.7** | +`edge0-145b` y `edge0-120b` (**negrita** = mejor de la columna; consulta la [página del modelo](docs/models/edge0-145b.md#quality)): + +| Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | +| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | +| Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | +| Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | +| Claude-Opus-4.6 (Max) | / | 78.2 | 78.0 | 62.5 | 91.3 | **40.0** | + ### Benchmark `edge0-35b` con enrutamiento `predicted`, medido con edge0-infer (cada rango va del mínimo al máximo de tres procesos): @@ -141,6 +157,33 @@ Máximo 100: Las métricas de memoria difieren entre plataformas y no se pueden comparar directamente. El método de medición, las cifras con enrutamiento `actual` y los comandos para reproducirlas están en [Benchmarking](edge0-infer/docs/benchmarking.md#reference-figures). +`edge0-145b` y `edge0-120b` en edge0-strata, una GPU NVIDIA (tok/s): + +#### edge0-145b + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 2169.9 | 3554.9 | 99.7 | **121.2** | +| RTX 5090 D 24 GB | NVFP4 | 1839.6 | 3165.7 | 79.6 | 98.3 | +| RTX 4090 24 GB | Q3_K | 2042.0 | 3179.4 | 69.9 | 91.7 | +| RTX Spark N1X 60 GiB | Q3_K | 674.0 | 770.3 | 37.7 | 36.7 | +| RTX Spark N1X 60 GiB | NVFP4 | 821.2 | 922.4 | 40.1 | 46.0 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 249 | 335 | 36.8 | 44.9 | + +#### edge0-120b + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 3236.7 | 4576.5 | 130.2 | **144.6** | +| RTX 5090 D 24 GB | NVFP4 | 2668.5 | 4357.1 | 124.9 | 135.5 | +| RTX 4090 24 GB | Q3_K | 2637.5 | 3551.1 | 95.0 | 104.9 | +| RTX Spark N1X 60 GiB | Q3_K | 987.0 | 1081.7 | 47.2 | 68.4 | +| RTX Spark N1X 60 GiB | NVFP4 | 996.4 | 1066.9 | 45.1 | 68.9 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 430 | 778 | 62.4 | 73.3 | +| 16 GiB VRAM + 20 GiB RAM | NVFP4 | 361.1 | 619.7 | 51.3 | 59.9 | + +Las filas `16 GiB VRAM + 20 GiB RAM` son la misma RTX 5090 D 24 GB, limitada por cgroup a 16 GiB de memoria de GPU y 20 GiB de memoria del sistema. Los detalles están en la [página del modelo](docs/models/edge0-145b.md#speed-measured-one-card). + ## Primeros pasos Ejecuta los siguientes comandos en `edge0-infer/`. Necesitas CMake 3.21 o @@ -159,7 +202,9 @@ build/mac-metal/bin/edge0-infer info # debe mostrar una línea: model.0: En máquinas sin Metal usa `--preset cpu`, `build/cpu/bin/edge0-infer` y `--device cpu` en los comandos siguientes. -### 2) Descargar y convertir los recursos del modelo +### 2) Obtener un modelo + +**edge0-35b (motor nativo).** Descarga y convierte los recursos del modelo: ```bash export MOE_SOURCE_35B= @@ -176,6 +221,8 @@ assets_py tools/assets/check_assets.py check "$MOE_ASSETS_35B" Se descarga `Edge0/Edge0-35B-A3B-preview` de Hugging Face y se verifica con la revisión y los resúmenes fijados. +**edge0-145b y edge0-120b (edge0-strata).** Compila edge0-strata como indica [`edge0-strata/EDGE0.md`](edge0-strata/EDGE0.md) y usa una carpeta de modelo publicada (consulta [edge0-145b y edge0-120b](docs/models/edge0-145b.md)). + ### 3) Ejecutar ```bash @@ -183,10 +230,12 @@ la revisión y los resúmenes fijados. build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "Preséntate en una frase." # chat de varios turnos build/mac-metal/bin/edge0-infer chat --model "$MOE_ASSETS_35B" --device metal -# servidor (compatible con OpenAI, por defecto http://127.0.0.1:8000/v1) +# servidor (compatible con OpenAI y Anthropic, por defecto http://127.0.0.1:8000) build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal ``` +Para edge0-145b y edge0-120b, `chat` y `serve` reciben la carpeta del modelo de la misma forma (`--model `); `run` es para el motor nativo. + Para las compilaciones de Android, Windows e iOS, y para usar el motor desde tu propio proyecto CMake, consulta [Getting started](edge0-infer/docs/getting-started.md). @@ -195,17 +244,22 @@ tu propio proyecto CMake, consulta [Getting started](edge0-infer/docs/getting-st **Modelos** - [Enrutamiento MoE y expertos](docs/moe.md) / [prerouter](docs/prerouter.md) -- [edge0-35b](docs/models/edge0-35b.md) / [edge0-8b](docs/models/edge0-8b.md) +- [edge0-35b](docs/models/edge0-35b.md) / [edge0-8b](docs/models/edge0-8b.md) / [edge0-145b y edge0-120b](docs/models/edge0-145b.md) - Informe técnico: [The Other Half of the Memory Wall](https://arxiv.org/abs/2609.18063) ([PDF](paper/main.pdf)) -**Motor (edge0-infer)** +**edge0-infer** - [Getting started](edge0-infer/docs/getting-started.md): compilación, recursos del modelo, ejecución, chat y servidor en cada plataforma -- [Configuration](edge0-infer/docs/configuration.md) / [Native API](edge0-infer/docs/native-api.md) / [Servidor compatible con OpenAI](edge0-infer/docs/openai-compatible-server.md) +- [Configuration](edge0-infer/docs/configuration.md) / [Native API](edge0-infer/docs/native-api.md) / [Servidor local](edge0-infer/docs/local-server.md) - [Architecture](edge0-infer/docs/architecture.md) / [Known limitations](edge0-infer/docs/limitations.md) - [Benchmarking](edge0-infer/docs/benchmarking.md) / [Testing](edge0-infer/docs/testing.md) - [Herramientas de recursos del modelo y cómo añadir un modelo](edge0-infer/tools/assets/README.md) +**Motores** + +- Motor nativo: [Architecture](edge0-infer/docs/architecture.md#native-engine) / [Native API](edge0-infer/docs/native-api.md) +- edge0-strata: [compilación y contrato de arranque](edge0-strata/EDGE0.md) / [edge0-145b](edge0-infer/docs/edge0-145b.md) y [edge0-120b](edge0-infer/docs/edge0-120b.md) en edge0-infer + ## Hoja de ruta ### Q4 2026 diff --git a/README_fr.md b/README_fr.md index 5f6b9345..e419cc49 100644 --- a/README_fr.md +++ b/README_fr.md @@ -6,7 +6,7 @@ **Un framework open source d'inférence MoE en streaming — SSD expert offload + Recover-LoRA + prédiction de routage prerouter.** -**macOS** · **iOS** · **Android** · **Windows** — un seul moteur, une seule API native. +**macOS** · **iOS** · **Android** · **Linux** · **Windows** — un seul programme pour tous les modèles Edge0. [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--35B--A3B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--8B--A1B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) @@ -23,6 +23,7 @@ ## Actualités +- **[2026-11-13]** Nous avons publié **`edge0-145b`** et **`edge0-120b`**, des modèles latent mixture-of-experts dérivés de Qwen3.8-Flash-Next pour un seul GPU NVIDIA sous Linux ou Windows. Ils s'exécutent sur [edge0-strata](edge0-strata/EDGE0.md), que `edge0-infer serve` et `edge0-infer chat` démarrent pour eux ; voir la [page du modèle](docs/models/edge0-145b.md). - **[2026-11-04]** Nous avons publié le **moteur d'inférence unifié [`edge0-infer`](edge0-infer/README_fr.md)** : construit sur [GGML](https://github.com/ggml-org/ggml), un seul moteur et une seule API C++ native exécutent le même modèle sur macOS, iOS, Android et Windows, avec une CLI, un serveur local compatible OpenAI et des exemples d'applications de discussion pour le bureau, Android et iOS. La première version prend en charge `edge0-35b` ; `edge0-8b` suivra dans une version ultérieure. Il remplace le framework Python et les moteurs propres à chaque plateforme ; l'ancien code est conservé au tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer). - **[2026-09-30]** Nous avons publié les **moteurs d'inférence edge0 pour quatre plateformes — iOS, macOS, Android et Windows** (code source au tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer)). - **[2026-09-16]** Notre rapport technique est sur arXiv : [The Other Half of the Memory Wall: Serving 35B MoEs from SSD with Trained Routing Prediction](https://arxiv.org/abs/2609.18063). @@ -55,34 +56,38 @@ matériel grand public : la mémoire de pointe est limitée par l'ensemble ### Plateformes -Un seul moteur et une seule API native, avec le même code sur chaque plateforme ; voir [`edge0-infer/`](edge0-infer/README_fr.md) : +[`edge0-infer`](edge0-infer/README_fr.md) est le programme que vous lancez (CLI, serveur local compatible OpenAI et Anthropic, applications de discussion), avec les mêmes commandes pour tous les modèles. Chaque modèle s'exécute sur le moteur conçu pour lui, et edge0-infer choisit le moteur d'après le dossier du modèle ; il n'y a rien à choisir : `edge0-35b` s'exécute sur le **moteur natif** de `edge0-infer/` (construit sur GGML, avec une API C++ native), et `edge0-145b` et `edge0-120b` sur **edge0-strata**, dans [`edge0-strata/`](edge0-strata/EDGE0.md), un fork de Strata. -| Plateforme | Backends | Vérifié sur | +| Plateforme | `edge0-35b` (moteur natif) | `edge0-145b`, `edge0-120b` (edge0-strata) | |---|---|---| -| **macOS** (Apple Silicon) | Metal, CPU | Apple M5, 16 Gio | -| **iOS** | Metal | iPhone 17 Pro Max | -| **Android** | CPU | OnePlus PKR110 (Snapdragon 8 Elite) | -| **Windows** (x86-64) | CPU, Vulkan | Intel Core i5-12400, NVIDIA GeForce RTX 4070 SUPER | +| **macOS** (Apple Silicon) | Metal, CPU | — | +| **iOS** | Metal | — | +| **Android** | CPU | — | +| **Linux** | CPU | GPU NVIDIA (CUDA 13) | +| **Windows** | CPU, Vulkan | GPU NVIDIA (CUDA 13) | -Le prefill et le decode peuvent s'exécuter sur des backends différents du même appareil, par exemple prefill sur Metal et decode sur CPU. Les exemples d'applications de discussion pour le bureau (macOS, Windows), Android et iOS se trouvent dans [`edge0-infer/examples/`](edge0-infer/examples/README.md#chat-applications). +Mesures : [Benchmarking](edge0-infer/docs/benchmarking.md#reference-figures) pour `edge0-35b`, et la [page du modèle](docs/models/edge0-145b.md) pour `edge0-145b` et `edge0-120b`. Sur le moteur natif, le prefill et le decode peuvent s'exécuter sur des backends différents du même appareil, par exemple prefill sur Metal et decode sur CPU. Les exemples d'applications de discussion pour le bureau (macOS, Windows), Android et iOS se trouvent dans [`edge0-infer/examples/`](edge0-infer/examples/README.md#chat-applications). ### Modèles -Deux gammes de modèles sont fournies avec le framework. Chaque gamme est +Deux gammes de modèles, `edge0-35b` et `edge0-8b`, sont fournies avec le framework. Chaque gamme est une publication complète : le checkpoint publié, les adaptateurs LoRA entraînés et les têtes prerouter entraînées fonctionnent ensemble comme une seule unité. -| Gamme | Checkpoint publié | Profil d'inférence | edge0-infer | +| Gamme | Checkpoint publié | Profil d'inférence | Moteur | |---|---|---|---| -| `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4-bit, 40 couches, 256 experts, prerouter K=4 | ✅ Pris en charge | -| `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4-bit, 24 couches, 128 experts, prerouter K=8 | Prévu | +| `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4-bit, 40 couches, 256 experts, prerouter K=4 | Moteur natif ✅ | +| `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4-bit, 24 couches, 128 experts, prerouter K=8 | Moteur natif (prévu) | +| [`edge0-145b`](docs/models/edge0-145b.md) | coming soon | experts latents Q3_K ou NVFP4 (1792), 48 couches, 512 experts, K=10 | edge0-strata ✅ (Linux ou Windows, GPU NVIDIA) | +| [`edge0-120b`](docs/models/edge0-145b.md) | coming soon | experts latents Q3_K ou NVFP4 (1280), 48 couches, 512 experts, K=10 | edge0-strata ✅ (Linux ou Windows, GPU NVIDIA) | -Les deux checkpoints reposent sur des modèles de base sparse-MoE ouverts +Les deux checkpoints publiés reposent sur des modèles de base sparse-MoE ouverts (respectivement Qwen3.6-35B-A3B et l'hybride bailing Ling 3.0) et intègrent l'entraînement LoRA et prerouter réalisé pour ce framework. edge0-infer télécharge le checkpoint publié avec ses outils de ressources et le convertit en ressources de modèle du moteur (voir [Démarrage rapide](#démarrage-rapide)). +`edge0-145b` et `edge0-120b` forment une gamme latent-MoE distincte pour un seul GPU NVIDIA ; ils s'exécutent avec edge0-strata, que `edge0-infer serve` et `edge0-infer chat` démarrent (voir [edge0-145b et edge0-120b](docs/models/edge0-145b.md)). ### Conception @@ -91,8 +96,8 @@ en ressources de modèle du moteur (voir [Démarrage rapide](#démarrage-rapide) fixe ; la compilation ne télécharge rien ; - **Une seule API native** : les applications se lient à la cible CMake `edge0::edge0` et incluent les en-têtes de `include/edge0/` pour utiliser le - moteur dans le processus ; la CLI de référence et le serveur compatible - OpenAI utilisent les mêmes paramètres et valeurs par défaut que l'API + moteur dans le processus ; la CLI de référence et le serveur local + utilisent les mêmes paramètres et valeurs par défaut que l'API native ; - **Experts à la demande** : les experts routés sont lus depuis le stockage à la demande dans un nombre fixe d'emplacements par couche, si bien qu'un @@ -127,6 +132,17 @@ au-dessus de la base). Maximum 100 : | IFBench | 57.9 | 61.7 | 53.9 | 60.6 | | **Moyenne** | **79.2** | **83.2** | **69.9** | **72.7** | +`edge0-145b` et `edge0-120b` (**gras** = meilleur de la colonne ; voir la [page du modèle](docs/models/edge0-145b.md#quality)) : + +| Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | +| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | +| Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | +| Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | +| Claude-Opus-4.6 (Max) | / | 78.2 | 78.0 | 62.5 | 91.3 | **40.0** | + ### Benchmark `edge0-35b` avec le routage `predicted`, mesuré avec edge0-infer (chaque plage va du minimum au maximum de trois processus) : @@ -142,6 +158,33 @@ au-dessus de la base). Maximum 100 : Les métriques mémoire diffèrent selon la plateforme et ne sont pas directement comparables. La méthode de mesure, les chiffres en routage `actual` et les commandes de reproduction se trouvent dans [Benchmarking](edge0-infer/docs/benchmarking.md#reference-figures). +`edge0-145b` et `edge0-120b` sur edge0-strata, un seul GPU NVIDIA (tok/s) : + +#### edge0-145b + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 2169.9 | 3554.9 | 99.7 | **121.2** | +| RTX 5090 D 24 GB | NVFP4 | 1839.6 | 3165.7 | 79.6 | 98.3 | +| RTX 4090 24 GB | Q3_K | 2042.0 | 3179.4 | 69.9 | 91.7 | +| RTX Spark N1X 60 GiB | Q3_K | 674.0 | 770.3 | 37.7 | 36.7 | +| RTX Spark N1X 60 GiB | NVFP4 | 821.2 | 922.4 | 40.1 | 46.0 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 249 | 335 | 36.8 | 44.9 | + +#### edge0-120b + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 3236.7 | 4576.5 | 130.2 | **144.6** | +| RTX 5090 D 24 GB | NVFP4 | 2668.5 | 4357.1 | 124.9 | 135.5 | +| RTX 4090 24 GB | Q3_K | 2637.5 | 3551.1 | 95.0 | 104.9 | +| RTX Spark N1X 60 GiB | Q3_K | 987.0 | 1081.7 | 47.2 | 68.4 | +| RTX Spark N1X 60 GiB | NVFP4 | 996.4 | 1066.9 | 45.1 | 68.9 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 430 | 778 | 62.4 | 73.3 | +| 16 GiB VRAM + 20 GiB RAM | NVFP4 | 361.1 | 619.7 | 51.3 | 59.9 | + +Les lignes `16 GiB VRAM + 20 GiB RAM` correspondent à la même RTX 5090 D 24 GB, limitée par cgroup à 16 Gio de mémoire GPU et 20 Gio de mémoire système. Les détails sont sur la [page du modèle](docs/models/edge0-145b.md#speed-measured-one-card). + ## Démarrage rapide Lancez les commandes suivantes dans `edge0-infer/`. Il faut CMake 3.21 ou @@ -160,7 +203,9 @@ build/mac-metal/bin/edge0-infer info # doit afficher une ligne : model.0: Sur les machines sans Metal, utilisez `--preset cpu`, `build/cpu/bin/edge0-infer` et `--device cpu` dans les commandes ci-dessous. -### 2) Télécharger et convertir les ressources du modèle +### 2) Obtenir un modèle + +**edge0-35b (moteur natif).** Téléchargez et convertissez les ressources du modèle : ```bash export MOE_SOURCE_35B= @@ -177,6 +222,8 @@ assets_py tools/assets/check_assets.py check "$MOE_ASSETS_35B" Le téléchargement récupère `Edge0/Edge0-35B-A3B-preview` sur Hugging Face et le vérifie avec la révision et les empreintes verrouillées. +**edge0-145b et edge0-120b (edge0-strata).** Compilez edge0-strata comme l'indique [`edge0-strata/EDGE0.md`](edge0-strata/EDGE0.md) et utilisez un dossier de modèle publié (voir [edge0-145b et edge0-120b](docs/models/edge0-145b.md)). + ### 3) Exécuter ```bash @@ -184,10 +231,12 @@ vérifie avec la révision et les empreintes verrouillées. build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "Présente-toi en une phrase." # discussion multi-tours build/mac-metal/bin/edge0-infer chat --model "$MOE_ASSETS_35B" --device metal -# serveur (compatible OpenAI, par défaut http://127.0.0.1:8000/v1) +# serveur (compatible OpenAI et Anthropic, par défaut http://127.0.0.1:8000) build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal ``` +Pour edge0-145b et edge0-120b, `chat` et `serve` prennent le dossier du modèle de la même façon (`--model `) ; `run` est réservé au moteur natif. + Pour les compilations Android, Windows et iOS, et pour utiliser le moteur depuis votre propre projet CMake, voir [Getting started](edge0-infer/docs/getting-started.md). @@ -196,17 +245,22 @@ votre propre projet CMake, voir [Getting started](edge0-infer/docs/getting-start **Modèles** - [Routage MoE et experts](docs/moe.md) / [prerouter](docs/prerouter.md) -- [edge0-35b](docs/models/edge0-35b.md) / [edge0-8b](docs/models/edge0-8b.md) +- [edge0-35b](docs/models/edge0-35b.md) / [edge0-8b](docs/models/edge0-8b.md) / [edge0-145b et edge0-120b](docs/models/edge0-145b.md) - Rapport technique : [The Other Half of the Memory Wall](https://arxiv.org/abs/2609.18063) ([PDF](paper/main.pdf)) -**Moteur (edge0-infer)** +**edge0-infer** - [Getting started](edge0-infer/docs/getting-started.md) : compilation, ressources du modèle, exécution, discussion et serveur sur chaque plateforme -- [Configuration](edge0-infer/docs/configuration.md) / [Native API](edge0-infer/docs/native-api.md) / [Serveur compatible OpenAI](edge0-infer/docs/openai-compatible-server.md) +- [Configuration](edge0-infer/docs/configuration.md) / [Native API](edge0-infer/docs/native-api.md) / [Serveur local](edge0-infer/docs/local-server.md) - [Architecture](edge0-infer/docs/architecture.md) / [Known limitations](edge0-infer/docs/limitations.md) - [Benchmarking](edge0-infer/docs/benchmarking.md) / [Testing](edge0-infer/docs/testing.md) - [Outils de ressources du modèle et ajout d'un modèle](edge0-infer/tools/assets/README.md) +**Moteurs** + +- Moteur natif : [Architecture](edge0-infer/docs/architecture.md#native-engine) / [Native API](edge0-infer/docs/native-api.md) +- edge0-strata : [compilation et contrat de lancement](edge0-strata/EDGE0.md) / [edge0-145b](edge0-infer/docs/edge0-145b.md) et [edge0-120b](edge0-infer/docs/edge0-120b.md) dans edge0-infer + ## Feuille de route ### T4 2026 diff --git a/README_ja.md b/README_ja.md index d6031932..dc2d46ed 100644 --- a/README_ja.md +++ b/README_ja.md @@ -6,7 +6,7 @@ **オープンソースのストリーミング MoE 推論フレームワーク — SSD expert offload + Recover-LoRA + prerouter ルーティング予測。** -**macOS** · **iOS** · **Android** · **Windows** — 1 つのエンジン、1 つのネイティブ API。 +**macOS** · **iOS** · **Android** · **Linux** · **Windows** — すべての Edge0 モデルを 1 つのプログラムで。 [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--35B--A3B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--8B--A1B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) @@ -23,6 +23,7 @@ ## ニュース +- **[2026-11-13]** **`edge0-145b`** と **`edge0-120b`** をリリースしました。Qwen3.8-Flash-Next から派生した latent MoE モデルで、Linux または Windows 上の NVIDIA GPU 1 枚で動作します。[edge0-strata](edge0-strata/EDGE0.md) で動作し、`edge0-infer serve` と `edge0-infer chat` がこれらのモデル用に起動します。[モデルページ](docs/models/edge0-145b.md)を参照してください。 - **[2026-11-04]** **統合推論エンジン [`edge0-infer`](edge0-infer/README_ja.md)** をリリースしました。[GGML](https://github.com/ggml-org/ggml) をベースに、1 つのエンジンと 1 つのネイティブ C++ API で、同じモデルを macOS、iOS、Android、Windows で実行します。CLI、OpenAI 互換のローカルサーバー、デスクトップ・Android・iOS 向けのチャットアプリのサンプルも付属します。最初のリリースは `edge0-35b` をサポートし、`edge0-8b` は今後のリリースでサポートします。これまでの Python フレームワークと各プラットフォームのエンジンを置き換えるもので、以前のコードは tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer) に残しています。 - **[2026-09-30]** **4 つのプラットフォーム — iOS、macOS、Android、Windows — 向けの edge0 推論エンジン**をリリースしました(ソースは tag [`pre-edge0-infer`](https://github.com/Edge0-AI/Edge0/tree/pre-edge0-infer) を参照)。 - **[2026-09-16]** 技術レポートが arXiv に掲載: [The Other Half of the Memory Wall: Serving 35B MoEs from SSD with Trained Routing Prediction](https://arxiv.org/abs/2609.18063)。 @@ -40,32 +41,36 @@ ### プラットフォーム -1 つのエンジンと 1 つのネイティブ API で、すべてのプラットフォームが同じコードを共有します。[`edge0-infer/`](edge0-infer/README_ja.md) を参照: +[`edge0-infer`](edge0-infer/README_ja.md) は実行するプログラム(CLI、OpenAI／Anthropic 互換のローカルサーバー、チャットアプリ)で、どのモデルも同じコマンドで使えます。各モデルはそれぞれ専用のエンジンで動作し、edge0-infer がモデルフォルダからエンジンを自動で選ぶため、選択する必要はありません。`edge0-35b` は `edge0-infer/` の**ネイティブエンジン**(GGML ベース、ネイティブ C++ API を提供)で、`edge0-145b` と `edge0-120b` は [`edge0-strata/`](edge0-strata/EDGE0.md) の **edge0-strata**(Strata のフォーク)で動作します。 -| プラットフォーム | バックエンド | 検証デバイス | +| プラットフォーム | `edge0-35b`(ネイティブエンジン) | `edge0-145b`、`edge0-120b`(edge0-strata) | |---|---|---| -| **macOS**(Apple Silicon) | Metal、CPU | Apple M5、16 GiB | -| **iOS** | Metal | iPhone 17 Pro Max | -| **Android** | CPU | OnePlus PKR110(Snapdragon 8 Elite) | -| **Windows**(x86-64) | CPU、Vulkan | Intel Core i5-12400、NVIDIA GeForce RTX 4070 SUPER | +| **macOS**(Apple Silicon) | Metal、CPU | — | +| **iOS** | Metal | — | +| **Android** | CPU | — | +| **Linux** | CPU | NVIDIA GPU(CUDA 13) | +| **Windows** | CPU、Vulkan | NVIDIA GPU(CUDA 13) | -同じデバイス上で prefill と decode に異なるバックエンドを使えます(例: Metal で prefill、CPU で decode)。デスクトップ(macOS、Windows)、Android、iOS 向けのチャットアプリのサンプルは [`edge0-infer/examples/`](edge0-infer/examples/README.md#chat-applications) にあります。 +測定結果：`edge0-35b` は [Benchmarking](edge0-infer/docs/benchmarking.md#reference-figures)、`edge0-145b` と `edge0-120b` は[モデルページ](docs/models/edge0-145b.md)を参照してください。ネイティブエンジンでは、同じデバイス上で prefill と decode に異なるバックエンドを使えます(例: Metal で prefill、CPU で decode)。デスクトップ(macOS、Windows)、Android、iOS 向けのチャットアプリのサンプルは [`edge0-infer/examples/`](edge0-infer/examples/README.md#chat-applications) にあります。 ### モデル -フレームワークには 2 つのモデルティアが付属します。各ティアはエンドツーエンドのリリースであり、公開されるチェックポイント、訓練済み LoRA アダプタ、訓練済み prerouter ヘッドが 1 つの単位として機能します。 +フレームワークには 2 つのモデルティア（`edge0-35b` と `edge0-8b`）が付属します。各ティアはエンドツーエンドのリリースであり、公開されるチェックポイント、訓練済み LoRA アダプタ、訓練済み prerouter ヘッドが 1 つの単位として機能します。 -| ティア | 公開チェックポイント | 推論プロファイル | edge0-infer | +| ティア | 公開チェックポイント | 推論プロファイル | エンジン | |---|---|---|---| -| `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4-bit、40 層、256 エキスパート、prerouter K=4 | ✅ サポート済み | -| `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4-bit、24 層、128 エキスパート、prerouter K=8 | 予定 | +| `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4-bit、40 層、256 エキスパート、prerouter K=4 | ネイティブエンジン ✅ | +| `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4-bit、24 層、128 エキスパート、prerouter K=8 | ネイティブエンジン(予定) | +| [`edge0-145b`](docs/models/edge0-145b.md) | coming soon | Q3_K または NVFP4 latent エキスパート（1792）、48 層、512 エキスパート、K=10 | edge0-strata ✅（Linux または Windows、NVIDIA GPU） | +| [`edge0-120b`](docs/models/edge0-145b.md) | coming soon | Q3_K または NVFP4 latent エキスパート（1280）、48 層、512 エキスパート、K=10 | edge0-strata ✅（Linux または Windows、NVIDIA GPU） | -両方のチェックポイントともオープンのスパース MoE ベースモデル(それぞれ Qwen3.6-35B-A3B と Ling 3.0 bailing hybrid)上に構築されており、本フレームワークのために実施された LoRA と prerouter の訓練成果物を同梱しています。edge0-infer は付属のアセットツールで公開チェックポイントをダウンロードし、エンジン用のモデルアセットに変換します([はじめに](#はじめに)を参照)。 +公開済みの 2 つのチェックポイントはいずれもオープンのスパース MoE ベースモデル(それぞれ Qwen3.6-35B-A3B と Ling 3.0 bailing hybrid)上に構築されており、本フレームワークのために実施された LoRA と prerouter の訓練成果物を同梱しています。edge0-infer は付属のアセットツールで公開チェックポイントをダウンロードし、エンジン用のモデルアセットに変換します([はじめに](#はじめに)を参照)。 +`edge0-145b` と `edge0-120b` は単一の NVIDIA GPU 向けの latent-MoE です。edge0-strata 上で動作し、`edge0-infer serve` と `edge0-infer chat` がそれを起動します([edge0-145b と edge0-120b](docs/models/edge0-145b.md) を参照)。 ### 設計 - **GGML ベース、全プラットフォームで同じコード**: ネイティブライブラリは、固定バージョンを無改変で同梱した GGML のみに依存し、ビルド時に何もダウンロードしません。 -- **1 つのネイティブ API**: アプリケーションは CMake ターゲット `edge0::edge0` にリンクし、`include/edge0/` のヘッダーをインクルードするだけで、プロセス内でエンジンを使えます。リファレンス CLI と OpenAI 互換サーバーは、ネイティブ API と同じ設定とデフォルト値を使います。 +- **1 つのネイティブ API**: アプリケーションは CMake ターゲット `edge0::edge0` にリンクし、`include/edge0/` のヘッダーをインクルードするだけで、プロセス内でエンジンを使えます。リファレンス CLI とローカルサーバーは、ネイティブ API と同じ設定とデフォルト値を使います。 - **エキスパートのオンデマンド読み込み**: ルーティングされるエキスパートは、ストレージから必要に応じて層ごとに固定数のエキスパートスロットへ読み込まれるため、デバイスのメモリ予算より大きなモデルでも全体を読み込まずに実行できます。 - **2 つのルーティングモード**: `predicted`(デフォルト)は prerouter を使い、現在のトークンを計算している間に次のトークンのエキスパートをバックグラウンドで準備します。`actual` はモデル本来の計算で、比較用です。 - **マルチターンの再利用**: マルチターン会話では通常、各ターンの新しいメッセージだけを計算し、それ以前の計算結果はそのまま再利用します。 @@ -85,6 +90,17 @@ | IFBench | 57.9 | 61.7 | 53.9 | 60.6 | | **平均** | **79.2** | **83.2** | **69.9** | **72.7** | +`edge0-145b` と `edge0-120b`(**太字** = 列内の最高値。[モデルページ](docs/models/edge0-145b.md#quality)を参照): + +| Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | +| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | +| Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | +| Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | +| Claude-Opus-4.6 (Max) | / | 78.2 | 78.0 | 62.5 | 91.3 | **40.0** | + ### ベンチマーク `edge0-35b`、`predicted` ルーティング、edge0-infer で測定(各値は 3 プロセスの最小値〜最大値): @@ -100,6 +116,33 @@ メモリの指標はプラットフォームごとに異なり、直接比較できません。測定方法、`actual` ルーティングの数値、再現コマンドは [Benchmarking](edge0-infer/docs/benchmarking.md#reference-figures) を参照してください。 +`edge0-145b` と `edge0-120b`、edge0-strata、NVIDIA GPU 1 枚(tok/s): + +#### edge0-145b + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 2169.9 | 3554.9 | 99.7 | **121.2** | +| RTX 5090 D 24 GB | NVFP4 | 1839.6 | 3165.7 | 79.6 | 98.3 | +| RTX 4090 24 GB | Q3_K | 2042.0 | 3179.4 | 69.9 | 91.7 | +| RTX Spark N1X 60 GiB | Q3_K | 674.0 | 770.3 | 37.7 | 36.7 | +| RTX Spark N1X 60 GiB | NVFP4 | 821.2 | 922.4 | 40.1 | 46.0 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 249 | 335 | 36.8 | 44.9 | + +#### edge0-120b + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 3236.7 | 4576.5 | 130.2 | **144.6** | +| RTX 5090 D 24 GB | NVFP4 | 2668.5 | 4357.1 | 124.9 | 135.5 | +| RTX 4090 24 GB | Q3_K | 2637.5 | 3551.1 | 95.0 | 104.9 | +| RTX Spark N1X 60 GiB | Q3_K | 987.0 | 1081.7 | 47.2 | 68.4 | +| RTX Spark N1X 60 GiB | NVFP4 | 996.4 | 1066.9 | 45.1 | 68.9 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 430 | 778 | 62.4 | 73.3 | +| 16 GiB VRAM + 20 GiB RAM | NVFP4 | 361.1 | 619.7 | 51.3 | 59.9 | + +`16 GiB VRAM + 20 GiB RAM` の行は同じ RTX 5090 D 24 GB を cgroup で GPU メモリ 16 GiB、システムメモリ 20 GiB に制限したものです。詳細は[モデルページ](docs/models/edge0-145b.md#speed-measured-one-card)を参照してください。 + ## はじめに 以下のコマンドはすべて `edge0-infer/` ディレクトリで実行します。CMake 3.21 以上、Ninja、C++17/C11 コンパイラ、uv、Git が必要です。35B の元の重みは約 18 GB、変換後のアセットは約 20 GB で、変換中はさらに同じくらいの空き容量が必要です。 @@ -114,7 +157,9 @@ build/mac-metal/bin/edge0-infer info # 次の行が出力されるはず: Metal のないマシンでは `--preset cpu`、`build/cpu/bin/edge0-infer` を使い、以下のコマンドでは `--device cpu` を指定してください。 -### 2) モデルアセットのダウンロードと変換 +### 2) モデルの準備 + +**edge0-35b(ネイティブエンジン)**:モデルアセットをダウンロードして変換します。 ```bash export MOE_SOURCE_35B=<元の重みを置くディレクトリ> @@ -130,6 +175,8 @@ assets_py tools/assets/check_assets.py check "$MOE_ASSETS_35B" ダウンロードされるのは Hugging Face 上の `Edge0/Edge0-35B-A3B-preview` で、固定されたリビジョンとダイジェストで検証されます。 +**edge0-145b と edge0-120b(edge0-strata)**:[`edge0-strata/EDGE0.md`](edge0-strata/EDGE0.md) に従って edge0-strata をビルドし、公開されたモデルフォルダを使います([edge0-145b と edge0-120b](docs/models/edge0-145b.md) を参照)。 + ### 3) 実行 ```bash @@ -137,10 +184,12 @@ assets_py tools/assets/check_assets.py check "$MOE_ASSETS_35B" build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "一文で自己紹介してください。" # マルチターンチャット build/mac-metal/bin/edge0-infer chat --model "$MOE_ASSETS_35B" --device metal -# サーバー起動(OpenAI 互換、デフォルト http://127.0.0.1:8000/v1) +# サーバー起動(OpenAI／Anthropic 互換、デフォルト http://127.0.0.1:8000) build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal ``` +edge0-145b と edge0-120b でも、`chat` と `serve` は同じく `--model <モデルフォルダ>` で使います。`run` はネイティブエンジン専用です。 + Android、Windows、iOS 向けのビルド、および自分の CMake プロジェクトからエンジンを使う方法は [Getting started](edge0-infer/docs/getting-started.md) を参照してください。 ## ドキュメント @@ -148,17 +197,22 @@ Android、Windows、iOS 向けのビルド、および自分の CMake プロジ **モデル** - [MoE のルーティングとエキスパート](docs/moe.md) / [prerouter](docs/prerouter.md) -- [edge0-35b](docs/models/edge0-35b.md) / [edge0-8b](docs/models/edge0-8b.md) +- [edge0-35b](docs/models/edge0-35b.md) / [edge0-8b](docs/models/edge0-8b.md) / [edge0-145b と edge0-120b](docs/models/edge0-145b.md) - 技術レポート: [The Other Half of the Memory Wall](https://arxiv.org/abs/2609.18063)([PDF](paper/main.pdf)) -**エンジン(edge0-infer)** +**edge0-infer** - [Getting started](edge0-infer/docs/getting-started.md): 各プラットフォームでのビルド、モデルアセット、実行、チャット、サーバー -- [Configuration](edge0-infer/docs/configuration.md) / [Native API](edge0-infer/docs/native-api.md) / [OpenAI 互換サーバー](edge0-infer/docs/openai-compatible-server.md) +- [Configuration](edge0-infer/docs/configuration.md) / [Native API](edge0-infer/docs/native-api.md) / [ローカルサーバー](edge0-infer/docs/local-server.md) - [Architecture](edge0-infer/docs/architecture.md) / [Known limitations](edge0-infer/docs/limitations.md) - [Benchmarking](edge0-infer/docs/benchmarking.md) / [Testing](edge0-infer/docs/testing.md) - [モデルアセットツールとモデルの追加](edge0-infer/tools/assets/README.md) +**エンジン** + +- ネイティブエンジン: [Architecture](edge0-infer/docs/architecture.md#native-engine) / [Native API](edge0-infer/docs/native-api.md) +- edge0-strata: [ビルドと起動の取り決め](edge0-strata/EDGE0.md) / edge0-infer での [edge0-145b](edge0-infer/docs/edge0-145b.md) と [edge0-120b](edge0-infer/docs/edge0-120b.md) + ## ロードマップ ### 2026 年 Q4 diff --git a/README_zh.md b/README_zh.md index 303e6d62..923e9adc 100644 --- a/README_zh.md +++ b/README_zh.md @@ -6,7 +6,7 @@ **开源流式 MoE 推理框架 —— SSD 专家 offload + Recover-LoRA + prerouter 路由预判** -**macOS** · **iOS** · **Android** · **Windows** —— 一个引擎，一套原生 API。 +**macOS** · **iOS** · **Android** · **Linux** · **Windows** —— 一个程序，运行所有 Edge0 模型。 [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--35B--A3B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Edge0--8B--A1B--preview-yellow?style=for-the-badge)](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) @@ -23,6 +23,7 @@ ## 新闻 +- **【2026-11-13】** 发布 **`edge0-145b`** 与 **`edge0-120b`**：由 Qwen3.8-Flash-Next 派生的 latent MoE 模型，在 Linux 或 Windows 上用单张 NVIDIA GPU 运行。它们运行在 [edge0-strata](edge0-strata/EDGE0.md) 上，由 `edge0-infer serve` 与 `edge0-infer chat` 为它们启动；见[模型页](docs/models/edge0-145b.md)。 - **【2026-11-04】** 发布**统一推理引擎 [`edge0-infer`](edge0-infer/README_zh.md)**： 基于 [GGML](https://github.com/ggml-org/ggml)，一个引擎、一套原生 C++ API， 在 macOS、iOS、Android、Windows 上运行同一个模型，并附带 CLI、OpenAI 兼容 @@ -60,41 +61,45 @@ MoE 模型跑在消费级硬件上——峰值内存由**激活**专家集而非 ### 平台 -一个引擎、一套原生 API，四端共用同一份代码，见 [`edge0-infer/`](edge0-infer/README_zh.md)： +[`edge0-infer`](edge0-infer/README_zh.md) 是你运行的程序（命令行、本地 OpenAI／Anthropic 兼容服务、聊天应用），所有模型使用相同的命令。每个模型运行在为它准备的引擎上，edge0-infer 根据模型目录自动选择，无需用户选择：`edge0-35b` 运行在 `edge0-infer/` 中的**原生引擎**上（基于 GGML，提供原生 C++ API）；`edge0-145b` 与 `edge0-120b` 运行在 [`edge0-strata/`](edge0-strata/EDGE0.md) 中的 **edge0-strata** 上（Strata 的分支）。 -| 平台 | 后端 | 验证设备 | +| 平台 | `edge0-35b`（原生引擎） | `edge0-145b`、`edge0-120b`（edge0-strata） | |---|---|---| -| **macOS**（Apple Silicon） | Metal、CPU | Apple M5，16 GiB | -| **iOS** | Metal | iPhone 17 Pro Max | -| **Android** | CPU | OnePlus PKR110（骁龙 8 Elite） | -| **Windows**（x86-64） | CPU、Vulkan | Intel Core i5-12400，NVIDIA GeForce RTX 4070 SUPER | +| **macOS**（Apple Silicon） | Metal、CPU | — | +| **iOS** | Metal | — | +| **Android** | CPU | — | +| **Linux** | CPU | NVIDIA GPU（CUDA 13） | +| **Windows** | CPU、Vulkan | NVIDIA GPU（CUDA 13） | -prefill 与 decode 可以用同一设备上的不同后端，例如 Metal prefill + CPU decode。 +测量结果：`edge0-35b` 见 [Benchmarking](edge0-infer/docs/benchmarking.md#reference-figures)，`edge0-145b` 与 `edge0-120b` 见[模型页](docs/models/edge0-145b.md)。在原生引擎上，prefill 与 decode 可以用同一设备上的不同后端，例如 Metal prefill + CPU decode。 桌面（macOS、Windows）、Android、iOS 的对话 App 示例见 [`edge0-infer/examples/`](edge0-infer/examples/README.md#chat-applications)。 ### 模型 -框架随附两个模型档位。每个档位是一个端到端发布：发布的 checkpoint、 +框架随附两个模型档位（`edge0-35b` 与 `edge0-8b`）。每个档位是一个端到端发布：发布的 checkpoint、 训练好的 LoRA 适配器与训练好的 prerouter 头作为整体协同工作。 -| 档位 | 发布 checkpoint | 推理档 | edge0-infer | +| 档位 | 发布 checkpoint | 推理档 | 引擎 | |---|---|---|---| -| `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4bit，40 层，256 专家，prerouter K=4 | ✅ 已支持 | -| `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4bit，24 层，128 专家，prerouter K=8 | 计划中 | +| `edge0-35b` | [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0/Edge0-35B-A3B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-35B-A3B-preview) | 4bit，40 层，256 专家，prerouter K=4 | 原生引擎 ✅ | +| `edge0-8b` | [`Edge0/Edge0-8B-A1B-preview`](https://huggingface.co/Edge0/Edge0-8B-A1B-preview) · [ModelScope](https://www.modelscope.cn/models/Edge0/Edge0-8B-A1B-preview) | 4bit，24 层，128 专家，prerouter K=8 | 原生引擎（计划中） | +| [`edge0-145b`](docs/models/edge0-145b.md) | coming soon | Q3_K 或 NVFP4 latent 专家（1792），48 层，512 专家，K=10 | edge0-strata ✅（Linux 或 Windows，NVIDIA GPU） | +| [`edge0-120b`](docs/models/edge0-145b.md) | coming soon | Q3_K 或 NVFP4 latent 专家（1280），48 层，512 专家，K=10 | edge0-strata ✅（Linux 或 Windows，NVIDIA GPU） | -两个 checkpoint 均基于开源稀疏 MoE 基座（分别为 Qwen3.6-35B-A3B +两个已发布的 checkpoint 均基于开源稀疏 MoE 基座（分别为 Qwen3.6-35B-A3B 与 Ling 3.0 混合架构），并携带为本框架训练的 LoRA 与 prerouter 权重。 edge0-infer 用自带的资产工具下载发布的 checkpoint，并转换成引擎的模型 资产（见[快速开始](#快速开始)）。 +`edge0-145b` 与 `edge0-120b` 是面向单块 NVIDIA GPU 的 latent-MoE；它们在 edge0-strata 上运行，由 `edge0-infer serve` 与 `edge0-infer chat` 启动（见 [edge0-145b 与 edge0-120b](docs/models/edge0-145b.md)）。 ### 设计 - **基于 GGML、四端同一份代码**：原生库只依赖固定版本、原样内置的 GGML， 构建时不下载任何东西； - **一套原生 API**：应用链接 CMake 目标 `edge0::edge0`、包含 - `include/edge0/` 下的头文件即可在进程内使用引擎；参考 CLI 与 OpenAI - 兼容服务使用与原生 API 相同的配置与默认值； + `include/edge0/` 下的头文件即可在进程内使用引擎；参考 CLI 与本地 + 服务使用与原生 API 相同的配置与默认值； - **专家按需读入**：路由专家从存储按需读入每层固定数量的专家槽，比 设备内存预算更大的模型无需整体载入内存即可运行； - **两种路由模式**：`predicted`（默认）用 prerouter 在当前 token 计算时 @@ -120,6 +125,17 @@ edge0-infer 用自带的资产工具下载发布的 checkpoint，并转换成引 | IFBench | 57.9 | 61.7 | 53.9 | 60.6 | | **平均** | **79.2** | **83.2** | **69.9** | **72.7** | +`edge0-145b` 与 `edge0-120b`（**粗体**为该列最高；见[模型页](docs/models/edge0-145b.md#quality)）： + +| Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | +| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | +| Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | +| Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | +| Claude-Opus-4.6 (Max) | / | 78.2 | 78.0 | 62.5 | 91.3 | **40.0** | + ### 性能实测 `edge0-35b`、`predicted` 路由，在 edge0-infer 上实测（每项为三个进程的最小值到最大值）： @@ -136,6 +152,33 @@ edge0-infer 用自带的资产工具下载发布的 checkpoint，并转换成引 各平台的内存指标不同，不能直接横向比较。测量方法、`actual` 路由的数据与 复现命令见 [Benchmarking](edge0-infer/docs/benchmarking.md#reference-figures)。 +`edge0-145b` 与 `edge0-120b`，运行在 edge0-strata 上，单张 NVIDIA GPU（tok/s）： + +#### edge0-145b + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 2169.9 | 3554.9 | 99.7 | **121.2** | +| RTX 5090 D 24 GB | NVFP4 | 1839.6 | 3165.7 | 79.6 | 98.3 | +| RTX 4090 24 GB | Q3_K | 2042.0 | 3179.4 | 69.9 | 91.7 | +| RTX Spark N1X 60 GiB | Q3_K | 674.0 | 770.3 | 37.7 | 36.7 | +| RTX Spark N1X 60 GiB | NVFP4 | 821.2 | 922.4 | 40.1 | 46.0 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 249 | 335 | 36.8 | 44.9 | + +#### edge0-120b + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 3236.7 | 4576.5 | 130.2 | **144.6** | +| RTX 5090 D 24 GB | NVFP4 | 2668.5 | 4357.1 | 124.9 | 135.5 | +| RTX 4090 24 GB | Q3_K | 2637.5 | 3551.1 | 95.0 | 104.9 | +| RTX Spark N1X 60 GiB | Q3_K | 987.0 | 1081.7 | 47.2 | 68.4 | +| RTX Spark N1X 60 GiB | NVFP4 | 996.4 | 1066.9 | 45.1 | 68.9 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 430 | 778 | 62.4 | 73.3 | +| 16 GiB VRAM + 20 GiB RAM | NVFP4 | 361.1 | 619.7 | 51.3 | 59.9 | + +`16 GiB VRAM + 20 GiB RAM` 这几行仍是同一张 RTX 5090 D 24 GB，用 cgroup 限制为 16 GiB 显存与 20 GiB 内存。详见[模型页](docs/models/edge0-145b.md#speed-measured-one-card)。 + ## 快速开始 以下命令都在 `edge0-infer/` 目录下运行。需要 CMake 3.21+、Ninja、C++17/C11 @@ -153,7 +196,9 @@ build/mac-metal/bin/edge0-infer info # 应输出一行：model.0: edge0_3 没有 Metal 的机器用 `--preset cpu`、`build/cpu/bin/edge0-infer`，并在下面的命令里 用 `--device cpu`。 -### 2) 下载并转换模型资产 +### 2) 获取模型 + +**edge0-35b（原生引擎）**：下载并转换模型资产。 ```bash export MOE_SOURCE_35B=<存放原始权重的目录> @@ -170,6 +215,8 @@ assets_py tools/assets/check_assets.py check "$MOE_ASSETS_35B" 下载的是 Hugging Face 上的 `Edge0/Edge0-35B-A3B-preview`，按锁定的版本与 摘要校验。 +**edge0-145b 与 edge0-120b（edge0-strata）**：按 [`edge0-strata/EDGE0.md`](edge0-strata/EDGE0.md) 构建 edge0-strata，使用发布的模型目录（见 [edge0-145b 与 edge0-120b](docs/models/edge0-145b.md)）。 + ### 3) 运行 ```bash @@ -177,10 +224,12 @@ assets_py tools/assets/check_assets.py check "$MOE_ASSETS_35B" build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "用一句话介绍你自己。" # 多轮对话 build/mac-metal/bin/edge0-infer chat --model "$MOE_ASSETS_35B" --device metal -# 起服务（OpenAI 兼容，默认 http://127.0.0.1:8000/v1） +# 起服务（OpenAI／Anthropic 兼容，默认 http://127.0.0.1:8000） build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal ``` +edge0-145b 与 edge0-120b 的 `chat` 和 `serve` 同样用 `--model <模型目录>`；`run` 只用于原生引擎。 + Android、Windows、iOS 的构建，以及在自己的 CMake 工程中使用引擎，见 [Getting started](edge0-infer/docs/getting-started.md)。 @@ -189,17 +238,22 @@ Android、Windows、iOS 的构建，以及在自己的 CMake 工程中使用引 **模型** - [MoE 路由与专家](docs/moe.md) / [prerouter](docs/prerouter.md) -- [edge0-35b](docs/models/edge0-35b.md) / [edge0-8b](docs/models/edge0-8b.md) +- [edge0-35b](docs/models/edge0-35b.md) / [edge0-8b](docs/models/edge0-8b.md) / [edge0-145b 与 edge0-120b](docs/models/edge0-145b.md) - 技术报告：[The Other Half of the Memory Wall](https://arxiv.org/abs/2609.18063)（[PDF](paper/main.pdf)） -**引擎（edge0-infer）** +**edge0-infer** - [Getting started](edge0-infer/docs/getting-started.md)：各平台的构建、模型资产、运行、对话与服务 -- [Configuration](edge0-infer/docs/configuration.md) / [Native API](edge0-infer/docs/native-api.md) / [OpenAI 兼容服务](edge0-infer/docs/openai-compatible-server.md) +- [Configuration](edge0-infer/docs/configuration.md) / [Native API](edge0-infer/docs/native-api.md) / [本地服务](edge0-infer/docs/local-server.md) - [Architecture](edge0-infer/docs/architecture.md) / [Known limitations](edge0-infer/docs/limitations.md) - [Benchmarking](edge0-infer/docs/benchmarking.md) / [Testing](edge0-infer/docs/testing.md) - [模型资产工具与接入新模型](edge0-infer/tools/assets/README.md) +**引擎** + +- 原生引擎：[Architecture](edge0-infer/docs/architecture.md#native-engine) / [Native API](edge0-infer/docs/native-api.md) +- edge0-strata：[构建与启动约定](edge0-strata/EDGE0.md) / 在 edge0-infer 中运行 [edge0-145b](edge0-infer/docs/edge0-145b.md) 与 [edge0-120b](edge0-infer/docs/edge0-120b.md) + ## 路线图 ### 2026 Q4 diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index d43fc1d3..584e6ac8 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -1,6 +1,6 @@ # Benchmarks -Measured speed and memory of the edge0 models on each supported platform, the +Measured speed and memory of the models on the native engine on each supported platform, the devices they were measured on, and how to measure your own device are in the engine's [Benchmarking guide](../../edge0-infer/docs/benchmarking.md) ([reference figures](../../edge0-infer/docs/benchmarking.md#reference-figures)). @@ -8,3 +8,6 @@ engine's [Benchmarking guide](../../edge0-infer/docs/benchmarking.md) To share a result, post the `edge0-infer bench` summary with your device, memory and storage type in [GitHub Discussions](https://github.com/Edge0-AI/Edge0/discussions). + +The speed and quality of edge0-145b and edge0-120b, which run on edge0-strata, are on their +[model page](../models/edge0-145b.md). diff --git a/docs/models/edge0-145b.md b/docs/models/edge0-145b.md new file mode 100644 index 00000000..7086c988 --- /dev/null +++ b/docs/models/edge0-145b.md @@ -0,0 +1,76 @@ +# Edge0 145B and Edge0 120B + +Two latent mixture-of-experts models for a single NVIDIA GPU, derived from Qwen3.8-Flash-Next: Edge0 145B and Edge0 120B. Their experts read compressed inputs, and both models run on [edge0-strata](../../edge0-strata/EDGE0.md), a fork of [Strata](https://github.com/Niko1221/Strata) whose scheduling of CPU, GPU and PCIe work they depend on. [edge0-infer](../../edge0-infer/README.md) starts edge0-strata for `serve` and `chat`, so the commands and endpoints are those of the other models. + +Model card: coming soon. + +## Model profile + +| Item | Value | +| --- | --- | +| Total parameters | Edge0 145B: 145B, 6.6B active per token; Edge0 120B: 120B, 6.1B active per token | +| Number of layers | 48: 36 linear-attention layers, and one indexed-attention layer in every 4 | +| Number of experts | 512 routed experts per layer, plus one shared expert | +| top_k (K) | 10 | +| Latent compression | Expert inputs compressed from 2560 to 1792 (Edge0 145B) or 1280 (Edge0 120B) dimensions; each layer has 8 groups of down/up (D/U) projections, chosen by each expert's group label | +| Expert formats | Q3_K (Q3_K gate/up and IQ4_NL down, with column scales) or NVFP4, released as separate model folders | +| D/U formats | Q8 or Q4; default Q8 for Edge0 145B, Q4 for Edge0 120B | +| Other parts | Hyper-connections, a per-layer n-gram embedding (PLE) table, one multi-token-prediction (MTP) layer for speculative decoding | + +## Quality + +| Model | deepswe 1.1 | terminal-bench 2.1 | AA-LCR | IFBench | GPQA-Diamond | HLE | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| **Edge0-145B** | **64.6** | **86.5** | 79.3 | 80.6 | 90.4 | 35.4 | +| **Edge0-120B** | 50.0 | 75.0 | 81.0 | 79.3 | 87.4 | 31.0 | +| DeepSeek-V4-Flash-0731 | 54.4 | 82.7 | 79.7 | 79.2 | 90.8 | 33.8 | +| Qwen3.8-Flash-Next | 58.7 | 83.3 | 79.1 | **81.3** | **91.7** | 35.9 | +| Qwen3.8-27B | 42.2 | 73.0 | **82.0** | 79.5 | 89.2 | 30.8 | +| Claude-Opus-4.6 (Max) | / | 78.2 | 78.0 | 62.5 | 91.3 | **40.0** | + +**Bold** = best in column. + +## Speed (measured, one card) + +### Edge0 145B + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 2169.9 | 3554.9 | 99.7 | **121.2** | +| RTX 5090 D 24 GB | NVFP4 | 1839.6 | 3165.7 | 79.6 | 98.3 | +| RTX 4090 24 GB | Q3_K | 2042.0 | 3179.4 | 69.9 | 91.7 | +| RTX Spark N1X 60 GiB | Q3_K | 674.0 | 770.3 | 37.7 | 36.7 | +| RTX Spark N1X 60 GiB | NVFP4 | 821.2 | 922.4 | 40.1 | 46.0 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 249 | 335 | 36.8 | 44.9 | + +`16 GiB VRAM + 20 GiB RAM` rows are the RTX 5090 D 24 GB again, constrained by cgroup to a 16 GiB VRAM and 20 GiB system-RAM budget — not a different card. With residency on, the same card holds **Q3_K** (40.81 GiB of weights) as 23.14 GiB VRAM + 31.28 GiB RAM, and **NVFP4** (47.79 GiB) as 23.25 GiB VRAM + 38.27 GiB RAM — disjoint expert sets, nothing counted twice. + +### Edge0 120B + +| Machine | Build | 4K prefill | 16K prefill | Chat decode | Agent decode | +| --- | --- | ---: | ---: | ---: | ---: | +| RTX 5090 D 24 GB | Q3_K | 3236.7 | 4576.5 | 130.2 | **144.6** | +| RTX 5090 D 24 GB | NVFP4 | 2668.5 | 4357.1 | 124.9 | 135.5 | +| RTX 4090 24 GB | Q3_K | 2637.5 | 3551.1 | 95.0 | 104.9 | +| RTX Spark N1X 60 GiB | Q3_K | 987.0 | 1081.7 | 47.2 | 68.4 | +| RTX Spark N1X 60 GiB | NVFP4 | 996.4 | 1066.9 | 45.1 | 68.9 | +| 16 GiB VRAM + 20 GiB RAM | Q3_K | 430 | 778 | 62.4 | 73.3 | +| 16 GiB VRAM + 20 GiB RAM | NVFP4 | 361.1 | 619.7 | 51.3 | 59.9 | + +`16 GiB VRAM + 20 GiB RAM` rows are the RTX 5090 D 24 GB again, constrained by cgroup to a 16 GiB VRAM and 20 GiB system-RAM budget — not a different card. With residency on, the same card holds **Q3_K** (27.98 GiB of weights) as 22.93 GiB VRAM + 18.40 GiB RAM, and **NVFP4** (32.96 GiB) as 23.02 GiB VRAM + 23.40 GiB RAM — disjoint expert sets, nothing counted twice. + +## Model folders + +Each model folder holds one model in one expert format: `Edge0-145B-q3k`, `Edge0-145B-fp4`, `Edge0-120B-q3k` or `Edge0-120B-fp4` (`-q3k` is Q3_K, `-fp4` is NVFP4). Use a folder as it is provided; `edge0-infer` identifies the model and the expert format by `model_name` in its `model.json`. The files a folder must hold are listed in Model directory for [Edge0 145B](../../edge0-infer/docs/edge0-145b.md#model-directory) and [Edge0 120B](../../edge0-infer/docs/edge0-120b.md#model-directory) in the edge0-infer documentation; `edge0-infer inspect --model --json` checks them. + +## Requirements + +Linux or Windows with an NVIDIA GPU and CUDA 13, at least 16 GiB of GPU memory and 20 GiB of system memory. + +## Running + +Both models run on edge0-strata, the engine built for them, which edge0-infer starts for these models (see [Platforms](../../README.md#platforms)). Build edge0-strata as [Build (Linux, NVIDIA)](../../edge0-strata/EDGE0.md#build-linux-nvidia) or [Build (Windows, NVIDIA)](../../edge0-strata/EDGE0.md#build-windows-nvidia) in `edge0-strata/EDGE0.md` describes, build edge0-infer as in its [Getting started](../../edge0-infer/docs/getting-started.md#build), and start a model folder with `edge0-infer serve --model ` or `edge0-infer chat --model `. The pages [Edge0 145B](../../edge0-infer/docs/edge0-145b.md) and [Edge0 120B](../../edge0-infer/docs/edge0-120b.md) in the edge0-infer documentation cover `serve`, `chat` and `inspect` for these models, how `edge0-infer` finds edge0-strata, the options (`--device`, `du`, `ple-io`, `--context`, `--strata-arg`) and the known limitations. + +## Acknowledgements + +Qwen3.8-Flash-Next is the base model. edge0-strata is a fork of [Strata](https://github.com/Niko1221/Strata) by Niko1221 and the Strata contributors, released under the MIT license, with latent MoE support added on top. These models run at this speed because of the engine they built. diff --git a/docs/models/edge0-35b.md b/docs/models/edge0-35b.md index f56344dd..d5f655a1 100644 --- a/docs/models/edge0-35b.md +++ b/docs/models/edge0-35b.md @@ -19,4 +19,4 @@ Release checkpoint: [`Edge0/Edge0-35B-A3B-preview`](https://huggingface.co/Edge0 ## Running with edge0-infer -[Getting started](../../edge0-infer/docs/getting-started.md) downloads the release checkpoint, converts it into the engine's assets and runs a prompt, a chat or an OpenAI-compatible server. The model's load options (routing mode, LoRA, expert slots, how experts are read) are listed under [Model options](../../edge0-infer/docs/configuration.md#model-options); measured speed and memory on each platform are under [Benchmarking](../../edge0-infer/docs/benchmarking.md#reference-figures). +[Getting started](../../edge0-infer/docs/getting-started.md) downloads the release checkpoint, converts it into the engine's assets and runs a prompt, a chat or an OpenAI- and Anthropic-compatible server. The model's load options (routing mode, LoRA, expert slots, how experts are read) are listed under [Model options](../../edge0-infer/docs/configuration.md#model-options); measured speed and memory on each platform are under [Benchmarking](../../edge0-infer/docs/benchmarking.md#reference-figures). diff --git a/edge0-infer/CMakeLists.txt b/edge0-infer/CMakeLists.txt index ff33e6bf..8e3eda52 100644 --- a/edge0-infer/CMakeLists.txt +++ b/edge0-infer/CMakeLists.txt @@ -6,7 +6,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND CMAKE_TOOLCHAIN_FILE MATCHES "android.toolchain.cmake$" AND "$ENV{ANDROID_NDK}" STREQUAL "") message(FATAL_ERROR "ANDROID_NDK must point to the Android NDK") endif() -project(edge0 VERSION 0.1.0 LANGUAGES C CXX) +project(edge0 VERSION 0.2.0 LANGUAGES C CXX) set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED ON) @@ -22,6 +22,7 @@ endif() option(MOE_BUILD_CLI "Build the reference CLI" ${PROJECT_IS_TOP_LEVEL}) option(MOE_BUILD_TESTS "Build and register tests" ${PROJECT_IS_TOP_LEVEL}) option(MOE_BUILD_EXAMPLES "Build public native examples" ${PROJECT_IS_TOP_LEVEL}) +option(MOE_WARNINGS_AS_ERRORS "Treat compiler warnings in this project's own targets as errors" OFF) include(cmake/MoeTargets.cmake) include(cmake/BuildIdentity.cmake) file(STRINGS "${PROJECT_SOURCE_DIR}/third_party/GGML_ORIGIN" origin) diff --git a/edge0-infer/README.md b/edge0-infer/README.md index f1fed673..fdbaa9e3 100644 --- a/edge0-infer/README.md +++ b/edge0-infer/README.md @@ -2,20 +2,22 @@ English | [Chinese](README_zh.md) | [Japanese](README_ja.md) | [Español](README_es.md) | [Français](README_fr.md) -An on-device inference engine for mixture-of-experts (MoE) language models, built on [GGML](https://github.com/ggml-org/ggml). One engine and one native C++ API run the same model on macOS, iOS, Android and Windows. Expert weights are loaded from storage on demand into a fixed pool of slots, so a model larger than the device memory budget runs without loading all of its weights. +`edge0-infer` runs mixture-of-experts (MoE) language models on your own hardware: a CLI, a local OpenAI- and Anthropic-compatible server and chat applications, with the same commands for every model. Each model runs on the engine built for it, and `edge0-infer` picks the engine from the model folder; there is nothing to choose: -The first supported model is **Edge0 35B** (Qwen3.6-35B-A3B with a LoRA and a trained pre-router). On an Apple M5 Mac it decodes at about 25 tokens per second with a peak physical footprint of about 3.4 GiB; figures for every platform are in [Benchmarking](docs/benchmarking.md#reference-figures). +- **Edge0 35B** (Qwen3.6-35B-A3B with a LoRA and a trained pre-router) runs on the **native engine** in this directory, built on [GGML](https://github.com/ggml-org/ggml) and with a native C++ API, on macOS, iOS, Android, Linux and Windows. Expert weights are loaded from storage on demand into a fixed pool of slots, so a model larger than the device memory budget runs without loading all of its weights. On an Apple M5 Mac it decodes at about 25 tokens per second with a peak physical footprint of about 3.4 GiB; figures for every platform are in [Benchmarking](docs/benchmarking.md#reference-figures). +- **Edge0 145B** and **Edge0 120B** run on **edge0-strata**, a fork of [Strata](https://github.com/Niko1221/Strata) in `edge0-strata/` of the Edge0 repository, on an NVIDIA GPU on Linux or Windows; see [Edge0 145B](docs/edge0-145b.md) and [Edge0 120B](docs/edge0-120b.md). ## Platforms -| Platform | Backend | Verified on | +| Platform | Edge0 35B (native engine) | Edge0 145B, Edge0 120B (edge0-strata) | | --- | --- | --- | -| macOS (Apple silicon) | Metal, CPU | Apple M5, 16 GiB | -| iOS | Metal | iPhone 17 Pro Max | -| Android | CPU | OnePlus PKR110 (Snapdragon 8 Elite) | -| Windows (x86-64) | CPU, Vulkan | Intel Core i5-12400, NVIDIA GeForce RTX 4070 SUPER | +| macOS (Apple silicon) | Metal, CPU | — | +| iOS | Metal | — | +| Android | CPU | — | +| Linux | CPU | NVIDIA GPU (CUDA 13) | +| Windows | CPU, Vulkan | NVIDIA GPU (CUDA 13) | -Prefill and decode may run on different backends of the same device, for example Metal prefill with CPU decode. The 35B offers two routing modes: `predicted` (default), where a pre-router lets the engine load the next token's experts while the current token computes, and `actual`, the model's original computation. See [Routing modes](docs/benchmarking.md#routing-modes). +Measurements: [Benchmarking](docs/benchmarking.md#reference-figures) for Edge0 35B, and the model page of Edge0 145B and Edge0 120B (`docs/models/edge0-145b.md` in the Edge0 repository). On the native engine, prefill and decode may run on different backends of the same device, for example Metal prefill with CPU decode. The 35B offers two routing modes: `predicted` (default), where a pre-router lets the engine load the next token's experts while the current token computes, and `actual`, the model's original computation. See [Routing modes](docs/benchmarking.md#routing-modes). ## Quick start @@ -26,7 +28,7 @@ cmake --preset mac-metal && cmake --build --preset mac-metal build/mac-metal/bin/edge0-infer info ``` -Then download and convert the model assets, and run a prompt, a chat or a local OpenAI-compatible server: +Then download and convert the model assets, and run a prompt, a chat or a local OpenAI- and Anthropic-compatible server: ```sh build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "Introduce yourself in one sentence." @@ -36,13 +38,17 @@ build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal [Getting started](docs/getting-started.md) walks through every step, including the asset download and conversion and the Android, Windows and iOS builds. +For Edge0 145B and Edge0 120B, build edge0-strata and run `edge0-infer serve --model `; [Getting started](docs/getting-started.md#edge0-145b-and-edge0-120b-edge0-strata) has the steps. + ## Documentation - [Getting started](docs/getting-started.md): build, get the model assets, run, chat and serve, on every platform +- [Edge0 145B](docs/edge0-145b.md): running this model on edge0-strata +- [Edge0 120B](docs/edge0-120b.md): running this model on edge0-strata - [Configuration reference](docs/configuration.md): runtime, load, model and generation settings, their defaults and the CLI flags - [Native API](docs/native-api.md): using the engine in process: lifecycle, streaming, multi-turn, concurrency and cancellation -- [OpenAI-compatible local server](docs/openai-compatible-server.md): endpoints, request fields and limits -- [Architecture](docs/architecture.md): how the engine is organized and how a request flows through it +- [Local server](docs/local-server.md): endpoints, request fields and limits +- [Architecture](docs/architecture.md): the two layers, how the native engine is organized and how a request flows through it - [Known limitations](docs/limitations.md): behavior to be aware of - [Benchmarking](docs/benchmarking.md): how to measure, recommended settings and reference figures - [Testing](docs/testing.md): test levels, what each needs and how to run them @@ -53,7 +59,7 @@ build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal ## Native API -Applications link the CMake target `edge0::edge0` and include the headers in [`include/edge0/`](include/edge0/), which are the only source of signatures; the types and functions are in the namespace `edge0`. [`examples/native_chat.cpp`](examples/native_chat.cpp) shows the whole lifecycle: open the runtime, load a model, open a session, generate two streamed turns and release everything. The [native API guide](docs/native-api.md) explains the lifecycle, streaming, multi-turn use, concurrency, cancellation and errors. +The native API belongs to the native engine. Applications link the CMake target `edge0::edge0` and include the headers in [`include/edge0/`](include/edge0/), which are the only source of signatures; the types and functions are in the namespace `edge0`. [`examples/native_chat.cpp`](examples/native_chat.cpp) shows the whole lifecycle: open the runtime, load a model, open a session, generate two streamed turns and release everything. The [native API guide](docs/native-api.md) explains the lifecycle, streaming, multi-turn use, concurrency, cancellation and errors. ## License and third-party code diff --git a/edge0-infer/README_es.md b/edge0-infer/README_es.md index 58f1509e..8f7a1c59 100644 --- a/edge0-infer/README_es.md +++ b/edge0-infer/README_es.md @@ -2,22 +2,24 @@ [English](README.md) | [中文](README_zh.md) | [日本語](README_ja.md) | Español | [Français](README_fr.md) -Un motor de inferencia en el dispositivo para modelos de lenguaje de mezcla de expertos (MoE), construido sobre [GGML](https://github.com/ggml-org/ggml). Un solo motor y una sola API nativa en C++ ejecutan el mismo modelo en macOS, iOS, Android y Windows. Los pesos de los expertos se cargan desde el almacenamiento bajo demanda en un conjunto fijo de ranuras, de modo que un modelo más grande que el presupuesto de memoria del dispositivo se ejecuta sin cargar todos sus pesos. +`edge0-infer` ejecuta modelos de lenguaje de mezcla de expertos (MoE) en tu propio hardware: una CLI, un servidor local compatible con OpenAI y Anthropic y aplicaciones de chat, con los mismos comandos para todos los modelos. Cada modelo se ejecuta en el motor preparado para él, y `edge0-infer` elige el motor a partir de la carpeta del modelo; no hay nada que elegir: -El primer modelo compatible es **Edge0 35B** (Qwen3.6-35B-A3B con un LoRA y un pre-router entrenado). En un Mac con Apple M5 decodifica a unos 25 tokens por segundo, con un pico de huella física de unos 3,4 GiB; las cifras de cada plataforma están en [Benchmarking](docs/benchmarking.md#reference-figures). +- **Edge0 35B** (Qwen3.6-35B-A3B con un LoRA y un pre-router entrenado) se ejecuta en el **motor nativo** de este directorio, construido sobre [GGML](https://github.com/ggml-org/ggml) y con una API nativa en C++, en macOS, iOS, Android, Linux y Windows. Los pesos de los expertos se cargan desde el almacenamiento bajo demanda en un conjunto fijo de ranuras, de modo que un modelo más grande que el presupuesto de memoria del dispositivo se ejecuta sin cargar todos sus pesos. En un Mac con Apple M5 decodifica a unos 25 tokens por segundo, con un pico de huella física de unos 3,4 GiB; las cifras de cada plataforma están en [Benchmarking](docs/benchmarking.md#reference-figures). +- **Edge0 145B** y **Edge0 120B** se ejecutan en **edge0-strata**, un fork de [Strata](https://github.com/Niko1221/Strata) en `edge0-strata/` del repositorio Edge0, en una GPU NVIDIA con Linux o Windows; consulta [Edge0 145B](docs/edge0-145b.md) y [Edge0 120B](docs/edge0-120b.md). > Esta página es una traducción del [README](README.md) en inglés; los documentos enlazados están en inglés. ## Plataformas -| Plataforma | Backend | Verificado en | +| Plataforma | Edge0 35B (motor nativo) | Edge0 145B, Edge0 120B (edge0-strata) | | --- | --- | --- | -| macOS (Apple silicon) | Metal, CPU | Apple M5, 16 GiB | -| iOS | Metal | iPhone 17 Pro Max | -| Android | CPU | OnePlus PKR110 (Snapdragon 8 Elite) | -| Windows (x86-64) | CPU, Vulkan | Intel Core i5-12400, NVIDIA GeForce RTX 4070 SUPER | +| macOS (Apple silicon) | Metal, CPU | — | +| iOS | Metal | — | +| Android | CPU | — | +| Linux | CPU | GPU NVIDIA (CUDA 13) | +| Windows | CPU, Vulkan | GPU NVIDIA (CUDA 13) | -El prefill y el decode pueden ejecutarse en backends distintos del mismo dispositivo, por ejemplo prefill en Metal y decode en CPU. El 35B ofrece dos modos de enrutamiento: `predicted` (por defecto), en el que un pre-router permite al motor cargar los expertos del siguiente token mientras se calcula el actual, y `actual`, el cálculo original del modelo. Consulta [Routing modes](docs/benchmarking.md#routing-modes). +Mediciones: [Benchmarking](docs/benchmarking.md#reference-figures) para Edge0 35B y la página del modelo Edge0 145B y Edge0 120B (`docs/models/edge0-145b.md` en el repositorio Edge0). En el motor nativo, el prefill y el decode pueden ejecutarse en backends distintos del mismo dispositivo, por ejemplo prefill en Metal y decode en CPU. El 35B ofrece dos modos de enrutamiento: `predicted` (por defecto), en el que un pre-router permite al motor cargar los expertos del siguiente token mientras se calcula el actual, y `actual`, el cálculo original del modelo. Consulta [Routing modes](docs/benchmarking.md#routing-modes). ## Inicio rápido @@ -28,7 +30,7 @@ cmake --preset mac-metal && cmake --build --preset mac-metal build/mac-metal/bin/edge0-infer info ``` -Después descarga y convierte los recursos del modelo, y ejecuta una instrucción, un chat o un servidor local compatible con OpenAI: +Después descarga y convierte los recursos del modelo, y ejecuta una instrucción, un chat o un servidor local compatible con OpenAI y Anthropic: ```sh build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "Preséntate en una frase." @@ -38,13 +40,17 @@ build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal [Getting started](docs/getting-started.md) recorre cada paso, incluidas la descarga y la conversión de los recursos y las compilaciones para Android, Windows e iOS. +Para Edge0 145B y Edge0 120B, compila edge0-strata y ejecuta `edge0-infer serve --model `; los pasos están en [Getting started](docs/getting-started.md#edge0-145b-and-edge0-120b-edge0-strata). + ## Documentación - [Getting started](docs/getting-started.md): compilar, obtener los recursos del modelo, ejecutar, chatear y servir, en cada plataforma +- [Edge0 145B](docs/edge0-145b.md): ejecutar este modelo en edge0-strata +- [Edge0 120B](docs/edge0-120b.md): ejecutar este modelo en edge0-strata - [Configuration reference](docs/configuration.md): ajustes de runtime, carga, modelo y generación, sus valores por defecto y las opciones de la CLI - [Native API](docs/native-api.md): uso del motor dentro del proceso: ciclo de vida, streaming, varios turnos, concurrencia y cancelación -- [OpenAI-compatible local server](docs/openai-compatible-server.md): endpoints, campos de la solicitud y límites -- [Architecture](docs/architecture.md): cómo está organizado el motor y cómo fluye una solicitud por él +- [Local server](docs/local-server.md): endpoints, campos de la solicitud y límites +- [Architecture](docs/architecture.md): las dos capas, cómo está organizado el motor nativo y cómo fluye una solicitud por él - [Known limitations](docs/limitations.md): comportamientos a tener en cuenta - [Benchmarking](docs/benchmarking.md): cómo medir, ajustes recomendados y cifras de referencia - [Testing](docs/testing.md): niveles de prueba, qué necesita cada uno y cómo ejecutarlos @@ -55,7 +61,7 @@ build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal ## API nativa -Las aplicaciones enlazan el target de CMake `edge0::edge0` e incluyen las cabeceras de [`include/edge0/`](include/edge0/), que son la única fuente de las firmas; los tipos y funciones están en el espacio de nombres `edge0`. [`examples/native_chat.cpp`](examples/native_chat.cpp) muestra el ciclo de vida completo: abrir el runtime, cargar un modelo, abrir una sesión, generar dos turnos en streaming y liberar todo. La [guía de la API nativa](docs/native-api.md) explica el ciclo de vida, el streaming, el uso con varios turnos, la concurrencia, la cancelación y los errores. +La API nativa corresponde al motor nativo. Las aplicaciones enlazan el target de CMake `edge0::edge0` e incluyen las cabeceras de [`include/edge0/`](include/edge0/), que son la única fuente de las firmas; los tipos y funciones están en el espacio de nombres `edge0`. [`examples/native_chat.cpp`](examples/native_chat.cpp) muestra el ciclo de vida completo: abrir el runtime, cargar un modelo, abrir una sesión, generar dos turnos en streaming y liberar todo. La [guía de la API nativa](docs/native-api.md) explica el ciclo de vida, el streaming, el uso con varios turnos, la concurrencia, la cancelación y los errores. ## Licencia y código de terceros diff --git a/edge0-infer/README_fr.md b/edge0-infer/README_fr.md index ee90b509..098152f0 100644 --- a/edge0-infer/README_fr.md +++ b/edge0-infer/README_fr.md @@ -2,22 +2,24 @@ [English](README.md) | [中文](README_zh.md) | [日本語](README_ja.md) | [Español](README_es.md) | Français -Un moteur d'inférence embarqué pour les modèles de langage à mélange d'experts (MoE), construit sur [GGML](https://github.com/ggml-org/ggml). Un seul moteur et une seule API C++ native exécutent le même modèle sur macOS, iOS, Android et Windows. Les poids des experts sont chargés depuis le stockage à la demande dans un ensemble fixe d'emplacements, si bien qu'un modèle plus grand que le budget mémoire de l'appareil s'exécute sans charger tous ses poids. +`edge0-infer` exécute des modèles de langage à mélange d'experts (MoE) sur votre propre matériel : une CLI, un serveur local compatible OpenAI et Anthropic et des applications de discussion, avec les mêmes commandes pour tous les modèles. Chaque modèle s'exécute sur le moteur conçu pour lui, et `edge0-infer` choisit le moteur d'après le dossier du modèle ; il n'y a rien à choisir : -Le premier modèle pris en charge est **Edge0 35B** (Qwen3.6-35B-A3B avec un LoRA et un pre-router entraîné). Sur un Mac Apple M5, il décode à environ 25 tokens par seconde, avec un pic d'empreinte physique d'environ 3,4 Gio ; les chiffres de chaque plateforme se trouvent dans [Benchmarking](docs/benchmarking.md#reference-figures). +- **Edge0 35B** (Qwen3.6-35B-A3B avec un LoRA et un pre-router entraîné) s'exécute sur le **moteur natif** de ce répertoire, construit sur [GGML](https://github.com/ggml-org/ggml) et doté d'une API C++ native, sous macOS, iOS, Android, Linux et Windows. Les poids des experts sont chargés depuis le stockage à la demande dans un ensemble fixe d'emplacements, si bien qu'un modèle plus grand que le budget mémoire de l'appareil s'exécute sans charger tous ses poids. Sur un Mac Apple M5, il décode à environ 25 tokens par seconde, avec un pic d'empreinte physique d'environ 3,4 Gio ; les chiffres de chaque plateforme se trouvent dans [Benchmarking](docs/benchmarking.md#reference-figures). +- **Edge0 145B** et **Edge0 120B** s'exécutent sur **edge0-strata**, un fork de [Strata](https://github.com/Niko1221/Strata) dans `edge0-strata/` du dépôt Edge0, sur un GPU NVIDIA sous Linux ou Windows ; voir [Edge0 145B](docs/edge0-145b.md) et [Edge0 120B](docs/edge0-120b.md). > Cette page est une traduction du [README](README.md) en anglais ; les documents liés sont en anglais. ## Plateformes -| Plateforme | Backend | Vérifié sur | +| Plateforme | Edge0 35B (moteur natif) | Edge0 145B, Edge0 120B (edge0-strata) | | --- | --- | --- | -| macOS (Apple silicon) | Metal, CPU | Apple M5, 16 Gio | -| iOS | Metal | iPhone 17 Pro Max | -| Android | CPU | OnePlus PKR110 (Snapdragon 8 Elite) | -| Windows (x86-64) | CPU, Vulkan | Intel Core i5-12400, NVIDIA GeForce RTX 4070 SUPER | +| macOS (Apple silicon) | Metal, CPU | — | +| iOS | Metal | — | +| Android | CPU | — | +| Linux | CPU | GPU NVIDIA (CUDA 13) | +| Windows | CPU, Vulkan | GPU NVIDIA (CUDA 13) | -Le prefill et le decode peuvent s'exécuter sur des backends différents du même appareil, par exemple prefill sur Metal et decode sur CPU. Le 35B propose deux modes de routage : `predicted` (par défaut), où un pre-router permet au moteur de charger les experts du token suivant pendant le calcul du token courant, et `actual`, le calcul d'origine du modèle. Voir [Routing modes](docs/benchmarking.md#routing-modes). +Mesures : [Benchmarking](docs/benchmarking.md#reference-figures) pour Edge0 35B, et la page du modèle Edge0 145B et Edge0 120B (`docs/models/edge0-145b.md` dans le dépôt Edge0). Sur le moteur natif, le prefill et le decode peuvent s'exécuter sur des backends différents du même appareil, par exemple prefill sur Metal et decode sur CPU. Le 35B propose deux modes de routage : `predicted` (par défaut), où un pre-router permet au moteur de charger les experts du token suivant pendant le calcul du token courant, et `actual`, le calcul d'origine du modèle. Voir [Routing modes](docs/benchmarking.md#routing-modes). ## Démarrage rapide @@ -28,7 +30,7 @@ cmake --preset mac-metal && cmake --build --preset mac-metal build/mac-metal/bin/edge0-infer info ``` -Téléchargez et convertissez ensuite les ressources du modèle, puis lancez une requête, une discussion ou un serveur local compatible OpenAI : +Téléchargez et convertissez ensuite les ressources du modèle, puis lancez une requête, une discussion ou un serveur local compatible OpenAI et Anthropic : ```sh build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "Présente-toi en une phrase." @@ -38,13 +40,17 @@ build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal [Getting started](docs/getting-started.md) détaille chaque étape, y compris le téléchargement et la conversion des ressources, ainsi que les compilations pour Android, Windows et iOS. +Pour Edge0 145B et Edge0 120B, compilez edge0-strata puis lancez `edge0-infer serve --model ` ; les étapes sont dans [Getting started](docs/getting-started.md#edge0-145b-and-edge0-120b-edge0-strata). + ## Documentation - [Getting started](docs/getting-started.md) : compiler, obtenir les ressources du modèle, exécuter, discuter et servir, sur chaque plateforme +- [Edge0 145B](docs/edge0-145b.md) : exécuter ce modèle sur edge0-strata +- [Edge0 120B](docs/edge0-120b.md) : exécuter ce modèle sur edge0-strata - [Configuration reference](docs/configuration.md) : paramètres du runtime, du chargement, du modèle et de la génération, leurs valeurs par défaut et les options de la CLI - [Native API](docs/native-api.md) : utiliser le moteur dans le processus : cycle de vie, streaming, multi-tours, concurrence et annulation -- [OpenAI-compatible local server](docs/openai-compatible-server.md) : endpoints, champs de requête et limites -- [Architecture](docs/architecture.md) : organisation du moteur et parcours d'une requête +- [Local server](docs/local-server.md) : endpoints, champs de requête et limites +- [Architecture](docs/architecture.md) : les deux couches, l'organisation du moteur natif et le parcours d'une requête - [Known limitations](docs/limitations.md) : comportements à connaître - [Benchmarking](docs/benchmarking.md) : comment mesurer, réglages recommandés et chiffres de référence - [Testing](docs/testing.md) : niveaux de test, ce dont chacun a besoin et comment les lancer @@ -55,7 +61,7 @@ build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal ## API native -Les applications se lient à la cible CMake `edge0::edge0` et incluent les en-têtes de [`include/edge0/`](include/edge0/), seule source des signatures ; les types et fonctions se trouvent dans l'espace de noms `edge0`. [`examples/native_chat.cpp`](examples/native_chat.cpp) montre le cycle de vie complet : ouvrir le runtime, charger un modèle, ouvrir une session, générer deux tours en streaming et tout libérer. Le [guide de l'API native](docs/native-api.md) explique le cycle de vie, le streaming, l'usage multi-tours, la concurrence, l'annulation et les erreurs. +L'API native correspond au moteur natif. Les applications se lient à la cible CMake `edge0::edge0` et incluent les en-têtes de [`include/edge0/`](include/edge0/), seule source des signatures ; les types et fonctions se trouvent dans l'espace de noms `edge0`. [`examples/native_chat.cpp`](examples/native_chat.cpp) montre le cycle de vie complet : ouvrir le runtime, charger un modèle, ouvrir une session, générer deux tours en streaming et tout libérer. Le [guide de l'API native](docs/native-api.md) explique le cycle de vie, le streaming, l'usage multi-tours, la concurrence, l'annulation et les erreurs. ## Licence et code tiers diff --git a/edge0-infer/README_ja.md b/edge0-infer/README_ja.md index 40512642..fa6b954d 100644 --- a/edge0-infer/README_ja.md +++ b/edge0-infer/README_ja.md @@ -2,22 +2,24 @@ [English](README.md) | [中文](README_zh.md) | 日本語 | [Español](README_es.md) | [Français](README_fr.md) -[GGML](https://github.com/ggml-org/ggml) をベースにした、混合エキスパート（MoE）言語モデル向けのオンデバイス推論エンジンです。1 つのエンジンと 1 つのネイティブ C++ API で、同じモデルを macOS、iOS、Android、Windows で実行します。エキスパートの重みはストレージから必要に応じて固定数のスロットに読み込まれるため、デバイスのメモリ予算より大きなモデルでも、すべての重みを読み込まずに実行できます。 +`edge0-infer` は、混合エキスパート（MoE）言語モデルを手元のハードウェアで実行します。CLI、ローカルの OpenAI／Anthropic 互換サーバー、チャットアプリを備え、どのモデルも同じコマンドで使えます。各モデルはそれぞれ専用のエンジンで動作し、`edge0-infer` がモデルフォルダからエンジンを自動で選ぶため、選択する必要はありません。 -最初にサポートするモデルは **Edge0 35B**（LoRA と学習済み pre-router を備えた Qwen3.6-35B-A3B）です。Apple M5 Mac ではデコードが毎秒約 25 トークン、物理メモリフットプリントのピークが約 3.4 GiB です。各プラットフォームの数値は [Benchmarking](docs/benchmarking.md#reference-figures) を参照してください。 +- **Edge0 35B**（LoRA と学習済み pre-router を備えた Qwen3.6-35B-A3B）は、このディレクトリの**ネイティブエンジン**で動作します。ネイティブエンジンは [GGML](https://github.com/ggml-org/ggml) をベースにネイティブ C++ API を備え、macOS、iOS、Android、Linux、Windows で動作します。エキスパートの重みはストレージから必要に応じて固定数のスロットに読み込まれるため、デバイスのメモリ予算より大きなモデルでも、すべての重みを読み込まずに実行できます。Apple M5 Mac ではデコードが毎秒約 25 トークン、物理メモリフットプリントのピークが約 3.4 GiB です。各プラットフォームの数値は [Benchmarking](docs/benchmarking.md#reference-figures) を参照してください。 +- **Edge0 145B** と **Edge0 120B** は **edge0-strata** で動作します。[Strata](https://github.com/Niko1221/Strata) のフォークで、Edge0 リポジトリの `edge0-strata/` にあり、Linux または Windows の NVIDIA GPU で動作します。[Edge0 145B](docs/edge0-145b.md) と [Edge0 120B](docs/edge0-120b.md) を参照してください。 > このページは英語版 [README](README.md) の翻訳です。リンク先のドキュメントはすべて英語です。 ## プラットフォーム -| プラットフォーム | バックエンド | 検証デバイス | +| プラットフォーム | Edge0 35B（ネイティブエンジン） | Edge0 145B、Edge0 120B（edge0-strata） | | --- | --- | --- | -| macOS（Apple シリコン） | Metal、CPU | Apple M5、16 GiB | -| iOS | Metal | iPhone 17 Pro Max | -| Android | CPU | OnePlus PKR110（Snapdragon 8 Elite） | -| Windows（x86-64） | CPU、Vulkan | Intel Core i5-12400、NVIDIA GeForce RTX 4070 SUPER | +| macOS（Apple シリコン） | Metal、CPU | — | +| iOS | Metal | — | +| Android | CPU | — | +| Linux | CPU | NVIDIA GPU（CUDA 13） | +| Windows | CPU、Vulkan | NVIDIA GPU（CUDA 13） | -同じデバイス上で、prefill と decode に異なるバックエンドを使えます（例：Metal で prefill、CPU で decode）。35B には 2 つのルーティングモードがあります。`predicted`（デフォルト）では pre-router により、現在のトークンを計算している間に次のトークンのエキスパートを読み込みます。`actual` はモデル本来の計算です。[Routing modes](docs/benchmarking.md#routing-modes) を参照してください。 +測定結果：Edge0 35B は [Benchmarking](docs/benchmarking.md#reference-figures)、Edge0 145B と Edge0 120B はモデルページ（Edge0 リポジトリの `docs/models/edge0-145b.md`）を参照してください。ネイティブエンジンでは、同じデバイス上で prefill と decode に異なるバックエンドを使えます（例：Metal で prefill、CPU で decode）。35B には 2 つのルーティングモードがあります。`predicted`（デフォルト）では pre-router により、現在のトークンを計算している間に次のトークンのエキスパートを読み込みます。`actual` はモデル本来の計算です。[Routing modes](docs/benchmarking.md#routing-modes) を参照してください。 ## クイックスタート @@ -28,7 +30,7 @@ cmake --preset mac-metal && cmake --build --preset mac-metal build/mac-metal/bin/edge0-infer info ``` -次に、モデルアセットをダウンロードして変換し、単発のプロンプト、チャット、またはローカルの OpenAI 互換サーバーを実行します。 +次に、モデルアセットをダウンロードして変換し、単発のプロンプト、チャット、またはローカルの OpenAI／Anthropic 互換サーバーを実行します。 ```sh build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "一文で自己紹介してください。" @@ -38,13 +40,17 @@ build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal [Getting started](docs/getting-started.md) では、アセットのダウンロードと変換、Android・Windows・iOS 向けのビルドを含め、すべての手順を説明しています。 +Edge0 145B と Edge0 120B では、edge0-strata をビルドしてから `edge0-infer serve --model <モデルフォルダ>` を実行します。手順は [Getting started](docs/getting-started.md#edge0-145b-and-edge0-120b-edge0-strata) にあります。 + ## ドキュメント - [Getting started](docs/getting-started.md)：各プラットフォームでのビルド、モデルアセットの取得、実行、チャット、サーバー +- [Edge0 145B](docs/edge0-145b.md)：edge0-strata でこのモデルを実行する +- [Edge0 120B](docs/edge0-120b.md)：edge0-strata でこのモデルを実行する - [Configuration reference](docs/configuration.md)：ランタイム、ロード、モデル、生成の設定とそのデフォルト値、CLI フラグ - [Native API](docs/native-api.md)：プロセス内でのエンジンの利用：ライフサイクル、ストリーミング、マルチターン、並行処理、キャンセル -- [OpenAI-compatible local server](docs/openai-compatible-server.md)：エンドポイント、リクエストフィールド、制限 -- [Architecture](docs/architecture.md)：エンジンの構成と、リクエストがエンジン内をどう流れるか +- [Local server](docs/local-server.md)：エンドポイント、リクエストフィールド、制限 +- [Architecture](docs/architecture.md)：2 つの層、ネイティブエンジンの構成、リクエストがエンジン内をどう流れるか - [Known limitations](docs/limitations.md)：注意すべき動作 - [Benchmarking](docs/benchmarking.md)：測定方法、推奨設定、参考値 - [Testing](docs/testing.md)：テストレベル、それぞれに必要なもの、実行方法 @@ -55,7 +61,7 @@ build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal ## ネイティブ API -アプリケーションは CMake ターゲット `edge0::edge0` にリンクし、[`include/edge0/`](include/edge0/) のヘッダーをインクルードします。シグネチャの唯一の根拠はこれらのヘッダーで、型と関数は名前空間 `edge0` にあります。[`examples/native_chat.cpp`](examples/native_chat.cpp) はライフサイクル全体を示します：ランタイムを開き、モデルを読み込み、セッションを開き、ストリーミングで 2 ターン生成し、すべてを解放します。[ネイティブ API ガイド](docs/native-api.md) では、ライフサイクル、ストリーミング、マルチターン、並行処理、キャンセル、エラーを説明しています。 +ネイティブ API はネイティブエンジンのものです。アプリケーションは CMake ターゲット `edge0::edge0` にリンクし、[`include/edge0/`](include/edge0/) のヘッダーをインクルードします。シグネチャの唯一の根拠はこれらのヘッダーで、型と関数は名前空間 `edge0` にあります。[`examples/native_chat.cpp`](examples/native_chat.cpp) はライフサイクル全体を示します：ランタイムを開き、モデルを読み込み、セッションを開き、ストリーミングで 2 ターン生成し、すべてを解放します。[ネイティブ API ガイド](docs/native-api.md) では、ライフサイクル、ストリーミング、マルチターン、並行処理、キャンセル、エラーを説明しています。 ## ライセンスとサードパーティコード diff --git a/edge0-infer/README_zh.md b/edge0-infer/README_zh.md index 7e86720b..638a2502 100644 --- a/edge0-infer/README_zh.md +++ b/edge0-infer/README_zh.md @@ -2,22 +2,24 @@ [English](README.md) | 中文 | [日本語](README_ja.md) | [Español](README_es.md) | [Français](README_fr.md) -基于 [GGML](https://github.com/ggml-org/ggml) 的端侧混合专家（MoE）语言模型推理引擎。一个引擎、一套原生 C++ API，在 macOS、iOS、Android 和 Windows 上运行同一个模型。专家权重从存储按需读入固定数量的专家槽，因此比设备内存预算更大的模型无需载入全部权重即可运行。 +`edge0-infer` 在你自己的硬件上运行混合专家（MoE）语言模型：命令行、本地 OpenAI／Anthropic 兼容服务与聊天应用，所有模型使用相同的命令。每个模型运行在为它准备的引擎上，`edge0-infer` 根据模型目录自动选择引擎，无需用户选择： -首个支持的模型是 **Edge0 35B**（Qwen3.6-35B-A3B，带 LoRA 与训练好的 pre-router）。在 Apple M5 Mac 上解码约每秒 25 个 token，物理内存占用峰值约 3.4 GiB；各平台的数据见 [Benchmarking](docs/benchmarking.md#reference-figures)。 +- **Edge0 35B**（Qwen3.6-35B-A3B，带 LoRA 与训练好的 pre-router）运行在本目录的**原生引擎**上。原生引擎基于 [GGML](https://github.com/ggml-org/ggml)，提供原生 C++ API，支持 macOS、iOS、Android、Linux 和 Windows。专家权重从存储按需读入固定数量的专家槽，因此比设备内存预算更大的模型无需载入全部权重即可运行。在 Apple M5 Mac 上解码约每秒 25 个 token，物理内存占用峰值约 3.4 GiB；各平台的数据见 [Benchmarking](docs/benchmarking.md#reference-figures)。 +- **Edge0 145B** 和 **Edge0 120B** 运行在 **edge0-strata** 上。它是 [Strata](https://github.com/Niko1221/Strata) 的分支，位于 Edge0 仓库的 `edge0-strata/`，在 Linux 或 Windows 的 NVIDIA GPU 上运行；见 [Edge0 145B](docs/edge0-145b.md) 与 [Edge0 120B](docs/edge0-120b.md)。 > 本页为英文版 [README](README.md) 的翻译；下面链接的文档均为英文。 ## 平台 -| 平台 | 后端 | 验证设备 | +| 平台 | Edge0 35B（原生引擎） | Edge0 145B、Edge0 120B（edge0-strata） | | --- | --- | --- | -| macOS（Apple Silicon） | Metal、CPU | Apple M5，16 GiB | -| iOS | Metal | iPhone 17 Pro Max | -| Android | CPU | OnePlus PKR110（骁龙 8 Elite） | -| Windows（x86-64） | CPU、Vulkan | Intel Core i5-12400，NVIDIA GeForce RTX 4070 SUPER | +| macOS（Apple Silicon） | Metal、CPU | — | +| iOS | Metal | — | +| Android | CPU | — | +| Linux | CPU | NVIDIA GPU（CUDA 13） | +| Windows | CPU、Vulkan | NVIDIA GPU（CUDA 13） | -同一设备上的 prefill 与 decode 可以使用不同后端，例如 Metal prefill + CPU decode。35B 提供两种路由模式：`predicted`（默认）由 pre-router 让引擎在当前 token 计算时加载下一个 token 的专家；`actual` 为模型的原始计算。见 [Routing modes](docs/benchmarking.md#routing-modes)。 +测量结果：Edge0 35B 见 [Benchmarking](docs/benchmarking.md#reference-figures)，Edge0 145B 与 Edge0 120B 见其模型页（Edge0 仓库中的 `docs/models/edge0-145b.md`）。在原生引擎上，同一设备上的 prefill 与 decode 可以使用不同后端，例如 Metal prefill + CPU decode。35B 提供两种路由模式：`predicted`（默认）由 pre-router 让引擎在当前 token 计算时加载下一个 token 的专家；`actual` 为模型的原始计算。见 [Routing modes](docs/benchmarking.md#routing-modes)。 ## 快速开始 @@ -28,7 +30,7 @@ cmake --preset mac-metal && cmake --build --preset mac-metal build/mac-metal/bin/edge0-infer info ``` -然后下载并转换模型资产，运行单轮、多轮对话或本地 OpenAI 兼容服务： +然后下载并转换模型资产，运行单轮、多轮对话或本地 OpenAI／Anthropic 兼容服务： ```sh build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "用一句话介绍你自己。" @@ -38,13 +40,17 @@ build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal [Getting started](docs/getting-started.md) 逐步说明每个环节，包括资产的下载与转换，以及 Android、Windows 和 iOS 的构建。 +Edge0 145B 和 Edge0 120B 需先构建 edge0-strata，再运行 `edge0-infer serve --model <模型目录>`；步骤见 [Getting started](docs/getting-started.md#edge0-145b-and-edge0-120b-edge0-strata)。 + ## 文档 - [Getting started](docs/getting-started.md)：各平台的构建、获取模型资产、运行、对话与服务 +- [Edge0 145B](docs/edge0-145b.md)：在 edge0-strata 上运行该模型 +- [Edge0 120B](docs/edge0-120b.md)：在 edge0-strata 上运行该模型 - [Configuration reference](docs/configuration.md)：运行时、加载、模型与生成设置，及其默认值和 CLI 参数 - [Native API](docs/native-api.md)：在进程内使用引擎：生命周期、流式输出、多轮、并发与取消 -- [OpenAI-compatible local server](docs/openai-compatible-server.md)：接口、请求字段与限制 -- [Architecture](docs/architecture.md)：引擎的组织方式，以及一个请求如何流经引擎 +- [Local server](docs/local-server.md)：接口、请求字段与限制 +- [Architecture](docs/architecture.md)：两层结构，原生引擎的组织方式，以及一个请求如何流经它 - [Known limitations](docs/limitations.md)：需要注意的行为 - [Benchmarking](docs/benchmarking.md)：如何测量、推荐设置与参考数据 - [Testing](docs/testing.md)：测试级别、各自需要什么、如何运行 @@ -55,7 +61,7 @@ build/mac-metal/bin/edge0-infer serve --model "$MOE_ASSETS_35B" --device metal ## 原生 API -应用链接 CMake 目标 `edge0::edge0`，并包含 [`include/edge0/`](include/edge0/) 中的头文件；签名以这些头文件为唯一依据，类型与函数位于命名空间 `edge0`。[`examples/native_chat.cpp`](examples/native_chat.cpp) 展示完整生命周期：打开运行时、加载模型、打开会话、流式生成两轮，并释放全部资源。[原生 API 指南](docs/native-api.md) 说明生命周期、流式输出、多轮使用、并发、取消与错误。 +原生 API 属于原生引擎。应用链接 CMake 目标 `edge0::edge0`，并包含 [`include/edge0/`](include/edge0/) 中的头文件；签名以这些头文件为唯一依据，类型与函数位于命名空间 `edge0`。[`examples/native_chat.cpp`](examples/native_chat.cpp) 展示完整生命周期：打开运行时、加载模型、打开会话、流式生成两轮，并释放全部资源。[原生 API 指南](docs/native-api.md) 说明生命周期、流式输出、多轮使用、并发、取消与错误。 ## 许可证与第三方代码 diff --git a/edge0-infer/cmake/MoeTargets.cmake b/edge0-infer/cmake/MoeTargets.cmake index 067a3b38..baedab68 100644 --- a/edge0-infer/cmake/MoeTargets.cmake +++ b/edge0-infer/cmake/MoeTargets.cmake @@ -1,23 +1,32 @@ function(moe_warnings target) + # Warnings are reported on our targets only (GGML and other third-party + # targets keep their own settings). They stop the build only with + # MOE_WARNINGS_AS_ERRORS=ON (default OFF): a compiler or version we do not + # build with may warn about code that is correct, and that must not stop a + # user's build. if(MSVC) - # Warnings as errors on our targets only (GGML and other third-party - # targets keep their own settings). /utf-8 compiles sources and runs - # with a UTF-8 execution charset (decision 2) so universal-character - # escapes are not rejected under code page 936 (C4566). The /wd - # suppressions cover the categories MSVC /W4 raises but our Clang/GCC - # baseline (-Wall -Wextra -Wpedantic) does not, so /WX stops new mistakes - # in Windows code without forcing a repo-wide cleanup of pre-existing - # portable code (decision 1, user 2026-10-26 option A; numbers confirmed - # by building under /WX): + # /utf-8 compiles sources and runs with a UTF-8 execution charset + # (decision 2) so universal-character escapes are not rejected under + # code page 936 (C4566). The /wd suppressions cover the categories MSVC + # /W4 raises but our Clang/GCC baseline (-Wall -Wextra -Wpedantic) does + # not, so warnings as errors stop new mistakes in Windows code without + # forcing a repo-wide cleanup of pre-existing portable code (decision 1, + # user 2026-10-26 option A; numbers confirmed by building under /WX): # 4244 4245 4267 4305 4310 integer, size and constant conversions # 4389 signed/unsigned comparison # 4456 local declaration shadows a previous one # 4996 CRT "unsafe" fopen/getenv/tmpfile target_compile_options(${target} PRIVATE - /W4 /WX /permissive- /utf-8 + /W4 /permissive- /utf-8 /wd4244 /wd4245 /wd4267 /wd4305 /wd4310 /wd4389 /wd4456 /wd4996) + if(MOE_WARNINGS_AS_ERRORS) + target_compile_options(${target} PRIVATE /WX) + endif() else() - target_compile_options(${target} PRIVATE -Wall -Wextra -Wpedantic -Werror) + target_compile_options(${target} PRIVATE -Wall -Wextra -Wpedantic) + if(MOE_WARNINGS_AS_ERRORS) + target_compile_options(${target} PRIVATE -Werror) + endif() endif() endfunction() diff --git a/edge0-infer/docs/architecture.md b/edge0-infer/docs/architecture.md index edc596b0..6f487e27 100644 --- a/edge0-infer/docs/architecture.md +++ b/edge0-infer/docs/architecture.md @@ -1,10 +1,25 @@ # Architecture -An overview of how the engine is organized. It names the parts and how they work together; the headers in [`include/edge0/`](../include/edge0/) and the [configuration reference](configuration.md) are the precise reference. +An overview of how `edge0-infer` is organized. It names the parts and how they work together; the headers in [`include/edge0/`](../include/edge0/) and the [configuration reference](configuration.md) are the precise reference. -## The pieces +## Two layers -The engine is a native library that applications call in process. The reference CLI `edge0-infer` and its `serve` command are applications of the same library; they add no engine logic of their own. +`edge0-infer` has two layers. The upper layer is the program you run: the commands `run`, `chat`, `serve` and `inspect`, the local server of `serve`, and the chat applications that use it. The lower layer is the engine that computes the model. Each model runs on the engine built for it: + +| Model | Engine | Hardware | How `edge0-infer` uses the engine | +| --- | --- | --- | --- | +| Edge0 35B | Native engine | GGML backends: Metal, CPU, Vulkan | A library in this directory, called in process; applications can also call it through the [native API](native-api.md) | +| Edge0 145B, Edge0 120B | edge0-strata | NVIDIA GPU with CUDA | A separate engine in `edge0-strata/` of the Edge0 repository, a fork of [Strata](https://github.com/Niko1221/Strata); `edge0-infer` starts it as a child process and talks to it over a loopback port | + +`edge0-infer` picks the engine from the model folder given with `--model`; there is nothing to choose. A `model.json` with an `architecture` field is an asset manifest of the native engine. A `model.json` whose `model_name` names an Edge0 145B or Edge0 120B folder selects edge0-strata: `edge0-infer` starts the engine with that model's configuration template from `edge0-strata/edge0/models/`, waits until it is ready, and then forwards generation requests to it. `serve` keeps answering `/health`, `/v1/models` and `/status` itself, and `chat` keeps its own commands, so clients see the same interface for every model. The pages [Edge0 145B](edge0-145b.md) and [Edge0 120B](edge0-120b.md) describe this path; edge0-strata's own build and its launch contract with `edge0-infer` are in `edge0-strata/EDGE0.md`. + +The rest of this page describes the native engine. + +## Native engine + +### The pieces + +The native engine is a library that applications call in process. For its models, `edge0-infer` and its `serve` command are applications of the same library; they add no engine logic of their own. | Part | Source | What it does | | --- | --- | --- | @@ -18,30 +33,31 @@ The engine is a native library that applications call in process. The reference Outside the library, the [asset tools](../tools/assets/README.md) convert and check model files before they are used. -## One request +### One request 1. The application opens a session and sends the whole conversation for each turn. 2. The engine renders and tokenizes the conversation and compares the tokens with what the session has already computed. Only the tokens after the common prefix are computed again (see [Multi-turn reuse](#multi-turn-reuse)). 3. Prefill computes the new prompt tokens in chunks. Decode then produces one token per step: the model implementation states which weights it needs, the weight manager makes them ready, and the execution layer runs the graph. 4. Each new token is sampled, decoded to text and streamed back to the application. The turn ends at an end token, a stop string, a length limit, a complete tool call, or a cancel request. -## Experts on demand +### Experts on demand A mixture-of-experts layer uses only a few of its experts for each token. The engine keeps a fixed number of expert slots per layer and reads the experts a token needs from the asset files into those slots; experts already in a slot are reused. Memory use therefore depends on the slot count, not on the model size. Prefill, which needs many experts at once, reads them through a staging area instead of the slots. With `predicted` routing, the 35B's pre-router predicts the experts the later layers will need for the next token, and the engine reads them in the background while the current token is still computing. With `actual` routing, every layer computes its router first and waits for the experts it chose. [Routing modes](benchmarking.md#routing-modes) compares the two. -## Multi-turn reuse +### Multi-turn reuse A session keeps the tokens it has computed. When the next turn arrives, the engine keeps the computed state up to the first token that differs and computes only the rest. Attention caches can be cut at any position; recurrent state, such as the 35B's linear-attention layers, cannot. For that state the engine saves one checkpoint per turn, just before the last prompt token, and restores it when the difference lies after it. A difference before the checkpoint, such as an edited earlier message, makes the engine compute the conversation again from the start. [Known limitations](limitations.md) explains when the tokens can differ. -## Backends and platforms +### Backends and platforms | Platform | Prefill | Decode | | --- | --- | --- | | macOS | Metal or CPU | Metal or CPU | | iOS | Metal | Metal | | Android | CPU | CPU | +| Linux | CPU | CPU | | Windows | CPU or Vulkan | CPU or Vulkan | By default both phases use the same backend; prefill can be moved to a different one (for example `--device cpu --prefill-device metal`). diff --git a/edge0-infer/docs/benchmarking.md b/edge0-infer/docs/benchmarking.md index cb626b38..b76115f7 100644 --- a/edge0-infer/docs/benchmarking.md +++ b/edge0-infer/docs/benchmarking.md @@ -2,6 +2,8 @@ How to measure the Edge0 35B on each supported platform: what is measured, a quick one-off measurement, and the standard measurement that produces a comparable report. Run every command from the engine's top-level directory (`edge0-infer/`) after building and getting the model assets as described in [Getting started](getting-started.md). +`bench` and the standard measurement run only on the native engine. The measured speed of Edge0 145B and Edge0 120B is on their model page, `docs/models/edge0-145b.md` in the Edge0 repository. + ## What is measured | Metric | Meaning | @@ -116,7 +118,7 @@ The report records the effective configuration printed by the engine, the measur ## Android -- Build with the `android-cpu` preset and push the binary and assets as in [Getting started](getting-started.md#android-cpu); set `ANDROID_SERIAL` and `MOE_ANDROID_ROOT`. Commands run on the device through `tests/platform/android-cli`. +- Build with the `android-cpu` preset and push the binary and assets as in [Getting started](getting-started.md#android); set `ANDROID_SERIAL` and `MOE_ANDROID_ROOT`. Commands run on the device through `tests/platform/android-cli`. - Keep the device charging and the screen on for the whole measurement; the script refuses to start a process otherwise. Turn on the system performance mode by hand: it cannot be queried, so note it with the results. - Prefill on the CPU is slow (about 34 tokens per second on a cool device, lower after long load), so the 627-token prompt is the default; the 3,084-token prompt takes over two minutes per process before decoding starts. - After long continuous load the system lowers the CPU frequency limits and decode slows down noticeably. The snapshots record the per-cluster frequency limits before and after each process; measure on a cool device. @@ -124,7 +126,7 @@ The report records the effective configuration printed by the engine, the measur ## iOS -- Build, install and push the assets as in [Getting started](getting-started.md#ios-metal); export `MOE_IOS_TEAM` and `MOE_IOS_DEVICE`. Commands run in the App on the device through `tests/platform/ios-cli`. +- Build, install and push the assets as in [Getting started](getting-started.md#ios); export `MOE_IOS_TEAM` and `MOE_IOS_DEVICE`. Commands run in the App on the device through `tests/platform/ios-cli`. - Connect the device with a USB data cable, keep it unlocked with the App in the foreground, and set Auto-Lock to Never; the script refuses to start a process when the device is not wired or is locked. Do not use the App by hand during a measurement: starting a command ends the running App. - iOS limits the memory of each App far below the device memory and terminates an App that exceeds the limit. On the reference iPhone the limit was about 3.29 GiB (`os_proc_available_memory` at App start); the snapshots record the available process memory before and after each process. A terminated process is a failure, not a result. - Battery and charging state, numeric temperature and CPU frequency are not available; the snapshots record the system thermal state instead. diff --git a/edge0-infer/docs/configuration.md b/edge0-infer/docs/configuration.md index 9b3749bc..c76de7c8 100644 --- a/edge0-infer/docs/configuration.md +++ b/edge0-infer/docs/configuration.md @@ -9,7 +9,7 @@ Every setting belongs to one of four scopes. The scope decides when the value is | [Model options](#model-options) | The model is loaded | One loaded model; each model declares its own | `LoadOverrides::model_options` | | [Generation](#generation) | Each request | One generation call | `GenerationOverrides` in `edge0/generation.h` | -The reference CLI (`edge0-infer`) and the OpenAI-compatible server use the same settings and defaults as the native API; the CLI flag names are given below. Settings that only exist in the CLI or the server are listed under [Command options](#command-options). +The reference CLI (`edge0-infer`) and the local server use the same settings and defaults as the native API; the CLI flag names are given below. Settings that only exist in the CLI or the server are listed under [Command options](#command-options). ## How a value is chosen @@ -86,7 +86,7 @@ Each model declares its own options with their values and defaults; the engine v Without generation flags the model's recommendation is used. The 35B recommends, with thinking off, temperature 0.7, top-k 20, top-p 0.8; with thinking on, 1.0, 20, 0.95; both use presence penalty 1.5 and at most 32768 generated tokens, and thinking is off by default. The platform defaults are temperature 0, top-k 0, top-p 1, repetition penalty 1, no penalties and 256 generated tokens. A maximum that comes from the model or the platform is shortened to the remaining capacity; an explicit maximum is not. For greedy output use `--temperature 0 --presence-penalty 0`. -Changing a generation setting between requests does not discard the conversation: the prompt is re-rendered and the reusable prefix is decided from the actual tokens. The server maps request fields to these settings as described in the [server guide](openai-compatible-server.md). +Changing a generation setting between requests does not discard the conversation: the prompt is re-rendered and the reusable prefix is decided from the actual tokens. The server maps request fields to these settings as described in the [server guide](local-server.md). ## Command options @@ -98,7 +98,7 @@ These flags select inputs and outputs of one command and are not part of the con | all model commands | `--model ` (the asset directory), `--verbose` (show GGML logs), `--help` | | `run` | `--user `, `--user-file `, `--system `, `--tokens `, `--turns ` (a scripted conversation), `--replay ` (repeat recorded blocks on their recorded devices: a prefill block must name a policy candidate, a decode block the decode device, and a missing device means the configured one), `--out ` (write tokens and results) | | `chat` | `--system `, `--no-warmup`, `--out ` | -| `serve` | `--host`, `--port`, `--alias`, `--no-warmup`, `--max-body-bytes`, `--write-timeout-seconds`, `--session-idle-seconds`, `--out`; see the [server guide](openai-compatible-server.md) | +| `serve` | `--host`, `--port`, `--alias`, `--no-warmup`, `--max-body-bytes`, `--write-timeout-seconds`, `--session-idle-seconds`, `--out`; see the [server guide](local-server.md) | | `bench` | `--user-file `, `--tokens `, `--turns `, `--warmup `, `--ntok `, `--runs `, `--breakdown`; see [Benchmarking](benchmarking.md) | | `tokenize`, `prompt` | `--user `, `--system `, `--thinking on\|off`, `--pieces`, `--parse-special`, `--tokens` | | `info` | `--probe` (initialize each device) | diff --git a/edge0-infer/docs/edge0-120b.md b/edge0-infer/docs/edge0-120b.md new file mode 100644 index 00000000..75501de3 --- /dev/null +++ b/edge0-infer/docs/edge0-120b.md @@ -0,0 +1,129 @@ +# Edge0 120B + +Edge0 120B is a mixture-of-experts model for a single NVIDIA GPU. It runs on **edge0-strata**, the engine built for it (see [Two layers](architecture.md#two-layers)): the `edge0-strata/` directory of the Edge0 repository, a fork of [Strata](https://github.com/Niko1221/Strata). `edge0-infer serve` and `edge0-infer chat` start edge0-strata for this model, wait until it is ready and talk to it on a loopback port, so clients use the same commands and endpoints as for every other model. [Edge0 145B](edge0-145b.md) runs the same way. + +This page is the reference for running Edge0 120B with `edge0-infer`; [Getting started](getting-started.md#edge0-145b-and-edge0-120b-edge0-strata) walks through the steps. The model itself (architecture and measured speed) is described on the model page of Edge0 145B and Edge0 120B, `docs/models/edge0-145b.md` in the Edge0 repository. + +## Requirements + +- Linux or Windows with an NVIDIA GPU and CUDA 13. +- At least 16 GiB of GPU memory and 20 GiB of system memory. +- `edge0-infer` built as in [Build](getting-started.md#build) (the `cpu` preset on Linux) or [Windows](getting-started.md#windows); it needs no GPU backend of its own for these models. +- edge0-strata built as described in `edge0-strata/EDGE0.md` in the Edge0 repository. `edge0-infer` uses the engine `edge0-strata/build/strata` (`strata.exe` on Windows) and the Python environment `edge0-strata/.venv` it produces. + +## Model directory + +Use each released model folder as provided: one model and one expert format per folder, with files laid out flat. `edge0-infer` reads only the `model_name` field of the folder's `model.json` and ignores its other fields (`architecture`, `args`, `env` and the rest); a `model.json` that contains `format_version` is an asset manifest of the native engine instead. The folder also holds a `start.sh` and a `model-bench.json`, which `edge0-infer` does not use. + +| model_name | Registration (default alias) | Expert format | +| --- | --- | --- | +| `edge0-120b-q3k` | `edge0_120b` | `q3k` | +| `edge0-120b-fp4` | `edge0_120b` | `fp4` | + +The expert format `q3k` is Q3_K (Q3_K gate/up, IQ4_NL down) and `fp4` is NVFP4. + +The commands below use the folder that contains `model.json`, for example `Edge0-120B-q3k`, not the folder that holds several models: + +```sh +export EDGE0_120B=/path/to/Edge0-120B-q3k +``` + +On Windows, set it with `$env:EDGE0_120B = "C:\path\to\Edge0-120B-q3k"` in PowerShell and run `build\windows\bin\edge0-infer.exe` instead of `build/cpu/bin/edge0-infer`. + +| Path | Content | Required | +| --- | --- | --- | +| `model.json` | Model identifier (above) | yes | +| `experts.bin` | Compressed experts in the directory's format, 48 layers x 512 | yes | +| `labels.bin` | The projection group of each expert, read beside the expert file | yes | +| `dense.bin`, `index.txt`, `native_experts.txt` | The rest of the expert pack | yes | +| `scales/` | Per-layer column scales | for Q3_K folders | +| `latent_dense_q8.bin` | Expert projections, Q8 | for `du=q8` | +| `latent_dense_q4.bin` | Expert projections, Q4 | for `du=q4` | +| `model-skeleton.gguf` | Attention, routing, shared experts, embeddings and output | yes | +| `ple-fp8.gguf` | Per-layer embedding table (FP8) | yes | +| `profile.bin` | Expert ranking used to choose which experts to keep on the GPU | yes | +| `mtp/` | Draft layer for speculative decoding | yes | +| `tokenizer/` | Tokenizer | yes | + +`edge0-infer`'s asset tools do not produce this directory. Check it with: + +```sh +build/cpu/bin/edge0-infer inspect --model "$EDGE0_120B" --json +``` + +The report lists files with `exists` and `bytes`; `ok` is `true` (exit code 0) when the selected configuration is complete. +The expert format comes from `model_name` and is not a model option; `--model-option experts=` is rejected. +D/U defaults to `q4`; add `--model-option du=q8` to check the other choice. +Only Q3_K folders list or require `scales/` (in the report its `required` value is `experts=q3k`). + +When files are missing, standard output remains valid JSON and standard error lists the missing files. +A hint suggests another D/U choice only when its files complete the directory. +Starting `serve` or `chat` uses the same defaults and reports missing files without falling back. + +## Serve + +```sh +build/cpu/bin/edge0-infer serve --model "$EDGE0_120B" +``` + +`edge0-infer` finds edge0-strata through `--strata-dir`, then `EDGE0_STRATA_DIR`, then an `edge0-strata/` directory beside an ancestor of the `edge0-infer` executable (`edge0-infer.exe` on Windows) (in the Edge0 repository layout, the sibling of `edge0-infer/`). The `Listening:` line appears when edge0-strata has loaded the model and is ready; `--strata-ready-timeout` bounds the wait (default 1800 seconds). While it loads, `serve` prints edge0-strata's server messages on its standard error; the engine process's own output is not shown or kept. The model name is its registration identifier (`--alias` changes it). + +Once `Listening:` appears (default port 8000): + +```sh +curl http://127.0.0.1:8000/v1/chat/completions -H "Content-Type: application/json" \ + -d '{"model": "edge0_120b", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +Generation requests are forwarded to edge0-strata unchanged, and `/health`, `/v1/models` and `/status` are answered by `edge0-infer serve` in its own formats (`/health` also carries edge0-strata's own fields, and edge0-strata's `/v1/status`, `/metrics` and `/props` are passed through, so its tools work against `edge0-infer serve`); see [Delegated models](local-server.md#delegated-models) for the options, the forwarding rules and what each `/status` field holds. + +## Chat + +```sh +build/cpu/bin/edge0-infer chat --model "$EDGE0_120B" +``` + +`chat` starts its own edge0-strata; it does not connect to a running `edge0-infer serve`. Do not run the two on the same GPU at the same time. To chat with a model that `serve` already runs, use an OpenAI- or Anthropic-compatible client against it. + +The commands are those of the native chat (`/file`, `"""`, `/think on|off`, `/new`, `/regen`, `/quit`). Thinking is on unless `/think off` is given. Reasoning is printed to standard error between `[thinking]` and `[/thinking]`, the answer to standard output, and after each turn a line with the reused and computed tokens, the first-token time, the decode speed and the stop reason. Ctrl-C stops the current answer; at the prompt it stops edge0-strata and exits (exit code 130). edge0-strata's messages, including those printed while it loads, are written to a temporary log file whose path chat prints at start (the engine process's own output is not kept); the file is removed when chat ends normally. If edge0-strata fails to start or exits during the session, chat prints the last lines of its log, keeps the log file, prints its path and exits with code 1; invalid options exit with code 2 before anything starts. + +## Options + +| Option | Values | Default | +| --- | --- | --- | +| `--device` | `cuda` or `cuda:` | `cuda:0` | +| `--model-option du=` | `q8` or `q4` | `q4` | +| `--model-option ple-io=` | `direct` (read the embedding table from disk as needed) or `ram` (load it into memory; to keep it locked there, raise the locked-memory limit, e.g. `ulimit -l unlimited`) | `direct` | +| `--context` | tokens | 131072 | +| `--strata-arg` | one word of edge0-strata's engine command line; repeat it for each word, e.g. `--strata-arg --pool-workers --strata-arg 24` | none | + +The model settings (automatic expert cache and prefill, speculative decoding, and int8 KV cache) are in `edge0-strata/edge0/models/edge0-120b.json`. CPU worker count, PCIe mode and fraction, VRAM reserve, and KV residency use the engine's defaults unless given with `--strata-arg`. This is meant for tuning to one machine, for example: + +```sh +build/cpu/bin/edge0-infer serve --model "$EDGE0_120B" \ + --strata-arg --pool-workers --strata-arg 24 --strata-arg --pcie-mode --strata-arg dma \ + --strata-arg --pcie-frac --strata-arg 0.1 --strata-arg --vram-reserve-mib --strata-arg 700 \ + --strata-arg --kv-resident --strata-arg 32768 +``` + +The words are appended in the order given, after the template's arguments; write each option and its value as two words (not `--option=value`). An option the template already sets (such as `--kv` or `--pack`), `--max-context` (use `--context`) and an empty value are rejected before anything starts. Other words are not checked by `edge0-infer`: edge0-strata decides, and since the engine process's own output is not shown, a word it does not accept appears as a failed start (run edge0-strata directly to see why). The template requires launch contract version 4; an older engine template is rejected with both versions in the error. + +## CUDA libraries on Windows + +The default interpreter is `edge0-strata/.venv/Scripts/python.exe` +on Windows and `edge0-strata/.venv/bin/python` on POSIX; `--strata-python` overrides it. +The template expands `${EXE_SUFFIX}` to `.exe` on Windows and to an empty string on POSIX. + +The CUDA toolkit sets `CUDA_PATH`. Existing `bin/x64` and `bin` directories under that path +replace the template's `${CUDA_LIB_DIR}` item, in that order, and are prepended to the +server's `PATH`. On POSIX that item expands to no directories and the inherited environment +is unchanged. If neither Windows directory contains `cublas64_13.dll`, the CLI warns, +lists the directories checked, and still launches: the inherited `PATH` may already provide +the DLL. A missing runtime DLL can produce exit code `0xC0000135`. + +## Known limitations + +- The native API does not support this model: `inspect_assets`, `resolve_load_configuration` and `load_model` return an `unsupported` error for a `model.json` that has no `format_version`, before reading any other file. Of the commands, only `serve`, `chat` and `inspect` (with or without `--json`, without `--digest`) accept it; `run`, `bench`, `describe`, `tokenize`, `prompt` and `forward` refuse it. +- Concurrent requests wait in edge0-strata's queue instead of receiving 429 as with the native server. +- `/status` holds no memory breakdown or expert statistics for this model; those fields are null. Progress of a request in flight is counted by `edge0-infer serve` from the text fragments it forwards (one token per fragment), and only for streaming requests; the finished records use the engine's own numbers (see the `Delegated` column of the [status fields](local-server.md#status-and-lan-authentication)). +- Identical requests at temperature 0 can produce different answers: edge0-strata's decoding is not bit-for-bit repeatable. diff --git a/edge0-infer/docs/edge0-145b.md b/edge0-infer/docs/edge0-145b.md new file mode 100644 index 00000000..3e70a430 --- /dev/null +++ b/edge0-infer/docs/edge0-145b.md @@ -0,0 +1,129 @@ +# Edge0 145B + +Edge0 145B is a mixture-of-experts model for a single NVIDIA GPU. It runs on **edge0-strata**, the engine built for it (see [Two layers](architecture.md#two-layers)): the `edge0-strata/` directory of the Edge0 repository, a fork of [Strata](https://github.com/Niko1221/Strata). `edge0-infer serve` and `edge0-infer chat` start edge0-strata for this model, wait until it is ready and talk to it on a loopback port, so clients use the same commands and endpoints as for every other model. [Edge0 120B](edge0-120b.md) runs the same way. + +This page is the reference for running Edge0 145B with `edge0-infer`; [Getting started](getting-started.md#edge0-145b-and-edge0-120b-edge0-strata) walks through the steps. The model itself (architecture and measured speed) is described on the model page of Edge0 145B and Edge0 120B, `docs/models/edge0-145b.md` in the Edge0 repository. + +## Requirements + +- Linux or Windows with an NVIDIA GPU and CUDA 13. +- At least 16 GiB of GPU memory and 20 GiB of system memory. +- `edge0-infer` built as in [Build](getting-started.md#build) (the `cpu` preset on Linux) or [Windows](getting-started.md#windows); it needs no GPU backend of its own for these models. +- edge0-strata built as described in `edge0-strata/EDGE0.md` in the Edge0 repository. `edge0-infer` uses the engine `edge0-strata/build/strata` (`strata.exe` on Windows) and the Python environment `edge0-strata/.venv` it produces. + +## Model directory + +Use each released model folder as provided: one model and one expert format per folder, with files laid out flat. `edge0-infer` reads only the `model_name` field of the folder's `model.json` and ignores its other fields (`architecture`, `args`, `env` and the rest); a `model.json` that contains `format_version` is an asset manifest of the native engine instead. The folder also holds a `start.sh` and a `model-bench.json`, which `edge0-infer` does not use. + +| model_name | Registration (default alias) | Expert format | +| --- | --- | --- | +| `edge0-145b-q3k` | `edge0_145b` | `q3k` | +| `edge0-145b-fp4` | `edge0_145b` | `fp4` | + +The expert format `q3k` is Q3_K (Q3_K gate/up, IQ4_NL down) and `fp4` is NVFP4. + +The commands below use the folder that contains `model.json`, for example `Edge0-145B-q3k`, not the folder that holds several models: + +```sh +export EDGE0_145B=/path/to/Edge0-145B-q3k +``` + +On Windows, set it with `$env:EDGE0_145B = "C:\path\to\Edge0-145B-q3k"` in PowerShell and run `build\windows\bin\edge0-infer.exe` instead of `build/cpu/bin/edge0-infer`. + +| Path | Content | Required | +| --- | --- | --- | +| `model.json` | Model identifier (above) | yes | +| `experts.bin` | Compressed experts in the directory's format, 48 layers x 512 | yes | +| `labels.bin` | The projection group of each expert, read beside the expert file | yes | +| `dense.bin`, `index.txt`, `native_experts.txt` | The rest of the expert pack | yes | +| `scales/` | Per-layer column scales | for Q3_K folders | +| `latent_dense_q8.bin` | Expert projections, Q8 | for `du=q8` | +| `latent_dense_q4.bin` | Expert projections, Q4 | for `du=q4` | +| `model-skeleton.gguf` | Attention, routing, shared experts, embeddings and output | yes | +| `ple-fp8.gguf` | Per-layer embedding table (FP8) | yes | +| `profile.bin` | Expert ranking used to choose which experts to keep on the GPU | yes | +| `mtp/` | Draft layer for speculative decoding | yes | +| `tokenizer/` | Tokenizer | yes | + +`edge0-infer`'s asset tools do not produce this directory. Check it with: + +```sh +build/cpu/bin/edge0-infer inspect --model "$EDGE0_145B" --json +``` + +The report lists files with `exists` and `bytes`; `ok` is `true` (exit code 0) when the selected configuration is complete. +The expert format comes from `model_name` and is not a model option; `--model-option experts=` is rejected. +D/U defaults to `q8`; add `--model-option du=q4` to check the other choice. +Only Q3_K folders list or require `scales/` (in the report its `required` value is `experts=q3k`). + +When files are missing, standard output remains valid JSON and standard error lists the missing files. +A hint suggests another D/U choice only when its files complete the directory. +Starting `serve` or `chat` uses the same defaults and reports missing files without falling back. + +## Serve + +```sh +build/cpu/bin/edge0-infer serve --model "$EDGE0_145B" +``` + +`edge0-infer` finds edge0-strata through `--strata-dir`, then `EDGE0_STRATA_DIR`, then an `edge0-strata/` directory beside an ancestor of the `edge0-infer` executable (`edge0-infer.exe` on Windows) (in the Edge0 repository layout, the sibling of `edge0-infer/`). The `Listening:` line appears when edge0-strata has loaded the model and is ready; `--strata-ready-timeout` bounds the wait (default 1800 seconds). While it loads, `serve` prints edge0-strata's server messages on its standard error; the engine process's own output is not shown or kept. The model name is its registration identifier (`--alias` changes it). + +Once `Listening:` appears (default port 8000): + +```sh +curl http://127.0.0.1:8000/v1/chat/completions -H "Content-Type: application/json" \ + -d '{"model": "edge0_145b", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +Generation requests are forwarded to edge0-strata unchanged, and `/health`, `/v1/models` and `/status` are answered by `edge0-infer serve` in its own formats (`/health` also carries edge0-strata's own fields, and edge0-strata's `/v1/status`, `/metrics` and `/props` are passed through, so its tools work against `edge0-infer serve`); see [Delegated models](local-server.md#delegated-models) for the options, the forwarding rules and what each `/status` field holds. + +## Chat + +```sh +build/cpu/bin/edge0-infer chat --model "$EDGE0_145B" +``` + +`chat` starts its own edge0-strata; it does not connect to a running `edge0-infer serve`. Do not run the two on the same GPU at the same time. To chat with a model that `serve` already runs, use an OpenAI- or Anthropic-compatible client against it. + +The commands are those of the native chat (`/file`, `"""`, `/think on|off`, `/new`, `/regen`, `/quit`). Thinking is on unless `/think off` is given. Reasoning is printed to standard error between `[thinking]` and `[/thinking]`, the answer to standard output, and after each turn a line with the reused and computed tokens, the first-token time, the decode speed and the stop reason. Ctrl-C stops the current answer; at the prompt it stops edge0-strata and exits (exit code 130). edge0-strata's messages, including those printed while it loads, are written to a temporary log file whose path chat prints at start (the engine process's own output is not kept); the file is removed when chat ends normally. If edge0-strata fails to start or exits during the session, chat prints the last lines of its log, keeps the log file, prints its path and exits with code 1; invalid options exit with code 2 before anything starts. + +## Options + +| Option | Values | Default | +| --- | --- | --- | +| `--device` | `cuda` or `cuda:` | `cuda:0` | +| `--model-option du=` | `q8` or `q4` | `q8` | +| `--model-option ple-io=` | `direct` (read the embedding table from disk as needed) or `ram` (load it into memory; to keep it locked there, raise the locked-memory limit, e.g. `ulimit -l unlimited`) | `direct` | +| `--context` | tokens | 131072 | +| `--strata-arg` | one word of edge0-strata's engine command line; repeat it for each word, e.g. `--strata-arg --pool-workers --strata-arg 24` | none | + +The model settings (automatic expert cache and prefill, speculative decoding, and int8 KV cache) are in `edge0-strata/edge0/models/edge0-145b.json`. CPU worker count, PCIe mode and fraction, VRAM reserve, and KV residency use the engine's defaults unless given with `--strata-arg`. This is meant for tuning to one machine, for example: + +```sh +build/cpu/bin/edge0-infer serve --model "$EDGE0_145B" \ + --strata-arg --pool-workers --strata-arg 24 --strata-arg --pcie-mode --strata-arg dma \ + --strata-arg --pcie-frac --strata-arg 0.1 --strata-arg --vram-reserve-mib --strata-arg 700 \ + --strata-arg --kv-resident --strata-arg 32768 +``` + +The words are appended in the order given, after the template's arguments; write each option and its value as two words (not `--option=value`). An option the template already sets (such as `--kv` or `--pack`), `--max-context` (use `--context`) and an empty value are rejected before anything starts. Other words are not checked by `edge0-infer`: edge0-strata decides, and since the engine process's own output is not shown, a word it does not accept appears as a failed start (run edge0-strata directly to see why). The template requires launch contract version 4; an older engine template is rejected with both versions in the error. + +## CUDA libraries on Windows + +The default interpreter is `edge0-strata/.venv/Scripts/python.exe` +on Windows and `edge0-strata/.venv/bin/python` on POSIX; `--strata-python` overrides it. +The template expands `${EXE_SUFFIX}` to `.exe` on Windows and to an empty string on POSIX. + +The CUDA toolkit sets `CUDA_PATH`. Existing `bin/x64` and `bin` directories under that path +replace the template's `${CUDA_LIB_DIR}` item, in that order, and are prepended to the +server's `PATH`. On POSIX that item expands to no directories and the inherited environment +is unchanged. If neither Windows directory contains `cublas64_13.dll`, the CLI warns, +lists the directories checked, and still launches: the inherited `PATH` may already provide +the DLL. A missing runtime DLL can produce exit code `0xC0000135`. + +## Known limitations + +- The native API does not support this model: `inspect_assets`, `resolve_load_configuration` and `load_model` return an `unsupported` error for a `model.json` that has no `format_version`, before reading any other file. Of the commands, only `serve`, `chat` and `inspect` (with or without `--json`, without `--digest`) accept it; `run`, `bench`, `describe`, `tokenize`, `prompt` and `forward` refuse it. +- Concurrent requests wait in edge0-strata's queue instead of receiving 429 as with the native server. +- `/status` holds no memory breakdown or expert statistics for this model; those fields are null. Progress of a request in flight is counted by `edge0-infer serve` from the text fragments it forwards (one token per fragment), and only for streaming requests; the finished records use the engine's own numbers (see the `Delegated` column of the [status fields](local-server.md#status-and-lan-authentication)). +- Identical requests at temperature 0 can produce different answers: edge0-strata's decoding is not bit-for-bit repeatable. diff --git a/edge0-infer/docs/getting-started.md b/edge0-infer/docs/getting-started.md index a0391184..37f19b22 100644 --- a/edge0-infer/docs/getting-started.md +++ b/edge0-infer/docs/getting-started.md @@ -1,16 +1,23 @@ # Getting started -From a fresh clone to a chat with the Edge0 35B model and a local OpenAI-compatible server. The engine is the `edge0-infer/` directory of the [Edge0 repository](https://github.com/Edge0-AI/Edge0): after cloning, `cd edge0-infer` and run every command from there. Paths in these documents are relative to that directory. +From a fresh clone to a chat and a local OpenAI- and Anthropic-compatible server. `edge0-infer` is the `edge0-infer/` directory of the [Edge0 repository](https://github.com/Edge0-AI/Edge0): after cloning, `cd edge0-infer` and run every command from there. Paths in these documents are relative to that directory. -## Status +## Two layers -One model, Edge0 35B, is supported. It is verified on macOS on Apple silicon with the Metal and CPU backends and on Android (CPU, tested on PKR110) and on Windows (x86-64: CPU, tested on an Intel Core i5-12400; Vulkan for prefill or for both phases, tested on an NVIDIA GeForce RTX 4070 SUPER) and on iOS (Metal, tested on an iPhone 17 Pro Max). Measured speed and memory on each platform are listed under [Benchmarking](benchmarking.md#reference-figures). +`edge0-infer` is the program you run: `run`, `chat`, `serve` (an OpenAI- and Anthropic-compatible server) and `inspect`, and the chat applications built on them, with the same commands for every model. Below it, each model runs on the engine built for it; `edge0-infer` picks the engine from the model folder given with `--model`, so there is nothing to choose: -## Requirements +| Model | Engine | Hardware | Where the engine is | +| --- | --- | --- | --- | +| Edge0 35B | Native engine | Metal, CPU or Vulkan on macOS, iOS, Android, Linux and Windows | this directory, also usable in process through the [native API](native-api.md) | +| Edge0 145B, Edge0 120B | edge0-strata | NVIDIA GPU (CUDA 13) on Linux and Windows | `edge0-strata/` in the Edge0 repository; `edge0-infer` starts it as a separate process | -CMake 3.21 or newer, Ninja, a C++17/C11 compiler, uv and Git. The `mac-metal` preset also needs macOS with the Xcode command line tools; the `cpu` preset needs no Apple frameworks. GGML is vendored at a fixed version, so the build downloads nothing. +[Architecture](architecture.md) describes how the two layers work together. -The 35B needs about 42 GB of free disk space: about 19.7 GB for the original weights and about 22 GB for the converted assets. The converter writes no other large files. Once converted, the engine needs only the assets; the tiny test fixtures need just three small text files of the original weights (see [Testing](testing.md#setup)). +Edge0 35B is verified on macOS on Apple silicon (Metal and CPU), on Android (CPU, PKR110), on Windows (CPU on an Intel Core i5-12400; Vulkan on an NVIDIA GeForce RTX 4070 SUPER) and on iOS (Metal, iPhone 17 Pro Max); measured speed and memory are under [Benchmarking](benchmarking.md#reference-figures). The measured speed of Edge0 145B and Edge0 120B is on their model page, `docs/models/edge0-145b.md` in the Edge0 repository. + +## Requirements + +CMake 3.21 or newer, Ninja, a C++17/C11 compiler, uv and Git. The `mac-metal` preset also needs macOS with the Xcode command line tools; the `cpu` preset needs no Apple frameworks. GGML is vendored at a fixed version, so the build downloads nothing. Each model has its own needs, listed under [Get a model](#get-a-model). ## Build @@ -20,9 +27,16 @@ cmake --build --preset mac-metal build/mac-metal/bin/edge0-infer info # expect a line: model.0: edge0_35b ``` -On machines without Metal use `--preset cpu`, `build/cpu/bin/edge0-infer` and `--device cpu` in the commands below. +On Linux and on other machines without Metal use `--preset cpu`, `build/cpu/bin/edge0-infer` and `--device cpu` in the commands below. For Edge0 145B and Edge0 120B, `edge0-infer` needs no GPU backend of its own: the `cpu` preset on Linux (or the `windows` preset on Windows) is enough. Android, Windows and iOS are built as described in their sections at the end of this page. + +Compiler warnings are printed but do not stop the build; configure with `-DMOE_WARNINGS_AS_ERRORS=ON` to treat warnings in this project's own code as errors. -## Get the model assets +## Get a model + + +### Edge0 35B (native engine) + +The 35B needs about 42 GB of free disk space: about 19.7 GB for the original weights and about 22 GB for the converted assets. The converter writes no other large files. Once converted, the engine needs only the assets; the tiny test fixtures need just three small text files of the original weights (see [Testing](testing.md#setup)). Weights never go into Git. Choose where to keep them and export two directories: @@ -70,12 +84,29 @@ assets_py() { uv run --project tools/assets --offline --no-sync python -I "$@"; The [asset tools guide](../tools/assets/README.md) describes the asset directory, comparing two copies, the tiny test fixtures and adding a model. +### Edge0 145B and Edge0 120B (edge0-strata) + +These models come as released model folders, one model and one expert format per folder (for example `Edge0-145B-q3k`); the asset tools above do not make them. They need an NVIDIA GPU with CUDA 13 on Linux or Windows. + +1. **Build edge0-strata**, the `edge0-strata/` directory beside `edge0-infer/` in the Edge0 repository, as its `EDGE0.md` describes (sections "Build (Linux, NVIDIA)" and "Build (Windows, NVIDIA)"). `edge0-infer` finds it there and uses the engine and the Python environment that the build produces. + +2. **Check the model folder**, the one that contains `model.json`: + + ```sh + export EDGE0_145B=/path/to/Edge0-145B-q3k + build/cpu/bin/edge0-infer inspect --model "$EDGE0_145B" --json + ``` + +[Edge0 145B](edge0-145b.md) and [Edge0 120B](edge0-120b.md) list the files of a folder, the options and the limits of each model. + ## Run one prompt ```sh build/mac-metal/bin/edge0-infer run --model "$MOE_ASSETS_35B" --device metal --user "Introduce yourself in one sentence." --max-tokens 256 ``` +`run` is for the native engine; for Edge0 145B and Edge0 120B use `chat` or `serve`. + `--max-tokens` caps the answer length; without it the model's limit applies (32768 tokens for the 35B). The answer streams to standard output; with thinking on, the thinking text goes to standard error, after a `[thinking]` line. Standard error ends with a JSON summary: `request_summary` (time to first token, decode speed, total time), `weight_activity` (expert demand, hits, loads and bytes read), `memory_snapshot` (budget ledger, RSS and platform physical footprint, reported separately) and the effective load configuration. On Metal the first run spends extra seconds compiling shaders. For Metal prefill and CPU decode, use `--device cpu --prefill-device metal`. Each request summary then reports its actual `prefill_device`, successful `handoffs`, `handoff_bytes` and `handoff_seconds`; with identical stage devices these counters are zero. Replay files include a `device` for each block, and replay uses that recorded device without choosing again. @@ -103,6 +134,12 @@ Each line is one turn. Answers go to standard output; thinking text and status l Ctrl-C stops the current answer; the conversation can continue. +For Edge0 145B and Edge0 120B, `chat` starts edge0-strata and takes the same commands; see Chat for [Edge0 145B](edge0-145b.md#chat) and [Edge0 120B](edge0-120b.md#chat): + +```sh +build/cpu/bin/edge0-infer chat --model "$EDGE0_145B" +``` + ## Serve @@ -116,7 +153,13 @@ The server loads and warms the model, then listens on `http://127.0.0.1:8000/v1` uv run --no-project --with openai==3.26.1 python examples/openai_client.py --model edge0_35b ``` -Endpoints, request fields and limits are in the [server guide](openai-compatible-server.md). +Endpoints, request fields and limits are in the [server guide](local-server.md). + +For Edge0 145B and Edge0 120B, `serve` starts edge0-strata and forwards generation requests to it; clients use the model name `edge0_145b` or `edge0_120b`. See Serve for [Edge0 145B](edge0-145b.md#serve) and [Edge0 120B](edge0-120b.md#serve): + +```sh +build/cpu/bin/edge0-infer serve --model "$EDGE0_145B" +``` ## Chat applications @@ -138,7 +181,7 @@ Example chat applications for desktop (macOS and Windows), Android and iOS use t ## Use from another CMake project -An application can build the engine as part of its own CMake project and link the static library `edge0::edge0`, +An application can build the native engine as part of its own CMake project and link the static library `edge0::edge0`, which carries the public include directory and the C++17 requirement: ```cmake @@ -188,9 +231,10 @@ Known interactions with the host build: ## Measure performance -See [Benchmarking](benchmarking.md): what is measured, a quick one-off `bench`, the recommended standard measurement for macOS and Android, and reference figures. +See [Benchmarking](benchmarking.md) for the native engine: what is measured, a quick one-off `bench`, the recommended standard measurement for macOS and Android, and reference figures. The speed of Edge0 145B and Edge0 120B is on their model page, `docs/models/edge0-145b.md` in the Edge0 repository. -## Android CPU + +## Android Use Android NDK r28c (`28.2.13676358`), CMake, Ninja, Python 3 and Android platform-tools on the host. Set `ANDROID_NDK` to the NDK directory, `ANDROID_SERIAL` to the authorized @@ -218,7 +262,7 @@ Text generation and tokenization run entirely on the device; Python is only host To measure performance on the device, see [Benchmarking](benchmarking.md#android). -For an OpenAI-compatible server: +For an OpenAI- and Anthropic-compatible server: ```sh tests/platform/android-cli serve --model "$MOE_ASSETS_35B" --device cpu --port 0 @@ -237,40 +281,54 @@ with reasons, not zero. Android thread defaults differ from macOS; see [Runtime settings](configuration.md#runtime). -## Windows CPU +## Windows -Windows uses the CPU backend on x86-64 (Intel or AMD); Windows on Arm is not supported. Requirements: Windows 10 version 1903 or newer (the executable asks for a UTF-8 code page, -which older versions ignore), Visual Studio 2022 or its Build Tools with the C++ workload -(MSVC), CMake, Ninja, uv and Git. Run the build in an x64 MSVC environment: the "x64 Native -Tools" prompt, or a shell after running `vcvars64.bat`. Build with the `windows` preset and -run the commands above with `build\windows\bin\edge0-infer.exe` and `--device cpu`. The -examples below use PowerShell; in PowerShell an environment variable is set with +which older versions ignore), CMake, Ninja, uv and Git, and a compiler that depends on the CPU of +the machine: + +- x64 CPU: Visual Studio 2022 or its Build Tools with the C++ workload (MSVC). Run the build in + the **x64 Native Tools Command Prompt for VS 2022**, or in a shell after running `vcvars64.bat`. +- Arm64 CPU: Visual Studio 2022 or its Build Tools with the **MSVC ARM64 build tools** and the + **Windows 11 SDK**, and LLVM for `clang-cl` (`winget install LLVM.LLVM`). ggml does not accept + MSVC as the compiler there, so the compiler is clang-cl and only the linker is MSVC. Run the + build in the **ARM64 Native Tools Command Prompt for VS 2022** with `clang-cl` on `PATH`. + +The examples use PowerShell; in PowerShell an environment variable is set with `$env:NAME = "value"` rather than `export`. +### Build + +With MSVC (x64 CPU): + ```powershell cmake --preset windows cmake --build --preset windows build\windows\bin\edge0-infer.exe info # expect a line: model.0: edge0_35b ``` +With clang-cl (Arm64 CPU): + ```powershell -$env:MOE_ASSETS_35B = "" -build\windows\bin\edge0-infer.exe run --model $env:MOE_ASSETS_35B --device cpu --user "Introduce yourself in one sentence." -build\windows\bin\edge0-infer.exe chat --model $env:MOE_ASSETS_35B --device cpu -build\windows\bin\edge0-infer.exe serve --model $env:MOE_ASSETS_35B --device cpu +cmake --preset windows -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_OSX_ARCHITECTURES=arm64 +cmake --build --preset windows +build\windows\bin\edge0-infer.exe info # expect a line: model.0: edge0_35b ``` -The asset tools from [Get the model assets](#get-the-model-assets) run the same way; define -the shortcut as a PowerShell function, for example -`function assets_py { uv run --project tools/assets --offline --no-sync python -I @args }`. +`CMAKE_OSX_ARCHITECTURES=arm64` makes ggml compile its Arm sources on Windows; without it the link +fails on a duplicate `ggml_quantize_mat_q8_K_4x8`. If `build\windows` holds a cache from an MSVC +configure, delete it first: CMake cannot change the compiler of an existing cache. For an +executable that runs without the Visual C++ redistributable and without OpenMP, add +`-DBUILD_SHARED_LIBS=OFF -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DGGML_OPENMP=OFF`; a missing +runtime DLL shows up as exit code `0xC0000135` before the program prints anything. -**Instruction set.** The `windows` preset optimizes for the CPU of the build machine -(`GGML_NATIVE`); GGML and the engine are linked statically into the executable. To build for other machines, turn -detection off and name the instruction set; with MSVC, AVX2 also enables FMA and F16C: +**Instruction set (x64 CPU).** The `windows` preset optimizes for the CPU of the build machine +(`GGML_NATIVE`); GGML and the engine are linked statically into the executable. To build for other +x64 machines, turn detection off and name the instruction set; with MSVC, AVX2 also enables FMA +and F16C: ```powershell -cmake --preset windows -DGGML_NATIVE=OFF -DGGML_AVX2=ON # most x86-64 CPUs since about 2013 +cmake --preset windows -DGGML_NATIVE=OFF -DGGML_AVX2=ON # most x64 CPUs since about 2013 cmake --preset windows -DGGML_NATIVE=OFF -DGGML_AVX512=ON # CPUs with AVX-512 ``` @@ -278,30 +336,36 @@ A binary built for instructions the CPU lacks stops with an illegal-instruction start-up. Reconfigure in a fresh build directory (or delete `build\windows`) when changing these options. Only the native AVX2 build was tested. -Windows specifics: +### Run -- **Default threads.** Decode and prefill default from the physical core count - `P = logical / 2`: decode uses `max(1, P - 2)` and prefill uses `min(logical, P + 2)`, - because decode is bandwidth-bound while prefill is compute-parallel. On a 12-logical, - 6-physical part this is decode four and prefill eight. The rule assumes two threads per - core and was measured only on such a CPU; on CPUs without simultaneous multithreading or - with efficiency cores, set the thread counts explicitly. Override either with `--threads` - and `--prefill-threads`; `info` reports the defaults and runtime summaries report the - resolved values and their source. -- **Expert reads.** `expert_read=direct` is unsupported on Windows (the GGUF expert - offsets are not aligned for unbuffered reads); experts are always read through the cache. -- **Memory reporting.** The budget ledger peak is always reported. The process footprint - (working-set current and peak) comes from the Windows process APIs. Values that the - platform does not provide are reported as JSON `null` with a reason, never as zero. -- **Measurement.** Windows Defender real-time scanning can read expert files on first - access and disturb a cold measurement; check its status before measuring and report it - with the figures. See [Benchmarking](benchmarking.md). +Edge0 35B on the native engine, with the commands described above: + +```powershell +$env:MOE_ASSETS_35B = "" +build\windows\bin\edge0-infer.exe run --model $env:MOE_ASSETS_35B --device cpu --user "Introduce yourself in one sentence." +build\windows\bin\edge0-infer.exe chat --model $env:MOE_ASSETS_35B --device cpu +build\windows\bin\edge0-infer.exe serve --model $env:MOE_ASSETS_35B --device cpu +``` + +The asset tools from [Get a model](#get-the-model-assets) run the same way; define +the shortcut as a PowerShell function, for example +`function assets_py { uv run --project tools/assets --offline --no-sync python -I @args }`. + +Edge0 145B and Edge0 120B on edge0-strata, after building it as the "Build (Windows, NVIDIA)" section +of `edge0-strata/EDGE0.md` describes: + +```powershell +$env:EDGE0_145B = "C:\path\to\Edge0-145B-q3k" +build\windows\bin\edge0-infer.exe serve --model $env:EDGE0_145B +``` + +How `edge0-infer` finds the CUDA libraries is under CUDA libraries on Windows for [Edge0 145B](edge0-145b.md#cuda-libraries-on-windows) and [Edge0 120B](edge0-120b.md#cuda-libraries-on-windows). ### Vulkan -The `windows-vulkan` preset is the `windows` preset with Vulkan enabled. Configure and -build it from the same x64 MSVC environment, and set `VULKAN_SDK` so `glslc` is available. -The first build compiles shaders and can take a long time. +The `windows-vulkan` preset is the `windows` preset with Vulkan enabled, for the native engine. +Configure and build it in the same developer prompt as the `windows` preset, and set `VULKAN_SDK` +so `glslc` is available. The first build compiles shaders and can take a long time. Keep the clone at a short path (for example `C:\src\edge0`): the shader generator's object paths can exceed the MSVC path limit and cause error C1041. Clone the repository there rather than copying its files, so the build still @@ -321,9 +385,29 @@ build\windows-vulkan\bin\edge0-infer.exe run --model $env:MOE_ASSETS_35B --devic build\windows-vulkan\bin\edge0-infer.exe run --model $env:MOE_ASSETS_35B --device vulkan --user "1+1=" --max-tokens 8 ``` -These paths were checked on one NVIDIA GPU. They do not cover other Vulkan implementations. +These paths were checked on one NVIDIA GPU with the MSVC build. They do not cover other Vulkan implementations. + +### Native engine notes + +- **Default threads.** Decode and prefill default from the physical core count + `P = logical / 2`: decode uses `max(1, P - 2)` and prefill uses `min(logical, P + 2)`, + because decode is bandwidth-bound while prefill is compute-parallel. On a 12-logical, + 6-physical part this is decode four and prefill eight. The rule assumes two threads per + core and was measured only on such a CPU; on CPUs without simultaneous multithreading or + with efficiency cores, set the thread counts explicitly. Override either with `--threads` + and `--prefill-threads`; `info` reports the defaults and runtime summaries report the + resolved values and their source. +- **Expert reads.** `expert_read=direct` is unsupported on Windows (the GGUF expert + offsets are not aligned for unbuffered reads); experts are always read through the cache. +- **Memory reporting.** The budget ledger peak is always reported. The process footprint + (working-set current and peak) comes from the Windows process APIs. Values that the + platform does not provide are reported as JSON `null` with a reason, never as zero. +- **Measurement.** Windows Defender real-time scanning can read expert files on first + access and disturb a cold measurement; check its status before measuring and report it + with the figures. See [Benchmarking](benchmarking.md). -## iOS (Metal) + +## iOS Use Xcode with the iOS SDK, a paired iPhone with Developer Mode enabled, and a USB data cable. Configure your own Apple account in Xcode. Export `MOE_IOS_TEAM` diff --git a/edge0-infer/docs/limitations.md b/edge0-infer/docs/limitations.md index 113597d9..d90a9aa8 100644 --- a/edge0-infer/docs/limitations.md +++ b/edge0-infer/docs/limitations.md @@ -7,3 +7,7 @@ Behavior of the current release that users and integrators should be aware of. **What you see.** In a multi-turn conversation the engine normally computes only the new message of each turn and reuses everything before it. Sometimes a turn instead restores a checkpoint and computes part of the previous answer again: its generation summary shows `"checkpoint_restored":true` and more `computed_tokens`, and the first token takes longer. The answer is still correct; only the extra prefill costs time. With thinking enabled, answers are longer, so this happens more often. **Why.** Reuse needs the tokens of the new turn to match the tokens already computed, one by one. For each turn the application sends the whole conversation as text, and the engine renders and tokenizes it again. The previous answer, however, was computed as the tokens the model generated one at a time. The same text can be split into tokens in more than one way, and the model does not always generate the split that the tokenizer produces from the text. Where the two sequences first differ, the computed state no longer matches the conversation, so the engine restores the checkpoint it saved at the end of the previous prompt and computes the previous answer and the new message again from there (see [Multi-turn reuse](architecture.md#multi-turn-reuse)). The longer an answer is, the more chances there are for such a difference. + +## Edge0 145B and Edge0 120B run only through serve and chat + +Edge0 145B and Edge0 120B run on the edge0-strata engine, not on the native engine (see [Two layers](architecture.md#two-layers)): only through `edge0-infer serve` and `edge0-infer chat` (and `inspect`), and not through the native API. Their server queues concurrent requests instead of answering 429, and part of `/status` is null for them (no memory breakdown or expert statistics; progress of a request in flight is counted from the forwarded text fragments). See Known limitations for [Edge0 145B](edge0-145b.md#known-limitations) and [Edge0 120B](edge0-120b.md#known-limitations). diff --git a/edge0-infer/docs/local-server.md b/edge0-infer/docs/local-server.md new file mode 100644 index 00000000..7120f828 --- /dev/null +++ b/edge0-infer/docs/local-server.md @@ -0,0 +1,535 @@ +# Local server + +The native server provides OpenAI Chat Completions and Responses, and Anthropic Messages. +Validated Python SDKs: `openai==3.26.1` and `anthropic==2.8.0`. + +Start the server and the example client as in [Getting started](getting-started.md#serve). + +The server loads and warms the model before listening. Default address: +`http://127.0.0.1:8000/v1`. Loopback clients may use any nonempty SDK API key. Non-loopback listeners require +`--api-key-file`, and non-loopback clients must supply its token. There is no CORS, +multimodal input, persistence, hosted tool execution, or constrained decoding. + +`--host`, `--port` (0 selects an ephemeral port), `--alias`, `--context`, +`--prefill-chunk`, `--model-option`, `--max-tokens`, `--no-warmup`, and the CLI +sampling options are supported. Defaults: CPU, port 8000, alias from the model +identifier (`architecture` in the asset `model.json`, such as `edge0_35b`), +`--max-body-bytes 16777216`, `--write-timeout-seconds 30`, and +`--session-idle-seconds 0` (retain indefinitely). Run `serve --help` for syntax. +Application sampling options provide defaults; each request can override them. +If another process is already listening on the same local address and port, a second server fails to start with a binding error instead of sharing that endpoint or choosing another port. + +## Endpoints + +| Method | Path | Result | +| --- | --- | --- | +| GET | `/health` | Readiness | +| GET | `/v1/models` | The single loaded alias | +| GET | `/v1/models/{id}` | Model or 404 | +| POST | `/v1/chat/completions` | Chat completion or SSE chunks | +| POST | `/v1/responses` | Response or typed SSE events | +| POST | `/v1/messages` | Anthropic message or typed SSE events | +| POST | `/v1/messages/count_tokens` | Input token count | + +Every request includes `model` equal to the alias. Send the complete conversation +on every turn. The native library renders templates, tokenizes, validates tools, +and decides prefix reuse. One inference may run at a time; another receives 429 +immediately. A disconnected client cancels generation. Output writes have a +bounded timeout. SIGINT/SIGTERM stop acceptance, cancel and wait for active work, +then release the session, unload the model, and finish the runtime. + +The token-count endpoint uses the same input rules as Messages, with optional `max_tokens` +and no `stream`. It returns `{"input_tokens": N}` without taking the generation gate, +creating a session, changing its state, or entering request counters (401s are recorded). + +## Delegated models + +A model directory whose `model.json` has no `format_version` (so it is not an Edge0 asset manifest) and has a registered delegated `model_name`, such +as [Edge0 145B](edge0-145b.md) (`edge0_145b`) and [Edge0 120B](edge0-120b.md) (`edge0_120b`), runs on the external engine edge0-strata +instead of the native library. `serve` then starts edge0-strata on a free loopback port, waits +until it is ready, and only then prints the `Listening:` line. It acts as a proxy: + +- `POST /v1/chat/completions`, `/v1/responses` and `/v1/messages` are forwarded + unchanged: the request body, and the response status, `Content-Type` and body + (server-sent events chunk by chunk, including the engine's `timings`). The request + rules and outputs are edge0-strata's, not the ones in the rest of this page. + Concurrent requests wait in edge0-strata's queue instead of receiving 429. A client + that disconnects also disconnects the forwarded request. +- `GET /health`, `/v1/models`, `/v1/models/{id}` and `/status` are answered by + `edge0-infer serve` in the formats on this page. `/health` also carries the fields of + edge0-strata's own `/health` answer (such as `service` and `loaded`); `status` is + `edge0-infer serve`'s. +- `GET /v1/status`, `/metrics` and `/props` are edge0-strata's own description + endpoints, passed through unchanged (status, `Content-Type` and body) for its tools, + such as its benchmark `tools/strata_bench.py`. They return 503 once + edge0-strata has exited. +- `POST /v1/messages/count_tokens` is forwarded unchanged, including its response + status and content. It does not enter the active request set, request history or totals. +- If the engine disconnects before completing a response, a non-streaming request + returns 503 (`engine_exited`) when the engine has exited, or 502 (`engine_unavailable`) + when it is still running; partial content is discarded. A stream already started + ends with one protocol error event (Chat error object, Responses `response.failed`, + or Messages `error`). The request record carries the failure status and code. + Complete engine error responses are forwarded unchanged. +- Authentication, `--api-key-file`, `--host`, `--port`, `--alias`, + `--max-body-bytes` and `--write-timeout-seconds` work as for native models. + `--context` sets edge0-strata's `--max-context`. `--device` accepts `cuda` or + `cuda:` (default `cuda:0`), and `--prefill-device` must equal it if given. + `--model-option` takes the model's options (see the model page). Other options of the + native engine, including sampling defaults, `--prefill-chunk` and `--no-warmup`, are + rejected; send sampling fields in each request instead. +- `--strata-dir ` locates edge0-strata (otherwise `EDGE0_STRATA_DIR`, then an + `edge0-strata/` directory beside an ancestor of the `edge0-infer` executable). + `--strata-python ` selects its Python (default `.venv` inside it). + `--strata-ready-timeout ` bounds the wait for it to load (default 1800). + `--strata-arg ` (repeatable) passes one word of edge0-strata's engine command + line, appended in order after the launch template's arguments; an option the template + already sets, `--max-context` (use `--context`) and an empty value are rejected before + anything starts. See Options for [Edge0 145B](edge0-145b.md#options) and [Edge0 120B](edge0-120b.md#options) for its uses and an example. +- If edge0-strata cannot be found, has an unsupported launch contract version, fails to + start, exits before it is ready or is not ready in time, `serve` exits with an error + before listening. If it exits later, generation requests and `/health` return 503. + SIGINT/SIGTERM stop acceptance, wait up to 60 seconds for forwarded requests to + finish, then stop edge0-strata (SIGTERM, then SIGKILL after 60 seconds). + +## Request compatibility + +Unknown fields are rejected with 400, including unknown nested control fields. +Tool parameter schemas are passed through as schema data. Fields marked ignored +are accepted without an effect. `store` never enables persistence. + +| Fields | Chat Completions | Responses | Messages | +| --- | --- | --- | --- | +| `model` | Required alias | Required alias | Required alias | +| `messages` | Nonempty; system, developer (system), user, assistant, tool | Not accepted | Nonempty user/assistant history; final role must be user | +| `content` | String or text parts; assistant may use null | Message string or input_text/output_text parts | String or text/tool_result (user), text/thinking/tool_use (assistant) blocks | +| `instructions`, `input` | Not accepted | Optional system instruction; required string or ordered items | Not accepted | +| `tool_calls`, `tool_call_id` | Assistant function calls; matched tool results | function_call and function_call_output items | Not accepted | +| `reasoning_content` | Assistant history extension | reasoning items with reasoning_text content | Not accepted | +| `tools` | Function wrapper, preserving key order | Flat function definition converted to a wrapper | name, description, input_schema; optional type custom; converted to function wrapper | +| `strict` | false or omitted only | false or omitted only | false or omitted only | +| `tool_choice` | auto or none | auto or none | Object: type auto or none; disable_parallel_tool_use inverts native parallel flag; any/tool rejected | +| `parallel_tool_calls` | Boolean | Boolean | Not accepted | +| `stream` | Boolean | Boolean | Boolean; rejected for count_tokens | +| `stream_options.include_usage` | Optional final usage chunk | Not accepted | Not accepted | +| `temperature`, `top_p` | Native range validation | Native range validation | Native range validation; SDK 2.8.0 uses extra_body | +| `max_tokens`, `max_completion_tokens` | Output limit including reasoning; must agree if both supplied | Not accepted | max_tokens required, includes thinking; max_completion_tokens rejected | +| `max_output_tokens` | Not accepted | Output limit including reasoning | Not accepted | +| `seed` | Unsigned integer | Not accepted | Not accepted | +| `stop` | String or string array | Not accepted | Not accepted | +| `presence_penalty`, `frequency_penalty` | Native range validation | Not accepted | Not accepted | +| `logit_bias` | Integer token ID keys and numeric biases | Not accepted | Not accepted | +| `logprobs` | Boolean | Use include | Not accepted | +| `top_logprobs` | Requires logprobs enabled | Requires include enabled | Not accepted | +| `include` | Not accepted | Only message.output_text.logprobs | Not accepted | +| `n` | Only 1 | Not accepted | Not accepted | +| `response_format` | Only type text | Not accepted | Not accepted | +| `text.format` | Not accepted | Only type text; verbosity rejected | Not accepted | +| `reasoning_effort` | none disables; minimal/low/medium/high enable | Use reasoning.effort | Not accepted | +| `chat_template_kwargs.enable_thinking` | Boolean extension; conflicting controls rejected | Not accepted | Not accepted | +| `reasoning.effort` | Not accepted | Same on/off mapping as Chat | Not accepted | +| `reasoning.summary` | Not accepted | null or auto; no summary is generated | Not accepted | +| `top_k`, `repetition_penalty` | Native extensions | Not accepted | top_k accepted (extra_body in SDK 2.8.0); repetition_penalty rejected | +| `truncation` | Not accepted | Only disabled | Not accepted | +| `background` | Not accepted | Only false; ignored | Not accepted | +| `user`, `metadata`, `store`, `service_tier`, `prompt_cache_key`, `safety_identifier` | Ignored | Ignored; store echoed false | metadata.user_id and service_tier ignored; remaining fields rejected | +| `promptCacheKey` | Alias of prompt_cache_key; string; ignored | Same as Chat | Not accepted | +| `previous_response_id`, `conversation` | Rejected | Rejected; send complete history | Not accepted | +| `functions`, `function_call`, `modalities`, `audio`, `prediction`, `web_search_options`, `verbosity` | Rejected | Rejected | Not accepted | +| Message `name`, `refusal`, `audio`; encrypted reasoning; nontext input | Rejected | Rejected | Not accepted | +| `system` | Use system message | Use instructions | String or text blocks, concatenated into the first system message | +| `tool_result` | Use tool message | Use function_call_output | tool_use_id required; content optional (empty text when omitted), otherwise string/text blocks; is_error accepted without rendering effect; results precede same-turn user text, in block order; result-only turns add no empty user message | +| `tool_use` | Use tool_calls | Use function_call | id, name, input object; input serialized compactly in original key order; streamed tool blocks appear only after the call ends, with at most one parameter delta; rejected calls emit no block | +| `thinking` history block | Use reasoning_content | Use reasoning item | thinking text; signature accepted without validation | +| `thinking` control | Use reasoning controls | Use reasoning controls | type enabled/adaptive enables, disabled disables; positive integer budget_tokens accepted without limiting thinking length; between_tools/display rejected | +| `stop_sequences` | Use stop | Not accepted | String array | +| `cache_control` | Not accepted | Not accepted | Accepted without effect; prefix reuse is automatic | +| `output_config`, `container`, `inference_geo`, `speed`, `diagnostics`, `user_profile_id`, `workspace_id`, `mcp_servers` | Rejected | Rejected | Rejected | +| Images, documents, search_result, redacted_thinking, server_tool_use and other nontext blocks | Rejected | Rejected | Rejected | + +Field names are matched exactly. `promptCacheKey` is the only accepted alias; +other camelCase spellings are unknown fields and are rejected. + +All enabled reasoning effort levels mean the same thing. The model default applies +when controls are absent. A model that cannot enable reasoning rejects the request. +Responses coalesces adjacent reasoning, assistant messages and function calls into +one native assistant message. User/system messages and tool results end that group. + +The server does not negotiate `anthropic-version` or `anthropic-beta` headers. +Messages has no prefill-prefix continuation, hosted tools, JSON output constraints, +Files, Batches or Anthropic Models endpoints. `GET /v1/models` keeps its OpenAI format. + +## Output and errors + +Chat emits `chat.completion` or `chat.completion.chunk`. Streaming begins with the +assistant role, then content/reasoning/tool argument deltas. Tool indices are +renumbered contiguously from zero, counting only calls with a begin event. +The finish chunk precedes optional usage and `[DONE]`. `reasoning_content` and +`rejected_tool_calls` are extensions. Rejected calls are never complete tool calls. + +Responses emits ordered reasoning, message, and function_call items. Streams begin +with `response.created` and `response.in_progress`. Each item is added before its +parts/deltas; done events close parts and items. `sequence_number` increases on +every event. The final event contains the complete response. Reasoning has an +empty `summary`; rejected started function calls have incomplete status. + +Messages returns a `message` with an `id` prefixed `msg_`, assistant role, model alias, +and content blocks in generation order: `thinking` (with empty `signature`), `text`, +and `tool_use` (with an object `input`). Thinking appears only when enabled. +Streaming emits `message_start`, then each block's `content_block_start`, +`content_block_delta` (`thinking_delta`, `text_delta`, or `input_json_delta`), and +`content_block_stop`, followed by `message_delta` and `message_stop`. Block indices +start at zero and match the complete content indices. No ping events are sent. +Tool blocks are buffered until the call ends, then sent as start, at most one delta +containing all parameters, and stop. Rejected or unfinished calls emit no blocks +and do not appear in complete content; they are listed in `rejected_tool_calls`, +also carried by `message_delta`. + +Messages `stop_reason` is `end_turn`, `tool_use`, `max_tokens`, +`model_context_window_exceeded`, or `stop_sequence`; only the last has a non-null +`stop_sequence`, containing the matched string. `tool_use` requires a tool block in +content; when all calls are rejected the reason is `end_turn`. Its usage reports uncached input +as `input_tokens`, reused input as `cache_read_input_tokens`, generated tokens +including thinking as `output_tokens`, and thinking tokens as +`output_tokens_details.thinking_tokens`. `cache_creation_input_tokens` is null. +At stream start, input and reuse come from live generation progress; if unavailable, +input uses the admission count and reuse is null. Initial output is zero; final +`message_delta.usage` contains the final counts and replaces the initial values. + +Messages errors (including 401, and errors under `/v1/messages/`) have the form +`{"type":"error","error":{"type":"invalid_request_error","message":"..."}}`. +HTTP 400/401/404/413/429/500/503 map respectively to `invalid_request_error`, +`authentication_error`, `not_found_error`, `request_too_large`, `rate_limit_error`, +`api_error`, and `overloaded_error`. Field errors end with `: `. +After streaming starts, failure or cancellation emits only an `error` event and +closes without `message_delta` or `message_stop`. Request logs retain the sent +failure's code, message and field path even though this error body has no code/param. + +Chat/Responses usage includes input, generated, cached, reasoning and total token counts from the +library. Logprobs cover answer tokens only, include raw byte arrays, and replace +invalid UTF-8 in their text representation. Pending logprobs accompany the next +answer delta or final content event. + +| Native outcome | Chat finish_reason | Responses status | +| --- | --- | --- | +| Normal end without tools; stop string | stop | completed | +| Normal end with tools; tool_calls | tool_calls | completed | +| Length or capacity | length | incomplete, max_output_tokens | +| Failure or cancellation | Error | failed | + +Chat/Responses errors have `error.message`, `error.type`, `error.param`, and `error.code`. +When an error concerns a specific field, `error.param` holds its path and +`error.message` ends with it, as `: ` (for example +`Unknown field: promptCacheKey` or `Unknown field: messages.name`), so clients +that display only the message still show the field. Errors without a field keep +the plain description. Streaming error objects and `response.failed` use the +same message. + +| Condition | HTTP status | Code | +| --- | --- | --- | +| Invalid/unknown fields or mismatched tool results | 400 | invalid_parameter / invalid_argument / unsupported_parameter | +| Unsupported capability | 400 | unsupported_value | +| Input exceeds capacity | 400 | context_length_exceeded | +| Wrong alias | 404 | model_not_found | +| Body too large | 413 | request_too_large | +| Active inference | 429 | busy | +| Shutting down | 503 | shutting_down | +| Other native failure or invalid session | 500 | Native category | + +After streaming starts, HTTP status cannot change. Chat sends an error object and +closes without `[DONE]`; Responses sends `response.failed`. Nonstreaming failure +returns 500, or 503 during shutdown, without partial output. A failed session is +released and rebuilt before later work; requests already executed are not retried. + +Default stderr logs contain request metadata and timing, not conversation text. +An inference request rejected before generation starts logs one JSON line with +`endpoint`, `http_status`, `code`, and `message`, plus `param` when the error +concerns a field; these come from the same failure used to format the response. +`--out` explicitly writes per-request tokens and results for development. +`--hold-file` is a development option that pauses after the first nonempty output +while that file exists; cancellation and disconnect detection remain active. + +Compatibility tests target pinned official Python SDKs `openai==3.26.1` and +`anthropic==2.8.0`, including their stream accumulators and Messages token counting. Missing offline SDK packages are reported as +skipped tests. See [examples](../examples/README.md) for native and HTTP clients. + +## Engine extensions + +Both model endpoints (`GET /v1/models` and `GET /v1/models/{id}`) include `context_length` (the loaded context capacity) and a `capabilities` object in each model entry. `thinking` is whether the loaded model supports enabling thinking; `thinking_default` is its default. Applications can use these booleans to enable and initialize a thinking switch. + +Completed generation output includes a top-level `metrics` object alongside `usage`: in full Chat Completions, the final Chat streaming usage chunk when `stream_options.include_usage` is true, and the final Responses response object (including streaming completion or token-limit termination). + +| Field | Engine source and units | +| --- | --- | +| `first_token_seconds` | Request summary time to first token, in seconds | +| `decode_tokens_per_second` | Request summary decode speed | +| `memory_peak_bytes` | Process physical memory peak, falling back to resident memory peak | +| `memory_peak_metric` | `physical` or `resident`, identifying the available peak | + +Unavailable fields are omitted. Process peaks cover the entire process lifetime, not just this request. The snapshot is taken after generation, from the same runtime. Clients may derive prefill speed as `(prompt_tokens - prompt_tokens_details.cached_tokens) / first_token_seconds` from Chat usage (or the corresponding `input_tokens` and `input_tokens_details` in Responses), when both counts are available, their difference is positive and the time is positive. Generated tokens do not contribute. These are engine measurements, not HTTP or UI timings. Errors, cancellations and Chat streams without requested usage do not include metrics. + +Messages includes the same `metrics` object in the complete response and final +`message_delta`. Its prefill rate is `usage.input_tokens / first_token_seconds` +when both are positive; the input count already excludes reused tokens. + +## Status and LAN authentication + +`GET /status` is available during generation and does not take the generation gate. +`--api-key-file ` reads a token, trimming surrounding whitespace. An absent, +unreadable or empty file fails before model loading. A non-loopback listener +requires a token. Every non-loopback request, including health, model and status +queries, must send `Authorization: Bearer ` or `x-api-key: `. +Either matching header authorizes the request, even if the other is incorrect; +neither header is forwarded to edge0-strata. IPv4 loopback (127/8), IPv6 +loopback and IPv4-mapped loopback clients are exempt, even with an incorrect header. +Tokens and conversation contents are never included in status or request logs. + +Each inference request and every 401 produces one JSON record in stderr and in +`recent_requests`, newest first, capped at 100. Successful health, status and token-count queries +are excluded. Messages is an inference endpoint, just like Chat and Responses. `last` is the last request that entered generation; rejected requests +do not replace it. Times use UTC ISO 8601 with millisecond precision. `total_seconds` +for generated requests is the native request duration; for rejected requests it is +HTTP handling duration. Native final timing values match the development `--out` +result summary. Counters reset on restart; `instance.id` distinguishes instances. + +Totals sum entered requests, with separate counts for 429, other pre-generation +rejections, and 401. Prefill speed is the sum of `(input_tokens - reused_tokens)` for +requests with first-token timing, divided by their summed first-token times. +Mean first-token time is the arithmetic mean. Decode speed is summed decode +forwards divided by summed decode duration. Zero denominators omit the rate. +Expert counts combine prefill and decode and are omitted when no applicable request +has occurred. Continued turns use keep, truncate or checkpoint restore. + +The table below is the field-name contract. `[]` denotes array elements and `*` +denotes a GGML device name or `host`. Optional fields may be absent; nullable +objects have no children when null. Counts and memory sizes are nonnegative. +Steady-clock timestamps are process-local seconds and must not be compared across +service instances. Final computed counts include decode forwards; live progress +computed counts cover input only. `generated_tokens` in final records excludes end +markers, matching native `generated_count`; live progress counts accepted tokens. + +For a delegated model the `Delegated` column states where each field comes from: +`portal` fields are `edge0-infer serve`'s own values, as for native models (`system.thermal` +and `system.app_memory_limit` describe the machine); `mapped` fields come from +edge0-strata's `/v1/status` (`engine.version`, the GPU in `engine.devices` named after +`--device`, `model.context_length`, and `system.memory` from its machine reading, with +source `edge0-strata`) and are null when it does not report them; `recorded` fields are +recorded by the proxy from what it forwarded and are null when the response does not +carry them; `n/a` fields do not apply and are null when required (an object then has no +children) and absent when optional. Recorded values come from the forwarded response: +`stop_reason` from the finish reason (Chat `finish_reason`, the Responses object's +`status`, Messages `stop_reason`), `input_tokens` and `generated_tokens` from `usage` +(for `/v1/messages`, `input_tokens` is the request's whole input, `input_tokens` + +`cache_creation_input_tokens` + `cache_read_input_tokens` with a missing one counted as +0, taken from `message_start` when streaming), `reused_tokens` from `timings.cache_n`, +`first_token_seconds` and `decode_seconds` from `timings.prompt_ms` and +`timings.predicted_ms`, and the per-request rates from `timings.prompt_per_second` and +`timings.predicted_per_second`. `total_seconds` is the proxy's handling time. In totals, +prefill speed is summed `timings.prompt_n` over summed prompt time and decode speed is +summed `timings.predicted_n` over summed decode time. A forwarded request answered with +429 counts as busy, with another 4xx status as an invalid request; `current` is the +oldest forwarded request still in progress. + +While a delegated request runs, `current` is counted from what the proxy has forwarded +so far, one token per streamed fragment that carries text (Chat `content`, +`reasoning_content` and tool-call `arguments`; Responses text, reasoning and tool-input +`delta` events; Messages `text_delta`, `thinking_delta` and `input_json_delta`). A +fragment is not always exactly one token. For a streaming request, `phase` is `prefill` +until the first such fragment and `decode` after it, `generated_tokens` is the number +of fragments forwarded (null before the first), and `first_token_seconds` is the +request's age when the first one was forwarded (absent before it). For a non-streaming +request nothing is forwarded until it ends, so `phase` and `generated_tokens` are null. +`input_tokens` and `reused_tokens` are filled in once the forwarded data carries them +(Messages `message_start` carries the input at the start; Chat reports both only in its +last chunk) and are null until then. Finished records (`last`, `recent_requests`, +`totals`) use only the numbers the engine reports, never the forwarded count. + + +| Field | Type | Presence | Delegated | Meaning | +| --- | --- | --- | --- | --- | +| `instance` | object | required | portal | instance | +| `listen` | object | required | portal | listen | +| `engine` | object | required | portal | engine | +| `model` | object | required | portal | model | +| `totals` | object | required | recorded | totals | +| `memory` | object | required | portal | memory | +| `system` | object | required | portal | system | +| `current` | object or null | required | recorded | current | +| `last` | object or null | required | recorded | last | +| `recent_requests` | array | required | recorded | recent requests | +| `recent_requests[]` | object | required | recorded | recent requests[] | +| `instance.id` | string | required | portal | id | +| `instance.started_at` | string | required | portal | started at | +| `instance.pid` | integer | required | portal | pid | +| `instance.uptime_seconds` | number | required | portal | uptime seconds | +| `listen.host` | string | required | portal | host | +| `listen.port` | integer | required | portal | port | +| `listen.auth` | boolean | required | portal | auth | +| `engine.version` | string | required | mapped | version | +| `engine.source_commit` | string | required | n/a | source commit | +| `engine.source_modified` | string | required | n/a | source modified | +| `engine.source_reason` | string | required | n/a | source reason | +| `engine.ggml_origin` | string | required | n/a | ggml origin | +| `engine.ggml_version` | string | required | n/a | ggml version | +| `engine.backends` | array | required | n/a | backends | +| `engine.backends[]` | object | required | n/a | backends[] | +| `engine.devices` | array | required | mapped | devices | +| `engine.devices[]` | object | required | mapped | devices[] | +| `engine.backends[].name` | string | required | n/a | name | +| `engine.backends[].compiled` | string | required | n/a | compiled | +| `engine.backends[].initializable` | string | required | n/a | initializable | +| `engine.backends[].verified` | string | required | n/a | verified | +| `engine.backends[].reason` | string | required | n/a | reason | +| `engine.devices[].name` | string | required | mapped | name | +| `engine.devices[].backend` | string | required | mapped | backend | +| `engine.devices[].description` | string | required | mapped | description | +| `engine.devices[].memory_total_bytes` | integer | required | mapped | memory total bytes | +| `engine.devices[].memory_free_bytes` | integer | required | mapped | memory free bytes | +| `model.alias` | string | required | portal | alias | +| `model.directory` | string | required | portal | directory | +| `model.device` | string | required | portal | device | +| `model.prefill_device` | string | required | portal | prefill device | +| `model.context_length` | integer | required | mapped | context length | +| `current.endpoint` | string | required | recorded | endpoint | +| `current.source` | string | required | recorded | source | +| `current.phase` | string | required | recorded | phase | +| `current.started_steady_seconds` | number | required | recorded | started steady seconds | +| `current.elapsed_seconds` | number | required | recorded | elapsed seconds | +| `current.input_tokens` | integer | required | recorded | input tokens | +| `current.reused_tokens` | integer | required | recorded | reused tokens | +| `current.computed_tokens` | integer | required | n/a | computed tokens | +| `current.pending_tokens` | integer | required | n/a | pending tokens | +| `current.generated_tokens` | integer | required | recorded | generated tokens | +| `current.first_token_seconds` | number | optional | recorded | first token seconds | +| `current.decode_elapsed_seconds` | number | optional | n/a | decode elapsed seconds | +| `current.prefill_tokens_per_second` | number or null | optional | n/a | prefill tokens per second | +| `current.decode_tokens_per_second` | number or null | optional | n/a | decode tokens per second | +| `last.time` | string | required | recorded | time | +| `last.endpoint` | string | required | recorded | endpoint | +| `last.source` | string | required | recorded | source | +| `last.http_status` | integer | required | recorded | http status | +| `last.stream` | boolean | required | recorded | stream | +| `last.total_seconds` | number | required | recorded | total seconds | +| `last.stop_reason` | string | required | recorded | stop reason | +| `last.reuse_method` | string or null | required | n/a | reuse method | +| `last.error_stage` | integer | required | n/a | error stage | +| `last.input_tokens` | integer | required | recorded | input tokens | +| `last.reused_tokens` | integer or null | required | recorded | reused tokens | +| `last.computed_tokens` | integer or null | required | n/a | computed tokens | +| `last.generated_tokens` | integer or null | required | recorded | generated tokens | +| `last.first_token_seconds` | number or null | required | recorded | first token seconds | +| `last.decode_tokens_per_second` | number or null | required | recorded | decode tokens per second | +| `last.prefill_tokens_per_second` | number or null | required | recorded | prefill tokens per second | +| `last.decode_seconds` | number or null | required | recorded | decode seconds | +| `last.decode_forwards` | integer or null | required | n/a | decode forwards | +| `last.checkpoint_restored` | boolean or null | required | n/a | checkpoint restored | +| `last.experts` | object | optional | n/a | experts | +| `last.experts.demand` | integer | required | n/a | demand | +| `last.experts.hits` | integer | required | n/a | hits | +| `last.experts.loads` | integer | required | n/a | loads | +| `last.experts.bytes` | integer | required | n/a | bytes | +| `last.ended_at` | string | required | recorded | ended at | +| `recent_requests[].time` | string | required | recorded | time | +| `recent_requests[].endpoint` | string | required | recorded | endpoint | +| `recent_requests[].source` | string | required | recorded | source | +| `recent_requests[].http_status` | integer | required | recorded | http status | +| `recent_requests[].stream` | boolean | required | recorded | stream | +| `recent_requests[].total_seconds` | number | required | recorded | total seconds | +| `recent_requests[].stop_reason` | string | optional | recorded | stop reason | +| `recent_requests[].reuse_method` | string or null | optional | n/a | reuse method | +| `recent_requests[].error_stage` | integer | optional | n/a | error stage | +| `recent_requests[].input_tokens` | integer | optional | recorded | input tokens | +| `recent_requests[].reused_tokens` | integer or null | optional | recorded | reused tokens | +| `recent_requests[].computed_tokens` | integer or null | optional | n/a | computed tokens | +| `recent_requests[].generated_tokens` | integer or null | optional | recorded | generated tokens | +| `recent_requests[].first_token_seconds` | number or null | optional | recorded | first token seconds | +| `recent_requests[].decode_tokens_per_second` | number or null | optional | recorded | decode tokens per second | +| `recent_requests[].prefill_tokens_per_second` | number or null | optional | recorded | prefill tokens per second | +| `recent_requests[].decode_seconds` | number or null | optional | recorded | decode seconds | +| `recent_requests[].decode_forwards` | integer or null | optional | n/a | decode forwards | +| `recent_requests[].checkpoint_restored` | boolean or null | optional | n/a | checkpoint restored | +| `recent_requests[].experts` | object | optional | n/a | experts | +| `recent_requests[].experts.demand` | integer | required | n/a | demand | +| `recent_requests[].experts.hits` | integer | required | n/a | hits | +| `recent_requests[].experts.loads` | integer | required | n/a | loads | +| `recent_requests[].experts.bytes` | integer | required | n/a | bytes | +| `recent_requests[].code` | string | optional | recorded | code | +| `recent_requests[].message` | string | optional | recorded | message | +| `recent_requests[].param` | string | optional | recorded | param | +| `totals.requests` | integer | required | recorded | requests | +| `totals.rejected_busy` | integer | required | recorded | rejected busy | +| `totals.rejected_invalid` | integer | required | recorded | rejected invalid | +| `totals.unauthorized` | integer | required | recorded | unauthorized | +| `totals.input_tokens` | integer | required | recorded | input tokens | +| `totals.reused_tokens` | integer | required | recorded | reused tokens | +| `totals.computed_tokens` | integer | required | n/a | computed tokens | +| `totals.generated_tokens` | integer | required | recorded | generated tokens | +| `totals.checkpoint_restores` | integer | required | n/a | checkpoint restores | +| `totals.continued_turns` | integer | required | n/a | continued turns | +| `totals.decode_seconds` | number | required | recorded | decode seconds | +| `totals.first_token_seconds_mean` | number | optional | recorded | first token seconds mean | +| `totals.prefill_tokens_per_second` | number | optional | recorded | prefill tokens per second | +| `totals.decode_tokens_per_second` | number | optional | recorded | decode tokens per second | +| `totals.experts` | object | optional | n/a | experts | +| `totals.experts.demand` | integer | required | n/a | demand | +| `totals.experts.hits` | integer | required | n/a | hits | +| `totals.experts.loads` | integer | required | n/a | loads | +| `totals.experts.bytes` | integer | required | n/a | bytes | +| `memory.ledger` | object | required | n/a | ledger | +| `memory.ledger.limit_bytes` | integer | required | n/a | limit bytes | +| `memory.ledger.current_bytes` | integer | required | n/a | current bytes | +| `memory.ledger.peak_bytes` | integer | required | n/a | peak bytes | +| `memory.ledger.scope` | string | required | n/a | scope | +| `memory.ledger.by_purpose` | object | required | n/a | by purpose | +| `memory.ledger.by_purpose.resident_weights` | integer | required | n/a | resident weights | +| `memory.ledger.by_purpose.experts` | integer | required | n/a | experts | +| `memory.ledger.by_purpose.session_state` | integer | required | n/a | session state | +| `memory.ledger.by_purpose.workspace` | integer | required | n/a | workspace | +| `memory.ledger.by_purpose.transfer` | integer | required | n/a | transfer | +| `memory.ledger.by_purpose.staging` | integer | required | n/a | staging | +| `memory.ledger.by_purpose.metadata` | integer | required | n/a | metadata | +| `memory.ledger.by_device` | object | required | n/a | by device | +| `memory.ledger.by_device.*` | object | optional | n/a | * | +| `memory.ledger.by_device.*.current_bytes` | integer | required | n/a | current bytes | +| `memory.ledger.by_device.*.peak_bytes` | integer | required | n/a | peak bytes | +| `memory.ledger.by_device.*.memory` | string | required | n/a | `system` or `dedicated`; callers aggregate these groups separately | +| `memory.resident` | object | required | n/a | resident | +| `memory.resident.scope` | string | required | n/a | scope | +| `memory.resident.reason` | string | required | n/a | reason | +| `memory.resident.source` | string | required | n/a | source | +| `memory.resident.values` | object or null | required | n/a | values | +| `memory.resident.values.current_bytes` | integer or null | required | n/a | current bytes | +| `memory.resident.values.peak_bytes` | integer or null | required | n/a | peak bytes | +| `memory.resident.values.current_reason` | string | required | n/a | current reason | +| `memory.resident.values.peak_reason` | string | required | n/a | peak reason | +| `memory.physical` | object | required | n/a | physical | +| `memory.physical.scope` | string | required | n/a | scope | +| `memory.physical.reason` | string | required | n/a | reason | +| `memory.physical.source` | string | required | n/a | source | +| `memory.physical.values` | object or null | required | n/a | values | +| `memory.physical.values.current_bytes` | integer or null | required | n/a | current bytes | +| `memory.physical.values.peak_bytes` | integer or null | required | n/a | peak bytes | +| `memory.physical.values.current_reason` | string | required | n/a | current reason | +| `memory.physical.values.peak_reason` | string | required | n/a | peak reason | +| `system.memory` | object | required | mapped | memory | +| `system.memory.total` | object | required | mapped | total | +| `system.memory.total.bytes` | integer or null | required | mapped | bytes | +| `system.memory.total.source` | string | required | mapped | source | +| `system.memory.total.reason` | string | required | mapped | reason | +| `system.memory.available` | object | required | mapped | available | +| `system.memory.available.bytes` | integer or null | required | mapped | bytes | +| `system.memory.available.source` | string | required | mapped | source | +| `system.memory.available.reason` | string | required | mapped | reason | +| `system.app_memory_limit` | object | required | portal | app memory limit | +| `system.app_memory_limit.bytes` | integer or null | required | portal | bytes | +| `system.app_memory_limit.source` | string | required | portal | source | +| `system.app_memory_limit.reason` | string | required | portal | reason | +| `system.thermal` | object | required | portal | thermal | +| `system.thermal.state` | string or null | required | portal | state | +| `system.thermal.source` | string | required | portal | source | +| `system.thermal.reason` | string | required | portal | reason | +| `system.app_memory_limit.current_physical_bytes` | integer or null | required | portal | iOS phys_footprint reading used in the limit calculation; null elsewhere | + +Memory ledger purpose and device totals each equal current bytes. Device peaks are independent and do not sum to the overall peak. Physical and resident process metrics are distinct from ledger reservations and must not be added together. System values carry their actual platform source and an unavailability reason. + +If generation fails before returning a native result, its record retains the known input count and HTTP handling duration; unavailable generation counts are null and do not enter numeric totals. The request itself is still counted. diff --git a/edge0-infer/docs/native-api.md b/edge0-infer/docs/native-api.md index d301b6aa..0db87141 100644 --- a/edge0-infer/docs/native-api.md +++ b/edge0-infer/docs/native-api.md @@ -90,7 +90,7 @@ To make a turn stoppable, create a token with `create_cancel_token(runtime)` and - A token serves one generation; create a new one for each turn. A token that was cancelled before the turn starts ends the turn before any computation, leaving the session unchanged; a cancel requested after `generate` returns has no effect. - `request_cancel` is a single lock-free atomic store and may be called from a signal handler. Keep the token alive during the call, and stop requesting cancels before finishing the runtime. -The OpenAI-compatible server has no separate cancel endpoint: closing the HTTP request cancels the generation ([server guide](openai-compatible-server.md)). +The local server has no separate cancel endpoint: closing the HTTP request cancels the generation ([server guide](local-server.md)). ## Diagnostics diff --git a/edge0-infer/docs/openai-compatible-server.md b/edge0-infer/docs/openai-compatible-server.md deleted file mode 100644 index 77c90202..00000000 --- a/edge0-infer/docs/openai-compatible-server.md +++ /dev/null @@ -1,390 +0,0 @@ -# OpenAI-compatible local server - -Start the server and the example client as in [Getting started](getting-started.md#serve). - -The server loads and warms the model before listening. Default address: -`http://127.0.0.1:8000/v1`. Loopback clients may use any nonempty SDK API key. Non-loopback listeners require -`--api-key-file`, and non-loopback clients must supply its token. There is no CORS, -multimodal input, persistence, hosted tool execution, or constrained decoding. - -`--host`, `--port` (0 selects an ephemeral port), `--alias`, `--context`, -`--prefill-chunk`, `--model-option`, `--max-tokens`, `--no-warmup`, and the CLI -sampling options are supported. Defaults: CPU, port 8000, alias from the model -identifier (`architecture` in the asset `model.json`, such as `edge0_35b`), -`--max-body-bytes 16777216`, `--write-timeout-seconds 30`, and -`--session-idle-seconds 0` (retain indefinitely). Run `serve --help` for syntax. -Application sampling options provide defaults; each request can override them. -If another process is already listening on the same local address and port, a second server fails to start with a binding error instead of sharing that endpoint or choosing another port. - -## Endpoints - -| Method | Path | Result | -| --- | --- | --- | -| GET | `/health` | Readiness | -| GET | `/v1/models` | The single loaded alias | -| GET | `/v1/models/{id}` | Model or 404 | -| POST | `/v1/chat/completions` | Chat completion or SSE chunks | -| POST | `/v1/responses` | Response or typed SSE events | - -Every request includes `model` equal to the alias. Send the complete conversation -on every turn. The native library renders templates, tokenizes, validates tools, -and decides prefix reuse. One inference may run at a time; another receives 429 -immediately. A disconnected client cancels generation. Output writes have a -bounded timeout. SIGINT/SIGTERM stop acceptance, cancel and wait for active work, -then release the session, unload the model, and finish the runtime. - -## Request compatibility - -Unknown fields are rejected with 400, including unknown nested control fields. -Tool parameter schemas are passed through as schema data. Fields marked ignored -are accepted without an effect. `store` never enables persistence. - -| Fields | Chat Completions | Responses | -| --- | --- | --- | -| `model` | Required alias | Required alias | -| `messages` | Nonempty; system, developer (system), user, assistant, tool | Not accepted | -| `content` | String or text parts; assistant may use null | Message string or input_text/output_text parts | -| `instructions`, `input` | Not accepted | Optional system instruction; required string or ordered items | -| `tool_calls`, `tool_call_id` | Assistant function calls; matched tool results | function_call and function_call_output items | -| `reasoning_content` | Assistant history extension | reasoning items with reasoning_text content | -| `tools` | Function wrapper, preserving key order | Flat function definition converted to a wrapper | -| `strict` | false or omitted only | false or omitted only | -| `tool_choice` | auto or none | auto or none | -| `parallel_tool_calls` | Boolean | Boolean | -| `stream` | Boolean | Boolean | -| `stream_options.include_usage` | Optional final usage chunk | Not accepted | -| `temperature`, `top_p` | Native range validation | Native range validation | -| `max_tokens`, `max_completion_tokens` | Output limit including reasoning; must agree if both supplied | Not accepted | -| `max_output_tokens` | Not accepted | Output limit including reasoning | -| `seed` | Unsigned integer | Not accepted | -| `stop` | String or string array | Not accepted | -| `presence_penalty`, `frequency_penalty` | Native range validation | Not accepted | -| `logit_bias` | Integer token ID keys and numeric biases | Not accepted | -| `logprobs` | Boolean | Use include | -| `top_logprobs` | Requires logprobs enabled | Requires include enabled | -| `include` | Not accepted | Only message.output_text.logprobs | -| `n` | Only 1 | Not accepted | -| `response_format` | Only type text | Not accepted | -| `text.format` | Not accepted | Only type text; verbosity rejected | -| `reasoning_effort` | none disables; minimal/low/medium/high enable | Use reasoning.effort | -| `chat_template_kwargs.enable_thinking` | Boolean extension; conflicting controls rejected | Not accepted | -| `reasoning.effort` | Not accepted | Same on/off mapping as Chat | -| `reasoning.summary` | Not accepted | null or auto; no summary is generated | -| `top_k`, `repetition_penalty` | Native extensions | Not accepted | -| `truncation` | Not accepted | Only disabled | -| `background` | Not accepted | Only false; ignored | -| `user`, `metadata`, `store`, `service_tier`, `prompt_cache_key`, `safety_identifier` | Ignored | Ignored; store echoed false | -| `promptCacheKey` | Alias of prompt_cache_key; string; ignored | Same as Chat | -| `previous_response_id`, `conversation` | Rejected | Rejected; send complete history | -| `functions`, `function_call`, `modalities`, `audio`, `prediction`, `web_search_options`, `verbosity` | Rejected | Rejected | -| Message `name`, `refusal`, `audio`; encrypted reasoning; nontext input | Rejected | Rejected | - -Field names are matched exactly. `promptCacheKey` is the only accepted alias; -other camelCase spellings are unknown fields and are rejected. - -All enabled reasoning effort levels mean the same thing. The model default applies -when controls are absent. A model that cannot enable reasoning rejects the request. -Responses coalesces adjacent reasoning, assistant messages and function calls into -one native assistant message. User/system messages and tool results end that group. - -## Output and errors - -Chat emits `chat.completion` or `chat.completion.chunk`. Streaming begins with the -assistant role, then content/reasoning/tool argument deltas. Tool indices are -renumbered contiguously from zero, counting only calls with a begin event. -The finish chunk precedes optional usage and `[DONE]`. `reasoning_content` and -`rejected_tool_calls` are extensions. Rejected calls are never complete tool calls. - -Responses emits ordered reasoning, message, and function_call items. Streams begin -with `response.created` and `response.in_progress`. Each item is added before its -parts/deltas; done events close parts and items. `sequence_number` increases on -every event. The final event contains the complete response. Reasoning has an -empty `summary`; rejected started function calls have incomplete status. - -Usage includes input, generated, cached, reasoning and total token counts from the -library. Logprobs cover answer tokens only, include raw byte arrays, and replace -invalid UTF-8 in their text representation. Pending logprobs accompany the next -answer delta or final content event. - -| Native outcome | Chat finish_reason | Responses status | -| --- | --- | --- | -| Normal end without tools; stop string | stop | completed | -| Normal end with tools; tool_calls | tool_calls | completed | -| Length or capacity | length | incomplete, max_output_tokens | -| Failure or cancellation | Error | failed | - -Errors have `error.message`, `error.type`, `error.param`, and `error.code`. -When an error concerns a specific field, `error.param` holds its path and -`error.message` ends with it, as `: ` (for example -`Unknown field: promptCacheKey` or `Unknown field: messages.name`), so clients -that display only the message still show the field. Errors without a field keep -the plain description. Streaming error objects and `response.failed` use the -same message. - -| Condition | HTTP status | Code | -| --- | --- | --- | -| Invalid/unknown fields or mismatched tool results | 400 | invalid_parameter / invalid_argument / unsupported_parameter | -| Unsupported capability | 400 | unsupported_value | -| Input exceeds capacity | 400 | context_length_exceeded | -| Wrong alias | 404 | model_not_found | -| Body too large | 413 | request_too_large | -| Active inference | 429 | busy | -| Shutting down | 503 | shutting_down | -| Other native failure or invalid session | 500 | Native category | - -After streaming starts, HTTP status cannot change. Chat sends an error object and -closes without `[DONE]`; Responses sends `response.failed`. Nonstreaming failure -returns 500, or 503 during shutdown, without partial output. A failed session is -released and rebuilt before later work; requests already executed are not retried. - -Default stderr logs contain request metadata and timing, not conversation text. -An inference request rejected before generation starts logs one JSON line with -`endpoint`, `http_status`, `code`, and `message`, plus `param` when the error -concerns a field; these match the error body returned to the client. -`--out` explicitly writes per-request tokens and results for development. -`--hold-file` is a development option that pauses after the first nonempty output -while that file exists; cancellation and disconnect detection remain active. - -Compatibility tests target the pinned official Python SDK `openai==3.26.1`, -including both stream accumulators. Missing offline SDK packages are reported as -skipped tests. See [examples](../examples/README.md) for native and HTTP clients. - -## Engine extensions - -Both model endpoints (`GET /v1/models` and `GET /v1/models/{id}`) include `context_length` (the loaded context capacity) and a `capabilities` object in each model entry. `thinking` is whether the loaded model supports enabling thinking; `thinking_default` is its default. Applications can use these booleans to enable and initialize a thinking switch. - -Completed generation output includes a top-level `metrics` object alongside `usage`: in full Chat Completions, the final Chat streaming usage chunk when `stream_options.include_usage` is true, and the final Responses response object (including streaming completion or token-limit termination). - -| Field | Engine source and units | -| --- | --- | -| `first_token_seconds` | Request summary time to first token, in seconds | -| `decode_tokens_per_second` | Request summary decode speed | -| `memory_peak_bytes` | Process physical memory peak, falling back to resident memory peak | -| `memory_peak_metric` | `physical` or `resident`, identifying the available peak | - -Unavailable fields are omitted. Process peaks cover the entire process lifetime, not just this request. The snapshot is taken after generation, from the same runtime. Clients may derive prefill speed as `(prompt_tokens - prompt_tokens_details.cached_tokens) / first_token_seconds` from Chat usage (or the corresponding `input_tokens` and `input_tokens_details` in Responses), when both counts are available, their difference is positive and the time is positive. Generated tokens do not contribute. These are engine measurements, not HTTP or UI timings. Errors, cancellations and Chat streams without requested usage do not include metrics. - -## Status and LAN authentication - -`GET /status` is available during generation and does not take the generation gate. -`--api-key-file ` reads a token, trimming surrounding whitespace. An absent, -unreadable or empty file fails before model loading. A non-loopback listener -requires a token. Every non-loopback request, including health, model and status -queries, must send `Authorization: Bearer `. IPv4 loopback (127/8), IPv6 -loopback and IPv4-mapped loopback clients are exempt, even with an incorrect header. -Tokens and conversation contents are never included in status or request logs. - -Each inference request and every 401 produces one JSON record in stderr and in -`recent_requests`, newest first, capped at 100. Successful health and status queries -are excluded. `last` is the last request that entered generation; rejected requests -do not replace it. Times use UTC ISO 8601 with millisecond precision. `total_seconds` -for generated requests is the native request duration; for rejected requests it is -HTTP handling duration. Native final timing values match the development `--out` -result summary. Counters reset on restart; `instance.id` distinguishes instances. - -Totals sum entered requests, with separate counts for 429, other pre-generation -rejections, and 401. Prefill speed is the sum of `(input_tokens - reused_tokens)` for -requests with first-token timing, divided by their summed first-token times. -Mean first-token time is the arithmetic mean. Decode speed is summed decode -forwards divided by summed decode duration. Zero denominators omit the rate. -Expert counts combine prefill and decode and are omitted when no applicable request -has occurred. Continued turns use keep, truncate or checkpoint restore. - -The table below is the field-name contract. `[]` denotes array elements and `*` -denotes a GGML device name or `host`. Optional fields may be absent; nullable -objects have no children when null. Counts and memory sizes are nonnegative. -Steady-clock timestamps are process-local seconds and must not be compared across -service instances. Final computed counts include decode forwards; live progress -computed counts cover input only. `generated_tokens` in final records excludes end -markers, matching native `generated_count`; live progress counts accepted tokens. - - -| Field | Type | Presence | Meaning | -| --- | --- | --- | --- | -| `instance` | object | required | instance | -| `listen` | object | required | listen | -| `engine` | object | required | engine | -| `model` | object | required | model | -| `totals` | object | required | totals | -| `memory` | object | required | memory | -| `system` | object | required | system | -| `current` | object or null | required | current | -| `last` | object or null | required | last | -| `recent_requests` | array | required | recent requests | -| `recent_requests[]` | object | required | recent requests[] | -| `instance.id` | string | required | id | -| `instance.started_at` | string | required | started at | -| `instance.pid` | integer | required | pid | -| `instance.uptime_seconds` | number | required | uptime seconds | -| `listen.host` | string | required | host | -| `listen.port` | integer | required | port | -| `listen.auth` | boolean | required | auth | -| `engine.version` | string | required | version | -| `engine.source_commit` | string | required | source commit | -| `engine.source_modified` | string | required | source modified | -| `engine.source_reason` | string | required | source reason | -| `engine.ggml_origin` | string | required | ggml origin | -| `engine.ggml_version` | string | required | ggml version | -| `engine.backends` | array | required | backends | -| `engine.backends[]` | object | required | backends[] | -| `engine.devices` | array | required | devices | -| `engine.devices[]` | object | required | devices[] | -| `engine.backends[].name` | string | required | name | -| `engine.backends[].compiled` | string | required | compiled | -| `engine.backends[].initializable` | string | required | initializable | -| `engine.backends[].verified` | string | required | verified | -| `engine.backends[].reason` | string | required | reason | -| `engine.devices[].name` | string | required | name | -| `engine.devices[].backend` | string | required | backend | -| `engine.devices[].description` | string | required | description | -| `engine.devices[].memory_total_bytes` | integer | required | memory total bytes | -| `engine.devices[].memory_free_bytes` | integer | required | memory free bytes | -| `model.alias` | string | required | alias | -| `model.directory` | string | required | directory | -| `model.device` | string | required | device | -| `model.prefill_device` | string | required | prefill device | -| `model.context_length` | integer | required | context length | -| `current.endpoint` | string | required | endpoint | -| `current.source` | string | required | source | -| `current.phase` | string | required | phase | -| `current.started_steady_seconds` | number | required | started steady seconds | -| `current.elapsed_seconds` | number | required | elapsed seconds | -| `current.input_tokens` | integer | required | input tokens | -| `current.reused_tokens` | integer | required | reused tokens | -| `current.computed_tokens` | integer | required | computed tokens | -| `current.pending_tokens` | integer | required | pending tokens | -| `current.generated_tokens` | integer | required | generated tokens | -| `current.first_token_seconds` | number | optional | first token seconds | -| `current.decode_elapsed_seconds` | number | optional | decode elapsed seconds | -| `current.prefill_tokens_per_second` | number or null | optional | prefill tokens per second | -| `current.decode_tokens_per_second` | number or null | optional | decode tokens per second | -| `last.time` | string | required | time | -| `last.endpoint` | string | required | endpoint | -| `last.source` | string | required | source | -| `last.http_status` | integer | required | http status | -| `last.stream` | boolean | required | stream | -| `last.total_seconds` | number | required | total seconds | -| `last.stop_reason` | string | required | stop reason | -| `last.reuse_method` | string or null | required | reuse method | -| `last.error_stage` | integer | required | error stage | -| `last.input_tokens` | integer | required | input tokens | -| `last.reused_tokens` | integer or null | required | reused tokens | -| `last.computed_tokens` | integer or null | required | computed tokens | -| `last.generated_tokens` | integer or null | required | generated tokens | -| `last.first_token_seconds` | number or null | required | first token seconds | -| `last.decode_tokens_per_second` | number or null | required | decode tokens per second | -| `last.prefill_tokens_per_second` | number or null | required | prefill tokens per second | -| `last.decode_seconds` | number or null | required | decode seconds | -| `last.decode_forwards` | integer or null | required | decode forwards | -| `last.checkpoint_restored` | boolean or null | required | checkpoint restored | -| `last.experts` | object | optional | experts | -| `last.experts.demand` | integer | required | demand | -| `last.experts.hits` | integer | required | hits | -| `last.experts.loads` | integer | required | loads | -| `last.experts.bytes` | integer | required | bytes | -| `last.ended_at` | string | required | ended at | -| `recent_requests[].time` | string | required | time | -| `recent_requests[].endpoint` | string | required | endpoint | -| `recent_requests[].source` | string | required | source | -| `recent_requests[].http_status` | integer | required | http status | -| `recent_requests[].stream` | boolean | required | stream | -| `recent_requests[].total_seconds` | number | required | total seconds | -| `recent_requests[].stop_reason` | string | optional | stop reason | -| `recent_requests[].reuse_method` | string or null | optional | reuse method | -| `recent_requests[].error_stage` | integer | optional | error stage | -| `recent_requests[].input_tokens` | integer | optional | input tokens | -| `recent_requests[].reused_tokens` | integer or null | optional | reused tokens | -| `recent_requests[].computed_tokens` | integer or null | optional | computed tokens | -| `recent_requests[].generated_tokens` | integer or null | optional | generated tokens | -| `recent_requests[].first_token_seconds` | number or null | optional | first token seconds | -| `recent_requests[].decode_tokens_per_second` | number or null | optional | decode tokens per second | -| `recent_requests[].prefill_tokens_per_second` | number or null | optional | prefill tokens per second | -| `recent_requests[].decode_seconds` | number or null | optional | decode seconds | -| `recent_requests[].decode_forwards` | integer or null | optional | decode forwards | -| `recent_requests[].checkpoint_restored` | boolean or null | optional | checkpoint restored | -| `recent_requests[].experts` | object | optional | experts | -| `recent_requests[].experts.demand` | integer | required | demand | -| `recent_requests[].experts.hits` | integer | required | hits | -| `recent_requests[].experts.loads` | integer | required | loads | -| `recent_requests[].experts.bytes` | integer | required | bytes | -| `recent_requests[].code` | string | optional | code | -| `recent_requests[].message` | string | optional | message | -| `recent_requests[].param` | string | optional | param | -| `totals.requests` | integer | required | requests | -| `totals.rejected_busy` | integer | required | rejected busy | -| `totals.rejected_invalid` | integer | required | rejected invalid | -| `totals.unauthorized` | integer | required | unauthorized | -| `totals.input_tokens` | integer | required | input tokens | -| `totals.reused_tokens` | integer | required | reused tokens | -| `totals.computed_tokens` | integer | required | computed tokens | -| `totals.generated_tokens` | integer | required | generated tokens | -| `totals.checkpoint_restores` | integer | required | checkpoint restores | -| `totals.continued_turns` | integer | required | continued turns | -| `totals.decode_seconds` | number | required | decode seconds | -| `totals.first_token_seconds_mean` | number | optional | first token seconds mean | -| `totals.prefill_tokens_per_second` | number | optional | prefill tokens per second | -| `totals.decode_tokens_per_second` | number | optional | decode tokens per second | -| `totals.experts` | object | optional | experts | -| `totals.experts.demand` | integer | required | demand | -| `totals.experts.hits` | integer | required | hits | -| `totals.experts.loads` | integer | required | loads | -| `totals.experts.bytes` | integer | required | bytes | -| `memory.ledger` | object | required | ledger | -| `memory.ledger.limit_bytes` | integer | required | limit bytes | -| `memory.ledger.current_bytes` | integer | required | current bytes | -| `memory.ledger.peak_bytes` | integer | required | peak bytes | -| `memory.ledger.scope` | string | required | scope | -| `memory.ledger.by_purpose` | object | required | by purpose | -| `memory.ledger.by_purpose.resident_weights` | integer | required | resident weights | -| `memory.ledger.by_purpose.experts` | integer | required | experts | -| `memory.ledger.by_purpose.session_state` | integer | required | session state | -| `memory.ledger.by_purpose.workspace` | integer | required | workspace | -| `memory.ledger.by_purpose.transfer` | integer | required | transfer | -| `memory.ledger.by_purpose.staging` | integer | required | staging | -| `memory.ledger.by_purpose.metadata` | integer | required | metadata | -| `memory.ledger.by_device` | object | required | by device | -| `memory.ledger.by_device.*` | object | optional | * | -| `memory.ledger.by_device.*.current_bytes` | integer | required | current bytes | -| `memory.ledger.by_device.*.peak_bytes` | integer | required | peak bytes | -| `memory.ledger.by_device.*.memory` | string | required | `system` or `dedicated`; callers aggregate these groups separately | -| `memory.resident` | object | required | resident | -| `memory.resident.scope` | string | required | scope | -| `memory.resident.reason` | string | required | reason | -| `memory.resident.source` | string | required | source | -| `memory.resident.values` | object or null | required | values | -| `memory.resident.values.current_bytes` | integer or null | required | current bytes | -| `memory.resident.values.peak_bytes` | integer or null | required | peak bytes | -| `memory.resident.values.current_reason` | string | required | current reason | -| `memory.resident.values.peak_reason` | string | required | peak reason | -| `memory.physical` | object | required | physical | -| `memory.physical.scope` | string | required | scope | -| `memory.physical.reason` | string | required | reason | -| `memory.physical.source` | string | required | source | -| `memory.physical.values` | object or null | required | values | -| `memory.physical.values.current_bytes` | integer or null | required | current bytes | -| `memory.physical.values.peak_bytes` | integer or null | required | peak bytes | -| `memory.physical.values.current_reason` | string | required | current reason | -| `memory.physical.values.peak_reason` | string | required | peak reason | -| `system.memory` | object | required | memory | -| `system.memory.total` | object | required | total | -| `system.memory.total.bytes` | integer or null | required | bytes | -| `system.memory.total.source` | string | required | source | -| `system.memory.total.reason` | string | required | reason | -| `system.memory.available` | object | required | available | -| `system.memory.available.bytes` | integer or null | required | bytes | -| `system.memory.available.source` | string | required | source | -| `system.memory.available.reason` | string | required | reason | -| `system.app_memory_limit` | object | required | app memory limit | -| `system.app_memory_limit.bytes` | integer or null | required | bytes | -| `system.app_memory_limit.source` | string | required | source | -| `system.app_memory_limit.reason` | string | required | reason | -| `system.thermal` | object | required | thermal | -| `system.thermal.state` | string or null | required | state | -| `system.thermal.source` | string | required | source | -| `system.thermal.reason` | string | required | reason | -| `system.app_memory_limit.current_physical_bytes` | integer or null | required | iOS phys_footprint reading used in the limit calculation; null elsewhere | - - -Memory ledger purpose and device totals each equal current bytes. Device peaks are independent and do not sum to the overall peak. Physical and resident process metrics are distinct from ledger reservations and must not be added together. System values carry their actual platform source and an unavailability reason. - -If generation fails before returning a native result, its record retains the known input count and HTTP handling duration; unavailable generation counts are null and do not enter numeric totals. The request itself is still counted. diff --git a/edge0-infer/docs/testing.md b/edge0-infer/docs/testing.md index acec0487..b831f671 100644 --- a/edge0-infer/docs/testing.md +++ b/edge0-infer/docs/testing.md @@ -23,7 +23,7 @@ The tests are CTest tests. Each one carries a level label, `L0` to `L4`, ordered ``` 2. For `L2`, generate the tiny fixtures as described in the [asset tools guide](../tools/assets/README.md#tiny-fixtures) and export their directory as `MOE_TINY`. This needs only three small text files of the 35B source, not the full weights; keep `MOE_SOURCE_35B` exported: the asset tools' fixture test needs it and is otherwise skipped. -3. For `L3` and `L4`, convert the 35B as described in [Getting started](getting-started.md#get-the-model-assets) and export `MOE_ASSETS_35B`. +3. For `L3` and `L4`, convert the 35B as described in [Getting started](getting-started.md#get-a-model) and export `MOE_ASSETS_35B`. ## Running diff --git a/edge0-infer/examples/README.md b/edge0-infer/examples/README.md index 905a5726..f89d9110 100644 --- a/edge0-infer/examples/README.md +++ b/edge0-infer/examples/README.md @@ -27,4 +27,4 @@ Start `build/cpu/bin/edge0-infer serve --model "$MOE_ASSETS_35B"`, then run: uv run --no-project --with openai==3.26.1 python examples/openai_client.py --model edge0_35b ``` -See the [server guide](../docs/openai-compatible-server.md) for supported fields. +See the [server guide](../docs/local-server.md) for supported fields. diff --git a/edge0-infer/examples/android/app/build.gradle.kts b/edge0-infer/examples/android/app/build.gradle.kts index 1d0ec601..aea10782 100644 --- a/edge0-infer/examples/android/app/build.gradle.kts +++ b/edge0-infer/examples/android/app/build.gradle.kts @@ -16,7 +16,7 @@ android { minSdk = 28 targetSdk = 36 versionCode = 1 - versionName = "1.0" + versionName = "0.2.0" ndk { abiFilters += "arm64-v8a" } externalNativeBuild { cmake { arguments += "-DCMAKE_BUILD_TYPE=Release" } } } diff --git a/edge0-infer/examples/desktop/README.md b/edge0-infer/examples/desktop/README.md index 65671a8b..a300a3d8 100644 --- a/edge0-infer/examples/desktop/README.md +++ b/edge0-infer/examples/desktop/README.md @@ -99,7 +99,11 @@ identifier/directory labels and unavailable entries, server-port validation, Stopping/Stopped transitions, CSS spinner markup, 429 backoff and cancellation, UTF-8, incomplete streams, normal HTTP request bodies without generation overrides, page-header actions, bounded scrolling layouts, all four editable/locked On/Off -switch states, and default/override application identifiers. +switch states, and default/override application identifiers. For a delegated model +they render the Status page from a `/status` that model returned (null commit and +backends) and its current-request card while streaming (prefill, decode with the +forwarded fragment count) and while a non-streaming request runs (In progress only), +never showing `null`, and read the metrics line from the final chunk's `timings`. Rust tests compile a small fixture executable with `rustc`. It does not load the engine or use a port. Tests cover readiness parsing, argument boundaries, settings, @@ -108,7 +112,8 @@ loading cancellation before child publication, a newer start replacing a pending start, recovery serialized with startup, preservation of replaced PID records, terminal shutdown rejecting queued starts, automatic/explicit ports, compatible legacy settings, inspection identifiers, conversation and totals storage, LAN arguments, token -format and file permissions. Each test owns its temporary +format and file permissions, and `EDGE0_HOME` holding the settings, archive and PID +record (unset or blank keeps the platform directories). Each test owns its temporary directory and child processes. A Mac can additionally check the Windows Rust source. Install Homebrew's `llvm` @@ -136,7 +141,7 @@ Windows checklist below. App tests are independent of the engine's ctest suite. ## Assets and first launch -Prepare assets using the [asset guide](../../docs/getting-started.md#get-the-model-assets). +Prepare assets using the [asset guide](../../docs/getting-started.md#get-a-model). Build `edge0-infer` using the [build guide](../../docs/getting-started.md). On Choose a model, expand Other location… to enter the full executable path and one asset directory per line. This section starts collapsed, including on @@ -159,6 +164,10 @@ The default device is `metal` on macOS and `cpu` on Windows. Settings also allow `vulkan` and an optional prefill device; the selected executable must support the chosen backend. Model loading can take several minutes. Cancel stops a pending load. Changing model or saving settings ends the old service and clears history. +A model that `inspect --json` reports as `delegated` is started without the device +settings: it runs on the GPU its external engine uses. Its `edge0-infer` finds edge0-strata through +`EDGE0_STRATA_DIR` in the app's environment or an `edge0-strata/` directory beside an ancestor of the +selected executable. Server port is optional: leave it blank for an automatically assigned port, or enter an integer from 1 to 65535. Invalid input cannot be saved. The app passes @@ -178,7 +187,7 @@ Choose a model header with loaded-only Back to chat and Refresh actions. Normal chat requests omit `max_tokens`, leaving the output limit to the engine. Requests explicitly include only model, complete messages, streaming, final usage, the selected thinking flag and the temperature of the generation style; other -sampling parameters keep the model's recommended values. Service startup passes only model, device, port and optional prefill device; +sampling parameters keep the model's recommended values. Service startup passes only model, device, port and optional prefill device (no devices for a delegated model); it does not override generation settings. The four empty-chat suggestions, and the system prompt used when the setting is @@ -213,6 +222,26 @@ and Service pages comes from the engine. Math is shown verbatim as code within Markdown; it is not typeset. Model errors and unexpected service exit are shown above the conversation; inspect the error before retrying. +### Environment variables + +None of these is needed for normal use; when unset, the App behaves as described above. + +| Variable | Effect | +| --- | --- | +| `EDGE0_HOME` | A directory that holds all of the App's state instead of the per-user directories: settings, API token, conversation archive, totals and `serve-pid.json`. Use it when the per-user directories are not writable for the App (a locked-down profile, a sandboxed launcher) or for a portable copy. Unset or blank means the per-user directories. | +| `EDGE0_DEBUG_LOG` | A file the App appends one line to for each step of loading a model (inspection, start, process attachment, PID record) and its outcome, so a failing load shows which step failed. | +| `EDGE0_AUTOLOAD` | `1` loads the first configured asset directory about three seconds after start-up, without using the UI; with `EDGE0_DEBUG_LOG` the load can be followed unattended. | + +On Windows the App puts the service it starts into a Job Object so that the service +ends when the App is ended outright. When the App itself runs inside a job that does not +allow this (some launchers, agent shells and CI runners), loading continues without the +Job Object. Stopping, switching models and quitting the App normally still stop the +service (through the stored PID, CTRL_BREAK and forced termination), but if the App is +then ended outright, for example from Task Manager, the service can keep running until +the App is started again or it is ended by hand. When the App cannot create its hidden +console, loading also continues; CTRL_BREAK then cannot reach the service, and it is +stopped by the forced termination that follows the 15-second wait. + ## Pages and settings The sidebar holds New chat, Chat, Models, Status, Service, Settings and the @@ -320,6 +349,9 @@ A checkmark below is a procedure, not a claim that it has passed on this machine group; the host sends CTRL_BREAK before the 15-second forced-termination fallback. 2. Start again and end the App process in Task Manager. Its Job Object has `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`; confirm the service immediately exits too. + Start the App from Explorer or a plain console for this check: started inside + a job that forbids nesting, the App has no Job Object (see + [Environment variables](#environment-variables)). 3. Repeat after switching models and after cancelling a pending load. Confirm the App remains usable and no service is left over. The GUI host creates a hidden console when it has none, so its child group can receive CTRL_BREAK without signalling the host. diff --git a/edge0-infer/examples/desktop/package-lock.json b/edge0-infer/examples/desktop/package-lock.json index b0c083a6..c1a0bf3e 100644 --- a/edge0-infer/examples/desktop/package-lock.json +++ b/edge0-infer/examples/desktop/package-lock.json @@ -1,12 +1,12 @@ { "name": "edge0-chat-desktop", - "version": "0.1.0", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "edge0-chat-desktop", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@tauri-apps/api": "2.12.4", "@tauri-apps/plugin-http": "2.8.3", diff --git a/edge0-infer/examples/desktop/package.json b/edge0-infer/examples/desktop/package.json index 1c24536f..2763d923 100644 --- a/edge0-infer/examples/desktop/package.json +++ b/edge0-infer/examples/desktop/package.json @@ -1,6 +1,6 @@ { "name": "edge0-chat-desktop", - "version": "0.1.0", + "version": "0.2.0", "private": true, "type": "module", "scripts": { diff --git a/edge0-infer/examples/desktop/src-tauri/Cargo.lock b/edge0-infer/examples/desktop/src-tauri/Cargo.lock index 6329ee36..927f8dab 100644 --- a/edge0-infer/examples/desktop/src-tauri/Cargo.lock +++ b/edge0-infer/examples/desktop/src-tauri/Cargo.lock @@ -702,7 +702,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "edge0-chat-desktop" -version = "0.1.0" +version = "0.2.0" dependencies = [ "libc", "serde", diff --git a/edge0-infer/examples/desktop/src-tauri/Cargo.toml b/edge0-infer/examples/desktop/src-tauri/Cargo.toml index 40e4c9f4..987ba912 100644 --- a/edge0-infer/examples/desktop/src-tauri/Cargo.toml +++ b/edge0-infer/examples/desktop/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "edge0-chat-desktop" -version = "0.1.0" +version = "0.2.0" edition = "2021" rust-version = "1.90" diff --git a/edge0-infer/examples/desktop/src-tauri/src/lib.rs b/edge0-infer/examples/desktop/src-tauri/src/lib.rs index 0c3db345..f24b9f8b 100644 --- a/edge0-infer/examples/desktop/src-tauri/src/lib.rs +++ b/edge0-infer/examples/desktop/src-tauri/src/lib.rs @@ -135,14 +135,55 @@ async fn service_status(state: tauri::State<'_, Arc>) -> Result( + home: Option, + data: impl FnOnce() -> Result, + config: impl FnOnce() -> Result, +) -> Result<(std::path::PathBuf, std::path::PathBuf), E> { + match home { + Some(path) if !path.to_string_lossy().trim().is_empty() => { + let directory = std::path::PathBuf::from(path); + Ok((directory.clone(), directory)) + } + _ => Ok((data()?, config()?)), + } +} pub fn run() { let app = tauri::Builder::default() .plugin(tauri_plugin_http::init()) .setup(|app| { - let config = app.path().app_config_dir()?; + let (home, config) = state_directories( + std::env::var_os("EDGE0_HOME"), + || app.path().app_data_dir(), + || app.path().app_config_dir(), + )?; + std::fs::create_dir_all(&home)?; + std::fs::create_dir_all(&config)?; let manager = Arc::new(Manager::new(config.clone())?); + // EDGE0_AUTOLOAD=1 loads the first configured asset directory shortly after start-up, without the UI, + // so a load can be run unattended; with EDGE0_DEBUG_LOG set, each step of it is written to that file. + if std::env::var("EDGE0_AUTOLOAD").as_deref() == Ok("1") { + let auto = manager.clone(); + std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_secs(3)); + match auto.settings() { + Ok(settings) => match settings.assets.first().cloned() { + Some(model) => match auto.start(&settings, &model) { + Ok(_) => service::trace_line("autoload: started"), + Err(e) => service::trace_line(&format!("autoload failed: {e}")), + }, + None => service::trace_line("autoload: no asset configured"), + }, + Err(e) => service::trace_line(&format!("autoload: settings failed: {e}")), + } + }); + } app.manage(manager); - app.manage(storage::Store::new(config, app.path().app_data_dir()?)); + app.manage(storage::Store::new(config, home)); Ok(()) }) .invoke_handler(tauri::generate_handler![ @@ -196,3 +237,55 @@ pub fn run() { } }); } +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + #[test] + fn edge0_home_holds_settings_archive_and_pid_record() { + let home = std::env::temp_dir().join(format!("edge0-home-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&home); + let platform = |name: &str| -> Result { + Err(format!("{name} must not be asked for")) + }; + let (data, config) = state_directories( + Some(home.clone().into_os_string()), + || platform("data"), + || platform("config"), + ) + .unwrap(); + assert_eq!((data.clone(), config.clone()), (home.clone(), home.clone())); + let manager = Manager::new(config.clone()).unwrap(); + manager.save_settings(&Settings::default()).unwrap(); + assert!(home.join("settings.json").is_file()); + assert_eq!(manager.record_path(), home.join("serve-pid.json")); + let store = storage::Store::new(config, data); + let conversation = serde_json::json!({"id": "c1", "createdAt": 1, "updatedAt": 2, "systemPrompt": "fixed", + "thinking": true, "messages": [{"role": "user", "content": "hi"}]}); + store.save("c1", &conversation).unwrap(); + assert!(home.join("conversations").join("c1.json").is_file()); + std::fs::remove_dir_all(&home).unwrap(); + } + #[test] + fn without_edge0_home_the_platform_directories_are_used() { + let platform = ( + PathBuf::from("platform-data"), + PathBuf::from("platform-config"), + ); + for home in [None, Some(""), Some(" ")] { + let chosen = state_directories::( + home.map(std::ffi::OsString::from), + || Ok(platform.0.clone()), + || Ok(platform.1.clone()), + ) + .unwrap(); + assert_eq!(chosen, platform, "{home:?}"); + } + let failed = state_directories::( + None, + || Err("no data directory".into()), + || Ok(PathBuf::new()), + ); + assert_eq!(failed, Err("no data directory".into())); + } +} diff --git a/edge0-infer/examples/desktop/src-tauri/src/platform/windows.rs b/edge0-infer/examples/desktop/src-tauri/src/platform/windows.rs index 74af0e34..f5280cbd 100644 --- a/edge0-infer/examples/desktop/src-tauri/src/platform/windows.rs +++ b/edge0-infer/examples/desktop/src-tauri/src/platform/windows.rs @@ -22,13 +22,17 @@ use windows_sys::Win32::{ }, UI::WindowsAndMessaging::{ShowWindow, SW_HIDE}, }; -pub struct Lifetime(HANDLE); +/// The job the engine is kept in, when one could be made. `None` means the app could not put the engine into a job +/// (see `attach`) and tracks it by pid instead, which is what `serve-pid.json`, `graceful` and `force` already do. +pub struct Lifetime(Option); // The handle is owned, is never aliased, and is only closed when its owner is dropped. unsafe impl Send for Lifetime {} impl Drop for Lifetime { fn drop(&mut self) { - unsafe { - CloseHandle(self.0); + if let Some(handle) = self.0 { + unsafe { + CloseHandle(handle); + } } } } @@ -41,10 +45,9 @@ pub fn configure(command: &mut Command) -> Result<(), String> { // A release GUI has no console. Create a hidden host console so the // child can inherit it while becoming a distinct, signallable group. // CREATE_NEW_CONSOLE would cause CREATE_NEW_PROCESS_GROUP to be ignored. - if GetConsoleProcessList(processes.as_mut_ptr(), 1) == 0 { - if AllocConsole() == 0 { - return Err(std::io::Error::last_os_error().to_string()); - } + // Best effort: without a console `graceful` has nothing to signal and the stop path falls back to + // TerminateProcess, so a host that refuses the console (a confined launcher) must not fail the load. + if GetConsoleProcessList(processes.as_mut_ptr(), 1) == 0 && AllocConsole() != 0 { ShowWindow(GetConsoleWindow(), SW_HIDE); } } @@ -52,25 +55,31 @@ pub fn configure(command: &mut Command) -> Result<(), String> { Ok(()) } pub fn attach(child: &Child) -> Result { + // Best effort. A job gives the engine "killed when the app dies" for free, but the app can only put a process + // into one when the process is not already inside a job that forbids nesting: AssignProcessToJobObject then + // fails with ERROR_ACCESS_DENIED (os error 5), which is what a launcher that itself runs processes in a job + // produces - an agent/sandbox shell, a CI runner, or any tool that starts this app inside its own job. That is + // not a reason to refuse to load the model: stopping the engine does not depend on the job (serve-pid.json + // records the pid, and `graceful`/`force` act on it), so the only thing given up is the automatic kill when the + // app is killed outright. unsafe { let handle = CreateJobObjectW(std::ptr::null(), std::ptr::null()); if handle.is_null() { - return Err(std::io::Error::last_os_error().to_string()); + return Ok(Lifetime(None)); } - let job = Lifetime(handle); let mut info: JOBOBJECT_EXTENDED_LIMIT_INFORMATION = std::mem::zeroed(); info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - if SetInformationJobObject( + let configured = SetInformationJobObject( handle, JobObjectExtendedLimitInformation, (&info as *const JOBOBJECT_EXTENDED_LIMIT_INFORMATION).cast(), std::mem::size_of_val(&info) as u32, - ) == 0 - || AssignProcessToJobObject(handle, child.as_raw_handle().cast()) == 0 - { - return Err(std::io::Error::last_os_error().to_string()); + ) != 0; + if !configured || AssignProcessToJobObject(handle, child.as_raw_handle().cast()) == 0 { + CloseHandle(handle); + return Ok(Lifetime(None)); } - Ok(job) + Ok(Lifetime(Some(handle))) } } pub fn graceful(pid: u32) -> Result<(), String> { diff --git a/edge0-infer/examples/desktop/src-tauri/src/service.rs b/edge0-infer/examples/desktop/src-tauri/src/service.rs index 05e737a0..2455581e 100644 --- a/edge0-infer/examples/desktop/src-tauri/src/service.rs +++ b/edge0-infer/examples/desktop/src-tauri/src/service.rs @@ -1,9 +1,9 @@ use crate::platform; use serde::{Deserialize, Serialize}; use std::{ - collections::{BTreeSet, VecDeque}, + collections::{BTreeMap, BTreeSet, VecDeque}, fs, - io::{BufRead, BufReader}, + io::{BufRead, BufReader, Write}, path::{Path, PathBuf}, process::{Child, Command, Stdio}, sync::{ @@ -13,6 +13,23 @@ use std::{ thread, time::{Duration, Instant}, }; + +/// When `EDGE0_DEBUG_LOG` names a file, every step of `Manager::start` and its outcome is appended there, so a load +/// that fails (for example with ERROR_ACCESS_DENIED, os error 5, where a directory is not writable) shows which call +/// failed. Silent when the variable is unset, so the default behaviour is unchanged. +fn trace(message: &str) { + let Ok(path) = std::env::var("EDGE0_DEBUG_LOG") else { + return; + }; + if let Ok(mut file) = fs::OpenOptions::new().create(true).append(true).open(&path) { + let _ = writeln!(file, "{message}"); + } +} + +/// `trace` for the rest of the crate (the autoload probe in lib.rs). +pub fn trace_line(message: &str) { + trace(message); +} #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(default)] pub struct Settings { @@ -107,6 +124,8 @@ pub struct Manager { epoch: AtomicU64, closing: AtomicBool, start_lock: Mutex<()>, + // Whether `inspect --json` reported each model directory as delegated (run by an external engine). + delegated: Mutex>, #[cfg(test)] publication_gate: Mutex>>, #[cfg(test)] @@ -118,20 +137,31 @@ struct PublicationGate { resume: std::sync::Barrier, } pub fn arguments(settings: &Settings, model: &str) -> Vec { - let mut args = vec![ - "serve".into(), - "--model".into(), - model.into(), - "--device".into(), - settings.device.clone(), - "--port".into(), - settings.port.unwrap_or(0).to_string(), - ]; - if !settings.prefill_device.is_empty() { + arguments_for(settings, model, false) +} +// A delegated model runs on the device its engine registers, so the device settings are not passed. +pub fn arguments_for(settings: &Settings, model: &str, delegated: bool) -> Vec { + let mut args = vec!["serve".into(), "--model".into(), model.into()]; + if !delegated { + args.extend(["--device".into(), settings.device.clone()]); + } + args.extend(["--port".into(), settings.port.unwrap_or(0).to_string()]); + if !delegated && !settings.prefill_device.is_empty() { args.extend(["--prefill-device".into(), settings.prefill_device.clone()]); } args } +fn is_delegated(report: Option<&serde_json::Value>) -> bool { + report + .and_then(|report| report.get("delegated")) + .and_then(serde_json::Value::as_bool) + == Some(true) +} +fn canonical(path: &str) -> PathBuf { + Path::new(path) + .canonicalize() + .unwrap_or_else(|_| PathBuf::from(path)) +} pub fn ready_url(line: &str) -> Option { let rest = line .strip_prefix("Listening: http://127.0.0.1:") @@ -146,8 +176,13 @@ pub fn ready_url(line: &str) -> Option { } Some(format!("http://127.0.0.1:{port}")) } -fn network_arguments(settings: &Settings, model: &str, token: &Path) -> Vec { - let mut args = arguments(settings, model); +fn network_arguments( + settings: &Settings, + model: &str, + token: &Path, + delegated: bool, +) -> Vec { + let mut args = arguments_for(settings, model, delegated); if settings.lan_access { args.extend([ "--host".into(), @@ -189,13 +224,14 @@ impl Manager { epoch: AtomicU64::new(0), closing: AtomicBool::new(false), start_lock: Mutex::new(()), + delegated: Mutex::new(BTreeMap::new()), #[cfg(test)] publication_gate: Mutex::new(None), #[cfg(test)] recovery_gate: Mutex::new(None), }) } - fn record_path(&self) -> PathBuf { + pub(crate) fn record_path(&self) -> PathBuf { self.directory.join("serve-pid.json") } pub fn settings(&self) -> Result { @@ -322,6 +358,9 @@ impl Manager { ) }; let report = serde_json::from_slice::(&output.stdout).ok(); + if let Ok(mut delegated) = self.delegated.lock() { + delegated.insert(canonical(path), is_delegated(report.as_ref())); + } let total_bytes = report.as_ref().and_then(|report| { report .get("files")? @@ -367,10 +406,17 @@ impl Manager { .collect() } pub fn start(&self, settings: &Settings, model: &str) -> Result { + trace(&format!( + "start: model={model} executable={} assets={:?} device={} port={:?}", + settings.executable, settings.assets, settings.device, settings.port + )); if self.closing.load(Ordering::SeqCst) { return Err("The application is shutting down".into()); } - settings.validate()?; + settings.validate().map_err(|e| { + trace(&format!("validate failed: {e}")); + e + })?; if !settings.assets.iter().any(|p| { p == model || Path::new(model) @@ -380,41 +426,89 @@ impl Manager { .zip(Path::new(p).canonicalize().ok()) .is_some_and(|(parent, root)| parent == root) }) { + trace("asset check failed: the model is not under a configured asset directory"); return Err("Select a configured asset directory".into()); } let executable = Path::new(&settings.executable) .canonicalize() - .map_err(|e| e.to_string())?; + .map_err(|e| { + trace(&format!("canonicalize executable failed: {e}")); + e.to_string() + })?; + // Decided before any lock is taken: an unknown directory is inspected once here. + let known = self + .delegated + .lock() + .map_err(|e| e.to_string())? + .get(&canonical(model)) + .copied(); + let delegated = match known { + Some(value) => value, + None => { + trace(&format!("running inspect: {}", executable.display())); + let output = Command::new(&executable) + .args(["inspect", "--model", model, "--json"]) + .output() + .map_err(|e| { + trace(&format!("inspect spawn failed: {e}")); + format!("Cannot run edge0-infer: {e}") + })?; + trace(&format!("inspect exited with {:?}", output.status.code())); + is_delegated(serde_json::from_slice(&output.stdout).ok().as_ref()) + } + }; + trace(&format!("delegated={delegated}")); let epoch = self.epoch.fetch_add(1, Ordering::SeqCst) + 1; - self.recover()?; + self.recover().map_err(|e| { + trace(&format!("recover failed: {e}")); + e + })?; let _starting = self.start_lock.lock().map_err(|e| e.to_string())?; if self.epoch.load(Ordering::SeqCst) != epoch || self.closing.load(Ordering::SeqCst) { return Err("Model loading cancelled".into()); } - self.stop_process(None)?; + self.stop_process(None).map_err(|e| { + trace(&format!("stop_process failed: {e}")); + e + })?; let mut publication = self.process.lock().map_err(|e| e.to_string())?; if self.epoch.load(Ordering::SeqCst) != epoch || self.closing.load(Ordering::SeqCst) { return Err("Model loading cancelled".into()); } let mut command = Command::new(&executable); - let args = network_arguments(settings, model, &self.directory.join("api-token")); + let args = network_arguments( + settings, + model, + &self.directory.join("api-token"), + delegated, + ); if settings.lan_access { if crate::storage::read_token(&self.directory)?.is_none() { return Err("Enable network access with an API token first".into()); } } + trace(&format!("spawning: {} {:?}", executable.display(), args)); command .args(args) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::piped()); - platform::configure(&mut command)?; - let mut child = command - .spawn() - .map_err(|e| format!("Cannot start edge0-infer: {e}"))?; + platform::configure(&mut command).map_err(|e| { + trace(&format!("platform::configure failed: {e}")); + e + })?; + let mut child = command.spawn().map_err(|e| { + trace(&format!("spawn failed: {e}")); + format!("Cannot start edge0-infer: {e}") + })?; + trace(&format!("spawned pid {}", child.id())); let lifetime = match platform::attach(&child) { - Ok(v) => v, + Ok(v) => { + trace("attach: ok (job or pid-only fallback)"); + v + } Err(e) => { + trace(&format!("attach failed: {e}")); let _ = child.kill(); let _ = child.wait(); return Err(e); @@ -423,10 +517,15 @@ impl Manager { let pid = child.id(); let record = PidRecord { pid, executable }; if let Err(e) = write_json(&self.record_path(), &record) { + trace(&format!( + "write_json({}) failed: {e}", + self.record_path().display() + )); let _ = child.kill(); let _ = child.wait(); return Err(e); } + trace("pid record written"); let stderr = match child.stderr.take() { Some(stderr) => stderr, None => { @@ -617,6 +716,7 @@ mod tests { &settings, "model with spaces", Path::new("config with spaces/api-token"), + false, ); assert_eq!( &args[args.len() - 4..], @@ -629,7 +729,8 @@ mod tests { ); settings.lan_access = false; assert!( - !network_arguments(&settings, "model", Path::new("unused")).contains(&"--host".into()) + !network_arguments(&settings, "model", Path::new("unused"), false) + .contains(&"--host".into()) ); } #[test] @@ -810,6 +911,73 @@ mod tests { .start(&fixture.settings, &fixture.directory.to_string_lossy()) .is_err()); } + fn check_delegated_directory(model_name: &str, identifier: &str) { + let mut fixture = Fixture::new(); + let root = fixture.directory.join("models"); + let big = root.join("big"); + for folder in ["mtp", "tokenizer"] { + fs::create_dir_all(big.join(folder)).unwrap(); + } + fs::write( + big.join("model.json"), + format!( + r#"{{"model_name":"{model_name}","args":["ignored"],"env":{{"ignored":"value"}}}}"# + ), + ) + .unwrap(); + let native = root.join("native"); + fs::create_dir_all(&native).unwrap(); + fs::write( + native.join("model.json"), + r#"{"architecture": "edge0_35b", "format_version": 1}"#, + ) + .unwrap(); + fixture.settings.device = "metal".into(); + fixture.settings.prefill_device = "metal".into(); + let big_path = big.to_string_lossy().into_owned(); + // As a location of its own and inside a parent location: the directory itself, its subdirectories not listed. + for location in [big_path.clone(), root.to_string_lossy().into_owned()] { + fixture.settings.assets = vec![location]; + let found = fixture.manager.inspect(&fixture.settings).unwrap(); + let paths: Vec<_> = found.iter().map(|a| a.path.clone()).collect(); + assert!(paths.contains(&big_path), "{paths:?}"); + assert!(paths + .iter() + .all(|p| !p.ends_with("mtp") && !p.ends_with("tokenizer"))); + let entry = found.iter().find(|a| a.path == big_path).unwrap(); + assert!(entry.ok && entry.model_id.as_deref() == Some(identifier)); + } + assert!(fixture.manager.delegated.lock().unwrap()[&big.canonicalize().unwrap()]); + assert!(!fixture.manager.delegated.lock().unwrap()[&native.canonicalize().unwrap()]); + // The fixture refuses device arguments for a delegated directory, so starting proves they were left out. + let service = fixture.manager.start(&fixture.settings, &big_path).unwrap(); + assert_eq!(service.url, "http://127.0.0.1:41325"); + fixture.manager.stop().unwrap(); + assert_eq!( + arguments_for(&fixture.settings, &big_path, true), + ["serve", "--model", big_path.as_str(), "--port", "0"] + ); + let native_args = arguments_for(&fixture.settings, &native.to_string_lossy(), false); + assert!( + native_args.contains(&"--device".into()) + && native_args.contains(&"--prefill-device".into()) + ); + // Without a prior inspection, start asks edge0-infer itself. + let fresh = Manager::new(fixture.directory.join("fresh")).unwrap(); + fresh.start(&fixture.settings, &big_path).unwrap(); + fresh.stop().unwrap(); + } + #[test] + fn delegated_directories_are_one_model_started_without_devices() { + for (name, identifier) in [ + ("edge0-145b-q3k", "edge0_145b"), + ("edge0-145b-fp4", "edge0_145b"), + ("edge0-120b-q3k", "edge0_120b"), + ("edge0-120b-fp4", "edge0_120b"), + ] { + check_delegated_directory(name, identifier); + } + } #[test] fn cancels_before_child_publication_without_leaking() { let fixture = Fixture::new(); @@ -1011,14 +1179,26 @@ mod tests { let mut empty = fixture.settings.clone(); empty.executable = " ".into(); let message = empty.validate().unwrap_err(); - assert!(message.starts_with("No edge0-infer executable is configured."), "{message}"); - assert!(message.contains("Settings > Engine > Executable"), "{message}"); + assert!( + message.starts_with("No edge0-infer executable is configured."), + "{message}" + ); + assert!( + message.contains("Settings > Engine > Executable"), + "{message}" + ); let mut missing = fixture.settings.clone(); - missing.executable = directory().join("edge0-infer").to_string_lossy().into_owned(); + missing.executable = directory() + .join("edge0-infer") + .to_string_lossy() + .into_owned(); let message = missing.validate().unwrap_err(); assert!(message.contains(&missing.executable), "{message}"); assert!(message.contains("moved or deleted"), "{message}"); - assert!(message.contains("Other location… on Choose a model"), "{message}"); + assert!( + message.contains("Other location… on Choose a model"), + "{message}" + ); let mut folder = fixture.settings.clone(); folder.executable = std::env::temp_dir().to_string_lossy().into_owned(); assert!(folder.validate().unwrap_err().contains("was not found at")); diff --git a/edge0-infer/examples/desktop/src-tauri/tauri.conf.json b/edge0-infer/examples/desktop/src-tauri/tauri.conf.json index a56c12f8..e9c1a7eb 100644 --- a/edge0-infer/examples/desktop/src-tauri/tauri.conf.json +++ b/edge0-infer/examples/desktop/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Edge0 Chat", - "version": "0.1.0", + "version": "0.2.0", "identifier": "dev.edge0.example.desktop", "build": { "beforeDevCommand": "npm run dev", diff --git a/edge0-infer/examples/desktop/src-tauri/tests/fixtures/serve.rs b/edge0-infer/examples/desktop/src-tauri/tests/fixtures/serve.rs index 2b736d6c..1fef55cf 100644 --- a/edge0-infer/examples/desktop/src-tauri/tests/fixtures/serve.rs +++ b/edge0-infer/examples/desktop/src-tauri/tests/fixtures/serve.rs @@ -1,11 +1,26 @@ -use std::{env, thread, time::Duration}; +use std::{env, fs, path::Path, thread, time::Duration}; fn main() { let args: Vec<_> = env::args().collect(); let model = args.windows(2).find(|a| a[0] == "--model").map(|a| a[1].as_str()).unwrap_or(""); + // The fixture recognizes the four release names, leaving native manifests to the native path. + let model_json = fs::read_to_string(Path::new(model).join("model.json")).unwrap_or_default(); + let names = [("edge0-145b-q3k", "edge0_145b"), ("edge0-145b-fp4", "edge0_145b"), + ("edge0-120b-q3k", "edge0_120b"), ("edge0-120b-fp4", "edge0_120b")]; + let found = names.iter().find(|(name, _)| model_json.contains(&format!("\"{name}\""))); + let delegated = !model_json.contains("\"architecture\"") && model_json.contains("\"model_name\"") && found.is_some(); + let architecture = found.map(|(_, id)| *id).unwrap_or("edge0_fixture"); if args.get(1).map(String::as_str) == Some("inspect") { if model == "missing" { eprintln!("Required asset is missing"); std::process::exit(1); } + if delegated { + println!("{{\"adapter\":\"{architecture}\",\"delegated\":true,\"engine\":\"edge0-strata\",\"ok\":true,\"files\":[{{\"bytes\":52}},{{\"bytes\":7}}]}}"); + return; + } println!("{{\"adapter\":\"edge0_fixture\",\"files\":[{{\"bytes\":12}},{{\"bytes\":34}}]}}"); return; } + if delegated && args.iter().any(|a| a == "--device" || a == "--prefill-device") { + eprintln!("invalid_argument: this model runs on a CUDA GPU through edge0-strata; --device must be cuda or cuda:"); + std::process::exit(2); + } if model == "error" { eprintln!("Model loading failed: invalid asset"); std::process::exit(1); } if model != "pending" { eprintln!("Listening: http://127.0.0.1:41325 model=fixture"); } loop { thread::sleep(Duration::from_millis(50)); } diff --git a/edge0-infer/examples/desktop/src/chat.ts b/edge0-infer/examples/desktop/src/chat.ts index b082e6c5..61f3552b 100644 --- a/edge0-infer/examples/desktop/src/chat.ts +++ b/edge0-infer/examples/desktop/src/chat.ts @@ -138,8 +138,11 @@ export class SseAccumulator { const delta = payload.choices?.[0]?.delta; if (typeof delta?.content === 'string') this.message.content += delta.content; if (typeof delta?.reasoning_content === 'string') this.message.reasoning_content += delta.reasoning_content; - // Metrics are accepted only on the final usage event, never inferred locally. - if (payload.usage && (!payload.choices || payload.choices.length === 0)) { + // The final event carries `usage`, and the numbers come with it: Edge0's own engine sends them as `metrics` in + // an event of its own (choices: []), while a delegated engine - edge0-strata - sends one last chunk that also + // holds the finishing choice and reports its figures as `timings`. Both are accepted here; nothing is inferred + // locally, every figure is the engine's own report. + if (payload.usage) { const metrics: Metrics = {}; if (valid(payload.usage.completion_tokens)) metrics.generated_count = payload.usage.completion_tokens; if (valid(payload.usage.prompt_tokens)) metrics.input_tokens = payload.usage.prompt_tokens; @@ -147,6 +150,16 @@ export class SseAccumulator { for (const key of ['first_token_seconds', 'decode_tokens_per_second', 'memory_peak_bytes'] as const) { if (valid(payload.metrics?.[key])) metrics[key] = payload.metrics[key]; } + const timings = payload.timings ?? {}; + if (!valid(metrics.first_token_seconds) && valid(timings.prompt_ms) && timings.prompt_ms > 0) { + metrics.first_token_seconds = timings.prompt_ms / 1000; + } + if (!valid(metrics.decode_tokens_per_second) && valid(timings.predicted_per_second)) { + metrics.decode_tokens_per_second = timings.predicted_per_second; + } + if (!valid(metrics.input_tokens) && valid(timings.prompt_n)) metrics.input_tokens = timings.prompt_n; + if (!valid(metrics.reused_tokens) && valid(timings.cache_n)) metrics.reused_tokens = timings.cache_n; + if (!valid(metrics.generated_count) && valid(timings.predicted_n)) metrics.generated_count = timings.predicted_n; this.message.metrics = metrics; } } diff --git a/edge0-infer/examples/desktop/src/delegated-metrics.test.ts b/edge0-infer/examples/desktop/src/delegated-metrics.test.ts new file mode 100644 index 00000000..a009374c --- /dev/null +++ b/edge0-infer/examples/desktop/src/delegated-metrics.test.ts @@ -0,0 +1,31 @@ +// The metrics a delegated model's stream yields: edge0-strata sends ONE final chunk that carries the finishing +// choice, `usage` and its own `timings`, where Edge0's engine sends a separate choices-less `metrics` event. +// The desktop app must read both, or a delegated model shows no prefill/decode figures. +import { expect, it } from 'vitest'; +import { formatMetrics, SseAccumulator } from './chat'; + +it('reads the figures from a delegated engine final chunk (usage + timings, with a finishing choice)', () => { + const accumulator = new SseAccumulator(); + accumulator.push('data: ' + JSON.stringify({ choices: [{ index: 0, delta: { content: 'one' }, finish_reason: null }] }) + '\n\n'); + accumulator.push('data: ' + JSON.stringify({ + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + usage: { prompt_tokens: 120, completion_tokens: 32, total_tokens: 152, prompt_tokens_details: { cached_tokens: 20 } }, + timings: { cache_n: 20, prompt_n: 120, prompt_ms: 2000, prompt_per_second: 50, predicted_n: 32, predicted_ms: 2560, predicted_per_second: 12.5 }, + }) + '\n\n'); + accumulator.push('data: [DONE]\n\n'); + const text = formatMetrics(accumulator.message.metrics!); + expect(text).toBe('32 tokens · TTFT 2.00 s · prefill 50.0 tok/s · decode 12.5 tok/s'); + expect(accumulator.done).toBe(true); +}); + +it('still prefers metrics when the engine sends them', () => { + const accumulator = new SseAccumulator(); + accumulator.push('data: ' + JSON.stringify({ + choices: [], usage: { prompt_tokens: 120, completion_tokens: 32, prompt_tokens_details: { cached_tokens: 20 } }, + metrics: { first_token_seconds: 4, decode_tokens_per_second: 9 }, + timings: { prompt_ms: 2000, predicted_per_second: 12.5 }, + }) + '\n\n'); + const metrics = accumulator.message.metrics!; + expect(metrics.first_token_seconds).toBe(4); + expect(metrics.decode_tokens_per_second).toBe(9); +}); diff --git a/edge0-infer/examples/desktop/src/live-status.test.tsx b/edge0-infer/examples/desktop/src/live-status.test.tsx new file mode 100644 index 00000000..a7107f09 --- /dev/null +++ b/edge0-infer/examples/desktop/src/live-status.test.tsx @@ -0,0 +1,57 @@ +// The Current-request card while a delegated model generates. Its /status counts what the proxy forwarded: a +// streaming request has a phase, a fragment count and a first-fragment time, while the prompt counts stay null +// until the engine reports them at the end; a non-streaming request has no phase at all. The card must show what +// is known and never print the word "null". +import { createElement } from 'react'; +import { renderToStaticMarkup } from 'react-dom/server'; +import { expect, it } from 'vitest'; +import realStatus from './real-status.json'; +import { Status } from './pages/Status'; +import { aggregateTotals, emptyTotals } from './state/totals'; +import type { EngineStatus } from './status'; + +const base = realStatus as unknown as EngineStatus; + +function render(current: unknown): string { + const status = { ...base, current } as EngineStatus; + return renderToStaticMarkup( + createElement(Status, { status, live: true, allTime: aggregateTotals(emptyTotals()), reset: () => {} }), + ); +} + +// The fields a delegated model's `current` carries (first_token_seconds only once the first fragment arrived). +const running = { + endpoint: '/v1/chat/completions', source: '127.0.0.1', started_steady_seconds: 0, elapsed_seconds: 4.2, + input_tokens: null, reused_tokens: null, computed_tokens: null, pending_tokens: null, +}; + +it('shows the live decode count without inventing the prompt size', () => { + const html = render({ ...running, phase: 'decode', generated_tokens: 141, first_token_seconds: 0.35 }); + expect(html).toContain('141 tokens'); + expect(html).toContain('Decoding'); + expect(html).toContain('3.9 s'); // decoding for elapsed - first token + expect(html).toContain('reported when the turn ends'); + expect(html).not.toContain('null'); +}); + +it('shows the prompt being read while prefill runs', () => { + const html = render({ ...running, phase: 'prefill', generated_tokens: null }); + expect(html).toContain('Reading the prompt'); + expect(html).toContain('4.2 s'); + expect(html).not.toContain('null'); +}); + +it('shows the input count once the stream carries it (Messages)', () => { + const html = render({ ...running, endpoint: '/v1/messages', phase: 'prefill', generated_tokens: null, input_tokens: 62 }); + expect(html).toContain('Computing 62 tokens'); + expect(html).not.toContain('null'); +}); + +it('shows only "In progress" and the elapsed time when the phase is unknown (non-streaming)', () => { + const html = render({ ...running, phase: null, generated_tokens: null }); + expect(html).toContain('In progress'); + expect(html).toContain('4.2 s'); + expect(html).not.toContain('Prefill'); + expect(html).not.toContain('Decode'); + expect(html).not.toContain('null'); +}); diff --git a/edge0-infer/examples/desktop/src/main.tsx b/edge0-infer/examples/desktop/src/main.tsx index 6e8ca4b7..806f0d32 100644 --- a/edge0-infer/examples/desktop/src/main.tsx +++ b/edge0-infer/examples/desktop/src/main.tsx @@ -137,7 +137,7 @@ export function App() { async function network(nextToken = false) { if (disabled) return; try { if (nextToken) { const value = generateToken(); await invoke('write_api_token', { token: value }); tokenRef.current = value; setToken(value); } else if (!settingsRef.current.lan_access) await ensureToken(); const next = { ...settingsRef.current, lan_access: nextToken || !settingsRef.current.lan_access }; await saveSettings(next); settingsRef.current = next; setSettings(next); if (service && activePath) { const current = conversationRef.current; const previousPage = page; await select(activePath, current, next); setPage(previousPage); } } catch (failure) { report(failure); } } const loaded = !!service && !!model; const nav: [Page, string, IconName][] = [['chat', 'Chat', 'chat'], ['models', 'Models', 'box'], ['status', 'Status', 'pulse'], ['service', 'Service', 'server']]; - return
{error &&
{error}
}{loading &&
Loading model… First load can take several minutes.
} + return
{error &&
{error}
}{loading &&
Loading model… First load can take several minutes.
} {page === 'chat' && changeConversation({ ...conversation, thinking })} reasoningExpanded={(index, expanded) => { const next = { ...conversationRef.current, messages: conversationRef.current.messages.map((message, i) => i === index ? { ...message, reasoningExpanded: expanded } : message) }; changeConversation(next); if (!busy) remember(next); }} edit={(index, text) => { if (!disabled) void generate(editAndResend(conversationRef.current.messages, index, text)); }} redo={() => { if (!disabled) void generate(regenerate(conversationRef.current.messages)); }} copy={text => void copy(text)} models={() => setPage('models')}/>} {page === 'models' && void select(path)} unload={() => void stopService()} chat={() => setPage('chat')} refresh={() => { void inspect().catch(report); }} add={() => void addLocation()} remove={path => void removeLocation(path)}/>} {page === 'status' && keepTotals(resetTotals(totalsRef.current))}/>} diff --git a/edge0-infer/examples/desktop/src/pages/Service.tsx b/edge0-infer/examples/desktop/src/pages/Service.tsx index 269ee73f..6acf0b34 100644 --- a/edge0-infer/examples/desktop/src/pages/Service.tsx +++ b/edge0-infer/examples/desktop/src/pages/Service.tsx @@ -9,5 +9,5 @@ export function Service({ status: s, running, disabled, lan, token, device, heal const available = health.info?.devices.find(d => d.backend === device || d.name === device); const selectedDevice = deviceLabel(available?.name ?? device, health.info?.devices ?? s?.engine.devices); const deviceCheck = !health.info ? 'Device not checked' : available ? `${selectedDevice} device available` : `${selectedDevice} device unavailable`; - return <>

Service

Local service

The OpenAI-compatible server this app runs. Other tools can use it too.

{running ? 'Running' : 'Stopped'}{running && Healthy}

{s && {url}{lan &&

Other devices connect with this computer's IP address and this port, and must send the token.

}], ['Model', s.model.alias], ['Device', s.model.prefill_device === s.model.device ? `${deviceLabel(s.model.device, s.engine.devices)} (prefill and decode)` : `${deviceLabel(s.model.prefill_device, s.engine.devices)} prefill · ${deviceLabel(s.model.device, s.engine.devices)} decode`], ['Uptime', duration(s.instance.uptime_seconds)], ['Process', `edge0-infer serve · pid ${s.instance.pid}`]]}/>}

Network access

{lan && <>
{maskToken(token)}
}
Recent requests
{['Time', 'Endpoint', 'From', 'Status', 'Tokens in / out', 'First token', 'Prefill', 'Decode', 'Total'].map(label => )}{s?.recent_requests.map((r, index) => )}
{label}
{time(r.time)}{r.endpoint}{sourceLabel(r.source)}{r.http_status}{r.input_tokens != null && r.generated_tokens != null ? `${count(r.input_tokens)} / ${count(r.generated_tokens)}` : '—'}{duration(r.first_token_seconds) ?? '—'}{rate(r.prefill_tokens_per_second) ?? '—'}{rate(r.decode_tokens_per_second) ?? '—'}{duration(r.total_seconds)}
Health check
{health.info ? 'Executable found and runnable' : 'Executable not checked'}{health.info ? `edge0-infer ${health.info.version ?? ''} · ${health.info.commit?.slice(0, 7) ?? ''}` : health.error}
{health.assets?.ok ? 'Model assets complete' : 'Model assets not checked'}{health.assets?.ok ? `${health.assets.file_count ?? ''} files present, sizes match` : health.assets?.detail}
{deviceCheck}{available?.description}
; + return <>

Service

Local service

The local server this app runs. Other tools can use it too.

{running ? 'Running' : 'Stopped'}{running && Healthy}

{s && {url}{lan &&

Other devices connect with this computer's IP address and this port, and must send the token.

}], ['Model', s.model.alias], ['Device', s.model.prefill_device === s.model.device ? `${deviceLabel(s.model.device, s.engine.devices)} (prefill and decode)` : `${deviceLabel(s.model.prefill_device, s.engine.devices)} prefill · ${deviceLabel(s.model.device, s.engine.devices)} decode`], ['Uptime', duration(s.instance.uptime_seconds)], ['Process', `edge0-infer serve · pid ${s.instance.pid}`]]}/>}

Network access

{lan && <>
{maskToken(token)}
}
Recent requests
{['Time', 'Endpoint', 'From', 'Status', 'Tokens in / out', 'First token', 'Prefill', 'Decode', 'Total'].map(label => )}{s?.recent_requests.map((r, index) => )}
{label}
{time(r.time)}{r.endpoint}{sourceLabel(r.source)}{r.http_status}{r.input_tokens != null && r.generated_tokens != null ? `${count(r.input_tokens)} / ${count(r.generated_tokens)}` : '—'}{duration(r.first_token_seconds) ?? '—'}{rate(r.prefill_tokens_per_second) ?? '—'}{rate(r.decode_tokens_per_second) ?? '—'}{duration(r.total_seconds)}
Health check
{health.info ? 'Executable found and runnable' : 'Executable not checked'}{health.info ? `edge0-infer ${health.info.version ?? ''} · ${health.info.commit?.slice(0, 7) ?? ''}` : health.error}
{health.assets?.ok ? 'Model assets complete' : 'Model assets not checked'}{health.assets?.ok ? `${health.assets.file_count ?? ''} files present, sizes match` : health.assets?.detail}
{deviceCheck}{available?.description}
; } diff --git a/edge0-infer/examples/desktop/src/pages/Settings.tsx b/edge0-infer/examples/desktop/src/pages/Settings.tsx index 05e69db7..08aa507e 100644 --- a/edge0-infer/examples/desktop/src/pages/Settings.tsx +++ b/edge0-infer/examples/desktop/src/pages/Settings.tsx @@ -3,5 +3,5 @@ import { appData } from '../chat'; import { validateServerPort } from '../conversation'; export function Settings({ settings, port, disabled, update, setPort, browse }: { settings: Preferences; port: string; disabled: boolean; update: (settings: Preferences) => void; setPort: (text: string) => void; browse: () => void }) { const validPort = validateServerPort(port); - return <>

Settings

Settings

Appearance
{(['system', 'light', 'dark'] as const).map(theme => )}
Conversation
{(['precise', 'balanced', 'creative'] as const).map((style, index) => )}
Sets the temperature for every request. Every other sampling setting follows the model.

" + + "
" + + "
"; + const label = tr("hist.editLabel", "Edit the answer"); + box.querySelector("label").textContent = label; + const input = box.querySelector("textarea"); + input.setAttribute("aria-label", label); + input.value = m.text || ""; + box.querySelector("p").textContent = (later ? tr("hist.editRemoves", {one: "Saving hides the {n} message after it. ", other: "Saving hides the {n} messages after it. "}, {n: later}) : "") + + tr("hist.editNote", "The answer's old thinking and tool calls are removed. No request is sent."); + const save = box.querySelector("[data-edit-save]"); + save.textContent = tr("hist.save", "Save"); + box.querySelector("[data-edit-cancel]").textContent = tr("cancel", "Cancel"); + save.onclick = () => saveAnswerEdit(i, input.value); + box.querySelector("[data-edit-cancel]").onclick = closePanel; + input.onkeydown = (e) => { + if (e.key === "Escape") { e.preventDefault(); e.stopPropagation(); closePanel(); } + if (e.key === "Enter" && (e.ctrlKey || e.metaKey) && !e.isComposing) { e.preventDefault(); save.click(); } + }; + el.dataset.editing = "true"; + el.appendChild(box); + panel = {kind: "edit", index: i, el}; + setBusy(!!busy); + input.focus(); + input.scrollIntoView({block: "nearest"}); + return true; +} +function saveAnswerEdit(i, text) { + if (busy || !panel || panel.kind !== "edit" || panel.index !== i) return false; + if (typeof text !== "string" || !text.trim()) { toast("warn", tr("hist.empty", "The answer is empty"), tr("hist.emptyText", "Write something before saving.")); return false; } + const plan = H.editAnswer(messages, i, text, tr("hist.edited", "Edited")); + if (!plan) return false; + closePanel(); + commit(plan.keep, plan.hidden ? tr("hist.editedHidden", {one: "Answer edited, {n} message hidden", other: "Answer edited, {n} messages hidden"}, {n: plan.hidden}) + : tr("hist.editedOnly", "Answer edited")); + return true; +} + +// "Rewind to here" asks what to do (#3823): go back HERE (this chat, with an Undo) or BRANCH (a new chat, this one stays) +function openRewind(i) { + if (!historyReady() || !messages[i]) return false; + const m = messages[i], el = $("chat").querySelector(`[data-i="${i}"]`); + const user = m.role === "user"; + const box = node("div", "history-panel"); + box.setAttribute("role", "group"); + box.setAttribute("aria-label", tr("rewind.title", "Rewind to here")); + const choice = (label, text, run, primary) => { + const row = node("div", "history-choice"); + const b = node("button", `st-btn ${primary ? "st-btn--primary" : "st-btn--secondary"}`, label); + b.type = "button"; b.onclick = run; + row.appendChild(b); + row.appendChild(node("span", "muted small", text)); + box.appendChild(row); + }; + choice(user ? tr("rewind.hereUser", "Edit and send again here") : tr("rewind.hereAnswer", "Continue from here"), + user ? tr("rewind.hereUserText", "Hides this message and the ones after it, and puts your message in the box below. Undo brings them back.") + : tr("rewind.hereAnswerText", "Hides the messages after this answer. Undo brings them back."), + () => rewindHere(i), true); + if (listOn) { + choice(tr("rewind.branch", "Branch from here"), + user ? tr("rewind.branchUserText", "Starts a new chat with the messages before this one, and puts your message in its box. This chat stays as it is.") + : tr("rewind.branchAnswerText", "Starts a new chat with the messages up to this answer. This chat stays as it is."), + () => branchHere(i), false); + } else { + box.appendChild(node("p", "muted small", tr("rewind.noList", "To start a new chat from here instead, turn on Saved chats in Chat settings."))); + } + const cancel = node("button", "st-btn st-btn--secondary", tr("cancel", "Cancel")); + cancel.type = "button"; cancel.onclick = closePanel; + box.appendChild(cancel); + box.onkeydown = (e) => { if (e.key === "Escape") { e.preventDefault(); closePanel(); } }; + el.appendChild(box); + panel = {kind: "rewind", index: i, el}; + setBusy(!!busy); + box.querySelector("button").focus(); + box.scrollIntoView({block: "nearest"}); + return true; +} +function fillBox(draft) { + $("input").value = draft.text; + attachments = [...draft.images, ...draft.files]; + renderAttachments(); + autosize(); + if (draft.lost) toast("warn", tr("hist.lostTitle", "Attachments need to be added again"), tr("hist.lostText2", "This browser kept their names, not their contents. Attach them again."), 7000); +} +function rewindHere(i) { + if (!settings.chatEdit || busy || pending || !panel || panel.kind !== "rewind") return false; // called from the open panel + const plan = H.rewind(messages, i); + if (!plan) return false; + const draftBefore = {text: $("input").value, attachments: attachments.slice()}; + closePanel(); + const u = commit(plan.keep, plan.hidden ? hiddenText(plan.hidden) : tr("hist.nothingHidden", "Back at this message")); + if (plan.draft) { u.draftBefore = draftBefore; fillBox(plan.draft); u.inserted = $("input").value; } + $("input").focus(); + return true; +} +function branchHere(i) { + if (!listOn || busy || pending || !panel || panel.kind !== "rewind") return false; + const plan = H.rewind(messages, i); + if (!plan) return false; + closePanel(); + const prev = chatId; + const c = H.branch(chat, plan.keep, plan.keep.length, tr("branch.title", "Branch of {title}", {title: H.chatTitle(chat, untitled())})); + chats.push(c); + showChat(c); + if (plan.draft) fillBox(plan.draft); + $("input").focus(); + toast("info", tr("branch.started", "Branch started"), tr("branch.startedText", "The chat you were in is unchanged, in the list on the left."), 8000, + {label: tr("branch.back", "Back to it"), run: () => openChat(prev)}); + return true; +} + +// ------------------------------------------------------------------ wiring (called once by app.js) +function initChats() { + $("sidebar-toggle").onclick = () => setSidebar(!sidebarOpen, true); + $("sidebar-new").onclick = startChat; + $("sidebar-search").addEventListener("input", () => { sidebarQuery = $("sidebar-search").value; renderSidebar(); }); + $("sidebar-list").addEventListener("click", (e) => { + // a click inside the rename field is caret placement, not a click on the row: the row's handler would open the chat, + // which re-renders the list and takes the field away mid-click + if (e.target.closest(".chat-item__input")) return; + const row = e.target.closest(".chat-item"); + if (!row) return; + const c = chats.find((x) => x.id === row.dataset.id); + if (!c) return; + if (e.target.closest("[data-delete]")) { deleteChat(c); return; } + if (e.target.closest("[data-rename]")) { renameChat(row, c); return; } + openChat(c.id); + }); + // a narrow screen starts with the list closed so the chat has room + setSidebar(store.get("sidebar", true) && !matchMedia("(max-width: 760px)").matches, false); +} diff --git a/edge0-strata/serve/web/chatstore.js b/edge0-strata/serve/web/chatstore.js new file mode 100644 index 00000000..fc5b682d --- /dev/null +++ b/edge0-strata/serve/web/chatstore.js @@ -0,0 +1,66 @@ +// serve/web/chatstore.js - the saved chats, kept in this browser (IndexedDB, database "strata", store "chats", one record per +// chat). Ported from PR #1787 by Scotty-Hudson. A chat can be several megabytes and localStorage holds about 5 MB for the whole +// page, so the chats live here and only small keys (which chat is open, the settings) stay in localStorage. Nothing goes to the +// server: the server has no endpoint for chats and never sees a title. +// +// Where the browser will not keep it (a private window, a blocked or full store) the list lives in memory for the visit, and +// onProblem(kind) tells the page once: "memory" (no store), "quota" (full), "failed" (a write the browser refused). +"use strict"; +(function (root) { + function createChatStore({indexedDB: idb = root.indexedDB, onProblem = () => {}} = {}) { + let db = null, told = false; + const problem = (kind) => { if (!told) { told = true; onProblem(kind); } }; + + function open() { + return new Promise((resolve) => { + let r; + try { r = idb.open("strata", 1); } catch (e) { resolve(null); return; } // private mode: memory only + r.onupgradeneeded = () => { + const d = r.result; + if (!d.objectStoreNames.contains("chats")) d.createObjectStore("chats", {keyPath: "id"}); + }; + // a database already at version 1 does not run onupgradeneeded, so the store may not be there + r.onsuccess = () => resolve(r.result.objectStoreNames.contains("chats") ? r.result : null); + r.onerror = () => resolve(null); + r.onblocked = () => resolve(null); // another tab holds an older version of the database + }); + } + // true when the store is there; false: memory only (and the page has been told) + async function start() { + db = await open(); + if (!db) problem("memory"); + return !!db; + } + function all() { + return new Promise((resolve) => { + if (!db) { resolve([]); return; } + // the result is on the request, not on the store: reading it at oncomplete gives nothing + const req = db.transaction("chats", "readonly").objectStore("chats").getAll(); + req.onsuccess = () => resolve(req.result || []); + req.onerror = () => resolve([]); + req.transaction.onabort = () => resolve([]); // an aborted read fires abort, not error: do not hang + }); + } + // a request that fails because the storage is full aborts the transaction, so it fires abort rather than error + function watch(t) { + const warn = () => problem(t.error && t.error.name === "QuotaExceededError" ? "quota" : "failed"); + t.onerror = warn; t.onabort = warn; + } + function put(chat) { // one record per chat: not the whole list for one message + if (!db) return; + const t = db.transaction("chats", "readwrite"); + t.objectStore("chats").put(chat); + watch(t); + } + function remove(id) { + if (!db) return; + const t = db.transaction("chats", "readwrite"); + t.objectStore("chats").delete(id); + watch(t); + } + return {start, all, put, remove, get usable() { return !!db; }}; + } + const api = {createChatStore}; + root.StrataChatStore = api; + if (typeof module !== "undefined" && module.exports) module.exports = api; +})(typeof globalThis !== "undefined" ? globalThis : this); diff --git a/edge0-strata/serve/web/components.css b/edge0-strata/serve/web/components.css new file mode 100644 index 00000000..899f5257 --- /dev/null +++ b/edge0-strata/serve/web/components.css @@ -0,0 +1,128 @@ +/* Strata UI — components. Requires tokens.css. */ +*,*::before,*::after{box-sizing:border-box} +html,body{margin:0;background:var(--st-bg);color:var(--st-ink);font-family:var(--st-font);font-size:var(--st-fs-base);line-height:var(--st-lh-body);-webkit-font-smoothing:antialiased} +a{color:var(--st-accent-text);text-decoration:none} a:hover{text-decoration:underline} +:focus-visible{outline:2px solid var(--st-focus);outline-offset:2px} + +/* icon — */ +.st-icon{width:20px;height:20px;flex:none;color:inherit} +.st-icon--sm{width:16px;height:16px} .st-icon--lg{width:24px;height:24px} + +/* header */ +.st-header{display:flex;align-items:center;gap:var(--st-s-4);height:60px;padding:0 var(--st-s-6);background:var(--st-surface);border-bottom:1px solid var(--st-line)} +.st-header__brand{font-weight:900;font-size:var(--st-fs-lg);letter-spacing:-.01em} +.st-header__spacer{flex:1} + +/* status pill */ +.st-pill{display:inline-flex;align-items:center;gap:8px;height:30px;padding:0 12px;border-radius:var(--st-r-pill);border:1px solid var(--st-line);background:var(--st-surface-2);font-size:var(--st-fs-sm);font-weight:500;color:var(--st-ink-soft);font-variant-numeric:tabular-nums} +.st-pill__dot{width:8px;height:8px;border-radius:50%;background:var(--st-ink-muted)} +.st-pill[data-state="generating"] .st-pill__dot{background:var(--st-accent);box-shadow:0 0 0 4px var(--st-accent-tint)} +.st-pill[data-state="reading"] .st-pill__dot{background:var(--st-info);box-shadow:0 0 0 4px var(--st-info-tint)} +.st-pill[data-state="queued"] .st-pill__dot{background:var(--st-warn)} + +/* tabs */ +.st-tabs{display:flex;gap:4px;padding:4px;background:var(--st-surface-2);border:1px solid var(--st-line);border-radius:var(--st-r-md)} +.st-tab{display:inline-flex;align-items:center;gap:8px;height:36px;padding:0 14px;border:0;border-radius:10px;background:transparent;color:var(--st-ink-muted);font:500 var(--st-fs-md)/1 var(--st-font);cursor:pointer} +.st-tab:hover{color:var(--st-ink)} +.st-tab[aria-selected="true"]{background:var(--st-surface);color:var(--st-ink);font-weight:700;box-shadow:var(--st-sh-card)} + +/* card + metric */ +.st-card{background:var(--st-surface);border:1px solid var(--st-line);border-radius:var(--st-r-lg);box-shadow:var(--st-sh-card);padding:var(--st-s-5)} +.st-metric{display:flex;flex-direction:column;gap:10px} +.st-metric__label{display:flex;align-items:center;gap:8px;font-size:var(--st-fs-sm);font-weight:500;color:var(--st-ink-muted)} +.st-metric__value{font-weight:900;font-size:var(--st-fs-stat);line-height:1;letter-spacing:-.01em;font-variant-numeric:tabular-nums} +.st-metric__unit{font-weight:500;font-size:var(--st-fs-md);color:var(--st-ink-muted);margin-left:4px} +.st-metric__spark{width:100%;height:32px;color:var(--st-accent)} + +/* progress + gauge */ +.st-progress{height:8px;border-radius:var(--st-r-pill);background:var(--st-surface-2);border:1px solid var(--st-line-soft);overflow:hidden} +.st-progress__bar{height:100%;border-radius:inherit;background:var(--st-accent);transition:width var(--st-dur) var(--st-ease)} +.st-progress[data-tone="info"] .st-progress__bar{background:var(--st-info)} +.st-progress[data-tone="warn"] .st-progress__bar{background:var(--st-warn)} +.st-progress[data-tone="danger"] .st-progress__bar{background:var(--st-danger)} +/* gauge: see README for SVG markup (stroke-dasharray on a 270° arc) */ +.st-gauge{position:relative;width:120px;height:120px} +.st-gauge__track{stroke:var(--st-surface-2)} .st-gauge__fill{stroke:var(--st-accent)} + +/* table */ +.st-table{width:100%;border-collapse:collapse;font-size:var(--st-fs-md);font-variant-numeric:tabular-nums} +.st-table th{text-align:left;font-weight:500;font-size:var(--st-fs-xs);letter-spacing:.04em;text-transform:uppercase;color:var(--st-ink-muted);padding:10px 12px;border-bottom:1px solid var(--st-line)} +.st-table td{padding:12px;border-bottom:1px solid var(--st-line-soft);color:var(--st-ink-soft)} +.st-table tr:hover td{background:var(--st-surface-2)} +.st-table .num{text-align:right} + +/* badges */ +.st-badge{display:inline-flex;align-items:center;gap:6px;height:24px;padding:0 10px;border-radius:var(--st-r-pill);font-size:var(--st-fs-xs);font-weight:700;background:var(--st-surface-2);color:var(--st-ink-muted);border:1px solid var(--st-line)} +.st-badge::before{content:"";width:6px;height:6px;border-radius:50%;background:currentColor} +.st-badge--reading{background:var(--st-info-tint);color:var(--st-info-text);border-color:transparent} +.st-badge--generating{background:var(--st-accent-tint);color:var(--st-accent-text);border-color:transparent} +.st-badge--queued{background:var(--st-warn-tint);color:var(--st-warn-text);border-color:transparent} +.st-badge--error{background:var(--st-danger-tint);color:var(--st-danger-text);border-color:transparent} + +/* chat */ +.st-msg{display:flex;flex-direction:column;gap:6px;max-width:720px} +.st-msg--user{align-self:flex-end;align-items:flex-end} +.st-bubble{padding:12px 16px;border-radius:var(--st-r-lg);font-size:var(--st-fs-base);line-height:1.55;color:var(--st-ink-soft)} +.st-msg--user .st-bubble{background:var(--st-accent);color:var(--st-accent-ink);border-bottom-right-radius:6px} +.st-msg--assistant .st-bubble{background:var(--st-surface);border:1px solid var(--st-line);border-bottom-left-radius:6px} +.st-msg__meta{font-size:var(--st-fs-xs);color:var(--st-ink-muted);display:flex;gap:8px;align-items:center} + +/* collapsible —
…
…
*/ +.st-collapse{border:1px solid var(--st-line);border-radius:var(--st-r-md);background:var(--st-surface-2)} +.st-collapse>summary{list-style:none;display:flex;align-items:center;gap:8px;padding:10px 14px;cursor:pointer;font-size:var(--st-fs-sm);font-weight:500;color:var(--st-ink-muted)} +.st-collapse>summary::-webkit-details-marker{display:none} +.st-collapse>summary .st-chev{margin-left:auto;transition:transform var(--st-dur) var(--st-ease)} +.st-collapse[open]>summary .st-chev{transform:rotate(180deg)} +.st-collapse__body{padding:0 14px 14px;font-size:var(--st-fs-sm);color:var(--st-ink-muted);line-height:1.6} + +/* code */ +.st-code{border-radius:var(--st-r-md);background:var(--st-code-bg);color:var(--st-code-ink);overflow:hidden} +.st-code__head{display:flex;align-items:center;justify-content:space-between;padding:8px 8px 8px 14px;font:500 var(--st-fs-xs)/1 var(--st-font);color:#9aa3ab;border-bottom:1px solid rgba(255,255,255,.06)} +.st-code pre{margin:0;padding:14px;overflow:auto;font:400 13px/1.6 var(--st-font-mono)} +.st-code .st-btn--icon{color:#9aa3ab} .st-code .st-btn--icon:hover{background:rgba(255,255,255,.08);color:#fff} + +/* composer */ +.st-composer{display:flex;flex-direction:column;gap:8px;padding:10px;background:var(--st-surface);border:1px solid var(--st-line);border-radius:var(--st-r-lg);box-shadow:var(--st-sh-bar)} +.st-composer:focus-within{border-color:var(--st-focus)} +.st-composer textarea{border:0;outline:0;resize:none;background:transparent;color:var(--st-ink);font:400 var(--st-fs-base)/1.5 var(--st-font);padding:6px 8px;min-height:48px} +.st-composer__bar{display:flex;align-items:center;gap:6px} +.st-composer__bar .spacer{flex:1} + +/* drawer */ +.st-drawer{position:fixed;top:0;right:0;bottom:0;width:360px;max-width:100vw;background:var(--st-surface);border-left:1px solid var(--st-line);box-shadow:var(--st-sh-pop);display:flex;flex-direction:column;transform:translateX(100%);transition:transform 220ms var(--st-ease)} +.st-drawer[data-open="true"]{transform:none} +.st-drawer__head{display:flex;align-items:center;justify-content:space-between;padding:18px 20px;border-bottom:1px solid var(--st-line)} +.st-drawer__body{padding:20px;display:flex;flex-direction:column;gap:22px;overflow:auto} + +/* buttons */ +.st-btn{display:inline-flex;align-items:center;justify-content:center;gap:8px;height:42px;padding:0 18px;border-radius:var(--st-r-md);border:0;font:700 var(--st-fs-md)/1 var(--st-font);cursor:pointer;transition:background var(--st-dur),transform 120ms} +.st-btn:active{transform:translateY(1px)} +.st-btn--primary{background:var(--st-accent);color:var(--st-accent-ink);box-shadow:0 6px 16px rgba(16,185,129,.24),inset 0 1px 0 rgba(255,255,255,.18)} +.st-btn--primary:hover{background:var(--st-accent-hover)} +.st-btn--secondary{background:transparent;color:var(--st-ink);border:1.5px solid var(--st-line)} +.st-btn--secondary:hover{background:var(--st-surface-2)} +.st-btn--danger{background:var(--st-danger-tint);color:var(--st-danger-text)} +.st-btn--icon{width:36px;height:36px;padding:0;border-radius:10px;background:transparent;color:var(--st-ink-muted)} +.st-btn--icon:hover{background:var(--st-surface-2);color:var(--st-ink)} +.st-btn:disabled{opacity:.45;cursor:not-allowed} + +/* form controls */ +.st-field{display:flex;flex-direction:column;gap:8px} +.st-label{display:flex;justify-content:space-between;font-size:var(--st-fs-sm);font-weight:500;color:var(--st-ink-soft)} +.st-label output{font-variant-numeric:tabular-nums;color:var(--st-ink-muted)} +.st-input{height:44px;padding:0 14px;border-radius:var(--st-r-md);border:1.5px solid var(--st-line);background:var(--st-surface);color:var(--st-ink);font:400 var(--st-fs-md) var(--st-font)} +.st-input:focus{outline:0;border-color:var(--st-focus);box-shadow:0 0 0 3px color-mix(in srgb,var(--st-focus) 20%,transparent)} +.st-slider{width:100%;accent-color:var(--st-accent)} +.st-toggle{position:relative;width:40px;height:24px;border-radius:var(--st-r-pill);border:0;background:var(--st-line);cursor:pointer;transition:background var(--st-dur)} +.st-toggle::after{content:"";position:absolute;top:3px;left:3px;width:18px;height:18px;border-radius:50%;background:#fff;box-shadow:0 1px 2px rgba(0,0,0,.2);transition:transform var(--st-dur) var(--st-ease)} +.st-toggle[aria-checked="true"]{background:var(--st-accent)} +.st-toggle[aria-checked="true"]::after{transform:translateX(16px)} + +/* toast / alert */ +.st-toast{display:flex;align-items:flex-start;gap:12px;padding:14px 16px;border-radius:var(--st-r-md);background:var(--st-surface);border:1px solid var(--st-line);box-shadow:var(--st-sh-pop);font-size:var(--st-fs-md);color:var(--st-ink-soft);max-width:420px} +.st-toast__title{font-weight:700;color:var(--st-ink)} +.st-toast--info .st-icon{color:var(--st-info)} +.st-toast--success .st-icon{color:var(--st-accent)} +.st-toast--warn .st-icon{color:var(--st-warn)} +.st-toast--error .st-icon{color:var(--st-danger)} +.st-toasts{position:fixed;right:20px;bottom:20px;display:flex;flex-direction:column;gap:10px;z-index:50} diff --git a/edge0-strata/serve/web/favicon.svg b/edge0-strata/serve/web/favicon.svg new file mode 100644 index 00000000..df7fbf73 --- /dev/null +++ b/edge0-strata/serve/web/favicon.svg @@ -0,0 +1,9 @@ + + + + + + diff --git a/edge0-strata/serve/web/fonts/OFL.txt b/edge0-strata/serve/web/fonts/OFL.txt new file mode 100644 index 00000000..723cd447 --- /dev/null +++ b/edge0-strata/serve/web/fonts/OFL.txt @@ -0,0 +1,93 @@ +Copyright 2021 The Outfit Project Authors (https://github.com/Outfitio/Outfit-Fonts) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +https://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/edge0-strata/serve/web/fonts/outfit-latin-ext-wght.woff2 b/edge0-strata/serve/web/fonts/outfit-latin-ext-wght.woff2 new file mode 100644 index 00000000..c54ceec8 Binary files /dev/null and b/edge0-strata/serve/web/fonts/outfit-latin-ext-wght.woff2 differ diff --git a/edge0-strata/serve/web/fonts/outfit-latin-wght.woff2 b/edge0-strata/serve/web/fonts/outfit-latin-wght.woff2 new file mode 100644 index 00000000..85e3332e Binary files /dev/null and b/edge0-strata/serve/web/fonts/outfit-latin-wght.woff2 differ diff --git a/edge0-strata/serve/web/history.js b/edge0-strata/serve/web/history.js new file mode 100644 index 00000000..8e5428d5 --- /dev/null +++ b/edge0-strata/serve/web/history.js @@ -0,0 +1,90 @@ +// serve/web/history.js - what the chat history does to a list of messages. Pure functions: no page, no storage, no clock of +// their own, so they can be tested alone (serve/test_web_history.js). app.js draws the controls and keeps the state. +// +// One design for five proposals (see docs/DETAILS.md, "The web page"): +// #2615 (uppet) edit your question, correct an answer, regenerate, go back, undo +// #1787 (Scotty-Hudson) saved chats in a list, kept in this browser (IndexedDB), renamed, deleted, searched +// #3823 (xvcezrqpbxtpzblrt) "Rewind to here" asks: edit and send again HERE, or BRANCH into a new chat +// #3550 / #4004 the header's Load / Unload; sending while an answer is being written (app.js) +// A message list is an array of {role, text, time, images, files, reasoning, tools, meta, ...}; an attachment that was read +// in this page has its contents (image url, file text), one that came back from the browser's store has its name only. +"use strict"; +(function (root) { + const TITLE_MAX = 60; + + const newChatId = (now, rnd) => `c${now.toString(36)}${Math.floor(rnd * 1e6).toString(36)}`; + function newChat(now = Date.now(), rnd = Math.random()) { + return {id: newChatId(now, rnd), title: "", time: now, messages: []}; + } + // the name in the list: what the user called it, else the first line of their first question, else the caller's word for "new" + function chatTitle(c, untitled = "New chat") { + if (c.title) return c.title; + const first = (c.messages.find((m) => m.role === "user") || {}).text || ""; + return first.split("\n")[0].trim().slice(0, TITLE_MAX) || untitled; + } + + // pictures and files are kept as their names only: a data URL would fill the browser's storage + const cleanMessages = (list) => list.map((m) => ({...m, images: (m.images || []).map((i) => ({name: i.name})), + files: (m.files || []).map((f) => ({name: f.name}))})); + const cleanChat = (c) => ({...c, messages: cleanMessages(c.messages)}); + + // "today" | "yesterday" | "week" | "month" | "older", by calendar days back from now + function groupKey(t, now = Date.now()) { + const days = Math.floor((now - t) / 86400000); + if (!days) return "today"; + if (days === 1) return "yesterday"; + if (days <= 7) return "week"; + if (days <= 30) return "month"; + return "older"; + } + // the list for a search box: titles and message text, newest first + function filterChats(chats, query, untitled) { + const q = String(query || "").trim().toLowerCase(); + return chats.filter((c) => !q || chatTitle(c, untitled).toLowerCase().includes(q) || + c.messages.some((m) => (m.text || "").toLowerCase().includes(q))) + .sort((a, b) => b.time - a.time); + } + + // Can a user turn be sent again as it was? A picture or file that came back from the browser's store has only its name. + const lostAttachments = (m) => ((m.images || []).filter((x) => !x.url).length + (m.files || []).filter((x) => typeof x.text !== "string").length); + // what goes back into the message box when a user turn is edited: its text and the attachments that still have contents + function draftOf(m) { + return {text: m.text || "", + images: (m.images || []).filter((x) => x.url).map((x) => ({kind: "image", name: x.name, url: x.url})), + files: (m.files || []).filter((x) => typeof x.text === "string").map((x) => ({kind: "file", name: x.name, text: x.text})), + lost: lostAttachments(m)}; + } + + // "Rewind to here" on message i. A user turn goes back into the box and leaves the list; an answer stays and the list ends + // with it. {keep, draft, hidden}: the messages that stay, the draft for the box (null for an answer), how many are hidden. + function rewind(messages, i) { + const m = messages[i]; + if (!m) return null; + if (m.role === "user") return {keep: messages.slice(0, i), draft: draftOf(m), hidden: messages.length - i}; + return {keep: messages.slice(0, i + 1), draft: null, hidden: messages.length - i - 1}; + } + // Regenerate the answer at i (or the last user turn): the list up to and including the user turn before it + function regenerate(messages, i) { + let u = i; + while (u >= 0 && messages[u] && messages[u].role !== "user") u--; + if (u < 0 || !messages[u]) return null; + return {keep: messages.slice(0, u + 1), hidden: messages.length - u - 1}; + } + // Edit the answer at i: its text replaced, its thinking, tool calls, error and stats dropped, everything after it hidden + function editAnswer(messages, i, text, meta = "Edited") { + const m = messages[i]; + if (!m || m.role !== "assistant") return null; + return {keep: [...messages.slice(0, i), {role: "assistant", text, time: m.time, meta}], hidden: messages.length - i - 1}; + } + // A branch: a new chat that starts with a copy of the first `end` messages. The original is not touched. The title says + // where it came from and stays: the first question does not rename it (the first-message case of #3823). + function branch(chat, messages, end, title, now = Date.now(), rnd = Math.random()) { + const copy = JSON.parse(JSON.stringify(messages.slice(0, end))); + return {...newChat(now, rnd), title, parent: {id: chat.id, title: chatTitle(chat)}, messages: copy}; + } + + const api = {TITLE_MAX, newChat, chatTitle, cleanMessages, cleanChat, groupKey, filterChats, lostAttachments, draftOf, rewind, + regenerate, editAnswer, branch}; + root.StrataHistory = api; + if (typeof module !== "undefined" && module.exports) module.exports = api; +})(typeof globalThis !== "undefined" ? globalThis : this); diff --git a/edge0-strata/serve/web/i18n.js b/edge0-strata/serve/web/i18n.js new file mode 100644 index 00000000..bdcebd7f --- /dev/null +++ b/edge0-strata/serve/web/i18n.js @@ -0,0 +1,77 @@ +// serve/web/i18n.js - the web page's translations. +// +// The English text sits in the code, next to the key: tr("chat.rewind", "Rewind to here") and +// tr("chat.hidden", {one: "{n} message hidden", other: "{n} messages hidden"}, {n: 3}). Static page text is marked +// data-i18n="key" (also data-i18n-title, data-i18n-label, data-i18n-placeholder) and holds its English text in the HTML. +// So the page reads fine with no catalog at all. Every other language is a catalog, serve/web/locales/.json +// (a flat {"key": "text"} object; a plural is {"one": "...", "other": "..."}, the CLDR names of the language). +// serve/test_web_i18n.py keeps the catalogs complete: same keys as the English text in the code, same {placeholders}. +// +// The language is the user's choice (About > Settings), kept in this browser. Nothing is chosen for them from the +// browser's language: a page that is translated in part would mix two languages. +"use strict"; +(function (root) { + // [code, the language's own name]. A new language is a locales/.json file plus one line here. + const LANGUAGES = [["en", "English"], ["de", "Deutsch"], ["fr", "Français"], ["es", "Español"], ["ja", "日本語"], ["ko", "한국어"], + ["pt", "Português"], ["ru", "Русский"], ["zh", "中文"]]; + let lang = "en", catalog = null, rules = null; + + const fill = (s, vars) => String(s).replace(/\{(\w+)\}/g, (m, k) => (vars && k in vars ? String(vars[k]) : m)); + function pick(entry, vars) { + if (entry && typeof entry === "object") { + const n = vars && vars.n != null ? Number(vars.n) : 1; + let cat = "other"; + try { cat = (rules || new Intl.PluralRules(lang === "en" ? "en" : lang)).select(n); } catch (e) { /* other */ } + return entry[cat] != null ? entry[cat] : entry.other != null ? entry.other : entry.one; + } + return entry; + } + // tr(key, english, vars): the catalog's text for the chosen language, else the English text in the code + function tr(key, english, vars) { + const own = catalog && Object.prototype.hasOwnProperty.call(catalog, key) ? catalog[key] : undefined; + const text = pick(own != null ? own : english, vars); + return fill(text == null ? key : text, vars); + } + + // marked page text: the English text is read once from the HTML and kept, so switching back restores it + const ATTRS = [["data-i18n", null], ["data-i18n-title", "title"], ["data-i18n-label", "aria-label"], + ["data-i18n-placeholder", "placeholder"]]; + function apply(scope) { + const doc = scope || root.document; + if (!doc || !doc.querySelectorAll) return; + for (const [mark, attr] of ATTRS) { + for (const el of doc.querySelectorAll(`[${mark}]`)) { + const key = el.getAttribute(mark); + const slot = "en_" + mark.replace(/[^a-z]/g, ""); + if (el.dataset[slot] === undefined) el.dataset[slot] = attr ? (el.getAttribute(attr) || "") : el.textContent; + const text = tr(key, el.dataset[slot]); + if (attr) el.setAttribute(attr, text); else el.textContent = text; + } + } + if (root.document && root.document.documentElement) root.document.documentElement.lang = lang; + } + + const known = (code) => LANGUAGES.some((l) => l[0] === code); + // load a catalog; English, or a catalog that cannot be read, leaves the English text in the code + async function setLanguage(code) { + code = known(code) ? code : "en"; + let next = null; + if (code !== "en") { + try { + const r = await root.fetch(`web/locales/${code}.json`); + const j = r.ok ? await r.json() : null; + if (j && typeof j === "object" && !Array.isArray(j)) next = j; + } catch (e) { /* the English text stays */ } + if (!next) code = "en"; + } + lang = code; catalog = next; rules = null; + try { rules = new Intl.PluralRules(lang); } catch (e) { /* other */ } + apply(); + return lang; + } + + const api = {LANGUAGES, tr, apply, setLanguage, get lang() { return lang; }, fill, pick}; + root.StrataI18n = api; + root.tr = tr; + if (typeof module !== "undefined" && module.exports) module.exports = api; +})(typeof globalThis !== "undefined" ? globalThis : this); diff --git a/edge0-strata/serve/web/index.html b/edge0-strata/serve/web/index.html new file mode 100644 index 00000000..4de2fc44 --- /dev/null +++ b/edge0-strata/serve/web/index.html @@ -0,0 +1,321 @@ + + + + + +Strata + + + + + + + +
+ Strata + + + + Connecting… + + +
+ +
+ +
+ + +
+ +
+
+
+
Ask anything
+
The model runs on this PC. Nothing leaves it.
+
+
+
+
+
+ + +
+ + + + + + + + + +
+
+
+
+
+ + + + + + + + + +
+ + + + +
+ + + + + + + + + + diff --git a/edge0-strata/serve/web/locales/de.json b/edge0-strata/serve/web/locales/de.json new file mode 100644 index 00000000..67a2c5cb --- /dev/null +++ b/edge0-strata/serve/web/locales/de.json @@ -0,0 +1,249 @@ +{ + "branch.back": "Zurück dorthin", + "branch.from": "Abgezweigt von „{title}“", + "branch.open": "Das Original öffnen", + "branch.started": "Verzweigung gestartet", + "branch.startedText": "Der Chat, in dem Sie waren, ist unverändert und steht in der Liste links.", + "branch.title": "Verzweigung von {title}", + "busy.stillWriting": "Es schreibt noch", + "busy.stopFirst": "Stoppen Sie zuerst die Antwort.", + "cancel": "Abbrechen", + "chat.clearedToast": "Der letzte Chat wurde geleert.", + "chat.copy": "Kopieren", + "chat.copyAnswer": "Antwort kopieren", + "chat.deleted": "Chat gelöscht", + "chat.empty": "Fragen Sie, was Sie möchten", + "chat.emptyModel": "{model} läuft auf diesem PC. Nichts verlässt ihn.", + "chat.image": "Bild", + "chat.interrupted": "Unterbrochen", + "chat.interruptedLower": "unterbrochen", + "chat.new": "Neuer Chat", + "chat.newToast": "Neuer Chat", + "chat.newToastText": "Der bisherige Chat steht in der Liste links.", + "chat.projectionOff": "Projektion aus", + "chat.projectionOn": "Projektion an", + "chat.stopped": "Gestoppt", + "chat.stoppedLower": "gestoppt", + "chat.tokens": "{n} Tokens", + "chat.toolCalls": {"one": "{n} Werkzeugaufruf", "other": "{n} Werkzeugaufrufe"}, + "chat.toolLimit": "bei der Grenze von {n} Werkzeugrunden gestoppt (mcp.max_rounds)", + "chat.untitled": "Neuer Chat", + "chat.you": "Sie", + "close": "Schließen", + "code.copy": "Code kopieren", + "code.label": "Code", + "composer.attach": "Textdatei anhängen (oder hierher ziehen)", + "composer.attachLabel": "Datei anhängen", + "composer.attachPictures": "Textdatei oder Bild anhängen (oder hierher ziehen)", + "composer.finishOpen": "Speichern oder abbrechen, was oben offen ist, um fortzufahren", + "composer.hintInterject": "Eingabe stoppt die Antwort hier und sendet sofort", + "composer.hintQueue": "Eingabe sendet, sobald die Antwort fertig ist", + "composer.message": "Nachricht", + "composer.newline": "Umschalt+Eingabe: neue Zeile", + "composer.placeholder": "Fragen Sie, was Sie möchten…", + "composer.save": "Diesen Chat als Markdown speichern", + "composer.saveLabel": "Diesen Chat speichern", + "composer.send": "Senden", + "composer.stop": "Stoppen", + "drawer.apply": "Anwenden", + "drawer.busy": "Wenn Sie senden, während es antwortet", + "drawer.busyInterject": "Unterbrechen", + "drawer.busyInterjectNote": "Die Antwort hört dort auf, wo sie ist (Geschriebenes bleibt), und Ihre Nachricht geht sofort los. Ein laufendes Werkzeug wird zuerst fertig.", + "drawer.busyQueue": "Danach senden", + "drawer.busyQueueNote": "Ihre Nachricht wartet und geht los, sobald die Antwort fertig ist.", + "drawer.busyWait": "Warten", + "drawer.busyWaitNote": "Senden ist aus, bis die Antwort fertig ist, wie bisher.", + "drawer.chats": "Chats", + "drawer.edit": "Nachrichten bearbeiten und zurückspulen", + "drawer.editNote": "eine Antwort bearbeiten, eine neue verlangen oder zu einer früheren Nachricht zurückgehen (mit Rückgängig)", + "drawer.greedy": "0 · deterministisch", + "drawer.high": "Hoch", + "drawer.list": "Liste gespeicherter Chats", + "drawer.listNote": "behält jeden Chat in einer Liste links, nur in diesem Browser. Aus: ein Chat, wie bisher", + "drawer.low": "Niedrig", + "drawer.math": "Formeln darstellen", + "drawer.mathNote": "Gleichungen in Antworten als Formeln zeichnen (funktioniert offline)", + "drawer.maxTokens": "Max. Tokens", + "drawer.maxTokensNote": "leer = bis fertig", + "drawer.mcp": "Werkzeuge von MCP-Servern nutzen", + "drawer.medium": "Mittel", + "drawer.off": "Aus", + "drawer.random": "Zufällig", + "drawer.reset": "Zurücksetzen", + "drawer.saved": "Chat-Einstellungen gespeichert", + "drawer.savedGreedy": "Deterministisch: dieselbe Frage gibt dieselbe Antwort.", + "drawer.savedHere": "Hier gespeichert, aber nicht für andere Apps", + "drawer.savedOwn": "Andere Apps nutzen wieder ihre eigenen Einstellungen.", + "drawer.savedShared": "Andere Apps (omp, API-Clients) nutzen diese Einstellungen ab ihrer nächsten Anfrage.", + "drawer.seed": "Seed", + "drawer.seedNote": "leer = zufällig", + "drawer.share": "Auch für andere Apps verwenden", + "drawer.shareNote": "omp und andere API-Clients bekommen diese Einstellungen für alles, was sie nicht selbst festlegen", + "drawer.short": "Einstellungen", + "drawer.showThinking": "Denken anzeigen", + "drawer.showThinkingNote": "aufgeklappt, solange es schreibt", + "drawer.temperature": "Temperatur", + "drawer.temperatureNote": "0 = immer das wahrscheinlichste Wort (exakt, wiederholbar)", + "drawer.thinkHigh": "gründlich (Standard)", + "drawer.thinkLow": "kurz", + "drawer.thinkMedium": "mittel", + "drawer.thinkNone": "antwortet sofort", + "drawer.thinking": "Denken", + "drawer.title": "Chat-Einstellungen", + "drawer.untilDone": "Bis fertig", + "export.nothing": "Noch nichts zu speichern", + "file.big": "Datei zu groß", + "file.bigText": "{name} ist größer als 512 KB.", + "file.binaryText": "{name} sieht nach einer Binärdatei aus.", + "file.notText": "Keine Textdatei", + "file.notTextOnly": "{name}: Hängen Sie Textdateien an (Code, Notizen, Protokolle, Daten).", + "file.notTextPictures": "{name}: Hängen Sie Textdateien (Code, Notizen, Protokolle, Daten) oder Bilder an.", + "file.pasted": "eingefügtes Bild", + "file.pictureBig": "Bild zu groß", + "file.pictureBigText": "{name} ist größer als 20 MB.", + "file.picturesOff": "Bilder sind aus", + "file.picturesOffText": "Dieses Modell wurde nur für Text eingerichtet.", + "file.remove": "Entfernen", + "group.month": "Letzte 30 Tage", + "group.older": "Älter", + "group.today": "Heute", + "group.week": "Letzte 7 Tage", + "group.yesterday": "Gestern", + "hist.editAnswer": "Antwort bearbeiten", + "hist.editAnswerTitle": "Den Text der Antwort korrigieren. Es wird keine Anfrage gesendet.", + "hist.editLabel": "Antwort bearbeiten", + "hist.editNote": "Das bisherige Denken und die Werkzeugaufrufe der Antwort werden entfernt. Es wird keine Anfrage gesendet.", + "hist.editRemoves": {"one": "Speichern blendet die {n} Nachricht danach aus. ", "other": "Speichern blendet die {n} Nachrichten danach aus. "}, + "hist.edited": "Bearbeitet", + "hist.editedHidden": {"one": "Antwort bearbeitet, {n} Nachricht ausgeblendet", "other": "Antwort bearbeitet, {n} Nachrichten ausgeblendet"}, + "hist.editedOnly": "Antwort bearbeitet", + "hist.empty": "Die Antwort ist leer", + "hist.emptyText": "Schreiben Sie etwas, bevor Sie speichern.", + "hist.finishFirst": "Schließen Sie zuerst ab, was offen ist", + "hist.finishFirstText": "Speichern Sie es oder drücken Sie Abbrechen.", + "hist.hidden": {"one": "{n} Nachricht ausgeblendet", "other": "{n} Nachrichten ausgeblendet"}, + "hist.lostText": "Dieser Browser hat nur die Namen behalten, nicht den Inhalt. Starten Sie einen neuen Chat und hängen Sie sie erneut an.", + "hist.lostText2": "Dieser Browser hat nur die Namen behalten, nicht den Inhalt. Hängen Sie sie erneut an.", + "hist.lostTitle": "Anhänge müssen erneut hinzugefügt werden", + "hist.nothingHidden": "Zurück bei dieser Nachricht", + "hist.pendingFirst": "Eine Nachricht wartet", + "hist.pendingFirstText": "Senden Sie sie oder nehmen Sie sie zuerst zurück.", + "hist.regenerate": "Neu erzeugen", + "hist.regenerateTitle": "Eine neue Antwort auf diese Frage verlangen. Werkzeuge können erneut laufen.", + "hist.replaced": "Die Antwort wurde ersetzt", + "hist.rewind": "Bis hier zurückgehen", + "hist.rewindTitle": "Zu diesem Punkt im Chat zurückgehen", + "hist.save": "Speichern", + "math.source": "Formelquelltext ({reason})", + "model.alreadyUnloaded": "Bereits entladen", + "model.busy": "Das Modell ist beschäftigt", + "model.confirmUnload": "Das Modell entladen und die GPU freigeben? Laden oder die nächste Nachricht holt es zurück.", + "model.fewMinutes": "Das kann einige Minuten dauern.", + "model.gpuFree": "Die GPU ist frei. Laden oder die nächste Nachricht holt das Modell zurück.", + "model.gpuInUse": "Die GPU wird genutzt.", + "model.imageBusy": "Ein Bild wird erzeugt. Es wird von selbst fertig.", + "model.imageLoading": "Ein Bild lädt das Modell bereits", + "model.imageLoadingDot": "Ein Bild lädt das Modell bereits.", + "model.load": "Laden", + "model.loadFailed": "Laden fehlgeschlagen", + "model.loadTitle": "Das Modell jetzt laden. Eine Nachricht lädt es auch.", + "model.loaded": "Modell geladen", + "model.loading": "Lädt…", + "model.loadingTitle": "Das Modell wird geladen", + "model.loadingToast": "Das Modell wird geladen", + "model.noAnswer": "Der Server hat nicht geantwortet.", + "model.replyLoading": "Eine Antwort lädt das Modell bereits", + "model.replyLoadingDot": "Eine Antwort lädt das Modell bereits.", + "model.stopFirst": "Antwort stoppen, dann entladen", + "model.stopFirstDot": "Stoppen Sie die Antwort, dann entladen.", + "model.unload": "Entladen", + "model.unloadFailed": "Entladen fehlgeschlagen", + "model.unloadTitle": "Das Modell entladen und die GPU freigeben", + "model.unloaded": "Modell entladen", + "model.unloading": "Entlädt…", + "model.unloadingTitle": "Das Modell wird entladen", + "model.unsupported": "Entladen ist nicht verfügbar", + "model.waiting": "Warte auf den Server", + "pending.afterTool": "Wartet, bis das Werkzeug fertig ist, dann wird gesendet: „{text}“", + "pending.cutting": "Die Antwort wird gestoppt, dann wird gesendet: „{text}“", + "pending.one": "Eine Nachricht wartet bereits", + "pending.oneText": "Nehmen Sie sie zurück oder warten Sie, bis sie gesendet ist.", + "pending.queued": "Wird gesendet, wenn diese Antwort fertig ist: „{text}“", + "pending.ready": "Noch nicht gesendet: „{text}“", + "pending.send": "Senden", + "pending.sendNow": "Jetzt senden", + "pending.takeBack": "Zurücknehmen", + "pill.connecting": "Verbinde…", + "pill.generating": "Schreibt · {rate} Tok/s", + "pill.idle": "Leerlauf", + "pill.image": "Erzeugt ein Bild…", + "pill.keyNeeded": "API-Schlüssel nötig", + "pill.queued": "{n} in der Warteschlange", + "pill.reading": "Liest die Eingabe", + "pill.readingPct": "Liest die Eingabe · {pct} %", + "pill.unloaded": "Entladen", + "pill.unreachable": "Server nicht erreichbar", + "preview.confirm": "Diesen Code mit Skripten ausführen?\n\nEr läuft in Ihrem Browser, abgeschottet von diesem Chat und vom Netzwerk, kann aber den Tab trotzdem einfrieren, und es ist Code, den das Modell geschrieben hat.\n\nNur dieser Block, und nur bis Sie die Vorschau schließen.", + "preview.runScripts": "Skripte ausführen", + "preview.showCode": "Den Code zeigen", + "preview.showPage": "Die Seite in der Vorschau zeigen", + "preview.showSvg": "Das Bild in der Vorschau zeigen", + "preview.stopScripts": "Skripte beenden", + "preview.titlePage": "Seitenvorschau", + "preview.titleSvg": "Bildvorschau", + "rewind.branch": "Ab hier verzweigen", + "rewind.branchAnswerText": "Startet einen neuen Chat mit den Nachrichten bis zu dieser Antwort. Dieser Chat bleibt, wie er ist.", + "rewind.branchUserText": "Startet einen neuen Chat mit den Nachrichten vor dieser und legt Ihre Nachricht in sein Eingabefeld. Dieser Chat bleibt, wie er ist.", + "rewind.hereAnswer": "Ab hier weitermachen", + "rewind.hereAnswerText": "Blendet die Nachrichten nach dieser Antwort aus. Rückgängig holt sie zurück.", + "rewind.hereUser": "Hier bearbeiten und erneut senden", + "rewind.hereUserText": "Blendet diese Nachricht und die danach aus und legt Ihre Nachricht ins Eingabefeld unten. Rückgängig holt sie zurück.", + "rewind.noList": "Um stattdessen von hier einen neuen Chat zu starten, schalten Sie in den Chat-Einstellungen die Liste gespeicherter Chats ein.", + "rewind.title": "Bis hier zurückgehen", + "settings.language": "Sprache", + "settings.languageNote": "Chat, seine Einstellungen und die Schaltflächen oben sind übersetzt. Die Ansichten Monitor, Bilder und Info sind noch auf Englisch.", + "sidebar.collapse": "Die Chatliste einklappen", + "sidebar.delete": "Diesen Chat löschen", + "sidebar.deleteShort": "Löschen", + "sidebar.expand": "Die Chatliste öffnen", + "sidebar.foot": {"one": "{n} Chat · in diesem Browser gespeichert", "other": "{n} Chats · in diesem Browser gespeichert"}, + "sidebar.nameFor": "Name für diesen Chat", + "sidebar.noMatch": "Kein Chat passt", + "sidebar.none": "Noch keine Chats", + "sidebar.rename": "Diesen Chat umbenennen", + "sidebar.renameShort": "Umbenennen", + "sidebar.search": "Chats durchsuchen", + "sidebar.title": "Chats", + "store.failed": "Die Chatliste konnte nicht gespeichert werden. Die Chats auf dem Bildschirm sind noch da.", + "store.memory": "Dieser Browser behält die Chatliste nicht. Sie bleibt im Speicher, bis Sie die Seite schließen.", + "store.quota": "Dieser Browser kann nicht mehr behalten. Die Chats auf dem Bildschirm sind noch da.", + "store.title": "Chatliste nicht gespeichert", + "tab.about": "Info", + "tab.chat": "Chat", + "tab.images": "Bilder", + "tab.monitor": "Monitor", + "tab.views": "Ansichten", + "theme.label": "Zwischen hell und dunkel wechseln", + "theme.title": "Hell / Dunkel", + "think.done": "Gedanken", + "think.for": "Nachgedacht: {s} s", + "think.now": "Denkt nach…", + "toast.copied": "In die Zwischenablage kopiert", + "toast.engineError": "die Engine hat einen Fehler gemeldet", + "toast.httpAnswer": "Der Server antwortete mit HTTP {code}.", + "toast.keyAbout": "Dieser Server braucht einen Schlüssel: Tragen Sie ihn unter Info > Einstellungen ein.", + "toast.keyAbout2": "API-Schlüssel nötig: Tragen Sie ihn unter Info > Einstellungen ein.", + "toast.keyNeeded": "Dieser Server braucht einen API-Schlüssel: Tragen Sie ihn unter Info > Einstellungen ein.", + "toast.requestFailed": "Die Anfrage ist fehlgeschlagen", + "tool.arguments": "Argumente", + "tool.beingWritten": "(wird geschrieben)", + "tool.chars": "{n} Zeichen", + "tool.cut": ", für das Modell gekürzt", + "tool.done": "Fertig", + "tool.error": "Fehler", + "tool.result": "Ergebnis", + "tool.running": "Läuft", + "tool.skipped": "Nicht ausgeführt", + "tool.writing": "Schreibt", + "undo": "Rückgängig" +} diff --git a/edge0-strata/serve/web/locales/en.json b/edge0-strata/serve/web/locales/en.json new file mode 100644 index 00000000..adcd6fd2 --- /dev/null +++ b/edge0-strata/serve/web/locales/en.json @@ -0,0 +1,264 @@ +{ + "branch.back": "Back to it", + "branch.from": "Branched from “{title}”", + "branch.open": "Open the original", + "branch.started": "Branch started", + "branch.startedText": "The chat you were in is unchanged, in the list on the left.", + "branch.title": "Branch of {title}", + "busy.stillWriting": "Still writing", + "busy.stopFirst": "Stop the answer first.", + "cancel": "Cancel", + "chat.clearedToast": "The last one was cleared.", + "chat.copy": "Copy", + "chat.copyAnswer": "Copy the answer", + "chat.deleted": "Chat deleted", + "chat.empty": "Ask anything", + "chat.emptyModel": "{model} runs on this PC. Nothing leaves it.", + "chat.image": "image", + "chat.interrupted": "Interrupted", + "chat.interruptedLower": "interrupted", + "chat.new": "New chat", + "chat.newToast": "New chat", + "chat.newToastText": "The one you were in is in the list on the left.", + "chat.projectionOff": "projection off", + "chat.projectionOn": "projection on", + "chat.stopped": "Stopped", + "chat.stoppedLower": "stopped", + "chat.tokens": "{n} tokens", + "chat.toolCalls": { + "one": "{n} tool call", + "other": "{n} tool calls" + }, + "chat.toolLimit": "stopped at the limit of {n} tool rounds (mcp.max_rounds)", + "chat.untitled": "New chat", + "chat.you": "You", + "close": "Close", + "code.copy": "Copy code", + "code.label": "code", + "composer.attach": "Attach a text file (or drop it here)", + "composer.attachLabel": "Attach a file", + "composer.attachPictures": "Attach a text file or a picture (or drop it here)", + "composer.finishOpen": "Save or cancel what is open above to continue", + "composer.hintInterject": "Enter stops the answer here and sends now", + "composer.hintQueue": "Enter sends when the answer is done", + "composer.message": "Message", + "composer.newline": "Shift+Enter: new line", + "composer.placeholder": "Ask anything…", + "composer.save": "Save this chat as Markdown", + "composer.saveLabel": "Save this chat", + "composer.send": "Send", + "composer.stop": "Stop", + "drawer.apply": "Apply", + "drawer.busy": "When you send while it is answering", + "drawer.busyInterject": "Interrupt", + "drawer.busyInterjectNote": "The answer stops where it is (what is written stays) and your message goes at once. A tool that is running finishes first.", + "drawer.busyQueue": "Send after", + "drawer.busyQueueNote": "Your message waits and goes as soon as the answer is done.", + "drawer.busyWait": "Wait", + "drawer.busyWaitNote": "Send is off until the answer is done, as before.", + "drawer.chats": "Chats", + "drawer.edit": "Edit and rewind messages", + "drawer.editNote": "edit an answer, ask for a new one, or go back to an earlier message (with an Undo)", + "drawer.greedy": "0 · greedy", + "drawer.high": "High", + "drawer.list": "Saved chats list", + "drawer.listNote": "keeps every chat in a list on the left, in this browser only. Off: one chat, as before", + "drawer.low": "Low", + "drawer.math": "Render math", + "drawer.mathNote": "draw equations in answers as formulas (works offline)", + "drawer.maxTokens": "Max tokens", + "drawer.maxTokensNote": "empty = until done", + "drawer.mcp": "Use tools from MCP servers", + "drawer.medium": "Medium", + "drawer.off": "Off", + "drawer.random": "Random", + "drawer.reset": "Reset", + "drawer.saved": "Chat settings saved", + "drawer.savedGreedy": "Greedy: the same question gives the same answer.", + "drawer.savedHere": "Saved here, but not for other apps", + "drawer.savedOwn": "Other apps use their own settings again.", + "drawer.savedShared": "Other apps (omp, API clients) use these settings from their next request.", + "drawer.seed": "Seed", + "drawer.seedNote": "empty = random", + "drawer.share": "Use for other apps too", + "drawer.shareNote": "omp and other API clients get these settings for anything they do not set themselves", + "drawer.short": "Settings", + "drawer.showThinking": "Show thinking", + "drawer.showThinkingNote": "expanded while it streams", + "drawer.temperature": "Temperature", + "drawer.temperatureNote": "0 = always the most likely word (exact, repeatable)", + "drawer.thinkHigh": "thorough (default)", + "drawer.thinkLow": "short", + "drawer.thinkMedium": "medium", + "drawer.thinkNone": "answers right away", + "drawer.thinking": "Thinking", + "drawer.title": "Chat settings", + "drawer.untilDone": "Until done", + "export.nothing": "Nothing to save yet", + "file.big": "File too large", + "file.bigText": "{name} is over 512 KB.", + "file.binaryText": "{name} looks like a binary file.", + "file.notText": "Not a text file", + "file.notTextOnly": "{name}: attach text files (code, notes, logs, data).", + "file.notTextPictures": "{name}: attach text files (code, notes, logs, data) or pictures.", + "file.pasted": "pasted image", + "file.pictureBig": "Picture too large", + "file.pictureBigText": "{name} is over 20 MB.", + "file.picturesOff": "Pictures are off", + "file.picturesOffText": "This model was set up for text only.", + "file.remove": "Remove", + "group.month": "Previous 30 days", + "group.older": "Older", + "group.today": "Today", + "group.week": "Previous 7 days", + "group.yesterday": "Yesterday", + "hist.editAnswer": "Edit answer", + "hist.editAnswerTitle": "Correct the answer's text. No request is sent.", + "hist.editLabel": "Edit the answer", + "hist.editNote": "The answer's old thinking and tool calls are removed. No request is sent.", + "hist.editRemoves": { + "one": "Saving hides the {n} message after it. ", + "other": "Saving hides the {n} messages after it. " + }, + "hist.edited": "Edited", + "hist.editedHidden": { + "one": "Answer edited, {n} message hidden", + "other": "Answer edited, {n} messages hidden" + }, + "hist.editedOnly": "Answer edited", + "hist.empty": "The answer is empty", + "hist.emptyText": "Write something before saving.", + "hist.finishFirst": "Finish what is open first", + "hist.finishFirstText": "Save it or press Cancel.", + "hist.hidden": { + "one": "{n} message hidden", + "other": "{n} messages hidden" + }, + "hist.lostText": "This browser kept their names, not their contents. Start a new chat and attach them again.", + "hist.lostText2": "This browser kept their names, not their contents. Attach them again.", + "hist.lostTitle": "Attachments need to be added again", + "hist.nothingHidden": "Back at this message", + "hist.pendingFirst": "A message is waiting", + "hist.pendingFirstText": "Send it or take it back first.", + "hist.regenerate": "Regenerate", + "hist.regenerateTitle": "Ask for a new answer to this question. Tools may run again.", + "hist.replaced": "The answer was replaced", + "hist.rewind": "Rewind to here", + "hist.rewindTitle": "Go back to this point in the chat", + "hist.save": "Save", + "math.source": "Math source ({reason})", + "model.alreadyUnloaded": "Already unloaded", + "model.busy": "Model is busy", + "model.confirmUnload": "Unload the model and free the GPU? Load, or the next message, brings it back.", + "model.fewMinutes": "This can take a few minutes.", + "model.gpuFree": "The GPU is free. Load, or the next message, brings it back.", + "model.gpuInUse": "The GPU is in use.", + "model.imageBusy": "An image is generating. It finishes on its own.", + "model.imageLoading": "An image is already loading the model", + "model.imageLoadingDot": "An image is already loading the model.", + "model.load": "Load", + "model.loadFailed": "Load failed", + "model.loadTitle": "Load the model now. A message also loads it.", + "model.loaded": "Model loaded", + "model.loading": "Loading…", + "model.loadingTitle": "The model is loading", + "model.loadingToast": "Loading the model", + "model.noAnswer": "The server did not answer.", + "model.replyLoading": "A reply is already loading the model", + "model.replyLoadingDot": "A reply is already loading the model.", + "model.stopFirst": "Stop the reply, then unload", + "model.stopFirstDot": "Stop the reply, then unload.", + "model.unload": "Unload", + "model.unloadFailed": "Unload failed", + "model.unloadTitle": "Unload the model and free the GPU", + "model.unloaded": "Model unloaded", + "model.unloading": "Unloading…", + "model.unloadingTitle": "The model is unloading", + "model.unsupported": "Unload is not available", + "model.waiting": "Waiting for the server", + "pending.afterTool": "Waiting for the tool to finish, then sending: “{text}”", + "pending.cutting": "Stopping the answer, then sending: “{text}”", + "pending.one": "A message is already waiting", + "pending.oneText": "Take it back or wait until it is sent.", + "pending.queued": "Sends when this answer is done: “{text}”", + "pending.ready": "Not sent yet: “{text}”", + "pending.send": "Send", + "pending.sendNow": "Send now", + "pending.takeBack": "Take back", + "pill.connecting": "Connecting…", + "pill.generating": "Generating · {rate} tok/s", + "pill.idle": "Idle", + "pill.image": "Generating image…", + "pill.keyNeeded": "API key needed", + "pill.queued": "{n} queued", + "pill.reading": "Reading prompt", + "pill.readingPct": "Reading prompt · {pct}%", + "pill.unloaded": "Unloaded", + "pill.unreachable": "Server not reachable", + "preview.confirm": "Run this code with scripts?\n\nIt runs in your browser, cut off from this chat and from the network, but it can still freeze the tab, and it is code the model wrote.\n\nOnly this block, and only until you close the preview.", + "preview.runScripts": "Run scripts", + "preview.showCode": "Show the code", + "preview.showPage": "Preview the page", + "preview.showSvg": "Preview the picture", + "preview.stopScripts": "Stop scripts", + "preview.titlePage": "Page preview", + "preview.titleSvg": "Picture preview", + "rewind.branch": "Branch from here", + "rewind.branchAnswerText": "Starts a new chat with the messages up to this answer. This chat stays as it is.", + "rewind.branchUserText": "Starts a new chat with the messages before this one, and puts your message in its box. This chat stays as it is.", + "rewind.hereAnswer": "Continue from here", + "rewind.hereAnswerText": "Hides the messages after this answer. Undo brings them back.", + "rewind.hereUser": "Edit and send again here", + "rewind.hereUserText": "Hides this message and the ones after it, and puts your message in the box below. Undo brings them back.", + "rewind.noList": "To start a new chat from here instead, turn on Saved chats in Chat settings.", + "rewind.title": "Rewind to here", + "settings.language": "Language", + "settings.languageNote": "Chat, its settings and the buttons above are translated. The Monitor, Images and About views are still in English.", + "sidebar.collapse": "Collapse the chat list", + "sidebar.delete": "Delete this chat", + "sidebar.deleteShort": "Delete", + "sidebar.expand": "Open the chat list", + "sidebar.foot": { + "one": "{n} chat · kept in this browser", + "other": "{n} chats · kept in this browser" + }, + "sidebar.nameFor": "Name for this chat", + "sidebar.noMatch": "No chat matches", + "sidebar.none": "No chats yet", + "sidebar.rename": "Rename this chat", + "sidebar.renameShort": "Rename", + "sidebar.search": "Search chats", + "sidebar.title": "Chats", + "store.failed": "The chat list could not be saved. The chats on the screen are still there.", + "store.memory": "This browser will not keep the chat list. It stays in memory until you close the page.", + "store.quota": "This browser cannot keep more. The chats on the screen are still there.", + "store.title": "Chat list not saved", + "tab.about": "About", + "tab.chat": "Chat", + "tab.images": "Images", + "tab.monitor": "Monitor", + "tab.views": "Views", + "theme.label": "Switch between light and dark", + "theme.title": "Light / dark", + "think.done": "Thoughts", + "think.for": "Thought for {s} s", + "think.now": "Thinking…", + "toast.copied": "Copied to clipboard", + "toast.engineError": "the engine reported an error", + "toast.httpAnswer": "The server answered HTTP {code}.", + "toast.keyAbout": "This server needs a key: add it under About > Settings.", + "toast.keyAbout2": "API key needed: add it under About > Settings.", + "toast.keyNeeded": "This server needs an API key: add it under About > Settings.", + "toast.requestFailed": "The request failed", + "tool.arguments": "Arguments", + "tool.beingWritten": "(being written)", + "tool.chars": "{n} characters", + "tool.cut": ", cut for the model", + "tool.done": "Done", + "tool.error": "Error", + "tool.result": "Result", + "tool.running": "Running", + "tool.skipped": "Not run", + "tool.writing": "Writing", + "undo": "Undo" +} diff --git a/edge0-strata/serve/web/locales/es.json b/edge0-strata/serve/web/locales/es.json new file mode 100644 index 00000000..9f9a2429 --- /dev/null +++ b/edge0-strata/serve/web/locales/es.json @@ -0,0 +1,249 @@ +{ + "branch.back": "Volver a ella", + "branch.from": "Rama creada desde «{title}»", + "branch.open": "Abrir la original", + "branch.started": "Rama creada", + "branch.startedText": "El chat en el que estabas no ha cambiado; está en la lista de la izquierda.", + "branch.title": "Rama de {title}", + "busy.stillWriting": "Todavía está escribiendo", + "busy.stopFirst": "Detén primero la respuesta.", + "cancel": "Cancelar", + "chat.clearedToast": "Se vació el último chat.", + "chat.copy": "Copiar", + "chat.copyAnswer": "Copiar la respuesta", + "chat.deleted": "Chat eliminado", + "chat.empty": "Pregunta lo que quieras", + "chat.emptyModel": "{model} se ejecuta en este PC. Nada sale de él.", + "chat.image": "imagen", + "chat.interrupted": "Interrumpida", + "chat.interruptedLower": "interrumpida", + "chat.new": "Nuevo chat", + "chat.newToast": "Nuevo chat", + "chat.newToastText": "El chat anterior está en la lista de la izquierda.", + "chat.projectionOff": "proyección desactivada", + "chat.projectionOn": "proyección activada", + "chat.stopped": "Detenida", + "chat.stoppedLower": "detenida", + "chat.tokens": "{n} tokens", + "chat.toolCalls": {"one": "{n} llamada a herramienta", "other": "{n} llamadas a herramientas"}, + "chat.toolLimit": "detenida en el límite de {n} rondas de herramientas (mcp.max_rounds)", + "chat.untitled": "Nuevo chat", + "chat.you": "Tú", + "close": "Cerrar", + "code.copy": "Copiar el código", + "code.label": "código", + "composer.attach": "Adjuntar un archivo de texto (o soltarlo aquí)", + "composer.attachLabel": "Adjuntar un archivo", + "composer.attachPictures": "Adjuntar un archivo de texto o una imagen (o soltarlos aquí)", + "composer.finishOpen": "Guarda o cancela lo que está abierto arriba para continuar", + "composer.hintInterject": "Intro detiene la respuesta aquí y envía ahora", + "composer.hintQueue": "Intro envía cuando termine la respuesta", + "composer.message": "Mensaje", + "composer.newline": "Mayús+Intro: nueva línea", + "composer.placeholder": "Pregunta lo que quieras…", + "composer.save": "Guardar este chat como Markdown", + "composer.saveLabel": "Guardar este chat", + "composer.send": "Enviar", + "composer.stop": "Detener", + "drawer.apply": "Aplicar", + "drawer.busy": "Si envías mientras responde", + "drawer.busyInterject": "Interrumpir", + "drawer.busyInterjectNote": "La respuesta se detiene donde está (lo escrito se queda) y tu mensaje sale enseguida. Una herramienta en marcha termina primero.", + "drawer.busyQueue": "Enviar después", + "drawer.busyQueueNote": "Tu mensaje espera y sale en cuanto termine la respuesta.", + "drawer.busyWait": "Esperar", + "drawer.busyWaitNote": "Enviar está desactivado hasta que termine la respuesta, como antes.", + "drawer.chats": "Chats", + "drawer.edit": "Editar y rebobinar mensajes", + "drawer.editNote": "editar una respuesta, pedir otra nueva o volver a un mensaje anterior (con Deshacer)", + "drawer.greedy": "0 · determinista", + "drawer.high": "Alto", + "drawer.list": "Lista de chats guardados", + "drawer.listNote": "guarda cada chat en una lista a la izquierda, solo en este navegador. Desactivado: un solo chat, como antes", + "drawer.low": "Bajo", + "drawer.math": "Mostrar fórmulas", + "drawer.mathNote": "dibuja las ecuaciones de las respuestas como fórmulas (funciona sin conexión)", + "drawer.maxTokens": "Máx. de tokens", + "drawer.maxTokensNote": "vacío = hasta terminar", + "drawer.mcp": "Usar herramientas de servidores MCP", + "drawer.medium": "Medio", + "drawer.off": "No", + "drawer.random": "Aleatoria", + "drawer.reset": "Restablecer", + "drawer.saved": "Ajustes del chat guardados", + "drawer.savedGreedy": "Determinista: la misma pregunta da la misma respuesta.", + "drawer.savedHere": "Guardado aquí, pero no para otras apps", + "drawer.savedOwn": "Las otras apps vuelven a usar sus propios ajustes.", + "drawer.savedShared": "Las otras apps (omp, clientes de la API) usan estos ajustes desde su próxima solicitud.", + "drawer.seed": "Semilla", + "drawer.seedNote": "vacío = aleatoria", + "drawer.share": "Usar también para otras apps", + "drawer.shareNote": "omp y otros clientes de la API reciben estos ajustes para todo lo que no definan ellos mismos", + "drawer.short": "Ajustes", + "drawer.showThinking": "Mostrar el razonamiento", + "drawer.showThinkingNote": "desplegado mientras se escribe", + "drawer.temperature": "Temperatura", + "drawer.temperatureNote": "0 = siempre la palabra más probable (exacto, repetible)", + "drawer.thinkHigh": "a fondo (predeterminado)", + "drawer.thinkLow": "breve", + "drawer.thinkMedium": "medio", + "drawer.thinkNone": "responde enseguida", + "drawer.thinking": "Razonamiento", + "drawer.title": "Ajustes del chat", + "drawer.untilDone": "Hasta terminar", + "export.nothing": "Todavía no hay nada que guardar", + "file.big": "Archivo demasiado grande", + "file.bigText": "{name} supera los 512 KB.", + "file.binaryText": "{name} parece un archivo binario.", + "file.notText": "No es un archivo de texto", + "file.notTextOnly": "{name}: adjunta archivos de texto (código, notas, registros, datos).", + "file.notTextPictures": "{name}: adjunta archivos de texto (código, notas, registros, datos) o imágenes.", + "file.pasted": "imagen pegada", + "file.pictureBig": "Imagen demasiado grande", + "file.pictureBigText": "{name} supera los 20 MB.", + "file.picturesOff": "Las imágenes están desactivadas", + "file.picturesOffText": "Este modelo se configuró solo para texto.", + "file.remove": "Quitar", + "group.month": "Últimos 30 días", + "group.older": "Más antiguos", + "group.today": "Hoy", + "group.week": "Últimos 7 días", + "group.yesterday": "Ayer", + "hist.editAnswer": "Editar respuesta", + "hist.editAnswerTitle": "Corrige el texto de la respuesta. No se envía ninguna solicitud.", + "hist.editLabel": "Editar la respuesta", + "hist.editNote": "Se quitan el razonamiento y las llamadas a herramientas anteriores de la respuesta. No se envía ninguna solicitud.", + "hist.editRemoves": {"one": "Al guardar se oculta el {n} mensaje posterior. ", "other": "Al guardar se ocultan los {n} mensajes posteriores. "}, + "hist.edited": "Editada", + "hist.editedHidden": {"one": "Respuesta editada, {n} mensaje oculto", "other": "Respuesta editada, {n} mensajes ocultos"}, + "hist.editedOnly": "Respuesta editada", + "hist.empty": "La respuesta está vacía", + "hist.emptyText": "Escribe algo antes de guardar.", + "hist.finishFirst": "Termina primero lo que está abierto", + "hist.finishFirstText": "Guárdalo o pulsa Cancelar.", + "hist.hidden": {"one": "{n} mensaje oculto", "other": "{n} mensajes ocultos"}, + "hist.lostText": "Este navegador guardó los nombres, no el contenido. Empieza un chat nuevo y adjúntalos de nuevo.", + "hist.lostText2": "Este navegador guardó los nombres, no el contenido. Adjúntalos de nuevo.", + "hist.lostTitle": "Hay que volver a adjuntar los archivos", + "hist.nothingHidden": "De vuelta en este mensaje", + "hist.pendingFirst": "Hay un mensaje esperando", + "hist.pendingFirstText": "Envíalo o recupéralo primero.", + "hist.regenerate": "Regenerar", + "hist.regenerateTitle": "Pedir una respuesta nueva a esta pregunta. Las herramientas pueden volver a ejecutarse.", + "hist.replaced": "Se reemplazó la respuesta", + "hist.rewind": "Volver hasta aquí", + "hist.rewindTitle": "Volver a este punto del chat", + "hist.save": "Guardar", + "math.source": "Código de la fórmula ({reason})", + "model.alreadyUnloaded": "Ya estaba descargado", + "model.busy": "El modelo está ocupado", + "model.confirmUnload": "¿Descargar el modelo y liberar la GPU? Cargar, o el siguiente mensaje, lo devuelve.", + "model.fewMinutes": "Puede tardar unos minutos.", + "model.gpuFree": "La GPU está libre. Cargar, o el siguiente mensaje, devuelve el modelo.", + "model.gpuInUse": "La GPU está en uso.", + "model.imageBusy": "Se está generando una imagen. Terminará por sí sola.", + "model.imageLoading": "Una imagen ya está cargando el modelo", + "model.imageLoadingDot": "Una imagen ya está cargando el modelo.", + "model.load": "Cargar", + "model.loadFailed": "No se pudo cargar", + "model.loadTitle": "Cargar el modelo ahora. Un mensaje también lo carga.", + "model.loaded": "Modelo cargado", + "model.loading": "Cargando…", + "model.loadingTitle": "El modelo se está cargando", + "model.loadingToast": "Cargando el modelo", + "model.noAnswer": "El servidor no respondió.", + "model.replyLoading": "Una respuesta ya está cargando el modelo", + "model.replyLoadingDot": "Una respuesta ya está cargando el modelo.", + "model.stopFirst": "Detén la respuesta y luego descarga", + "model.stopFirstDot": "Detén la respuesta y luego descarga.", + "model.unload": "Descargar", + "model.unloadFailed": "No se pudo descargar", + "model.unloadTitle": "Descargar el modelo y liberar la GPU", + "model.unloaded": "Modelo descargado", + "model.unloading": "Descargando…", + "model.unloadingTitle": "El modelo se está descargando", + "model.unsupported": "Descargar no está disponible", + "model.waiting": "Esperando al servidor", + "pending.afterTool": "Esperando a que termine la herramienta, luego se envía: «{text}»", + "pending.cutting": "Deteniendo la respuesta, luego se envía: «{text}»", + "pending.one": "Ya hay un mensaje esperando", + "pending.oneText": "Recupéralo o espera a que se envíe.", + "pending.queued": "Se envía cuando termine esta respuesta: «{text}»", + "pending.ready": "Aún no enviado: «{text}»", + "pending.send": "Enviar", + "pending.sendNow": "Enviar ahora", + "pending.takeBack": "Recuperar", + "pill.connecting": "Conectando…", + "pill.generating": "Escribiendo · {rate} tok/s", + "pill.idle": "Inactivo", + "pill.image": "Generando imagen…", + "pill.keyNeeded": "Falta la clave de API", + "pill.queued": "{n} en cola", + "pill.reading": "Leyendo el mensaje", + "pill.readingPct": "Leyendo el mensaje · {pct} %", + "pill.unloaded": "Descargado", + "pill.unreachable": "Servidor no disponible", + "preview.confirm": "¿Ejecutar este código con scripts?\n\nSe ejecuta en tu navegador, aislado de este chat y de la red, pero aun así puede bloquear la pestaña, y es código escrito por el modelo.\n\nSolo este bloque, y solo hasta que cierres la vista previa.", + "preview.runScripts": "Ejecutar scripts", + "preview.showCode": "Mostrar el código", + "preview.showPage": "Vista previa de la página", + "preview.showSvg": "Vista previa de la imagen", + "preview.stopScripts": "Detener scripts", + "preview.titlePage": "Vista previa de la página", + "preview.titleSvg": "Vista previa de la imagen", + "rewind.branch": "Crear una rama desde aquí", + "rewind.branchAnswerText": "Empieza un chat nuevo con los mensajes hasta esta respuesta. Este chat se queda como está.", + "rewind.branchUserText": "Empieza un chat nuevo con los mensajes anteriores a este y pone tu mensaje en su cuadro. Este chat se queda como está.", + "rewind.hereAnswer": "Continuar desde aquí", + "rewind.hereAnswerText": "Oculta los mensajes posteriores a esta respuesta. Deshacer los devuelve.", + "rewind.hereUser": "Editar y volver a enviar aquí", + "rewind.hereUserText": "Oculta este mensaje y los siguientes, y pone tu mensaje en el cuadro de abajo. Deshacer los devuelve.", + "rewind.noList": "Para empezar en su lugar un chat nuevo desde aquí, activa la lista de chats guardados en los ajustes del chat.", + "rewind.title": "Volver hasta aquí", + "settings.language": "Idioma", + "settings.languageNote": "El chat, sus ajustes y los botones de arriba están traducidos. Las vistas Monitor, Imágenes y Acerca de siguen en inglés.", + "sidebar.collapse": "Contraer la lista de chats", + "sidebar.delete": "Eliminar este chat", + "sidebar.deleteShort": "Eliminar", + "sidebar.expand": "Abrir la lista de chats", + "sidebar.foot": {"one": "{n} chat · guardado en este navegador", "other": "{n} chats · guardados en este navegador"}, + "sidebar.nameFor": "Nombre de este chat", + "sidebar.noMatch": "Ningún chat coincide", + "sidebar.none": "Aún no hay chats", + "sidebar.rename": "Cambiar el nombre de este chat", + "sidebar.renameShort": "Cambiar nombre", + "sidebar.search": "Buscar chats", + "sidebar.title": "Chats", + "store.failed": "No se pudo guardar la lista de chats. Los chats de la pantalla siguen ahí.", + "store.memory": "Este navegador no guardará la lista de chats. Se queda en memoria hasta que cierres la página.", + "store.quota": "Este navegador no puede guardar más. Los chats de la pantalla siguen ahí.", + "store.title": "Lista de chats sin guardar", + "tab.about": "Acerca de", + "tab.chat": "Chat", + "tab.images": "Imágenes", + "tab.monitor": "Monitor", + "tab.views": "Vistas", + "theme.label": "Cambiar entre claro y oscuro", + "theme.title": "Claro / oscuro", + "think.done": "Razonamiento", + "think.for": "Pensó durante {s} s", + "think.now": "Pensando…", + "toast.copied": "Copiado al portapapeles", + "toast.engineError": "el motor informó de un error", + "toast.httpAnswer": "El servidor respondió con HTTP {code}.", + "toast.keyAbout": "Este servidor necesita una clave: añádela en Acerca de > Ajustes.", + "toast.keyAbout2": "Falta la clave de API: añádela en Acerca de > Ajustes.", + "toast.keyNeeded": "Este servidor necesita una clave de API: añádela en Acerca de > Ajustes.", + "toast.requestFailed": "La solicitud falló", + "tool.arguments": "Argumentos", + "tool.beingWritten": "(se está escribiendo)", + "tool.chars": "{n} caracteres", + "tool.cut": ", recortado para el modelo", + "tool.done": "Hecho", + "tool.error": "Error", + "tool.result": "Resultado", + "tool.running": "En marcha", + "tool.skipped": "No ejecutada", + "tool.writing": "Escribiendo", + "undo": "Deshacer" +} diff --git a/edge0-strata/serve/web/locales/fr.json b/edge0-strata/serve/web/locales/fr.json new file mode 100644 index 00000000..7bbb7616 --- /dev/null +++ b/edge0-strata/serve/web/locales/fr.json @@ -0,0 +1,264 @@ +{ + "branch.back": "Y retourner", + "branch.from": "Créée à partir de « {title} »", + "branch.open": "Ouvrir l’originale", + "branch.started": "Branche créée", + "branch.startedText": "La conversation où vous étiez reste inchangée, dans la liste à gauche.", + "branch.title": "Branche de {title}", + "busy.stillWriting": "Écriture en cours", + "busy.stopFirst": "Arrêtez d’abord la réponse.", + "cancel": "Annuler", + "chat.clearedToast": "La dernière a été effacée.", + "chat.copy": "Copier", + "chat.copyAnswer": "Copier la réponse", + "chat.deleted": "Conversation supprimée", + "chat.empty": "Posez n’importe quelle question", + "chat.emptyModel": "{model} tourne sur ce PC. Rien n’en sort.", + "chat.image": "image", + "chat.interrupted": "Interrompue", + "chat.interruptedLower": "interrompue", + "chat.new": "Nouvelle conversation", + "chat.newToast": "Nouvelle conversation", + "chat.newToastText": "Celle où vous étiez est dans la liste à gauche.", + "chat.projectionOff": "projection désactivée", + "chat.projectionOn": "projection activée", + "chat.stopped": "Arrêtée", + "chat.stoppedLower": "arrêtée", + "chat.tokens": "{n} tokens", + "chat.toolCalls": { + "one": "{n} appel d’outil", + "other": "{n} appels d’outils" + }, + "chat.toolLimit": "arrêtée à la limite de {n} tours d’outils (mcp.max_rounds)", + "chat.untitled": "Nouvelle conversation", + "chat.you": "Vous", + "close": "Fermer", + "code.copy": "Copier le code", + "code.label": "code", + "composer.attach": "Joindre un fichier texte (ou le déposer ici)", + "composer.attachLabel": "Joindre un fichier", + "composer.attachPictures": "Joindre un fichier texte ou une image (ou la déposer ici)", + "composer.finishOpen": "Enregistrez ou annulez ce qui est ouvert au-dessus pour continuer", + "composer.hintInterject": "Entrée arrête la réponse ici et envoie tout de suite", + "composer.hintQueue": "Entrée envoie quand la réponse est terminée", + "composer.message": "Message", + "composer.newline": "Maj+Entrée : nouvelle ligne", + "composer.placeholder": "Posez une question…", + "composer.save": "Enregistrer cette conversation en Markdown", + "composer.saveLabel": "Enregistrer cette conversation", + "composer.send": "Envoyer", + "composer.stop": "Arrêter", + "drawer.apply": "Appliquer", + "drawer.busy": "Quand vous envoyez pendant qu’il répond", + "drawer.busyInterject": "Interrompre", + "drawer.busyInterjectNote": "La réponse s’arrête là où elle est (ce qui est écrit reste) et votre message part tout de suite. Un outil en cours se termine d’abord.", + "drawer.busyQueue": "Envoyer après", + "drawer.busyQueueNote": "Votre message attend et part dès que la réponse est terminée.", + "drawer.busyWait": "Attendre", + "drawer.busyWaitNote": "Envoyer est désactivé jusqu’à la fin de la réponse, comme avant.", + "drawer.chats": "Conversations", + "drawer.edit": "Modifier et revenir en arrière dans les messages", + "drawer.editNote": "modifier une réponse, en demander une nouvelle, ou revenir à un message précédent (avec Annuler)", + "drawer.greedy": "0 · glouton", + "drawer.high": "Élevé", + "drawer.list": "Liste des conversations enregistrées", + "drawer.listNote": "garde chaque conversation dans une liste à gauche, dans ce navigateur seulement. Désactivé : une seule conversation, comme avant", + "drawer.low": "Faible", + "drawer.math": "Afficher les maths", + "drawer.mathNote": "dessiner les équations des réponses sous forme de formules (fonctionne hors ligne)", + "drawer.maxTokens": "Tokens max.", + "drawer.maxTokensNote": "vide = jusqu’à la fin", + "drawer.mcp": "Utiliser les outils des serveurs MCP", + "drawer.medium": "Moyen", + "drawer.off": "Désactivé", + "drawer.random": "Aléatoire", + "drawer.reset": "Réinitialiser", + "drawer.saved": "Réglages de la conversation enregistrés", + "drawer.savedGreedy": "Glouton : la même question donne la même réponse.", + "drawer.savedHere": "Enregistré ici, mais pas pour les autres applications", + "drawer.savedOwn": "Les autres applications réutilisent leurs propres réglages.", + "drawer.savedShared": "Les autres applications (omp, clients API) utilisent ces réglages dès leur prochaine requête.", + "drawer.seed": "Graine", + "drawer.seedNote": "vide = aléatoire", + "drawer.share": "Utiliser aussi pour les autres applications", + "drawer.shareNote": "omp et les autres clients API reçoivent ces réglages pour tout ce qu’ils ne définissent pas eux-mêmes", + "drawer.short": "Réglages", + "drawer.showThinking": "Afficher la réflexion", + "drawer.showThinkingNote": "déployée pendant la diffusion", + "drawer.temperature": "Température", + "drawer.temperatureNote": "0 = toujours le mot le plus probable (exact, reproductible)", + "drawer.thinkHigh": "approfondie (par défaut)", + "drawer.thinkLow": "courte", + "drawer.thinkMedium": "moyenne", + "drawer.thinkNone": "répond tout de suite", + "drawer.thinking": "Réflexion", + "drawer.title": "Réglages de la conversation", + "drawer.untilDone": "Jusqu’à la fin", + "export.nothing": "Rien à enregistrer pour l’instant", + "file.big": "Fichier trop volumineux", + "file.bigText": "{name} dépasse 512 Ko.", + "file.binaryText": "{name} ressemble à un fichier binaire.", + "file.notText": "Ce n’est pas un fichier texte", + "file.notTextOnly": "{name} : joignez des fichiers texte (code, notes, journaux, données).", + "file.notTextPictures": "{name} : joignez des fichiers texte (code, notes, journaux, données) ou des images.", + "file.pasted": "image collée", + "file.pictureBig": "Image trop volumineuse", + "file.pictureBigText": "{name} dépasse 20 Mo.", + "file.picturesOff": "Les images sont désactivées", + "file.picturesOffText": "Ce modèle a été configuré pour le texte seul.", + "file.remove": "Retirer", + "group.month": "30 derniers jours", + "group.older": "Plus anciennes", + "group.today": "Aujourd’hui", + "group.week": "7 derniers jours", + "group.yesterday": "Hier", + "hist.editAnswer": "Modifier la réponse", + "hist.editAnswerTitle": "Corrigez le texte de la réponse. Aucune requête n’est envoyée.", + "hist.editLabel": "Modifier la réponse", + "hist.editNote": "L’ancienne réflexion et les anciens appels d’outils de la réponse sont retirés. Aucune requête n’est envoyée.", + "hist.editRemoves": { + "one": "Enregistrer masque le message qui la suit. ", + "other": "Enregistrer masque les {n} messages qui la suivent. " + }, + "hist.edited": "Modifiée", + "hist.editedHidden": { + "one": "Réponse modifiée, {n} message masqué", + "other": "Réponse modifiée, {n} messages masqués" + }, + "hist.editedOnly": "Réponse modifiée", + "hist.empty": "La réponse est vide", + "hist.emptyText": "Écrivez quelque chose avant d’enregistrer.", + "hist.finishFirst": "Terminez d’abord ce qui est ouvert", + "hist.finishFirstText": "Enregistrez-le ou appuyez sur Annuler.", + "hist.hidden": { + "one": "{n} message masqué", + "other": "{n} messages masqués" + }, + "hist.lostText": "Ce navigateur a gardé leurs noms, pas leur contenu. Ouvrez une nouvelle conversation et joignez-les à nouveau.", + "hist.lostText2": "Ce navigateur a gardé leurs noms, pas leur contenu. Joignez-les à nouveau.", + "hist.lostTitle": "Les pièces jointes doivent être ajoutées à nouveau", + "hist.nothingHidden": "De retour à ce message", + "hist.pendingFirst": "Un message attend", + "hist.pendingFirstText": "Envoyez-le ou reprenez-le d’abord.", + "hist.regenerate": "Régénérer", + "hist.regenerateTitle": "Demander une nouvelle réponse à cette question. Les outils peuvent se relancer.", + "hist.replaced": "La réponse a été remplacée", + "hist.rewind": "Revenir ici", + "hist.rewindTitle": "Revenir à ce point de la conversation", + "hist.save": "Enregistrer", + "math.source": "Source mathématique ({reason})", + "model.alreadyUnloaded": "Déjà déchargé", + "model.busy": "Le modèle est occupé", + "model.confirmUnload": "Décharger le modèle et libérer le GPU ? Charger, ou le prochain message, le ramène.", + "model.fewMinutes": "Cela peut prendre quelques minutes.", + "model.gpuFree": "Le GPU est libre. Charger, ou le prochain message, le ramène.", + "model.gpuInUse": "Le GPU est utilisé.", + "model.imageBusy": "Une image est en cours de génération. Elle se termine toute seule.", + "model.imageLoading": "Une image charge déjà le modèle", + "model.imageLoadingDot": "Une image charge déjà le modèle.", + "model.load": "Charger", + "model.loadFailed": "Échec du chargement", + "model.loadTitle": "Charger le modèle maintenant. Un message le charge aussi.", + "model.loaded": "Modèle chargé", + "model.loading": "Chargement…", + "model.loadingTitle": "Le modèle se charge", + "model.loadingToast": "Chargement du modèle", + "model.noAnswer": "Le serveur n’a pas répondu.", + "model.replyLoading": "Une réponse charge déjà le modèle", + "model.replyLoadingDot": "Une réponse charge déjà le modèle.", + "model.stopFirst": "Arrêtez la réponse, puis déchargez", + "model.stopFirstDot": "Arrêtez la réponse, puis déchargez.", + "model.unload": "Décharger", + "model.unloadFailed": "Échec du déchargement", + "model.unloadTitle": "Décharger le modèle et libérer le GPU", + "model.unloaded": "Modèle déchargé", + "model.unloading": "Déchargement…", + "model.unloadingTitle": "Le modèle se décharge", + "model.unsupported": "Le déchargement n’est pas disponible", + "model.waiting": "En attente du serveur", + "pending.afterTool": "Attente de la fin de l’outil, puis envoi : « {text} »", + "pending.cutting": "Arrêt de la réponse, puis envoi : « {text} »", + "pending.one": "Un message attend déjà", + "pending.oneText": "Reprenez-le ou attendez qu’il soit envoyé.", + "pending.queued": "Part quand cette réponse est terminée : « {text} »", + "pending.ready": "Pas encore envoyé : « {text} »", + "pending.send": "Envoyer", + "pending.sendNow": "Envoyer maintenant", + "pending.takeBack": "Reprendre", + "pill.connecting": "Connexion…", + "pill.generating": "Génération · {rate} tok/s", + "pill.idle": "Inactif", + "pill.image": "Génération de l’image…", + "pill.keyNeeded": "Clé API requise", + "pill.queued": "{n} en attente", + "pill.reading": "Lecture du prompt", + "pill.readingPct": "Lecture du prompt · {pct} %", + "pill.unloaded": "Déchargé", + "pill.unreachable": "Serveur injoignable", + "preview.confirm": "Exécuter ce code avec ses scripts ?\n\nIl s’exécute dans votre navigateur, coupé de cette conversation et du réseau, mais il peut quand même bloquer l’onglet, et c’est du code écrit par le modèle.\n\nSeulement ce bloc, et seulement jusqu’à la fermeture de l’aperçu.", + "preview.runScripts": "Exécuter les scripts", + "preview.showCode": "Afficher le code", + "preview.showPage": "Afficher un aperçu de la page", + "preview.showSvg": "Afficher un aperçu de l’image", + "preview.stopScripts": "Arrêter les scripts", + "preview.titlePage": "Aperçu de la page", + "preview.titleSvg": "Aperçu de l’image", + "rewind.branch": "Créer une branche ici", + "rewind.branchAnswerText": "Ouvre une nouvelle conversation avec les messages jusqu’à cette réponse. Cette conversation reste telle quelle.", + "rewind.branchUserText": "Ouvre une nouvelle conversation avec les messages avant celui-ci, et place votre message dans sa zone de saisie. Cette conversation reste telle quelle.", + "rewind.hereAnswer": "Continuer à partir d’ici", + "rewind.hereAnswerText": "Masque les messages après cette réponse. Annuler les ramène.", + "rewind.hereUser": "Modifier et renvoyer ici", + "rewind.hereUserText": "Masque ce message et ceux qui le suivent, et place votre message dans la zone ci-dessous. Annuler les ramène.", + "rewind.noList": "Pour plutôt ouvrir une nouvelle conversation à partir d’ici, activez Conversations enregistrées dans les réglages de la conversation.", + "rewind.title": "Revenir ici", + "settings.language": "Langue", + "settings.languageNote": "La conversation, ses réglages et les boutons ci-dessus sont traduits. Les vues Moniteur, Images et À propos sont encore en anglais.", + "sidebar.collapse": "Réduire la liste des conversations", + "sidebar.delete": "Supprimer cette conversation", + "sidebar.deleteShort": "Supprimer", + "sidebar.expand": "Ouvrir la liste des conversations", + "sidebar.foot": { + "one": "{n} conversation · gardée dans ce navigateur", + "other": "{n} conversations · gardées dans ce navigateur" + }, + "sidebar.nameFor": "Nom de cette conversation", + "sidebar.noMatch": "Aucune conversation ne correspond", + "sidebar.none": "Aucune conversation pour l’instant", + "sidebar.rename": "Renommer cette conversation", + "sidebar.renameShort": "Renommer", + "sidebar.search": "Rechercher dans les conversations", + "sidebar.title": "Conversations", + "store.failed": "La liste des conversations n’a pas pu être enregistrée. Les conversations à l’écran y sont toujours.", + "store.memory": "Ce navigateur ne gardera pas la liste des conversations. Elle reste en mémoire jusqu’à la fermeture de la page.", + "store.quota": "Ce navigateur ne peut pas en garder plus. Les conversations à l’écran y sont toujours.", + "store.title": "Liste des conversations non enregistrée", + "tab.about": "À propos", + "tab.chat": "Chat", + "tab.images": "Images", + "tab.monitor": "Moniteur", + "tab.views": "Vues", + "theme.label": "Basculer entre clair et sombre", + "theme.title": "Clair / sombre", + "think.done": "Réflexion", + "think.for": "A réfléchi pendant {s} s", + "think.now": "Réflexion…", + "toast.copied": "Copié dans le presse-papiers", + "toast.engineError": "le moteur a signalé une erreur", + "toast.httpAnswer": "Le serveur a répondu HTTP {code}.", + "toast.keyAbout": "Ce serveur a besoin d’une clé : ajoutez-la dans À propos > Paramètres.", + "toast.keyAbout2": "Clé API requise : ajoutez-la dans À propos > Paramètres.", + "toast.keyNeeded": "Ce serveur a besoin d’une clé API : ajoutez-la dans À propos > Paramètres.", + "toast.requestFailed": "La requête a échoué", + "tool.arguments": "Arguments", + "tool.beingWritten": "(en cours d’écriture)", + "tool.chars": "{n} caractères", + "tool.cut": ", coupé pour le modèle", + "tool.done": "Terminé", + "tool.error": "Erreur", + "tool.result": "Résultat", + "tool.running": "En cours", + "tool.skipped": "Non exécuté", + "tool.writing": "Écriture", + "undo": "Annuler" +} diff --git a/edge0-strata/serve/web/locales/ja.json b/edge0-strata/serve/web/locales/ja.json new file mode 100644 index 00000000..024050c1 --- /dev/null +++ b/edge0-strata/serve/web/locales/ja.json @@ -0,0 +1,249 @@ +{ + "branch.back": "元に戻る", + "branch.from": "「{title}」から分岐", + "branch.open": "元のチャットを開く", + "branch.started": "分岐を作成しました", + "branch.startedText": "元のチャットは変わらず、左の一覧にあります。", + "branch.title": "{title} の分岐", + "busy.stillWriting": "まだ書き込み中です", + "busy.stopFirst": "先に回答を停止してください。", + "cancel": "キャンセル", + "chat.clearedToast": "前のチャットを消去しました。", + "chat.copy": "コピー", + "chat.copyAnswer": "回答をコピー", + "chat.deleted": "チャットを削除しました", + "chat.empty": "なんでも聞いてください", + "chat.emptyModel": "{model} はこの PC で動いています。外部には何も送られません。", + "chat.image": "画像", + "chat.interrupted": "中断しました", + "chat.interruptedLower": "中断", + "chat.new": "新しいチャット", + "chat.newToast": "新しいチャット", + "chat.newToastText": "前のチャットは左の一覧にあります。", + "chat.projectionOff": "プロジェクションオフ", + "chat.projectionOn": "プロジェクションオン", + "chat.stopped": "停止しました", + "chat.stoppedLower": "停止", + "chat.tokens": "{n} トークン", + "chat.toolCalls": {"other": "ツール呼び出し {n} 回"}, + "chat.toolLimit": "ツールのラウンド上限 {n} 回で停止 (mcp.max_rounds)", + "chat.untitled": "新しいチャット", + "chat.you": "あなた", + "close": "閉じる", + "code.copy": "コードをコピー", + "code.label": "コード", + "composer.attach": "テキストファイルを添付 (ここにドロップも可)", + "composer.attachLabel": "ファイルを添付", + "composer.attachPictures": "テキストファイルまたは画像を添付 (ここにドロップも可)", + "composer.finishOpen": "続けるには、上で開いているものを保存するかキャンセルしてください", + "composer.hintInterject": "Enter で回答をここで止めてすぐに送信します", + "composer.hintQueue": "Enter で回答が終わってから送信します", + "composer.message": "メッセージ", + "composer.newline": "Shift+Enter: 改行", + "composer.placeholder": "なんでも聞いてください…", + "composer.save": "このチャットを Markdown で保存", + "composer.saveLabel": "このチャットを保存", + "composer.send": "送信", + "composer.stop": "停止", + "drawer.apply": "適用", + "drawer.busy": "回答中に送信したとき", + "drawer.busyInterject": "割り込む", + "drawer.busyInterjectNote": "回答はその場で止まり (書かれた分は残ります)、メッセージがすぐ送られます。実行中のツールは先に終わります。", + "drawer.busyQueue": "終了後に送信", + "drawer.busyQueueNote": "メッセージは待機し、回答が終わりしだい送られます。", + "drawer.busyWait": "待つ", + "drawer.busyWaitNote": "回答が終わるまで送信はできません (これまでどおり)。", + "drawer.chats": "チャット", + "drawer.edit": "メッセージの編集と巻き戻し", + "drawer.editNote": "回答の編集、新しい回答の再生成、以前のメッセージへの巻き戻し (元に戻せます)", + "drawer.greedy": "0 · 固定", + "drawer.high": "高", + "drawer.list": "保存したチャットの一覧", + "drawer.listNote": "すべてのチャットを左の一覧に、このブラウザー内だけに保存します。オフ: これまでどおり 1 つのチャット", + "drawer.low": "低", + "drawer.math": "数式を表示", + "drawer.mathNote": "回答の数式を整形して描画します (オフラインで動作)", + "drawer.maxTokens": "最大トークン数", + "drawer.maxTokensNote": "空欄 = 最後まで", + "drawer.mcp": "MCP サーバーのツールを使う", + "drawer.medium": "中", + "drawer.off": "オフ", + "drawer.random": "ランダム", + "drawer.reset": "リセット", + "drawer.saved": "チャット設定を保存しました", + "drawer.savedGreedy": "固定: 同じ質問には同じ回答が返ります。", + "drawer.savedHere": "ここには保存しましたが、他のアプリには反映されません", + "drawer.savedOwn": "他のアプリは再び独自の設定を使います。", + "drawer.savedShared": "他のアプリ (omp、API クライアント) は次のリクエストからこの設定を使います。", + "drawer.seed": "シード", + "drawer.seedNote": "空欄 = ランダム", + "drawer.share": "他のアプリにも使う", + "drawer.shareNote": "omp などの API クライアントが自分で指定しない項目に、この設定が使われます", + "drawer.short": "設定", + "drawer.showThinking": "思考を表示", + "drawer.showThinkingNote": "生成中は開いた状態にします", + "drawer.temperature": "温度", + "drawer.temperatureNote": "0 = 常に最も確率の高い単語 (正確で再現可能)", + "drawer.thinkHigh": "じっくり (既定)", + "drawer.thinkLow": "短め", + "drawer.thinkMedium": "中程度", + "drawer.thinkNone": "すぐに回答", + "drawer.thinking": "思考", + "drawer.title": "チャット設定", + "drawer.untilDone": "最後まで", + "export.nothing": "保存するものがまだありません", + "file.big": "ファイルが大きすぎます", + "file.bigText": "{name} は 512 KB を超えています。", + "file.binaryText": "{name} はバイナリファイルのようです。", + "file.notText": "テキストファイルではありません", + "file.notTextOnly": "{name}: テキストファイル (コード、メモ、ログ、データ) を添付してください。", + "file.notTextPictures": "{name}: テキストファイル (コード、メモ、ログ、データ) または画像を添付してください。", + "file.pasted": "貼り付けた画像", + "file.pictureBig": "画像が大きすぎます", + "file.pictureBigText": "{name} は 20 MB を超えています。", + "file.picturesOff": "画像はオフです", + "file.picturesOffText": "このモデルはテキスト専用に設定されています。", + "file.remove": "削除", + "group.month": "過去 30 日", + "group.older": "それ以前", + "group.today": "今日", + "group.week": "過去 7 日", + "group.yesterday": "昨日", + "hist.editAnswer": "回答を編集", + "hist.editAnswerTitle": "回答の文章を直します。リクエストは送られません。", + "hist.editLabel": "回答を編集", + "hist.editNote": "回答のこれまでの思考とツール呼び出しは削除されます。リクエストは送られません。", + "hist.editRemoves": {"other": "保存すると、これ以降の {n} 件のメッセージが非表示になります。"}, + "hist.edited": "編集済み", + "hist.editedHidden": {"other": "回答を編集しました。{n} 件のメッセージを非表示"}, + "hist.editedOnly": "回答を編集しました", + "hist.empty": "回答が空です", + "hist.emptyText": "保存する前に何か入力してください。", + "hist.finishFirst": "先に開いているものを終えてください", + "hist.finishFirstText": "保存するか、キャンセルを押してください。", + "hist.hidden": {"other": "{n} 件のメッセージを非表示"}, + "hist.lostText": "このブラウザーには名前だけが残り、内容は残っていません。新しいチャットを始めて、もう一度添付してください。", + "hist.lostText2": "このブラウザーには名前だけが残り、内容は残っていません。もう一度添付してください。", + "hist.lostTitle": "添付をもう一度追加してください", + "hist.nothingHidden": "このメッセージまで戻りました", + "hist.pendingFirst": "待機中のメッセージがあります", + "hist.pendingFirstText": "先に送信するか、取り下げてください。", + "hist.regenerate": "再生成", + "hist.regenerateTitle": "この質問に新しい回答を求めます。ツールがもう一度動くことがあります。", + "hist.replaced": "回答を置き換えました", + "hist.rewind": "ここまで戻す", + "hist.rewindTitle": "チャットのこの時点まで戻ります", + "hist.save": "保存", + "math.source": "数式のソース ({reason})", + "model.alreadyUnloaded": "すでに解放済みです", + "model.busy": "モデルは使用中です", + "model.confirmUnload": "モデルを解放して GPU を空けますか？「読み込む」または次のメッセージで元に戻ります。", + "model.fewMinutes": "数分かかることがあります。", + "model.gpuFree": "GPU は空きました。「読み込む」または次のメッセージでモデルが戻ります。", + "model.gpuInUse": "GPU を使用中です。", + "model.imageBusy": "画像を生成中です。自動で終わります。", + "model.imageLoading": "画像がすでにモデルを読み込んでいます", + "model.imageLoadingDot": "画像がすでにモデルを読み込んでいます。", + "model.load": "読み込む", + "model.loadFailed": "読み込みに失敗しました", + "model.loadTitle": "今すぐモデルを読み込みます。メッセージを送っても読み込まれます。", + "model.loaded": "モデルを読み込みました", + "model.loading": "読み込み中…", + "model.loadingTitle": "モデルを読み込んでいます", + "model.loadingToast": "モデルを読み込み中", + "model.noAnswer": "サーバーが応答しませんでした。", + "model.replyLoading": "回答がすでにモデルを読み込んでいます", + "model.replyLoadingDot": "回答がすでにモデルを読み込んでいます。", + "model.stopFirst": "回答を停止してから解放してください", + "model.stopFirstDot": "回答を停止してから解放してください。", + "model.unload": "解放", + "model.unloadFailed": "解放に失敗しました", + "model.unloadTitle": "モデルを解放して GPU を空けます", + "model.unloaded": "モデルを解放しました", + "model.unloading": "解放中…", + "model.unloadingTitle": "モデルを解放しています", + "model.unsupported": "解放は利用できません", + "model.waiting": "サーバーを待っています", + "pending.afterTool": "ツールの終了を待ってから送信します:「{text}」", + "pending.cutting": "回答を止めてから送信します:「{text}」", + "pending.one": "すでに待機中のメッセージがあります", + "pending.oneText": "取り下げるか、送信されるまでお待ちください。", + "pending.queued": "この回答が終わったら送信します:「{text}」", + "pending.ready": "未送信:「{text}」", + "pending.send": "送信", + "pending.sendNow": "今すぐ送信", + "pending.takeBack": "取り下げる", + "pill.connecting": "接続中…", + "pill.generating": "生成中 · {rate} tok/s", + "pill.idle": "待機中", + "pill.image": "画像を生成中…", + "pill.keyNeeded": "API キーが必要です", + "pill.queued": "待機 {n} 件", + "pill.reading": "プロンプトを読み込み中", + "pill.readingPct": "プロンプトを読み込み中 · {pct}%", + "pill.unloaded": "解放済み", + "pill.unreachable": "サーバーに接続できません", + "preview.confirm": "このコードをスクリプト付きで実行しますか？\n\nコードはお使いのブラウザー内で、このチャットとネットワークから隔離されて動きますが、タブがフリーズすることはあります。また、モデルが書いたコードです。\n\nこのブロックだけ、プレビューを閉じるまでの間だけ実行されます。", + "preview.runScripts": "スクリプトを実行", + "preview.showCode": "コードを表示", + "preview.showPage": "ページをプレビュー", + "preview.showSvg": "画像をプレビュー", + "preview.stopScripts": "スクリプトを停止", + "preview.titlePage": "ページのプレビュー", + "preview.titleSvg": "画像のプレビュー", + "rewind.branch": "ここから分岐", + "rewind.branchAnswerText": "この回答までのメッセージで新しいチャットを始めます。このチャットはそのまま残ります。", + "rewind.branchUserText": "このメッセージより前のメッセージで新しいチャットを始め、あなたのメッセージをその入力欄に入れます。このチャットはそのまま残ります。", + "rewind.hereAnswer": "ここから続ける", + "rewind.hereAnswerText": "この回答より後のメッセージを非表示にします。「元に戻す」で戻せます。", + "rewind.hereUser": "ここで編集して再送信", + "rewind.hereUserText": "このメッセージとそれ以降を非表示にし、あなたのメッセージを下の入力欄に入れます。「元に戻す」で戻せます。", + "rewind.noList": "代わりにここから新しいチャットを始めるには、チャット設定で「保存したチャットの一覧」をオンにしてください。", + "rewind.title": "ここまで戻す", + "settings.language": "言語", + "settings.languageNote": "チャットとその設定、上のボタンは翻訳されています。モニター、画像、情報の画面はまだ英語です。", + "sidebar.collapse": "チャット一覧を閉じる", + "sidebar.delete": "このチャットを削除", + "sidebar.deleteShort": "削除", + "sidebar.expand": "チャット一覧を開く", + "sidebar.foot": {"other": "{n} 件のチャット · このブラウザーに保存"}, + "sidebar.nameFor": "このチャットの名前", + "sidebar.noMatch": "一致するチャットはありません", + "sidebar.none": "チャットはまだありません", + "sidebar.rename": "このチャットの名前を変更", + "sidebar.renameShort": "名前を変更", + "sidebar.search": "チャットを検索", + "sidebar.title": "チャット", + "store.failed": "チャット一覧を保存できませんでした。画面上のチャットはそのまま残っています。", + "store.memory": "このブラウザーはチャット一覧を保存しません。ページを閉じるまでメモリ上に残ります。", + "store.quota": "このブラウザーはこれ以上保存できません。画面上のチャットはそのまま残っています。", + "store.title": "チャット一覧は保存されていません", + "tab.about": "情報", + "tab.chat": "チャット", + "tab.images": "画像", + "tab.monitor": "モニター", + "tab.views": "表示", + "theme.label": "ライトとダークを切り替え", + "theme.title": "ライト / ダーク", + "think.done": "思考", + "think.for": "{s} 秒考えました", + "think.now": "考え中…", + "toast.copied": "クリップボードにコピーしました", + "toast.engineError": "エンジンがエラーを報告しました", + "toast.httpAnswer": "サーバーが HTTP {code} で応答しました。", + "toast.keyAbout": "このサーバーにはキーが必要です。「情報」>「設定」で入力してください。", + "toast.keyAbout2": "API キーが必要です。「情報」>「設定」で入力してください。", + "toast.keyNeeded": "このサーバーには API キーが必要です。「情報」>「設定」で入力してください。", + "toast.requestFailed": "リクエストに失敗しました", + "tool.arguments": "引数", + "tool.beingWritten": "(書き込み中)", + "tool.chars": "{n} 文字", + "tool.cut": "、モデル向けに省略", + "tool.done": "完了", + "tool.error": "エラー", + "tool.result": "結果", + "tool.running": "実行中", + "tool.skipped": "未実行", + "tool.writing": "書き込み中", + "undo": "元に戻す" +} diff --git a/edge0-strata/serve/web/locales/ko.json b/edge0-strata/serve/web/locales/ko.json new file mode 100644 index 00000000..c14f4437 --- /dev/null +++ b/edge0-strata/serve/web/locales/ko.json @@ -0,0 +1,249 @@ +{ + "branch.back": "원래 채팅으로", + "branch.from": "“{title}”에서 분기됨", + "branch.open": "원본 열기", + "branch.started": "분기를 만들었습니다", + "branch.startedText": "이전 채팅은 그대로이며 왼쪽 목록에 있습니다.", + "branch.title": "{title}의 분기", + "busy.stillWriting": "아직 작성 중입니다", + "busy.stopFirst": "먼저 답변을 중지하세요.", + "cancel": "취소", + "chat.clearedToast": "지난 채팅을 비웠습니다.", + "chat.copy": "복사", + "chat.copyAnswer": "답변 복사", + "chat.deleted": "채팅을 삭제했습니다", + "chat.empty": "무엇이든 물어보세요", + "chat.emptyModel": "{model}은(는) 이 PC에서 실행됩니다. 밖으로 나가는 것은 없습니다.", + "chat.image": "이미지", + "chat.interrupted": "중단됨", + "chat.interruptedLower": "중단됨", + "chat.new": "새 채팅", + "chat.newToast": "새 채팅", + "chat.newToastText": "이전 채팅은 왼쪽 목록에 있습니다.", + "chat.projectionOff": "프로젝션 꺼짐", + "chat.projectionOn": "프로젝션 켜짐", + "chat.stopped": "중지됨", + "chat.stoppedLower": "중지됨", + "chat.tokens": "토큰 {n}개", + "chat.toolCalls": {"other": "도구 호출 {n}회"}, + "chat.toolLimit": "도구 라운드 한도 {n}회에서 중지됨 (mcp.max_rounds)", + "chat.untitled": "새 채팅", + "chat.you": "나", + "close": "닫기", + "code.copy": "코드 복사", + "code.label": "코드", + "composer.attach": "텍스트 파일 첨부 (여기로 끌어다 놓아도 됩니다)", + "composer.attachLabel": "파일 첨부", + "composer.attachPictures": "텍스트 파일 또는 이미지 첨부 (여기로 끌어다 놓아도 됩니다)", + "composer.finishOpen": "계속하려면 위에 열려 있는 것을 저장하거나 취소하세요", + "composer.hintInterject": "Enter를 누르면 답변을 여기서 멈추고 바로 보냅니다", + "composer.hintQueue": "Enter를 누르면 답변이 끝난 뒤에 보냅니다", + "composer.message": "메시지", + "composer.newline": "Shift+Enter: 줄바꿈", + "composer.placeholder": "무엇이든 물어보세요…", + "composer.save": "이 채팅을 Markdown으로 저장", + "composer.saveLabel": "이 채팅 저장", + "composer.send": "보내기", + "composer.stop": "중지", + "drawer.apply": "적용", + "drawer.busy": "답변 중에 보내면", + "drawer.busyInterject": "중단", + "drawer.busyInterjectNote": "답변이 그 자리에서 멈추고 (쓰인 내용은 남습니다) 메시지가 바로 전송됩니다. 실행 중인 도구는 먼저 끝납니다.", + "drawer.busyQueue": "끝난 뒤 보내기", + "drawer.busyQueueNote": "메시지가 기다렸다가 답변이 끝나는 즉시 전송됩니다.", + "drawer.busyWait": "대기", + "drawer.busyWaitNote": "답변이 끝날 때까지 보내기가 꺼져 있습니다. 이전과 같습니다.", + "drawer.chats": "채팅", + "drawer.edit": "메시지 편집 및 되감기", + "drawer.editNote": "답변을 편집하거나, 새 답변을 요청하거나, 이전 메시지로 되돌아갑니다 (실행 취소 가능)", + "drawer.greedy": "0 · 고정", + "drawer.high": "높음", + "drawer.list": "저장된 채팅 목록", + "drawer.listNote": "모든 채팅을 이 브라우저 안에서만 왼쪽 목록에 보관합니다. 끔: 이전처럼 채팅 하나", + "drawer.low": "낮음", + "drawer.math": "수식 표시", + "drawer.mathNote": "답변의 수식을 공식 형태로 그립니다 (오프라인에서 동작)", + "drawer.maxTokens": "최대 토큰", + "drawer.maxTokensNote": "비우면 끝까지", + "drawer.mcp": "MCP 서버의 도구 사용", + "drawer.medium": "보통", + "drawer.off": "끔", + "drawer.random": "무작위", + "drawer.reset": "초기화", + "drawer.saved": "채팅 설정을 저장했습니다", + "drawer.savedGreedy": "고정: 같은 질문에는 같은 답변이 나옵니다.", + "drawer.savedHere": "여기에는 저장했지만 다른 앱에는 적용되지 않았습니다", + "drawer.savedOwn": "다른 앱이 다시 자체 설정을 사용합니다.", + "drawer.savedShared": "다른 앱 (omp, API 클라이언트)은 다음 요청부터 이 설정을 사용합니다.", + "drawer.seed": "시드", + "drawer.seedNote": "비우면 무작위", + "drawer.share": "다른 앱에도 사용", + "drawer.shareNote": "omp 등 API 클라이언트가 직접 지정하지 않은 항목에 이 설정이 적용됩니다", + "drawer.short": "설정", + "drawer.showThinking": "추론 표시", + "drawer.showThinkingNote": "작성되는 동안 펼쳐 둡니다", + "drawer.temperature": "온도", + "drawer.temperatureNote": "0 = 항상 가장 가능성 높은 단어 (정확하고 반복 가능)", + "drawer.thinkHigh": "꼼꼼히 (기본값)", + "drawer.thinkLow": "짧게", + "drawer.thinkMedium": "보통", + "drawer.thinkNone": "바로 답변", + "drawer.thinking": "추론", + "drawer.title": "채팅 설정", + "drawer.untilDone": "끝까지", + "export.nothing": "아직 저장할 내용이 없습니다", + "file.big": "파일이 너무 큽니다", + "file.bigText": "{name}은(는) 512 KB를 넘습니다.", + "file.binaryText": "{name}은(는) 바이너리 파일 같습니다.", + "file.notText": "텍스트 파일이 아닙니다", + "file.notTextOnly": "{name}: 텍스트 파일(코드, 메모, 로그, 데이터)을 첨부하세요.", + "file.notTextPictures": "{name}: 텍스트 파일(코드, 메모, 로그, 데이터) 또는 이미지를 첨부하세요.", + "file.pasted": "붙여넣은 이미지", + "file.pictureBig": "이미지가 너무 큽니다", + "file.pictureBigText": "{name}은(는) 20 MB를 넘습니다.", + "file.picturesOff": "이미지가 꺼져 있습니다", + "file.picturesOffText": "이 모델은 텍스트 전용으로 설정되었습니다.", + "file.remove": "제거", + "group.month": "지난 30일", + "group.older": "이전", + "group.today": "오늘", + "group.week": "지난 7일", + "group.yesterday": "어제", + "hist.editAnswer": "답변 편집", + "hist.editAnswerTitle": "답변의 텍스트를 고칩니다. 요청은 전송되지 않습니다.", + "hist.editLabel": "답변 편집", + "hist.editNote": "답변의 이전 추론과 도구 호출이 제거됩니다. 요청은 전송되지 않습니다.", + "hist.editRemoves": {"other": "저장하면 이후 메시지 {n}개가 숨겨집니다. "}, + "hist.edited": "편집됨", + "hist.editedHidden": {"other": "답변을 편집했습니다. 메시지 {n}개 숨김"}, + "hist.editedOnly": "답변을 편집했습니다", + "hist.empty": "답변이 비어 있습니다", + "hist.emptyText": "저장하기 전에 내용을 입력하세요.", + "hist.finishFirst": "먼저 열려 있는 것을 마무리하세요", + "hist.finishFirstText": "저장하거나 취소를 누르세요.", + "hist.hidden": {"other": "메시지 {n}개 숨김"}, + "hist.lostText": "이 브라우저에는 이름만 남아 있고 내용은 없습니다. 새 채팅을 시작해 다시 첨부하세요.", + "hist.lostText2": "이 브라우저에는 이름만 남아 있고 내용은 없습니다. 다시 첨부하세요.", + "hist.lostTitle": "첨부 파일을 다시 추가해야 합니다", + "hist.nothingHidden": "이 메시지로 돌아왔습니다", + "hist.pendingFirst": "대기 중인 메시지가 있습니다", + "hist.pendingFirstText": "먼저 보내거나 되돌려 놓으세요.", + "hist.regenerate": "다시 생성", + "hist.regenerateTitle": "이 질문에 새 답변을 요청합니다. 도구가 다시 실행될 수 있습니다.", + "hist.replaced": "답변을 교체했습니다", + "hist.rewind": "여기로 되감기", + "hist.rewindTitle": "채팅의 이 지점으로 돌아갑니다", + "hist.save": "저장", + "math.source": "수식 원문 ({reason})", + "model.alreadyUnloaded": "이미 해제되어 있습니다", + "model.busy": "모델이 사용 중입니다", + "model.confirmUnload": "모델을 해제하고 GPU를 비울까요? 불러오기 또는 다음 메시지로 다시 돌아옵니다.", + "model.fewMinutes": "몇 분 걸릴 수 있습니다.", + "model.gpuFree": "GPU가 비었습니다. 불러오기 또는 다음 메시지로 모델이 돌아옵니다.", + "model.gpuInUse": "GPU를 사용 중입니다.", + "model.imageBusy": "이미지를 생성 중입니다. 저절로 끝납니다.", + "model.imageLoading": "이미지가 이미 모델을 불러오고 있습니다", + "model.imageLoadingDot": "이미지가 이미 모델을 불러오고 있습니다.", + "model.load": "불러오기", + "model.loadFailed": "불러오지 못했습니다", + "model.loadTitle": "지금 모델을 불러옵니다. 메시지를 보내도 불러옵니다.", + "model.loaded": "모델을 불러왔습니다", + "model.loading": "불러오는 중…", + "model.loadingTitle": "모델을 불러오는 중입니다", + "model.loadingToast": "모델을 불러오는 중", + "model.noAnswer": "서버가 응답하지 않았습니다.", + "model.replyLoading": "답변이 이미 모델을 불러오고 있습니다", + "model.replyLoadingDot": "답변이 이미 모델을 불러오고 있습니다.", + "model.stopFirst": "답변을 중지한 뒤 해제하세요", + "model.stopFirstDot": "답변을 중지한 뒤 해제하세요.", + "model.unload": "해제", + "model.unloadFailed": "해제하지 못했습니다", + "model.unloadTitle": "모델을 해제하고 GPU를 비웁니다", + "model.unloaded": "모델을 해제했습니다", + "model.unloading": "해제하는 중…", + "model.unloadingTitle": "모델을 해제하는 중입니다", + "model.unsupported": "해제를 사용할 수 없습니다", + "model.waiting": "서버를 기다리는 중", + "pending.afterTool": "도구가 끝나기를 기다린 뒤 보냅니다: “{text}”", + "pending.cutting": "답변을 멈춘 뒤 보냅니다: “{text}”", + "pending.one": "이미 대기 중인 메시지가 있습니다", + "pending.oneText": "되돌려 놓거나 전송될 때까지 기다리세요.", + "pending.queued": "이 답변이 끝나면 보냅니다: “{text}”", + "pending.ready": "아직 보내지 않음: “{text}”", + "pending.send": "보내기", + "pending.sendNow": "지금 보내기", + "pending.takeBack": "되돌리기", + "pill.connecting": "연결 중…", + "pill.generating": "작성 중 · {rate} tok/s", + "pill.idle": "유휴 상태", + "pill.image": "이미지 생성 중…", + "pill.keyNeeded": "API 키 필요", + "pill.queued": "대기 {n}건", + "pill.reading": "프롬프트 읽는 중", + "pill.readingPct": "프롬프트 읽는 중 · {pct}%", + "pill.unloaded": "해제됨", + "pill.unreachable": "서버에 연결할 수 없음", + "preview.confirm": "이 코드를 스크립트와 함께 실행할까요?\n\n코드는 이 채팅과 네트워크에서 격리된 채 브라우저 안에서 실행되지만, 탭이 멈출 수는 있으며 모델이 작성한 코드입니다.\n\n이 블록만, 미리보기를 닫을 때까지만 실행됩니다.", + "preview.runScripts": "스크립트 실행", + "preview.showCode": "코드 보기", + "preview.showPage": "페이지 미리보기", + "preview.showSvg": "이미지 미리보기", + "preview.stopScripts": "스크립트 중지", + "preview.titlePage": "페이지 미리보기", + "preview.titleSvg": "이미지 미리보기", + "rewind.branch": "여기서 분기", + "rewind.branchAnswerText": "이 답변까지의 메시지로 새 채팅을 시작합니다. 이 채팅은 그대로 남습니다.", + "rewind.branchUserText": "이 메시지 이전의 메시지로 새 채팅을 시작하고 내 메시지를 그 입력란에 넣습니다. 이 채팅은 그대로 남습니다.", + "rewind.hereAnswer": "여기서 계속", + "rewind.hereAnswerText": "이 답변 이후의 메시지를 숨깁니다. 실행 취소로 되돌릴 수 있습니다.", + "rewind.hereUser": "여기서 편집하고 다시 보내기", + "rewind.hereUserText": "이 메시지와 이후 메시지를 숨기고 내 메시지를 아래 입력란에 넣습니다. 실행 취소로 되돌릴 수 있습니다.", + "rewind.noList": "대신 여기서 새 채팅을 시작하려면 채팅 설정에서 저장된 채팅 목록을 켜세요.", + "rewind.title": "여기로 되감기", + "settings.language": "언어", + "settings.languageNote": "채팅과 채팅 설정, 위의 버튼은 번역되어 있습니다. 모니터, 이미지, 정보 화면은 아직 영어입니다.", + "sidebar.collapse": "채팅 목록 접기", + "sidebar.delete": "이 채팅 삭제", + "sidebar.deleteShort": "삭제", + "sidebar.expand": "채팅 목록 열기", + "sidebar.foot": {"other": "채팅 {n}개 · 이 브라우저에 저장됨"}, + "sidebar.nameFor": "이 채팅의 이름", + "sidebar.noMatch": "일치하는 채팅이 없습니다", + "sidebar.none": "아직 채팅이 없습니다", + "sidebar.rename": "이 채팅 이름 변경", + "sidebar.renameShort": "이름 변경", + "sidebar.search": "채팅 검색", + "sidebar.title": "채팅", + "store.failed": "채팅 목록을 저장하지 못했습니다. 화면의 채팅은 그대로 있습니다.", + "store.memory": "이 브라우저는 채팅 목록을 보관하지 않습니다. 페이지를 닫을 때까지 메모리에만 남습니다.", + "store.quota": "이 브라우저는 더 이상 보관할 수 없습니다. 화면의 채팅은 그대로 있습니다.", + "store.title": "채팅 목록이 저장되지 않음", + "tab.about": "정보", + "tab.chat": "채팅", + "tab.images": "이미지", + "tab.monitor": "모니터", + "tab.views": "보기", + "theme.label": "라이트와 다크 전환", + "theme.title": "라이트 / 다크", + "think.done": "추론", + "think.for": "{s}초 동안 생각함", + "think.now": "생각하는 중…", + "toast.copied": "클립보드에 복사했습니다", + "toast.engineError": "엔진이 오류를 보고했습니다", + "toast.httpAnswer": "서버가 HTTP {code}로 응답했습니다.", + "toast.keyAbout": "이 서버에는 키가 필요합니다. 정보 > 설정에 입력하세요.", + "toast.keyAbout2": "API 키가 필요합니다. 정보 > 설정에 입력하세요.", + "toast.keyNeeded": "이 서버에는 API 키가 필요합니다. 정보 > 설정에 입력하세요.", + "toast.requestFailed": "요청이 실패했습니다", + "tool.arguments": "인수", + "tool.beingWritten": "(작성 중)", + "tool.chars": "{n}자", + "tool.cut": ", 모델용으로 줄임", + "tool.done": "완료", + "tool.error": "오류", + "tool.result": "결과", + "tool.running": "실행 중", + "tool.skipped": "실행 안 됨", + "tool.writing": "작성 중", + "undo": "실행 취소" +} diff --git a/edge0-strata/serve/web/locales/pt.json b/edge0-strata/serve/web/locales/pt.json new file mode 100644 index 00000000..bc343a61 --- /dev/null +++ b/edge0-strata/serve/web/locales/pt.json @@ -0,0 +1,249 @@ +{ + "branch.back": "Voltar a ele", + "branch.from": "Ramificado de “{title}”", + "branch.open": "Abrir o original", + "branch.started": "Ramificação criada", + "branch.startedText": "O chat em que você estava não mudou; ele está na lista à esquerda.", + "branch.title": "Ramificação de {title}", + "busy.stillWriting": "Ainda está escrevendo", + "busy.stopFirst": "Pare a resposta primeiro.", + "cancel": "Cancelar", + "chat.clearedToast": "O último chat foi limpo.", + "chat.copy": "Copiar", + "chat.copyAnswer": "Copiar a resposta", + "chat.deleted": "Chat excluído", + "chat.empty": "Pergunte o que quiser", + "chat.emptyModel": "{model} roda neste PC. Nada sai dele.", + "chat.image": "imagem", + "chat.interrupted": "Interrompida", + "chat.interruptedLower": "interrompida", + "chat.new": "Novo chat", + "chat.newToast": "Novo chat", + "chat.newToastText": "O chat anterior está na lista à esquerda.", + "chat.projectionOff": "projeção desativada", + "chat.projectionOn": "projeção ativada", + "chat.stopped": "Parada", + "chat.stoppedLower": "parada", + "chat.tokens": "{n} tokens", + "chat.toolCalls": {"one": "{n} chamada de ferramenta", "other": "{n} chamadas de ferramenta"}, + "chat.toolLimit": "parada no limite de {n} rodadas de ferramentas (mcp.max_rounds)", + "chat.untitled": "Novo chat", + "chat.you": "Você", + "close": "Fechar", + "code.copy": "Copiar o código", + "code.label": "código", + "composer.attach": "Anexar um arquivo de texto (ou soltá-lo aqui)", + "composer.attachLabel": "Anexar um arquivo", + "composer.attachPictures": "Anexar um arquivo de texto ou uma imagem (ou soltá-los aqui)", + "composer.finishOpen": "Salve ou cancele o que está aberto acima para continuar", + "composer.hintInterject": "Enter interrompe a resposta aqui e envia agora", + "composer.hintQueue": "Enter envia quando a resposta terminar", + "composer.message": "Mensagem", + "composer.newline": "Shift+Enter: nova linha", + "composer.placeholder": "Pergunte o que quiser…", + "composer.save": "Salvar este chat como Markdown", + "composer.saveLabel": "Salvar este chat", + "composer.send": "Enviar", + "composer.stop": "Parar", + "drawer.apply": "Aplicar", + "drawer.busy": "Quando você envia enquanto ele responde", + "drawer.busyInterject": "Interromper", + "drawer.busyInterjectNote": "A resposta para onde está (o que foi escrito fica) e sua mensagem vai na hora. Uma ferramenta em execução termina primeiro.", + "drawer.busyQueue": "Enviar depois", + "drawer.busyQueueNote": "Sua mensagem espera e vai assim que a resposta terminar.", + "drawer.busyWait": "Esperar", + "drawer.busyWaitNote": "Enviar fica desativado até a resposta terminar, como antes.", + "drawer.chats": "Chats", + "drawer.edit": "Editar e voltar nas mensagens", + "drawer.editNote": "editar uma resposta, pedir uma nova ou voltar a uma mensagem anterior (com Desfazer)", + "drawer.greedy": "0 · determinístico", + "drawer.high": "Alto", + "drawer.list": "Lista de chats salvos", + "drawer.listNote": "guarda cada chat em uma lista à esquerda, só neste navegador. Desativado: um único chat, como antes", + "drawer.low": "Baixo", + "drawer.math": "Exibir fórmulas", + "drawer.mathNote": "desenha as equações das respostas como fórmulas (funciona offline)", + "drawer.maxTokens": "Máx. de tokens", + "drawer.maxTokensNote": "vazio = até terminar", + "drawer.mcp": "Usar ferramentas de servidores MCP", + "drawer.medium": "Médio", + "drawer.off": "Não", + "drawer.random": "Aleatória", + "drawer.reset": "Redefinir", + "drawer.saved": "Configurações do chat salvas", + "drawer.savedGreedy": "Determinístico: a mesma pergunta dá a mesma resposta.", + "drawer.savedHere": "Salvo aqui, mas não para outros apps", + "drawer.savedOwn": "Os outros apps voltam a usar as próprias configurações.", + "drawer.savedShared": "Os outros apps (omp, clientes da API) usam estas configurações a partir da próxima solicitação.", + "drawer.seed": "Semente", + "drawer.seedNote": "vazio = aleatória", + "drawer.share": "Usar também para outros apps", + "drawer.shareNote": "omp e outros clientes da API recebem estas configurações para tudo que eles mesmos não definirem", + "drawer.short": "Configurações", + "drawer.showThinking": "Mostrar o raciocínio", + "drawer.showThinkingNote": "aberto enquanto escreve", + "drawer.temperature": "Temperatura", + "drawer.temperatureNote": "0 = sempre a palavra mais provável (exato, repetível)", + "drawer.thinkHigh": "a fundo (padrão)", + "drawer.thinkLow": "curto", + "drawer.thinkMedium": "médio", + "drawer.thinkNone": "responde na hora", + "drawer.thinking": "Raciocínio", + "drawer.title": "Configurações do chat", + "drawer.untilDone": "Até terminar", + "export.nothing": "Ainda não há nada para salvar", + "file.big": "Arquivo grande demais", + "file.bigText": "{name} passa de 512 KB.", + "file.binaryText": "{name} parece ser um arquivo binário.", + "file.notText": "Não é um arquivo de texto", + "file.notTextOnly": "{name}: anexe arquivos de texto (código, notas, logs, dados).", + "file.notTextPictures": "{name}: anexe arquivos de texto (código, notas, logs, dados) ou imagens.", + "file.pasted": "imagem colada", + "file.pictureBig": "Imagem grande demais", + "file.pictureBigText": "{name} passa de 20 MB.", + "file.picturesOff": "As imagens estão desativadas", + "file.picturesOffText": "Este modelo foi configurado só para texto.", + "file.remove": "Remover", + "group.month": "Últimos 30 dias", + "group.older": "Mais antigos", + "group.today": "Hoje", + "group.week": "Últimos 7 dias", + "group.yesterday": "Ontem", + "hist.editAnswer": "Editar resposta", + "hist.editAnswerTitle": "Corrija o texto da resposta. Nenhuma solicitação é enviada.", + "hist.editLabel": "Editar a resposta", + "hist.editNote": "O raciocínio e as chamadas de ferramenta anteriores da resposta são removidos. Nenhuma solicitação é enviada.", + "hist.editRemoves": {"one": "Ao salvar, a {n} mensagem seguinte fica oculta. ", "other": "Ao salvar, as {n} mensagens seguintes ficam ocultas. "}, + "hist.edited": "Editada", + "hist.editedHidden": {"one": "Resposta editada, {n} mensagem oculta", "other": "Resposta editada, {n} mensagens ocultas"}, + "hist.editedOnly": "Resposta editada", + "hist.empty": "A resposta está vazia", + "hist.emptyText": "Escreva algo antes de salvar.", + "hist.finishFirst": "Termine primeiro o que está aberto", + "hist.finishFirstText": "Salve ou pressione Cancelar.", + "hist.hidden": {"one": "{n} mensagem oculta", "other": "{n} mensagens ocultas"}, + "hist.lostText": "Este navegador guardou os nomes, não o conteúdo. Comece um novo chat e anexe-os de novo.", + "hist.lostText2": "Este navegador guardou os nomes, não o conteúdo. Anexe-os de novo.", + "hist.lostTitle": "É preciso anexar os arquivos de novo", + "hist.nothingHidden": "De volta a esta mensagem", + "hist.pendingFirst": "Há uma mensagem esperando", + "hist.pendingFirstText": "Envie-a ou retire-a primeiro.", + "hist.regenerate": "Gerar de novo", + "hist.regenerateTitle": "Pedir uma nova resposta a esta pergunta. As ferramentas podem rodar de novo.", + "hist.replaced": "A resposta foi substituída", + "hist.rewind": "Voltar até aqui", + "hist.rewindTitle": "Voltar a este ponto do chat", + "hist.save": "Salvar", + "math.source": "Código da fórmula ({reason})", + "model.alreadyUnloaded": "Já estava descarregado", + "model.busy": "O modelo está ocupado", + "model.confirmUnload": "Descarregar o modelo e liberar a GPU? Carregar, ou a próxima mensagem, o traz de volta.", + "model.fewMinutes": "Isso pode levar alguns minutos.", + "model.gpuFree": "A GPU está livre. Carregar, ou a próxima mensagem, traz o modelo de volta.", + "model.gpuInUse": "A GPU está em uso.", + "model.imageBusy": "Uma imagem está sendo gerada. Ela termina sozinha.", + "model.imageLoading": "Uma imagem já está carregando o modelo", + "model.imageLoadingDot": "Uma imagem já está carregando o modelo.", + "model.load": "Carregar", + "model.loadFailed": "Falha ao carregar", + "model.loadTitle": "Carregar o modelo agora. Uma mensagem também o carrega.", + "model.loaded": "Modelo carregado", + "model.loading": "Carregando…", + "model.loadingTitle": "O modelo está sendo carregado", + "model.loadingToast": "Carregando o modelo", + "model.noAnswer": "O servidor não respondeu.", + "model.replyLoading": "Uma resposta já está carregando o modelo", + "model.replyLoadingDot": "Uma resposta já está carregando o modelo.", + "model.stopFirst": "Pare a resposta e depois descarregue", + "model.stopFirstDot": "Pare a resposta e depois descarregue.", + "model.unload": "Descarregar", + "model.unloadFailed": "Falha ao descarregar", + "model.unloadTitle": "Descarregar o modelo e liberar a GPU", + "model.unloaded": "Modelo descarregado", + "model.unloading": "Descarregando…", + "model.unloadingTitle": "O modelo está sendo descarregado", + "model.unsupported": "Descarregar não está disponível", + "model.waiting": "Esperando o servidor", + "pending.afterTool": "Esperando a ferramenta terminar, depois envia: “{text}”", + "pending.cutting": "Parando a resposta, depois envia: “{text}”", + "pending.one": "Já há uma mensagem esperando", + "pending.oneText": "Retire-a ou espere até ela ser enviada.", + "pending.queued": "Envia quando esta resposta terminar: “{text}”", + "pending.ready": "Ainda não enviada: “{text}”", + "pending.send": "Enviar", + "pending.sendNow": "Enviar agora", + "pending.takeBack": "Retirar", + "pill.connecting": "Conectando…", + "pill.generating": "Escrevendo · {rate} tok/s", + "pill.idle": "Ocioso", + "pill.image": "Gerando imagem…", + "pill.keyNeeded": "Falta a chave de API", + "pill.queued": "{n} na fila", + "pill.reading": "Lendo a mensagem", + "pill.readingPct": "Lendo a mensagem · {pct}%", + "pill.unloaded": "Descarregado", + "pill.unreachable": "Servidor inacessível", + "preview.confirm": "Executar este código com scripts?\n\nEle roda no seu navegador, isolado deste chat e da rede, mas ainda pode travar a aba, e é código escrito pelo modelo.\n\nSó este bloco, e só até você fechar a prévia.", + "preview.runScripts": "Executar scripts", + "preview.showCode": "Mostrar o código", + "preview.showPage": "Prévia da página", + "preview.showSvg": "Prévia da imagem", + "preview.stopScripts": "Parar scripts", + "preview.titlePage": "Prévia da página", + "preview.titleSvg": "Prévia da imagem", + "rewind.branch": "Criar ramificação a partir daqui", + "rewind.branchAnswerText": "Começa um novo chat com as mensagens até esta resposta. Este chat fica como está.", + "rewind.branchUserText": "Começa um novo chat com as mensagens anteriores a esta e põe sua mensagem no campo dele. Este chat fica como está.", + "rewind.hereAnswer": "Continuar daqui", + "rewind.hereAnswerText": "Oculta as mensagens depois desta resposta. Desfazer as traz de volta.", + "rewind.hereUser": "Editar e enviar de novo aqui", + "rewind.hereUserText": "Oculta esta mensagem e as seguintes e põe sua mensagem no campo abaixo. Desfazer as traz de volta.", + "rewind.noList": "Para começar um novo chat a partir daqui, ative a lista de chats salvos nas configurações do chat.", + "rewind.title": "Voltar até aqui", + "settings.language": "Idioma", + "settings.languageNote": "O chat, as configurações dele e os botões acima estão traduzidos. As telas Monitor, Imagens e Sobre ainda estão em inglês.", + "sidebar.collapse": "Recolher a lista de chats", + "sidebar.delete": "Excluir este chat", + "sidebar.deleteShort": "Excluir", + "sidebar.expand": "Abrir a lista de chats", + "sidebar.foot": {"one": "{n} chat · guardado neste navegador", "other": "{n} chats · guardados neste navegador"}, + "sidebar.nameFor": "Nome deste chat", + "sidebar.noMatch": "Nenhum chat corresponde", + "sidebar.none": "Ainda não há chats", + "sidebar.rename": "Renomear este chat", + "sidebar.renameShort": "Renomear", + "sidebar.search": "Buscar chats", + "sidebar.title": "Chats", + "store.failed": "Não foi possível salvar a lista de chats. Os chats na tela continuam lá.", + "store.memory": "Este navegador não vai guardar a lista de chats. Ela fica na memória até você fechar a página.", + "store.quota": "Este navegador não consegue guardar mais. Os chats na tela continuam lá.", + "store.title": "Lista de chats não salva", + "tab.about": "Sobre", + "tab.chat": "Chat", + "tab.images": "Imagens", + "tab.monitor": "Monitor", + "tab.views": "Telas", + "theme.label": "Alternar entre claro e escuro", + "theme.title": "Claro / escuro", + "think.done": "Raciocínio", + "think.for": "Pensou por {s} s", + "think.now": "Pensando…", + "toast.copied": "Copiado para a área de transferência", + "toast.engineError": "o motor informou um erro", + "toast.httpAnswer": "O servidor respondeu com HTTP {code}.", + "toast.keyAbout": "Este servidor precisa de uma chave: adicione-a em Sobre > Configurações.", + "toast.keyAbout2": "Falta a chave de API: adicione-a em Sobre > Configurações.", + "toast.keyNeeded": "Este servidor precisa de uma chave de API: adicione-a em Sobre > Configurações.", + "toast.requestFailed": "A solicitação falhou", + "tool.arguments": "Argumentos", + "tool.beingWritten": "(sendo escrito)", + "tool.chars": "{n} caracteres", + "tool.cut": ", cortado para o modelo", + "tool.done": "Concluído", + "tool.error": "Erro", + "tool.result": "Resultado", + "tool.running": "Em execução", + "tool.skipped": "Não executada", + "tool.writing": "Escrevendo", + "undo": "Desfazer" +} diff --git a/edge0-strata/serve/web/locales/ru.json b/edge0-strata/serve/web/locales/ru.json new file mode 100644 index 00000000..7b83de91 --- /dev/null +++ b/edge0-strata/serve/web/locales/ru.json @@ -0,0 +1,249 @@ +{ + "branch.back": "Вернуться к нему", + "branch.from": "Ответвление от «{title}»", + "branch.open": "Открыть оригинал", + "branch.started": "Ответвление создано", + "branch.startedText": "Чат, в котором вы были, не изменился: он в списке слева.", + "branch.title": "Ответвление от «{title}»", + "busy.stillWriting": "Ответ ещё пишется", + "busy.stopFirst": "Сначала остановите ответ.", + "cancel": "Отмена", + "chat.clearedToast": "Последний чат очищен.", + "chat.copy": "Копировать", + "chat.copyAnswer": "Копировать ответ", + "chat.deleted": "Чат удалён", + "chat.empty": "Спрашивайте что угодно", + "chat.emptyModel": "{model} работает на этом ПК. Ничто не уходит наружу.", + "chat.image": "изображение", + "chat.interrupted": "Прервано", + "chat.interruptedLower": "прервано", + "chat.new": "Новый чат", + "chat.newToast": "Новый чат", + "chat.newToastText": "Прежний чат остался в списке слева.", + "chat.projectionOff": "проекция выключена", + "chat.projectionOn": "проекция включена", + "chat.stopped": "Остановлено", + "chat.stoppedLower": "остановлено", + "chat.tokens": "{n} токенов", + "chat.toolCalls": {"one": "{n} вызов инструмента", "few": "{n} вызова инструмента", "many": "{n} вызовов инструмента", "other": "{n} вызова инструмента"}, + "chat.toolLimit": "остановлено на пределе в {n} раундов инструментов (mcp.max_rounds)", + "chat.untitled": "Новый чат", + "chat.you": "Вы", + "close": "Закрыть", + "code.copy": "Копировать код", + "code.label": "код", + "composer.attach": "Прикрепить текстовый файл (или перетащить сюда)", + "composer.attachLabel": "Прикрепить файл", + "composer.attachPictures": "Прикрепить текстовый файл или изображение (или перетащить сюда)", + "composer.finishOpen": "Сохраните или отмените то, что открыто выше, чтобы продолжить", + "composer.hintInterject": "Enter остановит ответ здесь и сразу отправит", + "composer.hintQueue": "Enter отправит, когда ответ закончится", + "composer.message": "Сообщение", + "composer.newline": "Shift+Enter: новая строка", + "composer.placeholder": "Спрашивайте что угодно…", + "composer.save": "Сохранить этот чат как Markdown", + "composer.saveLabel": "Сохранить этот чат", + "composer.send": "Отправить", + "composer.stop": "Остановить", + "drawer.apply": "Применить", + "drawer.busy": "Если отправить, пока идёт ответ", + "drawer.busyInterject": "Прервать", + "drawer.busyInterjectNote": "Ответ остановится там, где он есть (написанное останется), и ваше сообщение уйдёт сразу. Работающий инструмент сначала закончит.", + "drawer.busyQueue": "Отправить после", + "drawer.busyQueueNote": "Сообщение подождёт и уйдёт, как только ответ закончится.", + "drawer.busyWait": "Ждать", + "drawer.busyWaitNote": "Отправка выключена, пока ответ не закончится, как раньше.", + "drawer.chats": "Чаты", + "drawer.edit": "Правка и возврат по сообщениям", + "drawer.editNote": "править ответ, просить новый или вернуться к более раннему сообщению (с отменой)", + "drawer.greedy": "0 · детерминированно", + "drawer.high": "Высокий", + "drawer.list": "Список сохранённых чатов", + "drawer.listNote": "хранит каждый чат в списке слева, только в этом браузере. Выключено: один чат, как раньше", + "drawer.low": "Низкий", + "drawer.math": "Показывать формулы", + "drawer.mathNote": "рисовать уравнения в ответах как формулы (работает без сети)", + "drawer.maxTokens": "Макс. токенов", + "drawer.maxTokensNote": "пусто = до конца", + "drawer.mcp": "Использовать инструменты серверов MCP", + "drawer.medium": "Средний", + "drawer.off": "Выкл.", + "drawer.random": "Случайно", + "drawer.reset": "Сбросить", + "drawer.saved": "Настройки чата сохранены", + "drawer.savedGreedy": "Детерминированно: один и тот же вопрос даёт один и тот же ответ.", + "drawer.savedHere": "Сохранено здесь, но не для других приложений", + "drawer.savedOwn": "Другие приложения снова используют свои настройки.", + "drawer.savedShared": "Другие приложения (omp, клиенты API) используют эти настройки со следующего запроса.", + "drawer.seed": "Seed", + "drawer.seedNote": "пусто = случайно", + "drawer.share": "Использовать и для других приложений", + "drawer.shareNote": "omp и другие клиенты API получают эти настройки для всего, что не задают сами", + "drawer.short": "Настройки", + "drawer.showThinking": "Показывать рассуждение", + "drawer.showThinkingNote": "раскрыто, пока идёт ответ", + "drawer.temperature": "Температура", + "drawer.temperatureNote": "0 = всегда самое вероятное слово (точно, повторяемо)", + "drawer.thinkHigh": "тщательно (по умолчанию)", + "drawer.thinkLow": "коротко", + "drawer.thinkMedium": "средне", + "drawer.thinkNone": "отвечает сразу", + "drawer.thinking": "Рассуждение", + "drawer.title": "Настройки чата", + "drawer.untilDone": "До конца", + "export.nothing": "Пока нечего сохранять", + "file.big": "Файл слишком большой", + "file.bigText": "{name} больше 512 КБ.", + "file.binaryText": "{name} похож на двоичный файл.", + "file.notText": "Это не текстовый файл", + "file.notTextOnly": "{name}: прикрепляйте текстовые файлы (код, заметки, журналы, данные).", + "file.notTextPictures": "{name}: прикрепляйте текстовые файлы (код, заметки, журналы, данные) или изображения.", + "file.pasted": "вставленное изображение", + "file.pictureBig": "Изображение слишком большое", + "file.pictureBigText": "{name} больше 20 МБ.", + "file.picturesOff": "Изображения выключены", + "file.picturesOffText": "Эта модель настроена только на текст.", + "file.remove": "Удалить", + "group.month": "Последние 30 дней", + "group.older": "Раньше", + "group.today": "Сегодня", + "group.week": "Последние 7 дней", + "group.yesterday": "Вчера", + "hist.editAnswer": "Править ответ", + "hist.editAnswerTitle": "Исправить текст ответа. Запрос не отправляется.", + "hist.editLabel": "Правка ответа", + "hist.editNote": "Прежнее рассуждение и вызовы инструментов в ответе удаляются. Запрос не отправляется.", + "hist.editRemoves": {"one": "При сохранении скроется {n} следующее сообщение. ", "few": "При сохранении скроются {n} следующих сообщения. ", "many": "При сохранении скроются {n} следующих сообщений. ", "other": "При сохранении скроются {n} следующих сообщения. "}, + "hist.edited": "Изменено", + "hist.editedHidden": {"one": "Ответ изменён, скрыто {n} сообщение", "few": "Ответ изменён, скрыто {n} сообщения", "many": "Ответ изменён, скрыто {n} сообщений", "other": "Ответ изменён, скрыто {n} сообщения"}, + "hist.editedOnly": "Ответ изменён", + "hist.empty": "Ответ пуст", + "hist.emptyText": "Напишите что-нибудь перед сохранением.", + "hist.finishFirst": "Сначала закончите то, что открыто", + "hist.finishFirstText": "Сохраните это или нажмите «Отмена».", + "hist.hidden": {"one": "Скрыто {n} сообщение", "few": "Скрыто {n} сообщения", "many": "Скрыто {n} сообщений", "other": "Скрыто {n} сообщения"}, + "hist.lostText": "Браузер сохранил только имена, без содержимого. Начните новый чат и прикрепите их снова.", + "hist.lostText2": "Браузер сохранил только имена, без содержимого. Прикрепите их снова.", + "hist.lostTitle": "Вложения нужно добавить заново", + "hist.nothingHidden": "Возврат к этому сообщению", + "hist.pendingFirst": "Одно сообщение ждёт отправки", + "hist.pendingFirstText": "Сначала отправьте его или заберите обратно.", + "hist.regenerate": "Сгенерировать заново", + "hist.regenerateTitle": "Запросить новый ответ на этот вопрос. Инструменты могут запуститься снова.", + "hist.replaced": "Ответ заменён", + "hist.rewind": "Вернуться сюда", + "hist.rewindTitle": "Вернуться к этому месту в чате", + "hist.save": "Сохранить", + "math.source": "Исходный текст формулы ({reason})", + "model.alreadyUnloaded": "Уже выгружена", + "model.busy": "Модель занята", + "model.confirmUnload": "Выгрузить модель и освободить GPU? Кнопка «Загрузить» или следующее сообщение вернёт её.", + "model.fewMinutes": "Это может занять несколько минут.", + "model.gpuFree": "GPU свободен. Кнопка «Загрузить» или следующее сообщение вернёт модель.", + "model.gpuInUse": "GPU используется.", + "model.imageBusy": "Создаётся изображение. Оно закончится само.", + "model.imageLoading": "Изображение уже загружает модель", + "model.imageLoadingDot": "Изображение уже загружает модель.", + "model.load": "Загрузить", + "model.loadFailed": "Не удалось загрузить", + "model.loadTitle": "Загрузить модель сейчас. Сообщение тоже загрузит её.", + "model.loaded": "Модель загружена", + "model.loading": "Загрузка…", + "model.loadingTitle": "Модель загружается", + "model.loadingToast": "Загрузка модели", + "model.noAnswer": "Сервер не ответил.", + "model.replyLoading": "Ответ уже загружает модель", + "model.replyLoadingDot": "Ответ уже загружает модель.", + "model.stopFirst": "Остановите ответ, затем выгрузите", + "model.stopFirstDot": "Остановите ответ, затем выгрузите.", + "model.unload": "Выгрузить", + "model.unloadFailed": "Не удалось выгрузить", + "model.unloadTitle": "Выгрузить модель и освободить GPU", + "model.unloaded": "Модель выгружена", + "model.unloading": "Выгрузка…", + "model.unloadingTitle": "Модель выгружается", + "model.unsupported": "Выгрузка недоступна", + "model.waiting": "Ожидание сервера", + "pending.afterTool": "Ждёт, пока закончит инструмент, затем отправит: «{text}»", + "pending.cutting": "Останавливает ответ, затем отправит: «{text}»", + "pending.one": "Одно сообщение уже ждёт", + "pending.oneText": "Заберите его обратно или дождитесь отправки.", + "pending.queued": "Отправится, когда закончится этот ответ: «{text}»", + "pending.ready": "Ещё не отправлено: «{text}»", + "pending.send": "Отправить", + "pending.sendNow": "Отправить сейчас", + "pending.takeBack": "Забрать", + "pill.connecting": "Подключение…", + "pill.generating": "Пишет · {rate} ток/с", + "pill.idle": "Ожидание", + "pill.image": "Создаёт изображение…", + "pill.keyNeeded": "Нужен ключ API", + "pill.queued": "В очереди: {n}", + "pill.reading": "Читает запрос", + "pill.readingPct": "Читает запрос · {pct}%", + "pill.unloaded": "Выгружена", + "pill.unreachable": "Сервер недоступен", + "preview.confirm": "Запустить этот код со скриптами?\n\nОн работает в вашем браузере, отрезанный от этого чата и от сети, но всё равно может заморозить вкладку, и это код, написанный моделью.\n\nТолько этот блок и только пока вы не закроете предпросмотр.", + "preview.runScripts": "Запустить скрипты", + "preview.showCode": "Показать код", + "preview.showPage": "Предпросмотр страницы", + "preview.showSvg": "Предпросмотр изображения", + "preview.stopScripts": "Остановить скрипты", + "preview.titlePage": "Предпросмотр страницы", + "preview.titleSvg": "Предпросмотр изображения", + "rewind.branch": "Создать ветку отсюда", + "rewind.branchAnswerText": "Начинает новый чат с сообщениями до этого ответа. Этот чат остаётся как есть.", + "rewind.branchUserText": "Начинает новый чат с сообщениями до этого и кладёт ваше сообщение в его поле. Этот чат остаётся как есть.", + "rewind.hereAnswer": "Продолжить отсюда", + "rewind.hereAnswerText": "Скрывает сообщения после этого ответа. «Отменить» вернёт их.", + "rewind.hereUser": "Изменить и отправить заново здесь", + "rewind.hereUserText": "Скрывает это сообщение и следующие и кладёт ваше сообщение в поле ввода ниже. «Отменить» вернёт их.", + "rewind.noList": "Чтобы вместо этого начать новый чат отсюда, включите список сохранённых чатов в настройках чата.", + "rewind.title": "Вернуться сюда", + "settings.language": "Язык", + "settings.languageNote": "Чат, его настройки и кнопки выше переведены. Разделы «Монитор», «Изображения» и «О программе» пока на английском.", + "sidebar.collapse": "Свернуть список чатов", + "sidebar.delete": "Удалить этот чат", + "sidebar.deleteShort": "Удалить", + "sidebar.expand": "Открыть список чатов", + "sidebar.foot": {"one": "{n} чат · хранится в этом браузере", "few": "{n} чата · хранятся в этом браузере", "many": "{n} чатов · хранятся в этом браузере", "other": "{n} чата · хранятся в этом браузере"}, + "sidebar.nameFor": "Название этого чата", + "sidebar.noMatch": "Подходящих чатов нет", + "sidebar.none": "Чатов пока нет", + "sidebar.rename": "Переименовать этот чат", + "sidebar.renameShort": "Переименовать", + "sidebar.search": "Поиск по чатам", + "sidebar.title": "Чаты", + "store.failed": "Не удалось сохранить список чатов. Чаты на экране остаются на месте.", + "store.memory": "Этот браузер не сохранит список чатов. Он останется в памяти, пока вы не закроете страницу.", + "store.quota": "Этот браузер не может хранить больше. Чаты на экране остаются на месте.", + "store.title": "Список чатов не сохранён", + "tab.about": "О программе", + "tab.chat": "Чат", + "tab.images": "Изображения", + "tab.monitor": "Монитор", + "tab.views": "Разделы", + "theme.label": "Переключить светлую и тёмную тему", + "theme.title": "Светлая / тёмная", + "think.done": "Рассуждение", + "think.for": "Думал {s} с", + "think.now": "Думает…", + "toast.copied": "Скопировано в буфер обмена", + "toast.engineError": "движок сообщил об ошибке", + "toast.httpAnswer": "Сервер ответил HTTP {code}.", + "toast.keyAbout": "Этому серверу нужен ключ: укажите его в разделе «О программе» > «Настройки».", + "toast.keyAbout2": "Нужен ключ API: укажите его в разделе «О программе» > «Настройки».", + "toast.keyNeeded": "Этому серверу нужен ключ API: укажите его в разделе «О программе» > «Настройки».", + "toast.requestFailed": "Запрос не удался", + "tool.arguments": "Аргументы", + "tool.beingWritten": "(пишется)", + "tool.chars": "{n} симв.", + "tool.cut": ", сокращено для модели", + "tool.done": "Готово", + "tool.error": "Ошибка", + "tool.result": "Результат", + "tool.running": "Выполняется", + "tool.skipped": "Не выполнен", + "tool.writing": "Пишется", + "undo": "Отменить" +} diff --git a/edge0-strata/serve/web/locales/zh.json b/edge0-strata/serve/web/locales/zh.json new file mode 100644 index 00000000..773d5f60 --- /dev/null +++ b/edge0-strata/serve/web/locales/zh.json @@ -0,0 +1,249 @@ +{ + "branch.back": "回到原聊天", + "branch.from": "分支自“{title}”", + "branch.open": "打开原聊天", + "branch.started": "已创建分支", + "branch.startedText": "你之前所在的聊天没有改变，仍在左侧列表中。", + "branch.title": "{title} 的分支", + "busy.stillWriting": "仍在生成", + "busy.stopFirst": "请先停止回答。", + "cancel": "取消", + "chat.clearedToast": "上一个聊天已清空。", + "chat.copy": "复制", + "chat.copyAnswer": "复制回答", + "chat.deleted": "聊天已删除", + "chat.empty": "想问什么都可以", + "chat.emptyModel": "{model} 运行在这台电脑上，不会向外发送任何内容。", + "chat.image": "图片", + "chat.interrupted": "已打断", + "chat.interruptedLower": "已打断", + "chat.new": "新聊天", + "chat.newToast": "新聊天", + "chat.newToastText": "之前的聊天在左侧列表中。", + "chat.projectionOff": "投影关闭", + "chat.projectionOn": "投影开启", + "chat.stopped": "已停止", + "chat.stoppedLower": "已停止", + "chat.tokens": "{n} 个 token", + "chat.toolCalls": {"other": "{n} 次工具调用"}, + "chat.toolLimit": "已在 {n} 轮工具调用的上限处停止 (mcp.max_rounds)", + "chat.untitled": "新聊天", + "chat.you": "你", + "close": "关闭", + "code.copy": "复制代码", + "code.label": "代码", + "composer.attach": "附加文本文件 (也可拖到这里)", + "composer.attachLabel": "附加文件", + "composer.attachPictures": "附加文本文件或图片 (也可拖到这里)", + "composer.finishOpen": "请先保存或取消上方打开的内容，再继续", + "composer.hintInterject": "按 Enter 会在此处停止回答并立即发送", + "composer.hintQueue": "按 Enter 会在回答结束后发送", + "composer.message": "消息", + "composer.newline": "Shift+Enter: 换行", + "composer.placeholder": "想问什么都可以…", + "composer.save": "将此聊天保存为 Markdown", + "composer.saveLabel": "保存此聊天", + "composer.send": "发送", + "composer.stop": "停止", + "drawer.apply": "应用", + "drawer.busy": "回答过程中发送消息时", + "drawer.busyInterject": "打断", + "drawer.busyInterjectNote": "回答会在当前位置停止 (已写的内容保留)，你的消息立即发出。正在运行的工具会先完成。", + "drawer.busyQueue": "结束后发送", + "drawer.busyQueueNote": "你的消息会等待，回答一结束就发出。", + "drawer.busyWait": "等待", + "drawer.busyWaitNote": "回答结束前无法发送，和以前一样。", + "drawer.chats": "聊天", + "drawer.edit": "编辑和回退消息", + "drawer.editNote": "编辑回答、要求重新生成，或回到更早的消息 (可撤销)", + "drawer.greedy": "0 · 确定性", + "drawer.high": "高", + "drawer.list": "已保存聊天列表", + "drawer.listNote": "把每个聊天保存在左侧列表中，仅保存在此浏览器内。关闭: 和以前一样只有一个聊天", + "drawer.low": "低", + "drawer.math": "渲染公式", + "drawer.mathNote": "把回答中的公式绘制成数学公式 (可离线使用)", + "drawer.maxTokens": "最大 token 数", + "drawer.maxTokensNote": "留空 = 直到完成", + "drawer.mcp": "使用 MCP 服务器的工具", + "drawer.medium": "中", + "drawer.off": "关", + "drawer.random": "随机", + "drawer.reset": "重置", + "drawer.saved": "聊天设置已保存", + "drawer.savedGreedy": "确定性: 同一个问题得到同一个回答。", + "drawer.savedHere": "已保存在这里，但未用于其他应用", + "drawer.savedOwn": "其他应用重新使用它们自己的设置。", + "drawer.savedShared": "其他应用 (omp、API 客户端) 从下一次请求起使用这些设置。", + "drawer.seed": "种子", + "drawer.seedNote": "留空 = 随机", + "drawer.share": "也用于其他应用", + "drawer.shareNote": "omp 等 API 客户端未自行设置的项目会使用这些设置", + "drawer.short": "设置", + "drawer.showThinking": "显示思考过程", + "drawer.showThinkingNote": "生成时保持展开", + "drawer.temperature": "温度", + "drawer.temperatureNote": "0 = 总是选概率最高的词 (精确、可重复)", + "drawer.thinkHigh": "深入 (默认)", + "drawer.thinkLow": "简短", + "drawer.thinkMedium": "适中", + "drawer.thinkNone": "立即回答", + "drawer.thinking": "思考", + "drawer.title": "聊天设置", + "drawer.untilDone": "直到完成", + "export.nothing": "还没有可保存的内容", + "file.big": "文件太大", + "file.bigText": "{name} 超过 512 KB。", + "file.binaryText": "{name} 看起来是二进制文件。", + "file.notText": "不是文本文件", + "file.notTextOnly": "{name}: 请附加文本文件 (代码、笔记、日志、数据)。", + "file.notTextPictures": "{name}: 请附加文本文件 (代码、笔记、日志、数据) 或图片。", + "file.pasted": "粘贴的图片", + "file.pictureBig": "图片太大", + "file.pictureBigText": "{name} 超过 20 MB。", + "file.picturesOff": "图片功能已关闭", + "file.picturesOffText": "此模型只按文本方式设置。", + "file.remove": "移除", + "group.month": "过去 30 天", + "group.older": "更早", + "group.today": "今天", + "group.week": "过去 7 天", + "group.yesterday": "昨天", + "hist.editAnswer": "编辑回答", + "hist.editAnswerTitle": "修改回答的文字。不会发送请求。", + "hist.editLabel": "编辑回答", + "hist.editNote": "该回答之前的思考过程和工具调用会被移除。不会发送请求。", + "hist.editRemoves": {"other": "保存会隐藏其后的 {n} 条消息。"}, + "hist.edited": "已编辑", + "hist.editedHidden": {"other": "回答已编辑，隐藏了 {n} 条消息"}, + "hist.editedOnly": "回答已编辑", + "hist.empty": "回答是空的", + "hist.emptyText": "保存前请先写点内容。", + "hist.finishFirst": "请先完成打开着的内容", + "hist.finishFirstText": "保存它，或点击取消。", + "hist.hidden": {"other": "已隐藏 {n} 条消息"}, + "hist.lostText": "此浏览器只保留了名称，没有保留内容。请开始新聊天并重新附加。", + "hist.lostText2": "此浏览器只保留了名称，没有保留内容。请重新附加。", + "hist.lostTitle": "需要重新添加附件", + "hist.nothingHidden": "已回到这条消息", + "hist.pendingFirst": "有一条消息正在等待", + "hist.pendingFirstText": "请先发送它，或把它取回。", + "hist.regenerate": "重新生成", + "hist.regenerateTitle": "要求对这个问题给出新的回答。工具可能会再次运行。", + "hist.replaced": "回答已替换", + "hist.rewind": "回退到此处", + "hist.rewindTitle": "回到聊天中的这个位置", + "hist.save": "保存", + "math.source": "公式源码 ({reason})", + "model.alreadyUnloaded": "已经卸载", + "model.busy": "模型正忙", + "model.confirmUnload": "要卸载模型并释放 GPU 吗？点击“加载”或发送下一条消息即可恢复。", + "model.fewMinutes": "这可能需要几分钟。", + "model.gpuFree": "GPU 已空闲。点击“加载”或发送下一条消息即可恢复模型。", + "model.gpuInUse": "GPU 正在使用。", + "model.imageBusy": "正在生成图片，会自行结束。", + "model.imageLoading": "图片任务已在加载模型", + "model.imageLoadingDot": "图片任务已在加载模型。", + "model.load": "加载", + "model.loadFailed": "加载失败", + "model.loadTitle": "立即加载模型。发送消息也会加载。", + "model.loaded": "模型已加载", + "model.loading": "正在加载…", + "model.loadingTitle": "模型正在加载", + "model.loadingToast": "正在加载模型", + "model.noAnswer": "服务器没有响应。", + "model.replyLoading": "回答已在加载模型", + "model.replyLoadingDot": "回答已在加载模型。", + "model.stopFirst": "先停止回答，再卸载", + "model.stopFirstDot": "请先停止回答，再卸载。", + "model.unload": "卸载", + "model.unloadFailed": "卸载失败", + "model.unloadTitle": "卸载模型并释放 GPU", + "model.unloaded": "模型已卸载", + "model.unloading": "正在卸载…", + "model.unloadingTitle": "模型正在卸载", + "model.unsupported": "无法卸载", + "model.waiting": "正在等待服务器", + "pending.afterTool": "等工具完成后发送:“{text}”", + "pending.cutting": "停止回答后发送:“{text}”", + "pending.one": "已经有一条消息在等待", + "pending.oneText": "请取回它，或等它发送出去。", + "pending.queued": "本次回答结束后发送:“{text}”", + "pending.ready": "尚未发送:“{text}”", + "pending.send": "发送", + "pending.sendNow": "立即发送", + "pending.takeBack": "取回", + "pill.connecting": "正在连接…", + "pill.generating": "生成中 · {rate} tok/s", + "pill.idle": "空闲", + "pill.image": "正在生成图片…", + "pill.keyNeeded": "需要 API 密钥", + "pill.queued": "排队 {n} 个", + "pill.reading": "正在读取提示词", + "pill.readingPct": "正在读取提示词 · {pct}%", + "pill.unloaded": "已卸载", + "pill.unreachable": "无法连接服务器", + "preview.confirm": "要带脚本运行这段代码吗？\n\n它在你的浏览器中运行，与此聊天和网络隔离，但仍可能让标签页卡死，而且这是模型写的代码。\n\n仅限这一块，且只到你关闭预览为止。", + "preview.runScripts": "运行脚本", + "preview.showCode": "显示代码", + "preview.showPage": "预览页面", + "preview.showSvg": "预览图片", + "preview.stopScripts": "停止脚本", + "preview.titlePage": "页面预览", + "preview.titleSvg": "图片预览", + "rewind.branch": "从此处分支", + "rewind.branchAnswerText": "用到这条回答为止的消息开始一个新聊天。当前聊天保持原样。", + "rewind.branchUserText": "用这条消息之前的消息开始一个新聊天，并把你的消息放进它的输入框。当前聊天保持原样。", + "rewind.hereAnswer": "从此处继续", + "rewind.hereAnswerText": "隐藏这条回答之后的消息。点“撤销”可以恢复。", + "rewind.hereUser": "在此编辑并重新发送", + "rewind.hereUserText": "隐藏这条消息及其后的消息，并把你的消息放进下方输入框。点“撤销”可以恢复。", + "rewind.noList": "想改为从这里开始新聊天，请在聊天设置中打开“已保存聊天列表”。", + "rewind.title": "回退到此处", + "settings.language": "语言", + "settings.languageNote": "聊天、聊天设置和上方的按钮已翻译。监控、图片和关于页面仍是英文。", + "sidebar.collapse": "收起聊天列表", + "sidebar.delete": "删除此聊天", + "sidebar.deleteShort": "删除", + "sidebar.expand": "展开聊天列表", + "sidebar.foot": {"other": "{n} 个聊天 · 保存在此浏览器中"}, + "sidebar.nameFor": "此聊天的名称", + "sidebar.noMatch": "没有匹配的聊天", + "sidebar.none": "还没有聊天", + "sidebar.rename": "重命名此聊天", + "sidebar.renameShort": "重命名", + "sidebar.search": "搜索聊天", + "sidebar.title": "聊天", + "store.failed": "无法保存聊天列表。屏幕上的聊天仍在。", + "store.memory": "此浏览器不会保存聊天列表。它只留在内存中，关闭页面后就没有了。", + "store.quota": "此浏览器无法再保存更多内容。屏幕上的聊天仍在。", + "store.title": "聊天列表未保存", + "tab.about": "关于", + "tab.chat": "聊天", + "tab.images": "图片", + "tab.monitor": "监控", + "tab.views": "视图", + "theme.label": "在浅色和深色之间切换", + "theme.title": "浅色 / 深色", + "think.done": "思考", + "think.for": "思考了 {s} 秒", + "think.now": "正在思考…", + "toast.copied": "已复制到剪贴板", + "toast.engineError": "引擎报告了错误", + "toast.httpAnswer": "服务器返回了 HTTP {code}。", + "toast.keyAbout": "此服务器需要密钥: 请在“关于”>“设置”中填写。", + "toast.keyAbout2": "需要 API 密钥: 请在“关于”>“设置”中填写。", + "toast.keyNeeded": "此服务器需要 API 密钥: 请在“关于”>“设置”中填写。", + "toast.requestFailed": "请求失败", + "tool.arguments": "参数", + "tool.beingWritten": "(正在写入)", + "tool.chars": "{n} 个字符", + "tool.cut": "，已为模型截短", + "tool.done": "完成", + "tool.error": "错误", + "tool.result": "结果", + "tool.running": "运行中", + "tool.skipped": "未运行", + "tool.writing": "正在写入", + "undo": "撤销" +} diff --git a/edge0-strata/serve/web/math-adapter.js b/edge0-strata/serve/web/math-adapter.js new file mode 100644 index 00000000..7be74c8f --- /dev/null +++ b/edge0-strata/serve/web/math-adapter.js @@ -0,0 +1,34 @@ +// serve/web/math-adapter.js - the KaTeX renderer for math.js (PR #2974 by CC-David-CC). KaTeX 0.19.0 is vendored in +// serve/web/vendor/katex (MIT, see PROVENANCE.json there): no network, no CDN. Nothing is loaded until "Render math" is +// switched on and an answer holds a formula. +// +// What a formula may do: nothing but draw. `trust: false` turns off \href, \url, \includegraphics and \htmlClass-style +// commands, so a formula cannot make a link, load a picture or add an attribute; the macros are not shared between +// formulas; \def loops are cut by maxExpand and a huge \rule or \kern by maxSize. KaTeX builds DOM nodes itself (no +// innerHTML), the result is cloned into the page by math.js, and an error returns the source text instead. +"use strict"; +(function (root) { + const OPTIONS = {output: "htmlAndMathml", trust: false, throwOnError: true, strict: "ignore", maxExpand: 1000, maxSize: 20}; + const adapter = { + OPTIONS, + async initialize() { + if (root.katex && document.querySelector("link[data-katex-ready]")) return; + const css = document.createElement("link"); css.rel = "stylesheet"; + css.href = "web/vendor/katex/dist/katex.min.css"; + const loaded = new Promise((resolve, reject) => { css.onload = resolve; css.onerror = reject; }); + document.head.append(css); + await Promise.all([loaded, root.katex ? Promise.resolve() : root.StrataMath.script("web/vendor/katex/dist/katex.min.js")]); + css.dataset.katexReady = "1"; + }, + async render(source, {display, signal}) { + if (signal.aborted) return {ok: false, reason: "unavailable"}; + const element = document.createElement("span"); + try { + root.katex.render(source, element, {...OPTIONS, displayMode: display, macros: {}}); + return {ok: true, element: element.firstElementChild}; + } catch (_) { return {ok: false, reason: "invalid"}; } + }, + }; + root.StrataMathAdapter = adapter; + if (typeof module !== "undefined" && module.exports) module.exports = adapter; +})(typeof globalThis !== "undefined" ? globalThis : this); diff --git a/edge0-strata/serve/web/math.js b/edge0-strata/serve/web/math.js new file mode 100644 index 00000000..15e295f5 --- /dev/null +++ b/edge0-strata/serve/web/math.js @@ -0,0 +1,142 @@ +// serve/web/math.js - chat math that never changes the source. Engine-independent: a renderer (math-adapter.js) is plugged in. +// Ported from PR #2972 by CC-David-CC. The text of a message stays as the model wrote it (history, copying, the Markdown +// export and the requests all use it); math is only drawn over a placeholder in the page, and a formula that cannot be drawn +// stays as its source text. +"use strict"; +(function (root) { + const escape = (s) => String(s).replace(/[&<>"']/g, (c) => ({"&": "&", "<": "<", ">": ">", '"': """, "'": "'"}[c])); + const escaped = (s, i) => { let n = 0; while (i > 0 && s[--i] === "\\") n++; return n % 2 === 1; }; + + // split(text) -> [{raw}, {source, display, raw}, {raw, incomplete: true}, ...] + // \( \) and $x$ are inline, \[ \] and $$ $$ are display. Code (fences, `spans`), Markdown link targets and prices ($5 and + // $10) are not math. A formula that is still being written (no closing delimiter yet) stays plain text. + function split(text) { + const parts = []; let start = 0, i = 0; + const push = (end, math) => { if (start < i) parts.push({raw: text.slice(start, i)}); parts.push(math); start = i = end; }; + while (i < text.length) { + // Code takes precedence, including incomplete fences while streaming. + const fence = (i === 0 || text[i - 1] === "\n") && text.slice(i).match(/^ {0,3}(`{3,}|~{3,})[^\n]*\n/); + if (fence) { + const marker = fence[1][0], count = fence[1].length; + const end = new RegExp("^ {0,3}" + marker + "{" + count + ",}[ \\t]*(?:\\n|$)", "m").exec(text.slice(i + fence[0].length)); + i = end ? i + fence[0].length + end.index + end[0].length : text.length; continue; + } + if (text[i] === "`" && !escaped(text, i)) { + const ticks = text.slice(i).match(/^`+/)[0]; let end = text.indexOf(ticks, i + ticks.length); + while (end >= 0 && (text[end - 1] === "`" || text[end + ticks.length] === "`")) end = text.indexOf(ticks, end + ticks.length); + i = end < 0 ? text.length : end + ticks.length; continue; + } + if (escaped(text, i)) { i++; continue; } + // Never inject a math element into a Markdown link's href attribute. + if (text.startsWith("](http", i)) { + const end = text.indexOf(")", i + 2); + i = end < 0 ? text.length : end + 1; continue; + } + let open, close, display; + if (text.startsWith("\\(", i)) { open = "\\("; close = "\\)"; display = false; } + else if (text.startsWith("\\[", i)) { open = "\\["; close = "\\]"; display = true; } + else if (text.startsWith("$$", i)) { open = close = "$$"; display = true; } + else if (text[i] === "$" && !/\s/.test(text[i + 1] || " ") && text[i - 1] !== "$") { open = close = "$"; display = false; } + if (!open) { i++; continue; } + let end = text.indexOf(close, i + open.length); + while (end >= 0 && (escaped(text, end) || (close === "$" && (/\s/.test(text[end - 1]) || /[\d$]/.test(text[end + 1] || ""))))) end = text.indexOf(close, end + close.length); + const source = end >= 0 ? text.slice(i + open.length, end) : text.slice(i + open.length); + // Numeric prices are text, even when separated by a second dollar amount. + if (open === "$" && (/\n/.test(source) || /\$/.test(source) || + /^\d[\d,.]*(?:\s*[A-Za-z]|\s*$)/.test(source))) { i++; continue; } + if (end < 0) { push(text.length, {raw: text.slice(i), incomplete: true}); break; } + push(end + close.length, {source, display, raw: text.slice(i, end + close.length)}); + } + if (start < text.length) parts.push({raw: text.slice(start)}); + return parts; + } + + // markdown(text, format): format() is the page's Markdown, run on the text with every formula swapped for a placeholder + // word, then each word becomes holding the escaped source until it is drawn. + function markdown(text, format) { + let prefix = "STRATAMATHPLACEHOLDER"; while (text.includes(prefix)) prefix += "X"; + const slots = []; + const protectedText = split(text).map((p) => { + if (p.source === undefined && !p.incomplete) return p.raw; + const key = prefix + slots.length + "END"; + slots.push(p); return key; + }).join(""); + let html = format(protectedText); + slots.forEach((p, i) => { + const body = p.incomplete ? escape(p.raw) : `${escape(p.raw)}`; + html = html.replaceAll(prefix + i + "END", () => body); + }); + return html; + } + + // drawing: one formula at a time (an engine may keep state between formulas), 256 results cached, a 10 s deadline for the + // engine to start and 15 s for one formula. A formula over 16 KiB is not drawn. The adapter returns an element that is not + // in the page; the core clones it into the placeholder if that is still on the screen. + const cache = new Map(), done = new Map(); + let initialization = null, queue = Promise.resolve(), epoch = 0; + function reset() { cache.clear(); done.clear(); initialization = null; epoch++; } + const keyOf = (source, display) => JSON.stringify([source, display]); + async function result(source, display) { + if (new TextEncoder().encode(source).length > 16384) return {ok: false, reason: "limit"}; + const key = keyOf(source, display); + if (cache.has(key)) { const hit = cache.get(key); cache.delete(key); cache.set(key, hit); return hit; } + if (cache.size >= 256) { const old = cache.keys().next().value; cache.delete(old); done.delete(old); } + const current = epoch; let work; + const task = queue.then(async () => { + if (epoch !== current || cache.get(key) !== task) return {ok: false, reason: "unavailable"}; + try { + const adapter = root.StrataMathAdapter; + if (!adapter) return {ok: false, reason: "unavailable"}; + initialization ||= Promise.race([adapter.initialize(), new Promise((_, reject) => + setTimeout(() => reject(new Error("Math initialization timed out")), 10000))]); + await initialization; + const signal = AbortSignal.timeout(15000); + let onAbort; + const deadline = new Promise((resolve) => { + onAbort = () => resolve({ok: false, reason: "limit"}); + signal.addEventListener("abort", onAbort, {once: true}); + }); + work = Promise.resolve(adapter.render(source, {display, signal})); + try { return await Promise.race([work, deadline]); } + finally { signal.removeEventListener("abort", onAbort); } + } catch (_) { return {ok: false, reason: "unavailable"}; } + }); + // A browser engine may ignore cancellation. Return source at the deadline, + // but do not overlap conversions that mutate the engine's macro state. + queue = task.then(() => work, () => work).then(() => undefined, () => undefined); + cache.set(key, task); + task.then((r) => { if (cache.get(key) === task) done.set(key, r); }); + return task; + } + + function place(slot, r, tr) { + if (r.ok) slot.replaceChildren(r.element.cloneNode(true)); + else { slot.dataset.mathError = r.reason; slot.title = tr ? tr("math.source", "Math source ({reason})", {reason: r.reason}) : "Math source (" + r.reason + ")"; } + } + // hydrate(container): draw the placeholders in a container. A formula drawn before is placed at once, so a streaming + // answer that is rebuilt on every frame does not flicker back to source text. + function hydrate(container) { + const generation = container._mathGeneration = (container._mathGeneration || 0) + 1; + for (const slot of container.querySelectorAll("[data-math]")) { + const source = decodeURIComponent(slot.dataset.math), display = slot.dataset.display === "1"; + const known = done.get(keyOf(source, display)); + if (known && known.ok) { place(slot, known, root.tr); continue; } + result(source, display).then((r) => { + if (generation !== container._mathGeneration || !container.contains(slot)) return; + place(slot, r, root.tr); + }); + } + } + function script(url) { + return new Promise((resolve, reject) => { + const s = document.createElement("script"); s.src = url; + const timer = setTimeout(() => { s.remove(); reject(new Error("Math script timed out")); }, 10000); + s.onload = () => { clearTimeout(timer); resolve(); }; + s.onerror = () => { clearTimeout(timer); s.remove(); reject(new Error("Math script unavailable")); }; + document.head.append(s); + }); + } + const api = {split, markdown, hydrate, reset, script, result}; + root.StrataMath = api; + if (typeof module !== "undefined" && module.exports) module.exports = api; +})(typeof globalThis !== "undefined" ? globalThis : this); diff --git a/edge0-strata/serve/web/monitor.html b/edge0-strata/serve/web/monitor.html new file mode 100644 index 00000000..c1e87e7c --- /dev/null +++ b/edge0-strata/serve/web/monitor.html @@ -0,0 +1,38 @@ + + + + +Strata API Monitor + + + + + + +
Strata Local inference
+
+

API Monitor

Model status and every request, in one place.

+ + +
+
Model
Connecting…
127.0.0.1:5001/v1
+
Requests retained
—
Waiting for server
+
Last wall-clock
—
Includes queue and model loading
+
Last decode speed
—
Engine timing · tokens / second
+
+

Requests

+
+
Inspect a requestSelect a request to see its input, output, and timings.
+ +
+
Last 100 requests kept in memory until restart. Inputs and outputs stay on this machine.Connecting…
+
+ diff --git a/edge0-strata/serve/web/monitor.js b/edge0-strata/serve/web/monitor.js new file mode 100644 index 00000000..5543b94b --- /dev/null +++ b/edge0-strata/serve/web/monitor.js @@ -0,0 +1,111 @@ +"use strict"; +const $ = id => document.getElementById(id); +let selected = null, detail = null, pane = "input", records = [], state = null, operating = false, stamp = "", actionError = ""; +let apiKey = sessionStorage.getItem("strata.monitor.key") || ""; +document.documentElement.dataset.theme = localStorage.getItem("strata.theme") || "dark"; +const seconds = n => typeof n === "number" ? `${n.toFixed(2)} s` : "—"; +const speed = n => typeof n === "number" ? n.toFixed(1) : "—"; +const active = r => !["completed", "error", "disconnected"].includes(r.state); +function text(id, value) { const el = $(id); if (el.textContent !== String(value)) el.textContent = value; } +async function api(path, body) { + const response = await fetch(path, {cache:"no-store", headers:{...(apiKey ? {Authorization:`Bearer ${apiKey}`} : {}), ...(body !== undefined ? {"Content-Type":"application/json"} : {})}, ...(body !== undefined ? {method:"POST", body:JSON.stringify(body)} : {})}); + const value = await response.json(); + if (!response.ok) { + if (response.status === 401) $("auth").hidden = false; + throw new Error(value.error?.message || `HTTP ${response.status}`); + } + return value; +} +function notice(message) { $("error").hidden = !message; text("error", message || ""); } +function list() { + const query = $("search").value.toLowerCase(), filter = $("filter").value; + const visible = records.filter(r => `${r.id} ${r.path} ${r.model}`.toLowerCase().includes(query) && (filter === "all" || (filter === "active" ? active(r) : r.state === filter))); + const nextStamp = JSON.stringify([visible, selected]); + if (stamp === nextStamp) return; + stamp = nextStamp; + const focusId = document.activeElement?.dataset.request; + $("requests").replaceChildren(); + if (!visible.length) { + const empty = document.createElement("div"); empty.className = "empty"; + empty.textContent = records.length ? "No matching requests." : "No requests yet. API calls will appear here automatically."; + $("requests").append(empty); + } + for (const r of visible) { + const button = document.createElement("button"); button.className = `request${selected === r.id ? " selected" : ""}`; + button.dataset.request = r.id; button.setAttribute("aria-label", `Request ${r.id}`); button.setAttribute("aria-pressed", String(selected === r.id)); + const row = document.createElement("div"); row.className = "row"; + const time = document.createElement("span"); time.textContent = new Date(r.started_at * 1000).toLocaleTimeString(); + const tag = document.createElement("span"); tag.className = `tag ${r.state}`; tag.textContent = r.state; + row.append(time, tag); + const endpoint = document.createElement("div"); endpoint.className = "endpoint"; endpoint.textContent = `POST ${r.path}`; + const meta = document.createElement("div"); meta.className = "small"; + meta.textContent = `${r.id} · ${seconds(r.wallclock_s)} · ${r.stream ? "stream" : "JSON response"}${r.http_status ? ` · HTTP ${r.http_status}` : ""}`; + button.append(row, endpoint, meta); + button.onclick = async () => { selected = r.id; detail = null; list(); await showDetail(); }; + $("requests").append(button); + if (focusId === r.id) button.focus({preventScroll:true}); + } +} +function content() { + if (!detail) return; + let value = detail[pane] || (pane === "response" && detail.stream ? "Streaming response: see Output and the request error, if any." : "No content."); + if (pane === "output") { try { value = JSON.stringify(JSON.parse(value), null, 2); } catch (_) {} } + text("content", value); + const note = pane === "output" && detail.error ? "Raw model output retained for diagnosis. The API returned an error." : pane === "input" ? "Original request body." : pane === "output" ? "Model answer. JSON is formatted here for readability." : pane === "reasoning" ? "Separate reasoning content, when enabled." : "API response body."; + text("content-note", note + (detail[`${pane}_truncated`] ? " Monitor capture truncated at 256K characters; the API response was not shortened." : "")); +} +async function showDetail() { + if (!selected) return; + const id = selected; + const value = await api(`/api/requests?id=${encodeURIComponent(id)}`); + if (selected !== id) return; + detail = value; $("selection").hidden = false; $("no-selection").hidden = true; + text("detail-title", `${value.id} · ${value.path}`); + text("detail-status", value.state); $("detail-status").className = `tag ${value.state}`; + text("detail-time", new Date(value.started_at * 1000).toLocaleString()); + text("detail-wall", seconds(value.wallclock_s ?? ((Date.now() / 1000) - value.started_at))); + text("detail-load", seconds(value.load_s)); text("detail-queue", seconds(value.queue_s)); text("detail-first", seconds(value.first_token_s)); + text("detail-prompt", value.usage?.prompt_tokens ?? value.usage?.input_tokens ?? "—"); + text("detail-tokens", value.usage?.completion_tokens ?? value.usage?.output_tokens ?? "—"); + text("detail-speed", speed(value.timings?.predicted_per_second)); text("detail-format", value.response_format || "text"); + $("detail-error").hidden = !value.error; text("detail-error", value.error?.message || ""); content(); +} +async function refresh() { + try { + const [status, history] = await Promise.all([api("/v1/status"), api("/api/requests")]); + state = status; records = history.requests; + status.loaded = history.loaded; status.auto_load = history.auto_load; + if (selected && !records.some(r => r.id === selected)) { + selected = detail = null; $("selection").hidden = true; $("no-selection").hidden = false; + } + const inflight = Math.max(status.activity.in_flight, records.filter(active).length); + const loading = records.some(r => r.state === "loading"); + text("loaded", loading ? "Loading…" : inflight ? "In use" : status.loaded ? "Loaded" : "Unloaded"); + $("model-state").className = `value ${inflight || loading ? "busy" : status.loaded ? "loaded" : ""}`; + text("model", status.model); text("count", records.length); + text("active", inflight ? `${inflight} active / queued` : status.auto_load ? "Loads automatically on the next request" : "No active requests"); + const last = records.find(r => !active(r)); + text("wall", last ? seconds(last.wallclock_s) : "—"); text("speed", speed(last?.timings?.predicted_per_second)); + $("load").disabled = operating || Boolean(inflight) || status.loaded; $("unload").disabled = operating || Boolean(inflight) || !status.loaded; + list(); if (selected) await showDetail(); + text("updated", `Live · updated ${new Date().toLocaleTimeString()}`); + if (!operating) notice(actionError); + } catch (error) { notice(error.message); text("updated", "Disconnected · retrying"); $("load").disabled = $("unload").disabled = true; } +} +async function control(load) { + actionError = ""; + operating = true; $("load").disabled = $("unload").disabled = true; + text("loaded", load ? "Loading…" : "Unloading…"); + try { await api(load ? "/load" : "/unload", {}); notice(""); } + catch (error) { actionError = error.message; notice(actionError); } + finally { operating = false; await refresh(); } +} +$("load").onclick = () => control(true); $("unload").onclick = () => control(false); +$("key").onclick = () => { $("auth").hidden = !$("auth").hidden; if (!$("auth").hidden) $("api-key").focus(); }; +$("auth").onsubmit = async event => { event.preventDefault(); apiKey = $("api-key").value.trim(); sessionStorage.setItem("strata.monitor.key", apiKey); $("auth").hidden = true; await refresh(); }; +$("theme").onclick = () => { const theme = document.documentElement.dataset.theme === "dark" ? "light" : "dark"; document.documentElement.dataset.theme = theme; localStorage.setItem("strata.theme", theme); }; +$("search").oninput = list; $("filter").onchange = list; +document.querySelectorAll("[data-pane]").forEach(button => { button.onclick = () => { pane = button.dataset.pane; document.querySelectorAll("[data-pane]").forEach(b => b.setAttribute("aria-selected", String(b === button))); content(); }; }); +$("copy").onclick = async () => { try { await navigator.clipboard.writeText($("content").textContent); text("copy", "Copied"); setTimeout(() => text("copy", "Copy"), 1500); } catch (_) { notice("Clipboard unavailable; select the text to copy it."); } }; +async function poll() { await refresh(); setTimeout(poll, 2000); } +poll(); diff --git a/edge0-strata/serve/web/preview.js b/edge0-strata/serve/web/preview.js new file mode 100644 index 00000000..b6d26fb9 --- /dev/null +++ b/edge0-strata/serve/web/preview.js @@ -0,0 +1,81 @@ +// serve/web/preview.js - the preview of an ```svg or ```html block in an answer (PR #1628 by thoives). +// +// A block is shown in an