From 4bad1def54ea1df049e0347583638ecb8e970605 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Thu, 18 Jun 2026 00:19:43 +0800 Subject: [PATCH 01/19] docs: improve self-upgrade/model-switching sections + multi-session brainstorming roadmap --- README.md | 33 ++++++++- docs/roadmap/multi-session-multi-model.md | 86 +++++++++++++++++++++++ 2 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 docs/roadmap/multi-session-multi-model.md diff --git a/README.md b/README.md index 4505d90..d87a01c 100644 --- a/README.md +++ b/README.md @@ -40,10 +40,37 @@ Star the repo ⭐, fork to contribute, or open an issue for feedback. β†’ Full feature reference: [docs/GUIDE.md](docs/GUIDE.md#advanced-features) +### Agent Lifecycle & Runtime + +| Capability | Description | +|------------|-------------| +| **Self-Upgrade** | Trigger an in-place upgrade: pulls from git source or downloads the latest GitHub release binary. Auto-restarts after upgrade β€” no SSH, no manual steps. | +| **Model Switching** | Switch OpenRouter models at runtime via `/models`. Interactive picker lets you choose the best model per task: fast/cheap for simple queries, powerful for complex reasoning. | +| **Soul Files** | SOUL.md (persona), AGENTS.md (behaviour), USER.md (preferences) β€” persistent identity files auto-injected into every system prompt. Session-end self-reflection with `.bak` backups. | + --- -| πŸš€ **Self-Upgrade** | `/self-upgrade` slash command + `self_upgrade` tool, auto-restart after upgrade, git source or GitHub release binary | -| πŸ”€ **Model Switching** | `/models` slash command to switch OpenRouter LLM models at runtime, with interactive model picker | -| πŸ“ **Soul Files** | SOUL.md / AGENTS.md / USER.md persistent identity files, auto-injected into system prompt, session-end reflection with `.bak` backups | + +### πŸ’­ Multi-Session & Multi-Model (Brainstorming) + +> ⚠️ **Planning phase** β€” not yet implemented. This section captures ideas explored in the `feat/readme-improve-multi-session-brainstorm` branch. + +The vision: run multiple concurrent chat sessions, each with its own model and isolated context. + +| Use Case | Description | +|----------|-------------| +| **Parallel execution** | Run a cheap model for quick tasks while a powerful model tackles deep analysis β€” concurrently, not sequentially | +| **Per-user isolation** | Each Telegram user gets their own session with independent conversation context and model preference | +| **Sub-agent delegation** | Spawn sub-agents with different models (e.g., GPT-4o for code review, Claude for writing) without polluting the main session | + +**Topics to explore:** + +- Session lifecycle β€” create, switch, merge, archive +- Per-session model binding vs global default model +- Context isolation between sessions (independent or shared RAG?) +- Telegram UX for multi-session management (inline buttons? slash commands?) +- Persistence and RAG across session boundaries + +See [docs/roadmap/multi-session.md](docs/roadmap/multi-session.md) for detailed design notes. ## Quick Start diff --git a/docs/roadmap/multi-session-multi-model.md b/docs/roadmap/multi-session-multi-model.md new file mode 100644 index 0000000..376982c --- /dev/null +++ b/docs/roadmap/multi-session-multi-model.md @@ -0,0 +1,86 @@ +# Multi-Session & Multi-Model β€” Brainstorming Roadmap + +> ⚠️ **Planning phase** β€” not yet implemented. This document captures the vision, design constraints, and open questions. + +## Motivation + +Today RustFox runs a **single chat session** with **one model at a time**. You `/models` to switch, and the whole bot changes personality. + +But power users want: + +1. **Multiple concurrent sessions** β€” work on 3 different projects in 3 different chats, each with independent context. +2. **Different models per session** β€” code review chat uses Claude Opus, casual chat uses Haiku. +3. **Agent-to-agent delegation** (already partially solved) β€” one session can ask another session's model to do something. + +## Core Concepts + +### Session = Chat Thread + +Each Telegram chat (DM or group) is a **session**. Sessions are isolated: +- Each has its own conversation history (context window) +- Each can be assigned a different model +- Each can have different system prompts / soul files + +### Session Model Binding + +| Concept | Description | +|---------|-------------| +| **Default model** | The model used for new sessions (from config) | +| **Session model** | Override for a specific session via `/models` | +| **Session stickiness** | Model choice persists across bot restarts | + +### Context Isolation + +| Aspect | Shared | Per-Session | +|--------|--------|-------------| +| Soul files (SOUL.md, AGENTS.md, USER.md) | βœ… | ❌ | +| Conversation history | ❌ | βœ… | +| Scheduled tasks | βœ… | ❌ (scheduler is global) | +| Active model | ❌ | βœ… | +| Plans | ❌ | βœ… (per-session plan tracking) | +| Skills | βœ… | ❌ | + +## Potential Approach + +### Option A: Stateless Sessions + +- Session state lives entirely in Telegram (message history replayed on context window overflow) +- No per-session persistence β€” just model binding +- **Pros**: Simple, no new infrastructure +- **Cons**: Context window limited to recent messages, slow startup replay + +### Option B: Lightweight Session Store + +- SQLite table `sessions (chat_id, model_name, context_summary, last_active)` +- On each new message, load session state, inject context summary, respond +- **Pros**: Context summaries survive restarts, faster than full replay +- **Cons**: New persistence complexity, summary drift + +### Option C: Full Context Persistence + +- Store full conversation vector embeddings + message history in SQLite +- RAG-style retrieval at the start of each turn +- **Pros**: Maximum context recall +- **Cons**: Heavy infrastructure, token cost, complexity + +## Open Questions + +1. **Session lifecycle**: When does a session "expire"? After inactivity? Never? +2. **Cross-session memory**: Should sessions share learned facts (RLHF-style)? +3. **SQLite schema**: How does session state compose with the existing learning/scheduler tables? +4. **Cost tracking**: Per-session token usage & cost reporting? +5. **Billing model**: Some users may want per-session billing (e.g., `gpt-4` costs more than `haiku`). +6. **Migration path**: Users upgrading from single-model shouldn't lose their config. + +## Related Work + +- `invoke_agent` / `spawn_agents` β€” already supports subagent isolation +- `/models` command β€” already supports runtime model switching +- Soul files β€” already support persistent identity + +## Next Steps + +1. Choose an approach (A, B, or hybrid) +2. Design SQLite schema if needed +3. Scope implementation into trackable issues +4. Plan migration path for existing users \ No newline at end of file From 0c51d9cb010a8d9a989b7f02646a4fa667f0f94a Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 16:32:22 +0800 Subject: [PATCH 02/19] feat: add provider and fallback config types --- src/config.rs | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/config.rs b/src/config.rs index 424e295..34f5fe0 100644 --- a/src/config.rs +++ b/src/config.rs @@ -31,6 +31,15 @@ pub struct Config { pub supervisor: SupervisorConfig, #[serde(default)] pub subagents: SubagentsConfig, + /// Explicit provider sections (multi-provider mode). Optional β€” + /// when empty, `build_providers()` synthesizes a single OpenRouter + /// provider from the legacy `[openrouter]` section. + #[serde(default)] + pub provider: Vec, + /// Fallback chain β€” additional provider/model names tried when + /// the primary call fails. + #[serde(default)] + pub fallback: FallbackConfig, /// Absolute home root resolved at load time (not read from TOML). #[serde(skip)] pub resolved_home: Option, @@ -125,6 +134,46 @@ pub struct OpenRouterConfig { pub supports_vision: bool, } +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub enum ProviderType { + #[serde(rename = "openrouter")] + OpenRouter, + #[serde(rename = "openai_compatible")] + OpenAICompatible, + #[serde(rename = "ollama")] + Ollama, +} + +#[allow(clippy::derivable_impls)] +impl Default for ProviderType { + fn default() -> Self { + Self::OpenRouter + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ProviderSection { + pub name: String, + #[serde(rename = "type")] + pub provider_type: ProviderType, + pub base_url: String, + pub api_key: Option, + #[serde(default = "default_model")] + pub model: String, + #[serde(default)] + pub supports_vision: bool, + #[serde(default = "default_max_tokens")] + pub max_tokens: u32, + #[serde(default)] + pub discover_models: bool, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct FallbackConfig { + #[serde(default)] + pub chain: Vec, +} + #[derive(Debug, Deserialize, Clone)] pub struct OcrConfig { /// Directory where OCR model files are cached. @@ -540,6 +589,40 @@ impl Config { Ok(config) } + + /// Build the provider list from config, handling legacy [openrouter] backward compat. + /// Returns (providers, default_provider_name, fallback_chain). + pub fn build_providers(&self) -> (Vec, String, Vec) { + let mut providers: Vec = self.provider.clone(); + + // Backward compat: if [openrouter] section exists and no explicit provider named "openrouter" + let has_openrouter = providers.iter().any(|p| p.name == "openrouter"); + if has_openrouter { + tracing::warn!( + "Both [[provider]] name=\"openrouter\" and [openrouter] configured β€” explicit [[provider]] entry takes precedence" + ); + } else { + providers.push(ProviderSection { + name: "openrouter".to_string(), + provider_type: ProviderType::OpenRouter, + base_url: self.openrouter.base_url.clone(), + api_key: Some(self.openrouter.api_key.clone()), + model: self.openrouter.model.clone(), + supports_vision: self.openrouter.supports_vision, + max_tokens: self.openrouter.max_tokens, + discover_models: false, + }); + } + + let default = if providers.is_empty() { + "openrouter".to_string() + } else { + providers[0].name.clone() + }; + + let fallback = self.fallback.chain.clone(); + (providers, default, fallback) + } } #[cfg(test)] From e6276dc748d1c381773601fc9f1d75cb42f5a38f Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 16:41:18 +0800 Subject: [PATCH 03/19] feat: add Provider trait, ProviderRegistry, OpenRouter/OpenAICompatible/Ollama providers --- src/lib.rs | 1 + src/llm.rs | 31 ++- src/provider.rs | 694 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 713 insertions(+), 13 deletions(-) create mode 100644 src/provider.rs diff --git a/src/lib.rs b/src/lib.rs index 8d87714..8c1eac2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ pub mod llm; pub mod mcp; pub mod memory; pub mod platform; +pub mod provider; pub mod scheduler; pub mod setup; pub mod skills; diff --git a/src/llm.rs b/src/llm.rs index 5d55327..4651210 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -130,26 +130,31 @@ pub fn is_empty_assistant_response(message: &ChatMessage) -> bool { } #[derive(Debug, Serialize)] -struct ChatRequest { - model: String, - messages: Vec, +pub struct ChatRequest { + pub model: String, + pub messages: Vec, #[serde(skip_serializing_if = "Option::is_none")] - tools: Option>, + pub tools: Option>, #[serde(skip_serializing_if = "Option::is_none")] - tool_choice: Option, - max_tokens: u32, + pub tool_choice: Option, + pub max_tokens: u32, } #[derive(Debug, Deserialize)] -struct ChatResponse { - choices: Vec, +pub struct ChatResponse { + pub choices: Vec, } #[derive(Debug, Deserialize)] -struct Choice { - message: ChatMessage, +pub struct Choice { + pub message: ChatMessage, #[serde(default)] - finish_reason: Option, + pub finish_reason: Option, +} + +#[doc(hidden)] +pub mod internal { + pub use super::{ChatRequest, ChatResponse, Choice}; } /// Sanitize a JSON Schema parameter object so it is accepted by strict providers @@ -170,7 +175,7 @@ struct Choice { /// /// This function mutates the schema in-place and recurses into `properties`, /// `items`, `anyOf`, `oneOf`, and `allOf` sub-schemas. -fn sanitize_parameters(schema: &mut serde_json::Value) { +pub fn sanitize_parameters(schema: &mut serde_json::Value) { let obj = match schema.as_object_mut() { Some(o) => o, None => return, @@ -276,7 +281,7 @@ fn sanitize_parameters(schema: &mut serde_json::Value) { /// /// Returns `Some(Vec)` with at least one entry when the format is /// detected, or `None` if the content does not contain the Kimi markers. -fn parse_kimi_tool_calls(content: &str) -> Option> { +pub fn parse_kimi_tool_calls(content: &str) -> Option> { if !content.contains("<|tool_calls_section_begin|>") { return None; } diff --git a/src/provider.rs b/src/provider.rs new file mode 100644 index 0000000..cf53e18 --- /dev/null +++ b/src/provider.rs @@ -0,0 +1,694 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use async_trait::async_trait; + +use crate::config::{ProviderSection, ProviderType}; +use crate::llm::{ChatCompletion, ChatMessage, ToolDefinition}; + +/// Runtime configuration for a single LLM provider. +/// +/// This is the "live" view of a provider built from a [`ProviderSection`] in +/// the config file. The `From<&ProviderSection>` conversion is the canonical +/// way to construct one. +#[derive(Debug, Clone)] +pub struct ProviderConfig { + pub name: String, + pub provider_type: ProviderType, + pub base_url: String, + pub api_key: Option, + pub default_model: String, + pub supports_vision: bool, + pub max_tokens: u32, + pub discover_models: bool, +} + +impl From<&ProviderSection> for ProviderConfig { + fn from(s: &ProviderSection) -> Self { + Self { + name: s.name.clone(), + provider_type: s.provider_type.clone(), + base_url: s.base_url.clone(), + api_key: s.api_key.clone(), + default_model: s.model.clone(), + supports_vision: s.supports_vision, + max_tokens: s.max_tokens, + discover_models: s.discover_models, + } + } +} + +/// Unified LLM provider abstraction. +/// +/// Each implementation owns its own HTTP shaping (URL, headers, model list +/// endpoint) and delegates the request/response body shapes to the shared +/// types in [`crate::llm`]. +#[async_trait] +pub trait Provider: Send + Sync { + fn name(&self) -> &str; + fn default_model(&self) -> &str; + fn supports_vision(&self) -> bool; + fn config(&self) -> &ProviderConfig; + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result; + + async fn list_models(&self, client: &reqwest::Client) -> Result>; +} + +/// Holds the set of providers available to the agent and the default provider +/// name used when a model string does not carry an explicit `provider/` prefix. +pub struct ProviderRegistry { + providers: HashMap>, + default_provider: String, +} + +impl std::fmt::Debug for ProviderRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProviderRegistry") + .field("providers", &self.providers.keys().collect::>()) + .field("default_provider", &self.default_provider) + .finish() + } +} + +impl ProviderRegistry { + pub fn new(providers: HashMap>, default_provider: String) -> Self { + Self { + providers, + default_provider, + } + } + + /// Resolve a model string to (provider, stripped_model). + /// Never fails β€” unknown prefixes fall through to default provider. + pub fn resolve_model<'a>(&'a self, model: &'a str) -> (&'a dyn Provider, &'a str) { + if let Some((prefix, rest)) = model.split_once('/') { + if let Some(provider) = self.providers.get(prefix) { + return (provider.as_ref(), rest); + } + } + // Fall through to default provider with full string + let default = &self.providers[&self.default_provider]; + (default.as_ref(), model) + } + + pub fn get_provider(&self, name: &str) -> Option<&dyn Provider> { + self.providers.get(name).map(|p| p.as_ref()) + } + + pub fn providers(&self) -> impl Iterator { + self.providers.values().map(|p| p.as_ref()) + } + + pub fn default_provider_name(&self) -> &str { + &self.default_provider + } + + pub fn provider_count(&self) -> usize { + self.providers.len() + } + + pub fn provider_names(&self) -> Vec { + self.providers.keys().cloned().collect() + } +} + +/// Build a ProviderRegistry from config sections. +pub fn build_registry( + sections: &[ProviderSection], + default_name: &str, +) -> Result { + let mut providers: HashMap> = HashMap::new(); + + for section in sections { + let cfg: ProviderConfig = ProviderConfig::from(section); + let provider: Arc = match section.provider_type { + ProviderType::OpenRouter => Arc::new(OpenRouterProvider::new(cfg)), + ProviderType::OpenAICompatible => Arc::new(OpenAICompatibleProvider::new(cfg)), + ProviderType::Ollama => Arc::new(OllamaProvider::new(cfg)), + }; + providers.insert(section.name.clone(), provider); + } + + if providers.is_empty() { + anyhow::bail!("No LLM providers configured"); + } + + if !providers.contains_key(default_name) { + anyhow::bail!( + "Default provider '{}' not found in configured providers", + default_name + ); + } + + Ok(ProviderRegistry::new(providers, default_name.to_string())) +} + +// === OpenRouterProvider === + +pub struct OpenRouterProvider { + config: ProviderConfig, +} + +impl OpenRouterProvider { + pub fn new(config: ProviderConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl Provider for OpenRouterProvider { + fn name(&self) -> &str { + &self.config.name + } + fn default_model(&self) -> &str { + &self.config.default_model + } + fn supports_vision(&self) -> bool { + self.config.supports_vision + } + fn config(&self) -> &ProviderConfig { + &self.config + } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + let tools_param = if tools.is_empty() { + None + } else { + let sanitized: Vec = tools + .iter() + .map(|t| { + let mut t = t.clone(); + crate::llm::sanitize_parameters(&mut t.function.parameters); + t + }) + .collect(); + Some(sanitized) + }; + + let request = crate::llm::internal::ChatRequest { + model: model.to_string(), + messages: messages.to_vec(), + tools: tools_param, + tool_choice: None, + max_tokens, + }; + + let url = format!("{}/chat/completions", self.config.base_url); + let mut req = client.post(&url).json(&request); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + + let response = req.send().await.context("Failed to send request to OpenRouter")?; + + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("OpenRouter API error ({}): {}", status, body); + } + + let chat_response: crate::llm::internal::ChatResponse = response + .json() + .await + .context("Failed to parse OpenRouter response")?; + + let mut choice = chat_response + .choices + .into_iter() + .next() + .context("No response from OpenRouter")?; + + // Kimi tool-call fallback + let has_tool_calls = choice + .message + .tool_calls + .as_ref() + .is_some_and(|t| !t.is_empty()); + if !has_tool_calls { + if let Some(ref content) = choice.message.content { + if let Some(parsed) = crate::llm::parse_kimi_tool_calls(&content.as_text()) { + choice.message.tool_calls = Some(parsed); + choice.message.content = None; + } + } + } + + Ok(ChatCompletion { + message: choice.message, + finish_reason: choice.finish_reason, + model: model.to_string(), + }) + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = format!("{}/models", self.config.base_url); + let mut req = client.get(&url); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + let response = req.send().await.context("Failed to fetch models from OpenRouter")?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("OpenRouter models API error ({}): {}", status, body); + } + let list: serde_json::Value = response.json().await?; + let models = list["data"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|m| m["id"].as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + Ok(models) + } +} + +// === OpenAICompatibleProvider === + +pub struct OpenAICompatibleProvider { + config: ProviderConfig, +} + +impl OpenAICompatibleProvider { + pub fn new(config: ProviderConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl Provider for OpenAICompatibleProvider { + fn name(&self) -> &str { + &self.config.name + } + fn default_model(&self) -> &str { + &self.config.default_model + } + fn supports_vision(&self) -> bool { + self.config.supports_vision + } + fn config(&self) -> &ProviderConfig { + &self.config + } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + let tools_param = if tools.is_empty() { + None + } else { + Some(tools.to_vec()) + }; + + let request = crate::llm::internal::ChatRequest { + model: model.to_string(), + messages: messages.to_vec(), + tools: tools_param, + tool_choice: None, + max_tokens, + }; + + let url = format!("{}/chat/completions", self.config.base_url); + let mut req = client.post(&url).json(&request); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + + let response = req + .send() + .await + .context("Failed to send request to OpenAI-compatible provider")?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!( + "Provider '{}' error ({}): {}", + self.config.name, + status, + body + ); + } + + let chat_response: crate::llm::internal::ChatResponse = response + .json() + .await + .context("Failed to parse OpenAI-compatible provider response")?; + let choice = + chat_response.choices.into_iter().next().ok_or_else(|| { + anyhow::anyhow!("No response from provider '{}'", self.config.name) + })?; + + Ok(ChatCompletion { + message: choice.message, + finish_reason: choice.finish_reason, + model: model.to_string(), + }) + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = format!("{}/models", self.config.base_url); + let mut req = client.get(&url); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + let response = req + .send() + .await + .context("Failed to fetch models from OpenAI-compatible provider")?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!( + "Provider '{}' models API error ({}): {}", + self.config.name, + status, + body + ); + } + let list: serde_json::Value = response.json().await?; + let models = list["data"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|m| m["id"].as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + Ok(models) + } +} + +// === OllamaProvider === + +pub struct OllamaProvider { + config: ProviderConfig, + inner: OpenAICompatibleProvider, +} + +impl OllamaProvider { + pub fn new(config: ProviderConfig) -> Self { + let inner_cfg = config.clone(); + let inner = OpenAICompatibleProvider::new(inner_cfg); + Self { config, inner } + } + + /// Compute the native Ollama `/api/tags` model-discovery URL. + /// + /// Ollama's native endpoint lives at `/api/tags` on the server root + /// (not under `/v1`). Strip any common version suffix from the configured + /// base URL before appending the tags path. + fn discovery_url(&self) -> String { + let base = self.config.base_url.trim_end_matches('/'); + let base = base + .strip_suffix("/v1") + .or_else(|| base.strip_suffix("/v2")) + .or_else(|| base.strip_suffix("/v3")) + .unwrap_or(base); + format!("{}/api/tags", base.trim_end_matches('/')) + } +} + +#[async_trait] +impl Provider for OllamaProvider { + fn name(&self) -> &str { + &self.config.name + } + fn default_model(&self) -> &str { + &self.config.default_model + } + fn supports_vision(&self) -> bool { + self.config.supports_vision + } + fn config(&self) -> &ProviderConfig { + &self.config + } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + self.inner + .chat_completion(client, messages, tools, model, max_tokens) + .await + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = self.discovery_url(); + let mut req = client.get(&url); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + let response = req + .send() + .await + .context("Failed to fetch models from Ollama")?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("Ollama models API error ({}): {}", status, body); + } + let body: serde_json::Value = response.json().await?; + let models = body["models"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|m| m["name"].as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + Ok(models) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::ProviderSection; + + fn make_section( + name: &str, + provider_type: ProviderType, + base_url: &str, + model: &str, + ) -> ProviderSection { + ProviderSection { + name: name.to_string(), + provider_type, + base_url: base_url.to_string(), + api_key: Some("test-key".to_string()), + model: model.to_string(), + supports_vision: false, + max_tokens: 1024, + discover_models: false, + } + } + + #[test] + fn provider_config_from_section_copies_all_fields() { + let section = make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + ); + let cfg = ProviderConfig::from(§ion); + assert_eq!(cfg.name, "alpha"); + assert_eq!(cfg.provider_type, ProviderType::OpenRouter); + assert_eq!(cfg.base_url, "https://openrouter.ai/api/v1"); + assert_eq!(cfg.api_key.as_deref(), Some("test-key")); + assert_eq!(cfg.default_model, "anthropic/claude-sonnet-4-6"); + assert_eq!(cfg.max_tokens, 1024); + } + + #[test] + fn build_registry_creates_one_provider_per_section() { + let sections = vec![ + make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + ), + make_section( + "beta", + ProviderType::Ollama, + "http://localhost:11434/v1", + "llama3.1", + ), + ]; + let reg = build_registry(§ions, "alpha").unwrap(); + assert_eq!(reg.provider_count(), 2); + assert!(reg.get_provider("alpha").is_some()); + assert!(reg.get_provider("beta").is_some()); + assert_eq!(reg.default_provider_name(), "alpha"); + } + + #[test] + fn build_registry_fails_with_no_providers() { + let result = build_registry(&[], "alpha"); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("No LLM providers")); + } + + #[test] + fn build_registry_fails_when_default_missing() { + let sections = vec![make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + )]; + let result = build_registry(§ions, "missing"); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("Default provider 'missing'")); + } + + #[test] + fn resolve_model_uses_prefix_when_provider_known() { + let sections = vec![ + make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + ), + make_section( + "beta", + ProviderType::Ollama, + "http://localhost:11434/v1", + "llama3.1", + ), + ]; + let reg = build_registry(§ions, "alpha").unwrap(); + let (provider, model) = reg.resolve_model("beta/llama3.1:8b"); + assert_eq!(provider.name(), "beta"); + assert_eq!(model, "llama3.1:8b"); + } + + #[test] + fn resolve_model_falls_back_to_default_on_unknown_prefix() { + let sections = vec![make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + )]; + let reg = build_registry(§ions, "alpha").unwrap(); + let (provider, model) = reg.resolve_model("moonshotai/kimi-k2.5"); + assert_eq!(provider.name(), "alpha"); + assert_eq!(model, "moonshotai/kimi-k2.5"); + } + + #[test] + fn resolve_model_uses_default_for_bare_model() { + let sections = vec![make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + )]; + let reg = build_registry(§ions, "alpha").unwrap(); + let (provider, model) = reg.resolve_model("claude-sonnet-4-6"); + assert_eq!(provider.name(), "alpha"); + assert_eq!(model, "claude-sonnet-4-6"); + } + + #[test] + fn provider_names_returns_all_registered_names() { + let sections = vec![ + make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + ), + make_section( + "beta", + ProviderType::Ollama, + "http://localhost:11434/v1", + "llama3.1", + ), + ]; + let reg = build_registry(§ions, "alpha").unwrap(); + let mut names = reg.provider_names(); + names.sort(); + assert_eq!(names, vec!["alpha".to_string(), "beta".to_string()]); + } + + #[test] + fn ollama_discovery_url_strips_v1_suffix() { + let cfg = ProviderConfig { + name: "local".to_string(), + provider_type: ProviderType::Ollama, + base_url: "http://localhost:11434/v1".to_string(), + api_key: None, + default_model: "llama3.1".to_string(), + supports_vision: false, + max_tokens: 1024, + discover_models: true, + }; + let p = OllamaProvider::new(cfg); + assert_eq!(p.discovery_url(), "http://localhost:11434/api/tags"); + } + + #[test] + fn ollama_discovery_url_strips_trailing_slash() { + let cfg = ProviderConfig { + name: "local".to_string(), + provider_type: ProviderType::Ollama, + base_url: "http://localhost:11434/".to_string(), + api_key: None, + default_model: "llama3.1".to_string(), + supports_vision: false, + max_tokens: 1024, + discover_models: true, + }; + let p = OllamaProvider::new(cfg); + assert_eq!(p.discovery_url(), "http://localhost:11434/api/tags"); + } + + #[test] + fn ollama_discovery_url_appends_to_bare_host() { + let cfg = ProviderConfig { + name: "local".to_string(), + provider_type: ProviderType::Ollama, + base_url: "http://localhost:11434".to_string(), + api_key: None, + default_model: "llama3.1".to_string(), + supports_vision: false, + max_tokens: 1024, + discover_models: true, + }; + let p = OllamaProvider::new(cfg); + assert_eq!(p.discovery_url(), "http://localhost:11434/api/tags"); + } +} From 94c5935f217ef6c007bb7844554469c6e16095be Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 16:58:11 +0800 Subject: [PATCH 04/19] refactor: wire ProviderRegistry into LlmClient, Agent, and main.rs with fallback chain --- src/agent.rs | 137 +++++++++++++++++----- src/file_processor/mod.rs | 3 +- src/llm.rs | 236 ++++---------------------------------- src/main.rs | 17 ++- src/platform/telegram.rs | 113 ++---------------- src/provider.rs | 12 +- 6 files changed, 170 insertions(+), 348 deletions(-) diff --git a/src/agent.rs b/src/agent.rs index 483bf28..f1f1e63 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -39,6 +39,7 @@ pub struct ScheduledJobRequest { /// Platform-agnostic β€” receives IncomingMessage, returns response text. pub struct Agent { pub llm: LlmClient, + pub registry: Arc, pub config: Config, pub mcp: McpManager, pub memory: MemoryStore, @@ -100,6 +101,7 @@ impl Agent { #[allow(clippy::too_many_arguments)] pub fn new( config: Config, + registry: Arc, mcp: McpManager, memory: MemoryStore, skills: SkillRegistry, @@ -112,10 +114,18 @@ impl Agent { langsmith: Arc, config_path: PathBuf, ) -> Self { - let llm = LlmClient::new(config.openrouter.clone()); - let initial_model = config.openrouter.model.clone(); + let llm = LlmClient::new(registry.clone()); + let initial_model = format!( + "{}/{}", + registry.default_provider_name(), + registry + .resolve_model(registry.default_provider_name()) + .0 + .default_model() + ); Self { llm, + registry, config, mcp, memory, @@ -325,32 +335,53 @@ impl Agent { anyhow::bail!("Model ID cannot be empty"); } - let content = tokio::fs::read_to_string(&self.config_path) - .await - .context("Failed to read config.toml")?; - let mut doc: toml::value::Table = - toml::from_str(&content).context("Failed to parse config.toml")?; - - let openrouter_entry = doc - .entry("openrouter") - .or_insert_with(|| toml::Value::Table(toml::value::Table::new())); - if let Some(table) = openrouter_entry.as_table_mut() { - table.insert( - "model".to_string(), - toml::Value::String(model_id.to_string()), - ); + // Validate: resolve succeeds for any string + let (provider, actual_model) = self.registry.resolve_model(model_id); + if let Some((prefix, _)) = model_id.split_once('/') { + if self.registry.get_provider(prefix).is_none() { + tracing::warn!( + "Model '{}': prefix '{}' does not match any known provider \ + (falling through to default '{}')", + model_id, + prefix, + self.registry.default_provider_name() + ); + } } - let new_content = - toml::to_string_pretty(&doc).context("Failed to serialize config.toml")?; - tokio::fs::write(&self.config_path, &new_content) - .await - .with_context(|| format!("Failed to write {}", self.config_path.display()))?; + let content = tokio::fs::read_to_string(&self.config_path).await?; + let mut doc: toml::value::Table = toml::from_str(&content)?; + + let provider_name = provider.name().to_string(); + + if provider_name == "openrouter" && doc.contains_key("openrouter") { + if let Some(table) = doc.get_mut("openrouter").and_then(|v| v.as_table_mut()) { + table.insert( + "model".to_string(), + toml::Value::String(actual_model.to_string()), + ); + } + } else if let Some(provider_array) = doc.get_mut("provider").and_then(|v| v.as_array_mut()) + { + for entry in provider_array.iter_mut() { + if let Some(table) = entry.as_table_mut() { + if table.get("name").and_then(|v| v.as_str()) == Some(&provider_name) { + table.insert( + "model".to_string(), + toml::Value::String(actual_model.to_string()), + ); + } + } + } + } + + let new_content = toml::to_string_pretty(&doc)?; + tokio::fs::write(&self.config_path, &new_content).await?; let mut current = self.current_model.write().await; *current = model_id.to_string(); - tracing::info!(model = %model_id, "Model changed and persisted"); + tracing::info!(model = %model_id, provider = %provider_name, "Model changed and persisted"); Ok(()) } @@ -494,12 +525,19 @@ impl Agent { } // Process attachments (images β†’ vision parts or OCR text; PDFs/DOCXs β†’ extracted text) + let supports_vision = { + let current = self.current_model.read().await; + let (provider, _) = self.registry.resolve_model(¤t); + provider.supports_vision() + }; + let (attachment_text, image_parts) = if !incoming.attachments.is_empty() { crate::file_processor::process_attachments( &incoming.attachments, &incoming.text, &self.config, &self.memory, + supports_vision, // NEW parameter ) .await } else { @@ -652,12 +690,57 @@ impl Agent { start_time: llm_start, }); - // Call LLM with prepared prompt + // Call LLM with prepared prompt (with fallback chain) let model = self.current_model.read().await.clone(); - let completion_result = self - .llm - .chat_completion_with_model(&prompt.messages, &all_tools, &model) - .await; + let fallback_chain = &self.config.fallback.chain; + let mut last_error = None; + + let completion_result = 'fallback: { + for attempt in 0..=fallback_chain.len() { + let current_model = if attempt == 0 { + model.clone() + } else { + fallback_chain[attempt - 1].clone() + }; + + match self + .llm + .chat_completion_with_model( + &prompt.messages, + &all_tools, + ¤t_model, + ) + .await + { + Ok(c) => { + if attempt > 0 { + tracing::info!( + "Fallback succeeded: switched from '{}' to '{}'", + model, + current_model + ); + *self.current_model.write().await = current_model.clone(); + } + break 'fallback Ok(c); + } + Err(e) => { + tracing::warn!( + "Model '{}' failed (attempt {}/{}): {}", + current_model, + attempt, + fallback_chain.len(), + e + ); + last_error = Some(e); + if attempt == fallback_chain.len() { + break 'fallback Err(last_error.unwrap()); + } + continue; + } + } + } + Err(last_error.unwrap()) + }; // Handle LLM transport/API errors let completion = match completion_result { diff --git a/src/file_processor/mod.rs b/src/file_processor/mod.rs index 1edcd9d..a34bcb8 100644 --- a/src/file_processor/mod.rs +++ b/src/file_processor/mod.rs @@ -27,6 +27,7 @@ pub async fn process_attachments( user_query: &str, config: &Config, memory: &MemoryStore, + supports_vision: bool, ) -> (String, Vec) { let mut text_parts: Vec = Vec::new(); let mut image_parts: Vec = Vec::new(); @@ -37,7 +38,7 @@ pub async fn process_attachments( match process_image( &attachment.path, &attachment.mime_type, - config.openrouter.supports_vision, + supports_vision, &config.ocr.model_dir, ) .await diff --git a/src/llm.rs b/src/llm.rs index 4651210..9177867 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -1,8 +1,8 @@ -use anyhow::{Context, Result}; +use anyhow::Result; use serde::{Deserialize, Serialize}; -use tracing::{debug, warn}; +use tracing::debug; -use crate::config::OpenRouterConfig; +use std::sync::Arc; /// A single part in a multi-modal message #[derive(Debug, Clone, Serialize, Deserialize)] @@ -135,8 +135,6 @@ pub struct ChatRequest { pub messages: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub tools: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_choice: Option, pub max_tokens: u32, } @@ -353,174 +351,49 @@ pub fn parse_kimi_tool_calls(content: &str) -> Option> { #[derive(Clone)] pub struct LlmClient { - client: reqwest::Client, - config: OpenRouterConfig, + pub client: reqwest::Client, + pub registry: Arc, } impl LlmClient { - pub fn new(config: OpenRouterConfig) -> Self { + pub fn new(registry: Arc) -> Self { Self { client: reqwest::Client::new(), - config, + registry, } } /// Core chat method returning full completion metadata (message, finish_reason, model). /// - /// This is the lowest-level method that performs the HTTP request and returns - /// all available metadata from the LLM response. All other chat methods delegate - /// to this one. + /// Resolves the model string through the registry to pick the right provider, + /// then delegates the actual HTTP call to that provider. pub async fn chat_completion_with_model( &self, messages: &[ChatMessage], tools: &[ToolDefinition], model: &str, ) -> Result { - let tools_param = if tools.is_empty() { - None - } else { - let sanitized = tools - .iter() - .map(|t| { - let mut t = t.clone(); - sanitize_parameters(&mut t.function.parameters); - t - }) - .collect(); - Some(sanitized) - }; - - let tool_choice = if tools_param.is_some() { - Some("auto".to_string()) - } else { - None - }; - - let request = ChatRequest { - model: model.to_string(), - messages: messages.to_vec(), - tools: tools_param, - tool_choice, - max_tokens: self.config.max_tokens, - }; - - let url = format!("{}/chat/completions", self.config.base_url); - - debug!( - url = %url, - model = %model, - message_count = messages.len(), - tool_count = tools.len(), - "Sending request to OpenRouter" - ); - - let response = self - .client - .post(&url) - .header("Authorization", format!("Bearer {}", self.config.api_key)) - .header("Content-Type", "application/json") - .json(&request) - .send() - .await - .context("Failed to send request to OpenRouter")?; - - let status = response.status(); - if !status.is_success() { - let error_body = response.text().await.unwrap_or_default(); - anyhow::bail!("OpenRouter API error ({}): {}", status, error_body); - } - - let chat_response: ChatResponse = response - .json() - .await - .context("Failed to parse OpenRouter response")?; - - if let Some(choice) = chat_response.choices.first() { - debug!( - finish_reason = ?choice.finish_reason, - has_content = choice.message.content.is_some(), - tool_call_count = choice.message.tool_calls.as_ref().map_or(0, |t| t.len()), - "Received LLM response" - ); - if choice - .message - .content - .as_ref() - .is_none_or(MessageContent::is_empty) - && choice.message.tool_calls.as_ref().is_none_or(Vec::is_empty) - { - warn!( - finish_reason = ?choice.finish_reason, - "LLM returned no content and no tool calls" - ); - } - } - - let mut choice = chat_response - .choices - .into_iter() - .next() - .context("No response from OpenRouter")?; - - // Kimi-family models occasionally leak their native tool-call syntax into - // the `content` field instead of populating `tool_calls`. Detect and fix. - let has_tool_calls = choice - .message - .tool_calls - .as_ref() - .is_some_and(|t| !t.is_empty()); - if !has_tool_calls { - if let Some(ref content) = choice.message.content.clone() { - if let Some(parsed) = parse_kimi_tool_calls(&content.as_text()) { - warn!( - tool_count = parsed.len(), - "Kimi native tool-call format detected in content β€” \ - extracting tool calls and clearing content" - ); - choice.message.tool_calls = Some(parsed); - choice.message.content = None; - choice.finish_reason = Some("tool_calls".to_string()); - } - } - } - - Ok(ChatCompletion { - message: choice.message, - finish_reason: choice.finish_reason, - model: model.to_string(), - }) - } + let (provider, actual_model) = self.registry.resolve_model(model); + let max_tokens = provider.config().max_tokens; - /// Compatibility wrapper returning only the message (delegates to chat_completion_with_model). - pub async fn chat_with_model( - &self, - messages: &[ChatMessage], - tools: &[ToolDefinition], - model: &str, - ) -> Result { - Ok(self - .chat_completion_with_model(messages, tools, model) - .await? - .message) - } + let mut completion = provider + .chat_completion(&self.client, messages, tools, actual_model, max_tokens) + .await?; - /// Chat using an explicit model, returning full completion metadata. - pub async fn chat_completion( - &self, - messages: &[ChatMessage], - tools: &[ToolDefinition], - ) -> Result { - self.chat_completion_with_model(messages, tools, &self.config.model) - .await + completion.model = model.to_string(); + Ok(completion) } - /// Chat using the model configured in config.toml (delegates to chat_completion). + /// Convenience wrapper that uses the default provider's model. pub async fn chat( &self, messages: &[ChatMessage], tools: &[ToolDefinition], ) -> Result { - Ok(self.chat_completion(messages, tools).await?.message) + let default_model = self.registry.default_provider_name().to_string(); + self.chat_completion_with_model(messages, tools, &default_model) + .await + .map(|c| c.message) } /// Stream an already-complete `content` string through a token channel in small chunks. @@ -557,68 +430,6 @@ impl LlmClient { } /// Information about an OpenRouter model, deserialized from GET /api/v1/models. -#[derive(Debug, Clone, serde::Deserialize)] -pub struct ModelInfo { - pub id: String, - pub name: String, - #[serde(default)] - pub description: String, - #[serde(default)] - pub context_length: u64, - #[serde(default)] - pub pricing: ModelPricing, - #[serde(default)] - pub architecture: ModelArchitecture, -} - -#[derive(Debug, Clone, serde::Deserialize, Default)] -pub struct ModelPricing { - #[serde(default)] - pub prompt: String, - #[serde(default)] - pub completion: String, -} - -#[derive(Debug, Clone, serde::Deserialize, Default)] -pub struct ModelArchitecture { - #[serde(default)] - pub modality: String, -} - -#[derive(Debug, serde::Deserialize)] -struct ModelsListResponse { - data: Vec, -} - -impl LlmClient { - /// Fetch the list of available models from OpenRouter. - /// The endpoint is public (no auth required for GET /api/v1/models), - /// but we send the API key in case of rate-limiting. - pub async fn fetch_models(&self) -> anyhow::Result> { - let url = format!("{}/models", self.config.base_url); - let response = self - .client - .get(&url) - .header("Authorization", format!("Bearer {}", self.config.api_key)) - .send() - .await - .context("Failed to fetch model list from OpenRouter")?; - - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("OpenRouter models API error ({}): {}", status, body); - } - - let list: ModelsListResponse = response - .json() - .await - .context("Failed to parse OpenRouter model list response")?; - - Ok(list.data) - } -} - #[cfg(test)] mod tests { use super::*; @@ -663,7 +474,6 @@ mod tests { model: "anthropic/claude-sonnet-4-6".to_string(), messages: vec![], tools: None, - tool_choice: None, max_tokens: 100, }; let json = serde_json::to_value(&req).unwrap(); @@ -672,19 +482,17 @@ mod tests { #[test] fn test_chat_request_default_model_is_different_from_override() { - // Ensures chat_with_model can use a different model than the config default + // Ensures chat_completion_with_model can use a different model than the config default let default_req = ChatRequest { model: "moonshotai/kimi-k2.5".to_string(), messages: vec![], tools: None, - tool_choice: None, max_tokens: 100, }; let override_req = ChatRequest { model: "anthropic/claude-sonnet-4-6".to_string(), messages: vec![], tools: None, - tool_choice: None, max_tokens: 100, }; let json_default = serde_json::to_value(&default_req).unwrap(); diff --git a/src/main.rs b/src/main.rs index ee3eedd..e7c95da 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ use rustfox::config::Config; use rustfox::mcp::McpManager; use rustfox::memory::MemoryStore; use rustfox::platform; +use rustfox::provider; use rustfox::scheduler::tasks::register_builtin_tasks; use rustfox::scheduler::Scheduler; use rustfox::setup; @@ -61,6 +62,19 @@ async fn main() -> Result<()> { let config = Config::load(&config_path) .with_context(|| format!("Failed to load config from {}", config_path.display()))?; + // Build provider registry from config + let (provider_sections, default_provider, fallback_chain) = config.build_providers(); + let registry = Arc::new( + provider::build_registry(&provider_sections, &default_provider) + .context("Failed to build LLM provider registry")?, + ); + info!( + " Providers: {} (default: {}, fallback: {} model(s))", + registry.provider_count(), + registry.default_provider_name(), + fallback_chain.len() + ); + info!("Configuration loaded successfully"); info!(" Model: {}", config.openrouter.model); info!(" Sandbox: {}", config.sandbox.allowed_directory.display()); @@ -164,6 +178,7 @@ async fn main() -> Result<()> { let agent = Arc::new_cyclic(|weak| { Agent::new( config.clone(), + registry.clone(), mcp_manager, memory.clone(), skills, @@ -257,7 +272,7 @@ async fn main() -> Result<()> { register_builtin_tasks( &scheduler, memory.clone(), - rustfox::llm::LlmClient::new(config.openrouter.clone()), + rustfox::llm::LlmClient::new(registry.clone()), config.memory.summarize_cron.clone(), config.memory.summarize_threshold, config.learning.user_model_cron.clone(), diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index f597194..e1cfd3f 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -8,7 +8,6 @@ use teloxide::types::ParseMode; use tracing::{error, info, warn}; use crate::agent::Agent; -use crate::llm::ModelInfo; use crate::platform::{Attachment, AttachmentKind, IncomingMessage}; use crate::utils::markdown_entities::{markdown_to_entities, split_entities}; use crate::utils::telegram_markdown::escape_text; @@ -228,114 +227,26 @@ async fn handle_model_search( agent: &Arc, query: &str, ) -> ResponseResult<()> { - let models = match agent.llm.fetch_models().await { - Ok(list) => list, + // TODO(Task 9): /models interactive picker will be reworked against the new + // ProviderRegistry (per-provider list_models, fuzzy match across providers). + // For now, accept the model string verbatim via set_model(). + let _ = agent; + match agent.set_model(query).await { + Ok(()) => { + let reply = format!("βœ… Model changed to `{}`", query); + bot.send_message(chat_id, escape_text(&reply)) + .parse_mode(ParseMode::MarkdownV2) + .await?; + } Err(e) => { bot.send_message( chat_id, - escape_text(&format!("Failed to fetch model list: {:#}", e)), + escape_text(&format!("Failed to save model: {:#}", e)), ) .parse_mode(ParseMode::MarkdownV2) .await?; - return Ok(()); } - }; - - // Try exact match first. - if let Some(model) = models.iter().find(|m| m.id == query) { - match agent.set_model(&model.id).await { - Ok(()) => { - let reply = format!("βœ… Model changed to `{}` ({})", model.id, model.name); - bot.send_message(chat_id, escape_text(&reply)) - .parse_mode(ParseMode::MarkdownV2) - .await?; - } - Err(e) => { - bot.send_message( - chat_id, - escape_text(&format!("Failed to save model: {:#}", e)), - ) - .parse_mode(ParseMode::MarkdownV2) - .await?; - } - } - return Ok(()); } - - // Fuzzy search: case-insensitive match on id or name. - let q = query.to_lowercase(); - let mut matches: Vec<&ModelInfo> = models - .iter() - .filter(|m| m.id.to_lowercase().contains(&q) || m.name.to_lowercase().contains(&q)) - .collect(); - matches.sort_by(|a, b| { - let a_name = a.name.to_lowercase().contains(&q); - let b_name = b.name.to_lowercase().contains(&q); - b_name.cmp(&a_name).then(a.id.cmp(&b.id)) - }); - matches.truncate(10); - - if matches.is_empty() { - bot.send_message( - chat_id, - escape_text(&format!( - "No models found matching '{}'. Try a different search term.", - query - )), - ) - .parse_mode(ParseMode::MarkdownV2) - .await?; - return Ok(()); - } - - if matches.len() == 1 { - let model = &matches[0]; - match agent.set_model(&model.id).await { - Ok(()) => { - let reply = format!("βœ… Model changed to `{}` ({})", model.id, model.name); - bot.send_message(chat_id, escape_text(&reply)) - .parse_mode(ParseMode::MarkdownV2) - .await?; - } - Err(e) => { - bot.send_message( - chat_id, - escape_text(&format!("Failed to save model: {:#}", e)), - ) - .parse_mode(ParseMode::MarkdownV2) - .await?; - } - } - return Ok(()); - } - - // Show top 5 results with inline keyboard buttons. - use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; - let mut keyboard: Vec> = Vec::new(); - for model in matches.iter().take(5) { - let label = if model.name.len() > 35 { - format!("{}…", &model.name[..35]) - } else { - model.name.clone() - }; - keyboard.push(vec![InlineKeyboardButton::callback( - label, - format!("model_select:{}", model.id), - )]); - } - keyboard.push(vec![InlineKeyboardButton::callback( - "❌ Cancel", - "model_select:cancel", - )]); - - let reply = format!( - "Found {} models matching '{}'. Select one:", - matches.len(), - query - ); - bot.send_message(chat_id, &reply) - .reply_markup(InlineKeyboardMarkup::new(keyboard)) - .await?; Ok(()) } diff --git a/src/provider.rs b/src/provider.rs index cf53e18..7e73131 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -205,7 +205,6 @@ impl Provider for OpenRouterProvider { model: model.to_string(), messages: messages.to_vec(), tools: tools_param, - tool_choice: None, max_tokens, }; @@ -215,7 +214,10 @@ impl Provider for OpenRouterProvider { req = req.header("Authorization", format!("Bearer {}", key)); } - let response = req.send().await.context("Failed to send request to OpenRouter")?; + let response = req + .send() + .await + .context("Failed to send request to OpenRouter")?; let status = response.status(); if !status.is_success() { @@ -262,7 +264,10 @@ impl Provider for OpenRouterProvider { if let Some(ref key) = self.config.api_key { req = req.header("Authorization", format!("Bearer {}", key)); } - let response = req.send().await.context("Failed to fetch models from OpenRouter")?; + let response = req + .send() + .await + .context("Failed to fetch models from OpenRouter")?; let status = response.status(); if !status.is_success() { let body = response.text().await.unwrap_or_default(); @@ -326,7 +331,6 @@ impl Provider for OpenAICompatibleProvider { model: model.to_string(), messages: messages.to_vec(), tools: tools_param, - tool_choice: None, max_tokens, }; From 8c268d8b5cfbe3017fe2f9fd4f0a9e12096a7a46 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 17:04:46 +0800 Subject: [PATCH 05/19] feat: redesign /models with multi-step provider selection and model discovery --- src/platform/telegram.rs | 195 +++++++++++++++++++++++++++++++++------ 1 file changed, 169 insertions(+), 26 deletions(-) diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index e1cfd3f..14ea9e8 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -9,6 +9,7 @@ use tracing::{error, info, warn}; use crate::agent::Agent; use crate::platform::{Attachment, AttachmentKind, IncomingMessage}; +use crate::provider::Provider; use crate::utils::markdown_entities::{markdown_to_entities, split_entities}; use crate::utils::telegram_markdown::escape_text; @@ -219,6 +220,98 @@ fn is_verbose_enabled(value: Option<&str>) -> bool { value.map(|v| v == "true").unwrap_or(false) } +/// Show models for a selected provider, or prompt for text search. +/// When prompting for text search, stores pending state in memory so the next +/// user message from this user routes to `handle_model_search` scoped to this provider. +async fn handle_provider_model_select( + bot: Bot, + chat_id: ChatId, + agent: &Arc, + provider_name: &str, + provider: &dyn Provider, + user_id: &str, +) -> ResponseResult<()> { + + let set_pending = |agent: &Arc, user_id: &str| { + let agent = agent.clone(); + let user_id = user_id.to_string(); + let provider_name = provider_name.to_string(); + Box::pin(async move { + agent.memory.remember( + "settings", + &format!("model_search_pending_{}", user_id), + "true", + None, + ).await.ok(); + agent.memory.remember( + "settings", + &format!("model_search_provider_{}", user_id), + &provider_name, + None, + ).await.ok(); + }) + }; + + if !provider.config().discover_models { + let prompt = format!( + "Send me a model name or ID to search for on **{provider_name}**.\n\ + Example: `{}`", + provider.default_model() + ); + bot.send_message(chat_id, &prompt).await?; + set_pending(agent, user_id).await; + return Ok(()); + } + + match provider.list_models(&agent.llm.client).await { + Ok(models) if models.len() <= 20 => { + use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; + let mut keyboard: Vec> = models + .iter() + .map(|m| { + let qualified = format!("{}/{}", provider_name, m); + vec![InlineKeyboardButton::callback( + m.clone(), + format!("model_select:{}", qualified), + )] + }) + .collect(); + keyboard.push(vec![InlineKeyboardButton::callback( + "\u{1F50D} Search all", + "model_search_prompt", + )]); + keyboard.push(vec![InlineKeyboardButton::callback( + "\u{274C} Cancel", + "model_select:cancel", + )]); + + let reply = format!("Models on **{provider_name}** ({}):", models.len()); + bot.send_message(chat_id, &reply) + .reply_markup(InlineKeyboardMarkup::new(keyboard)) + .await?; + } + Ok(models) => { + let prompt = format!( + "**{provider_name}** has {} models available.\n\ + Send me a model name or ID to search for.", + models.len() + ); + bot.send_message(chat_id, &prompt).await?; + set_pending(agent, user_id).await; + } + Err(e) => { + let prompt = format!( + "Could not load model list from **{provider_name}**: {e}\n\ + Send a model name or ID directly." + ); + bot.send_message(chat_id, &prompt).await?; + set_pending(agent, user_id).await; + } + } + + Ok(()) +} + /// Shared model search logic: fetch models, try exact match, fall back to fuzzy search, /// show top 5 results as inline keyboard buttons. async fn handle_model_search( @@ -227,10 +320,11 @@ async fn handle_model_search( agent: &Arc, query: &str, ) -> ResponseResult<()> { - // TODO(Task 9): /models interactive picker will be reworked against the new - // ProviderRegistry (per-provider list_models, fuzzy match across providers). - // For now, accept the model string verbatim via set_model(). - let _ = agent; + // Direct model set via qualified ID (e.g. "ollama/llama3" or + // "openrouter/moonshotai/kimi-k2.6"). The agent.set_model handles validation + // through the registry; the multi-step providerβ†’model picker is in + // `handle_provider_model_select` and is invoked from the `/models` command + // and the `provider_select:` callback. match agent.set_model(query).await { Ok(()) => { let reply = format!("βœ… Model changed to `{}`", query); @@ -581,30 +675,61 @@ async fn handle_message(bot: Bot, msg: Message, agent: Arc) -> ResponseRe return Ok(()); } "models" => { - if arg.is_empty() { - let current = agent.current_model.read().await; - let reply = format!( - "Current model: `{}`\n\nTap the button below to search for a model.", - *current - ); - use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; - let keyboard = InlineKeyboardMarkup::new(vec![ - vec![InlineKeyboardButton::callback( - "πŸ” Search models", - "model_search_prompt", - )], - vec![InlineKeyboardButton::callback( - "❌ Cancel", - "model_select:cancel", - )], - ]); - bot.send_message(msg.chat.id, &reply) - .reply_markup(keyboard) - .await?; - return Ok(()); + if !arg.is_empty() { + return handle_model_search(bot, msg.chat.id, &agent, &arg).await; + } + + let providers = agent.registry.provider_names(); + + if providers.len() == 1 { + // Single provider: jump straight to model search + let provider_name = providers[0].clone(); + let provider = match agent.registry.get_provider(&provider_name) { + Some(p) => p, + None => { + bot.send_message( + msg.chat.id, + escape_text(&format!("Provider '{}' not found.", provider_name)), + ) + .parse_mode(ParseMode::MarkdownV2) + .await?; + return Ok(()); + } + }; + let user_id = user_id.to_string(); + return handle_provider_model_select( + bot, + msg.chat.id, + &agent, + &provider_name, + provider, + &user_id, + ) + .await; } - return handle_model_search(bot, msg.chat.id, &agent, &arg).await; + // Multiple providers: show inline keyboard + let current = agent.current_model.read().await; + let reply = format!("Active model: `{}`\n\nSelect a provider:", *current); + use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; + let mut keyboard: Vec> = providers + .iter() + .map(|name| { + vec![InlineKeyboardButton::callback( + name.clone(), + format!("provider_select:{}", name), + )] + }) + .collect(); + keyboard.push(vec![InlineKeyboardButton::callback( + "❌ Cancel", + "model_select:cancel", + )]); + + bot.send_message(msg.chat.id, &reply) + .reply_markup(InlineKeyboardMarkup::new(keyboard)) + .await?; + return Ok(()); } _ => {} // ignore unknown commands for now } @@ -885,6 +1010,24 @@ async fn handle_model_callback( bot.answer_callback_query(callback_id).await?; + if let Some(provider_name) = data.strip_prefix("provider_select:") { + if let Some(provider) = agent.registry.get_provider(provider_name) { + if let Some(ref m) = msg { + let user_id = q.from.id.0.to_string(); + return handle_provider_model_select( + bot, + m.chat.id, + &agent, + provider_name, + provider, + &user_id, + ) + .await; + } + } + return Ok(()); + } + if data == "model_search_prompt" { if let Some(m) = msg { let prompt = "Send me a model name or ID to search for. Examples: claude, kimi, gpt, or a full model ID like openrouter/o3-mini."; From 2f7f4cc9f5f2e9adee6afeaab689d25e9706a46f Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 17:07:12 +0800 Subject: [PATCH 06/19] docs: add [[provider]] and [fallback] examples to config.example.toml --- config.example.toml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/config.example.toml b/config.example.toml index dc90b6f..9ef3b1c 100644 --- a/config.example.toml +++ b/config.example.toml @@ -25,6 +25,37 @@ Use the available tools to help the user with their tasks. \ When using file or terminal tools, operate only within the allowed sandbox directory. \ Be concise and helpful.""" +# ── Multi-Provider Configuration (optional) ──────────────────────────── +# Define additional LLM providers alongside OpenRouter. +# Each [[provider]] block adds a new provider. Model strings use the +# format `provider_name/model_id` (e.g., `ollama/llama3.1`). +# The first provider becomes the default. + +# Example: Ollama (local models) +# [[provider]] +# name = "ollama" +# type = "ollama" +# base_url = "http://localhost:11434/v1" +# model = "llama3.1" +# discover_models = true + +# Example: Any OpenAI-compatible endpoint (LM Studio, vLLM, llama.cpp, etc.) +# [[provider]] +# name = "lmstudio" +# type = "openai_compatible" +# base_url = "http://localhost:1234/v1" +# model = "qwen2.5-7b-instruct" + +# ── Fallback Chain (optional) ────────────────────────────────────────── +# When the primary model fails, RustFox tries each fallback in order. +# Each entry is a full provider-prefixed model string. +# [fallback] +# chain = [ +# "openrouter/moonshotai/kimi-k2.6", +# "ollama/llama3.1", +# "lmstudio/qwen2.5-7b-instruct", +# ] + # ── Home directory & paths ────────────────────────────────────────────── # By default RustFox stores everything under ~/.rustfox: # ~/.rustfox/config.toml, rustfox.db, skills/, agents/, workspace/, artifacts/ From 42aba599cc291d86fe9b404fbdb072056a7a2d6d Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 17:10:27 +0800 Subject: [PATCH 07/19] feat: redesign /models with multi-step provider selection and model discovery --- src/agent.rs | 32 ++++++------- src/config.rs | 98 ++++++++++++++++++++++++++++++++++++++++ src/main.rs | 9 +++- src/platform/telegram.rs | 85 ++++++++++++++++++++++++---------- src/provider.rs | 6 +++ 5 files changed, 187 insertions(+), 43 deletions(-) diff --git a/src/agent.rs b/src/agent.rs index f1f1e63..f2b4e31 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -115,14 +115,7 @@ impl Agent { config_path: PathBuf, ) -> Self { let llm = LlmClient::new(registry.clone()); - let initial_model = format!( - "{}/{}", - registry.default_provider_name(), - registry - .resolve_model(registry.default_provider_name()) - .0 - .default_model() - ); + let initial_model = registry.default_qualified_model(); Self { llm, registry, @@ -354,15 +347,9 @@ impl Agent { let provider_name = provider.name().to_string(); - if provider_name == "openrouter" && doc.contains_key("openrouter") { - if let Some(table) = doc.get_mut("openrouter").and_then(|v| v.as_table_mut()) { - table.insert( - "model".to_string(), - toml::Value::String(actual_model.to_string()), - ); - } - } else if let Some(provider_array) = doc.get_mut("provider").and_then(|v| v.as_array_mut()) - { + // Try explicit [[provider]] array first + let mut found_in_array = false; + if let Some(provider_array) = doc.get_mut("provider").and_then(|v| v.as_array_mut()) { for entry in provider_array.iter_mut() { if let Some(table) = entry.as_table_mut() { if table.get("name").and_then(|v| v.as_str()) == Some(&provider_name) { @@ -370,11 +357,22 @@ impl Agent { "model".to_string(), toml::Value::String(actual_model.to_string()), ); + found_in_array = true; } } } } + // Fall back to legacy [openrouter] section if not found in [[provider]] array + if !found_in_array && provider_name == "openrouter" && doc.contains_key("openrouter") { + if let Some(table) = doc.get_mut("openrouter").and_then(|v| v.as_table_mut()) { + table.insert( + "model".to_string(), + toml::Value::String(actual_model.to_string()), + ); + } + } + let new_content = toml::to_string_pretty(&doc)?; tokio::fs::write(&self.config_path, &new_content).await?; diff --git a/src/config.rs b/src/config.rs index 34f5fe0..1edd8ea 100644 --- a/src/config.rs +++ b/src/config.rs @@ -994,4 +994,102 @@ mod tests { assert_eq!(cfg.agent.empty_response_retry_limit, 0); assert_eq!(cfg.empty_response_retry_limit(), 0); } + + #[test] + fn test_provider_section_parses_ollama() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + model = "moonshotai/kimi-k2.6" + [[provider]] + name = "ollama" + type = "ollama" + base_url = "http://localhost:11434/v1" + model = "llama3.1" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert_eq!(cfg.provider.len(), 1); + assert_eq!(cfg.provider[0].name, "ollama"); + assert_eq!(cfg.provider[0].model, "llama3.1"); + } + + #[test] + fn test_legacy_openrouter_auto_creates_provider() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + model = "moonshotai/kimi-k2.6" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + let (providers, default_name, _) = cfg.build_providers(); + assert!(providers.iter().any(|p| p.name == "openrouter")); + assert_eq!(default_name, "openrouter"); + } + + #[test] + fn test_visible_provider_overrides_legacy() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "old-key" + model = "moonshotai/kimi-k2.6" + [[provider]] + name = "openrouter" + type = "openrouter" + base_url = "https://openrouter.ai/api/v1" + api_key = "new-key" + model = "anthropic/claude-sonnet-4-6" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + let (providers, _default, _) = cfg.build_providers(); + // Should not have duplicate openrouter providers + let or_count = providers.iter().filter(|p| p.name == "openrouter").count(); + assert_eq!( + or_count, 1, + "should not duplicate openrouter when explicit [[provider]] exists" + ); + let or = providers.iter().find(|p| p.name == "openrouter").unwrap(); + assert_eq!( + or.api_key.as_deref(), + Some("new-key"), + "explicit [[provider]] should win" + ); + } + + #[test] + fn test_fallback_config_parses() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + [fallback] + chain = ["openrouter/model-a", "ollama/model-b"] + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert_eq!(cfg.fallback.chain.len(), 2); + assert_eq!(cfg.fallback.chain[0], "openrouter/model-a"); + } + + #[test] + fn test_fallback_defaults_empty() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert!(cfg.fallback.chain.is_empty()); + } } diff --git a/src/main.rs b/src/main.rs index e7c95da..94a4dcf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -76,7 +76,14 @@ async fn main() -> Result<()> { ); info!("Configuration loaded successfully"); - info!(" Model: {}", config.openrouter.model); + let default_provider_obj = registry + .get_provider(registry.default_provider_name()) + .expect("default provider must exist"); + info!( + " Model: {}/{}", + registry.default_provider_name(), + default_provider_obj.default_model() + ); info!(" Sandbox: {}", config.sandbox.allowed_directory.display()); if let Some(home) = &config.resolved_home { info!(" Home: {}", home.display()); diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index 14ea9e8..56cc247 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -231,24 +231,31 @@ async fn handle_provider_model_select( provider: &dyn Provider, user_id: &str, ) -> ResponseResult<()> { - let set_pending = |agent: &Arc, user_id: &str| { let agent = agent.clone(); let user_id = user_id.to_string(); let provider_name = provider_name.to_string(); Box::pin(async move { - agent.memory.remember( - "settings", - &format!("model_search_pending_{}", user_id), - "true", - None, - ).await.ok(); - agent.memory.remember( - "settings", - &format!("model_search_provider_{}", user_id), - &provider_name, - None, - ).await.ok(); + agent + .memory + .remember( + "settings", + &format!("model_search_pending_{}", user_id), + "true", + None, + ) + .await + .ok(); + agent + .memory + .remember( + "settings", + &format!("model_search_provider_{}", user_id), + &provider_name, + None, + ) + .await + .ok(); }) }; @@ -311,23 +318,43 @@ async fn handle_provider_model_select( Ok(()) } - -/// Shared model search logic: fetch models, try exact match, fall back to fuzzy search, -/// show top 5 results as inline keyboard buttons. +/// Accept a text model query and attempt to set the active model. +/// If the user previously selected a provider (via the inline keyboard), +/// the bare query is prefixed with that provider's name to form a +/// qualified model ID. async fn handle_model_search( bot: Bot, chat_id: ChatId, agent: &Arc, query: &str, + user_id: &str, ) -> ResponseResult<()> { - // Direct model set via qualified ID (e.g. "ollama/llama3" or - // "openrouter/moonshotai/kimi-k2.6"). The agent.set_model handles validation - // through the registry; the multi-step providerβ†’model picker is in - // `handle_provider_model_select` and is invoked from the `/models` command - // and the `provider_select:` callback. - match agent.set_model(query).await { + // Check if user has a stored provider from the interactive picker + let stored_provider = agent + .memory + .recall("settings", &format!("model_search_provider_{}", user_id)) + .await + .unwrap_or(None); + + let qualified = if let Some(ref provider_name) = stored_provider { + // Clear the stored provider so it doesn't affect future searches + agent + .memory + .forget("settings", &format!("model_search_provider_{}", user_id)) + .await + .ok(); + if query.contains('/') { + query.to_string() + } else { + format!("{}/{}", provider_name, query) + } + } else { + query.to_string() + }; + + match agent.set_model(&qualified).await { Ok(()) => { - let reply = format!("βœ… Model changed to `{}`", query); + let reply = format!("βœ… Model changed to `{}`", qualified); bot.send_message(chat_id, escape_text(&reply)) .parse_mode(ParseMode::MarkdownV2) .await?; @@ -341,6 +368,7 @@ async fn handle_model_search( .await?; } } + Ok(()) } @@ -426,7 +454,7 @@ async fn handle_message(bot: Bot, msg: Message, agent: Arc) -> ResponseRe .await .ok(); // Treat message as a model search query β€” dispatch to shared model search logic. - return handle_model_search(bot, msg.chat.id, &agent, &text).await; + return handle_model_search(bot, msg.chat.id, &agent, &text, &user_id.to_string()).await; } info!( @@ -676,7 +704,14 @@ async fn handle_message(bot: Bot, msg: Message, agent: Arc) -> ResponseRe } "models" => { if !arg.is_empty() { - return handle_model_search(bot, msg.chat.id, &agent, &arg).await; + return handle_model_search( + bot, + msg.chat.id, + &agent, + &arg, + &user_id.to_string(), + ) + .await; } let providers = agent.registry.provider_names(); diff --git a/src/provider.rs b/src/provider.rs index 7e73131..6663861 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -119,6 +119,12 @@ impl ProviderRegistry { pub fn provider_names(&self) -> Vec { self.providers.keys().cloned().collect() } + + /// Return the fully-qualified default model string, e.g. `"openrouter/moonshotai/kimi-k2.6"`. + pub fn default_qualified_model(&self) -> String { + let provider = &self.providers[&self.default_provider]; + format!("{}/{}", self.default_provider, provider.default_model()) + } } /// Build a ProviderRegistry from config sections. From e33419de3049b5b55bb10ab25b7e05732940f0ad Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Tue, 23 Jun 2026 11:44:04 +0800 Subject: [PATCH 08/19] docs: add multi-provider design spec and implementation plan --- .../plans/2026-06-22-multi-provider-plan.md | 1502 +++++++++++++++++ .../specs/2026-06-22-multi-provider-design.md | 464 +++++ 2 files changed, 1966 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-22-multi-provider-plan.md create mode 100644 docs/superpowers/specs/2026-06-22-multi-provider-design.md diff --git a/docs/superpowers/plans/2026-06-22-multi-provider-plan.md b/docs/superpowers/plans/2026-06-22-multi-provider-plan.md new file mode 100644 index 0000000..136b8d8 --- /dev/null +++ b/docs/superpowers/plans/2026-06-22-multi-provider-plan.md @@ -0,0 +1,1502 @@ +# Multi-Provider LLM Architecture β€” Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add multiple LLM provider support (Ollama, LM Studio, etc.), fallback chains, and per-subagent provider mixing. + +**Architecture:** Define a `Provider` trait with concrete impls (OpenRouter, OpenAICompatible, Ollama). `ProviderRegistry` resolves model strings (`ollama/llama3` β†’ Ollama provider + model `llama3`). `LlmClient` routes calls through the registry. Fallback chains in the agent loop try alternative models on failure. + +**Tech Stack:** Rust, reqwest, serde, async_trait, teloxide (inline keyboards) + +--- + +### Task 1: Add config types for providers and fallback + +**Files:** +- Modify: `src/config.rs` +- Read: `docs/superpowers/specs/2026-06-22-multi-provider-design.md` (for reference) + +- [ ] **Step 1: Add ProviderType enum to config.rs** + +Add this after the existing config structs (before `OcrConfig` or similar): + +```rust +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub enum ProviderType { + #[serde(rename = "openrouter")] + OpenRouter, + #[serde(rename = "openai_compatible")] + OpenAICompatible, + #[serde(rename = "ollama")] + Ollama, +} + +impl Default for ProviderType { + fn default() -> Self { + Self::OpenRouter + } +} +``` + +- [ ] **Step 2: Add ProviderSection and FallbackConfig structs** + +```rust +#[derive(Debug, Clone, Deserialize)] +pub struct ProviderSection { + pub name: String, + #[serde(rename = "type")] + pub provider_type: ProviderType, + pub base_url: String, + pub api_key: Option, + #[serde(default = "default_model")] + pub model: String, + #[serde(default)] + pub supports_vision: bool, + #[serde(default = "default_max_tokens")] + pub max_tokens: u32, + #[serde(default)] + pub discover_models: bool, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct FallbackConfig { + #[serde(default)] + pub chain: Vec, +} +``` + +- [ ] **Step 3: Add fields to Config struct** + +Add inside `pub struct Config`: +```rust + #[serde(default)] + pub provider: Vec, + #[serde(default)] + pub fallback: FallbackConfig, +``` + +- [ ] **Step 4: Add backward compat builder method to Config** + +```rust +/// Build the provider list from config, handling legacy [openrouter] backward compat. +/// Returns (providers, default_provider_name, fallback_chain). +pub fn build_providers(&self) -> (Vec, String, Vec) { + let mut providers: Vec = self.provider.clone(); + + // Backward compat: if [openrouter] section exists and no explicit provider named "openrouter" + let has_openrouter = providers.iter().any(|p| p.name == "openrouter"); + if has_openrouter { + tracing::warn!( + "Explicit [[provider]] name=\"openrouter\" found β€” legacy [openrouter] section will be ignored" + ); + } else { + providers.push(ProviderSection { + name: "openrouter".to_string(), + provider_type: ProviderType::OpenRouter, + base_url: self.openrouter.base_url.clone(), + api_key: Some(self.openrouter.api_key.clone()), + model: self.openrouter.model.clone(), + supports_vision: self.openrouter.supports_vision, + max_tokens: self.openrouter.max_tokens, + discover_models: false, + }); + } + + let default = if providers.is_empty() { + "openrouter".to_string() + } else { + providers[0].name.clone() + }; + + let fallback = self.fallback.chain.clone(); + (providers, default, fallback) +} +``` + +- [ ] **Step 5: Run tests to verify existing config parsing still works** + +```bash +cargo test -p rustfox -- config::tests --nocapture +``` +Expected: Existing config tests pass (backward compat). + +- [ ] **Step 6: Commit** + +```bash +git add src/config.rs +git commit -m "feat: add provider and fallback config types" +``` + +--- + +### Task 2: Create `src/provider.rs` β€” Provider trait, types, and registry + +**Files:** +- Create: `src/provider.rs` +- Modify: `src/lib.rs` + +- [ ] **Step 1: Add module declaration to lib.rs** + +```rust +pub mod provider; +``` + +- [ ] **Step 2: Create src/provider.rs with the core types** + +```rust +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; + +use crate::config::{ProviderSection, ProviderType}; +use crate::llm::{ChatCompletion, ChatMessage, ToolDefinition}; + +#[derive(Debug, Clone)] +pub struct ProviderConfig { + pub name: String, + pub provider_type: ProviderType, + pub base_url: String, + pub api_key: Option, + pub default_model: String, + pub supports_vision: bool, + pub max_tokens: u32, + pub discover_models: bool, +} + +impl From<&ProviderSection> for ProviderConfig { + fn from(s: &ProviderSection) -> Self { + Self { + name: s.name.clone(), + provider_type: s.provider_type.clone(), + base_url: s.base_url.clone(), + api_key: s.api_key.clone(), + default_model: s.model.clone(), + supports_vision: s.supports_vision, + max_tokens: s.max_tokens, + discover_models: s.discover_models, + } + } +} + +#[async_trait] +pub trait Provider: Send + Sync { + fn name(&self) -> &str; + fn default_model(&self) -> &str; + fn supports_vision(&self) -> bool; + fn config(&self) -> &ProviderConfig; + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result; + + async fn list_models(&self, client: &reqwest::Client) -> Result>; +} +``` + +- [ ] **Step 3: Add ProviderRegistry** + +```rust +pub struct ProviderRegistry { + providers: HashMap>, + default_provider: String, +} + +impl ProviderRegistry { + pub fn new( + providers: HashMap>, + default_provider: String, + ) -> Self { + Self { + providers, + default_provider, + } + } + + /// Resolve a model string to (provider, stripped_model). + /// Never fails β€” unknown prefixes fall through to default provider. + pub fn resolve_model(&self, model: &str) -> (&dyn Provider, &str) { + if let Some((prefix, rest)) = model.split_once('/') { + if let Some(provider) = self.providers.get(prefix) { + return (provider.as_ref(), rest); + } + } + // Fall through to default provider with full string + let default = &self.providers[&self.default_provider]; + (default.as_ref(), model) + } + + pub fn get_provider(&self, name: &str) -> Option<&dyn Provider> { + self.providers.get(name).map(|p| p.as_ref()) + } + + pub fn providers(&self) -> impl Iterator { + self.providers.values().map(|p| p.as_ref()) + } + + pub fn default_provider_name(&self) -> &str { + &self.default_provider + } + + pub fn provider_count(&self) -> usize { + self.providers.len() + } + + pub fn provider_names(&self) -> Vec { + self.providers.keys().cloned().collect() + } +} +``` + +- [ ] **Step 4: Add construction helper** + +```rust +use crate::config::ProviderSection; + +/// Build a ProviderRegistry from config sections. +pub fn build_registry( + sections: &[ProviderSection], + default_name: &str, +) -> Result { + let mut providers: HashMap> = HashMap::new(); + + for section in sections { + let cfg: ProviderConfig = ProviderConfig::from(section); + let provider: Arc = match section.provider_type { + ProviderType::OpenRouter => Arc::new(OpenRouterProvider::new(cfg)), + ProviderType::OpenAICompatible => Arc::new(OpenAICompatibleProvider::new(cfg)), + ProviderType::Ollama => Arc::new(OllamaProvider::new(cfg)), + }; + providers.insert(section.name.clone(), provider); + } + + if providers.is_empty() { + anyhow::bail!("No LLM providers configured"); + } + + // Ensure default exists + if !providers.contains_key(default_name) { + anyhow::bail!("Default provider '{}' not found in configured providers", default_name); + } + + Ok(ProviderRegistry::new(providers, default_name.to_string())) +} +``` + +- [ ] **Step 6: Commit** + +```bash +git add src/provider.rs src/lib.rs +git commit -m "feat: add Provider trait, ProviderRegistry, ProviderConfig types" +``` + +--- + +### Task 3: Implement OpenRouterProvider + +**Files:** +- Modify: `src/provider.rs` +- Modify: `src/llm.rs` + +- [ ] **Step 0: Export ChatRequest/ChatResponse from llm.rs for reuse by all providers** + +Add to `llm.rs`: +```rust +// These are shared with provider implementations in src/provider.rs +#[doc(hidden)] +pub mod internal { + pub use super::{ChatRequest, ChatResponse}; +} +``` + +Change `ChatRequest` and `ChatResponse` from `struct ChatRequest` / `struct ChatResponse` to `pub struct ChatRequest` / `pub struct ChatResponse`. + +- [ ] **Step 1: Add OpenRouterProvider struct and impl** + +Append to `src/provider.rs`: + +```rust +// === OpenRouterProvider === + +pub struct OpenRouterProvider { + config: ProviderConfig, +} + +impl OpenRouterProvider { + pub fn new(config: ProviderConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl Provider for OpenRouterProvider { + fn name(&self) -> &str { &self.config.name } + fn default_model(&self) -> &str { &self.config.default_model } + fn supports_vision(&self) -> bool { self.config.supports_vision } + fn config(&self) -> &ProviderConfig { &self.config } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + // Same logic as current LlmClient::chat_completion_with_model: + let tools_param = if tools.is_empty() { + None + } else { + let sanitized: Vec = tools + .iter() + .map(|t| { + let mut t = t.clone(); + sanitize_parameters(&mut t.function.parameters); + t + }) + .collect(); + Some(sanitized) + }; + + let request = crate::llm::internal::ChatRequest { + model: model.to_string(), + messages: messages.to_vec(), + tools: tools_param, + max_tokens, + }; + + let url = format!("{}/chat/completions", self.config.base_url); + let response = client + .post(&url) + .header("Authorization", format!("Bearer {}", + self.config.api_key.as_deref().unwrap_or(""))) + .json(&request) + .send() + .await + .context("Failed to send request to OpenRouter")?; + + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("OpenRouter API error ({}): {}", status, body); + } + + let chat_response: crate::llm::internal::ChatResponse = response + .json() + .await + .context("Failed to parse OpenRouter response")?; + + let mut choice = chat_response + .choices + .into_iter() + .next() + .context("No response from OpenRouter")?; + + // Kimi tool-call fallback + let has_tool_calls = choice + .message + .tool_calls + .as_ref() + .is_some_and(|t| !t.is_empty()); + if !has_tool_calls { + if let Some(ref content) = choice.message.content { + if let Some(parsed) = parse_kimi_tool_calls(&content.as_text()) { + choice.message.tool_calls = Some(parsed); + choice.message.content = None; + } + } + } + + Ok(ChatCompletion { + message: choice.message, + finish_reason: choice.finish_reason, + model: model.to_string(), + }) + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = format!("{}/models", self.config.base_url); + let response = client + .get(&url) + .header("Authorization", format!("Bearer {}", self.config.api_key.as_deref().unwrap_or(""))) + .send() + .await?; + let list: serde_json::Value = response.json().await?; + let models = list["data"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|m| m["id"].as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + Ok(models) + } +} +``` + +The `sanitize_parameters` and `parse_kimi_tool_calls` functions need to remain in `llm.rs` (they are referenced from within the OpenRouterProvider impl above via `use crate::llm::{sanitize_parameters, parse_kimi_tool_calls}`). + +- [ ] **Step 2: Add the import for sanitize_parameters at top of provider.rs** + +```rust +use crate::llm::{sanitize_parameters, parse_kimi_tool_calls}; +``` + +Make `sanitize_parameters` and `parse_kimi_tool_calls` `pub` in `llm.rs`. + +- [ ] **Step 3: Commit** + +```bash +git add src/provider.rs src/llm.rs +git commit -m "feat: add OpenRouterProvider with param sanitization and Kimi fallback" +``` + +--- + +### Task 4: Implement OpenAICompatibleProvider + +**Files:** +- Modify: `src/provider.rs` + +- [ ] **Step 1: Add OpenAICompatibleProvider** + +```rust +// === OpenAICompatibleProvider === + +pub struct OpenAICompatibleProvider { + config: ProviderConfig, +} + +impl OpenAICompatibleProvider { + pub fn new(config: ProviderConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl Provider for OpenAICompatibleProvider { + fn name(&self) -> &str { &self.config.name } + fn default_model(&self) -> &str { &self.config.default_model } + fn supports_vision(&self) -> bool { self.config.supports_vision } + fn config(&self) -> &ProviderConfig { &self.config } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + // Pure OpenAI-compatible endpoint: no param sanitization, no Kimi fallback. + let tools_param = if tools.is_empty() { None } else { Some(tools.to_vec()) }; + + let request = crate::llm::internal::ChatRequest { + model: model.to_string(), + messages: messages.to_vec(), + tools: tools_param, + max_tokens, + }; + + let url = format!("{}/chat/completions", self.config.base_url); + let mut req = client.post(&url).json(&request); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + + let response = req.send().await?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("Provider '{}' error ({}): {}", self.config.name, status, body); + } + + let chat_response = response.json::().await?; + let choice = chat_response.choices.into_iter().next() + .ok_or_else(|| anyhow::anyhow!("No response from provider '{}'", self.config.name))?; + + Ok(ChatCompletion { + message: choice.message, + finish_reason: choice.finish_reason, + model: model.to_string(), + }) + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = format!("{}/models", self.config.base_url); + let mut req = client.get(&url); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + let response = req.send().await?; + let list: serde_json::Value = response.json().await?; + let models = list["data"] + .as_array() + .map(|arr| arr.iter().filter_map(|m| m["id"].as_str().map(String::from)).collect()) + .unwrap_or_default(); + Ok(models) + } +} +``` + +(The `pub mod internal` export was already added in Task 3 Step 0. No additional llm.rs changes needed here.) + +- [ ] **Step 2: Commit** + +```bash +git add src/provider.rs src/llm.rs +git commit -m "feat: add OpenAICompatibleProvider for any OpenAI-compatible endpoint" +``` + +--- + +### Task 5: Implement OllamaProvider + +**Files:** +- Modify: `src/provider.rs` + +- [ ] **Step 1: Add OllamaProvider (delegates chat to OpenAICompatibleProvider)** + +```rust +// === OllamaProvider === + +pub struct OllamaProvider { + config: ProviderConfig, + inner: OpenAICompatibleProvider, +} + +impl OllamaProvider { + pub fn new(config: ProviderConfig) -> Self { + let inner_cfg = config.clone(); + let inner = OpenAICompatibleProvider::new(inner_cfg); + Self { config, inner } + } + + /// Build the discovery URL from the OpenAI-compatible base_url. + /// Trailing `/v1` or `/vN` is stripped and replaced with `/api/tags`. + /// e.g. "http://localhost:11434/v1" β†’ "http://localhost:11434/api/tags" + /// "http://localhost:11434" β†’ "http://localhost:11434/api/tags" + fn discovery_url(&self) -> String { + let base = self.config.base_url.trim_end_matches('/'); + // Strip versioned path suffix (/v1, /v2, etc.) + let base = base.strip_suffix("/v1") + .or_else(|| base.strip_suffix("/v2")) + .or_else(|| base.strip_suffix("/v3")) + .unwrap_or(base); + format!("{}/api/tags", base.trim_end_matches('/')) + } +} + +#[async_trait] +impl Provider for OllamaProvider { + fn name(&self) -> &str { &self.config.name } + fn default_model(&self) -> &str { &self.config.default_model } + fn supports_vision(&self) -> bool { self.config.supports_vision } + fn config(&self) -> &ProviderConfig { &self.config } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + // Delegate to OpenAICompatibleProvider (same API format) + self.inner.chat_completion(client, messages, tools, model, max_tokens).await + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = self.discovery_url(); + let mut req = client.get(&url); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + let response = req.send().await?; + let body: serde_json::Value = response.json().await?; + // Ollama /api/tags returns {"models": [{"name": "llama3.1:8b", ...}, ...]} + let models = body["models"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|m| m["name"].as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + Ok(models) + } +} +``` + +- [ ] **Step 2: Commit** + +```bash +git add src/provider.rs +git commit -m "feat: add OllamaProvider with /api/tags discovery" +``` + +--- + +### Task 6: Refactor LlmClient to use ProviderRegistry + +**Files:** +- Modify: `src/llm.rs` + +- [ ] **Step 1: Change LlmClient struct** + +```rust +pub struct LlmClient { + pub client: reqwest::Client, + pub registry: Arc, +} +``` + +- [ ] **Step 2: Update constructor** + +```rust +impl LlmClient { + pub fn new(registry: Arc) -> Self { + Self { + client: reqwest::Client::new(), + registry, + } + } +} +``` + +- [ ] **Step 3: Rewrite chat_completion_with_model as a routing method** + +```rust + pub async fn chat_completion_with_model( + &self, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + ) -> Result { + let (provider, actual_model) = self.registry.resolve_model(model); + let max_tokens = provider.config().max_tokens; + + let mut completion = provider + .chat_completion(&self.client, messages, tools, actual_model, max_tokens) + .await?; + + // Preserve the original model string in metadata + completion.model = model.to_string(); + Ok(completion) + } +``` + +- [ ] **Step 4: Keep `chat()` as a convenience, remove `chat_with_model()` and `chat_completion()`** + +Keep `chat()` as a thin wrapper for backward compat (e.g., soul reflection in agent.rs): +```rust +/// Convenience wrapper that uses the default provider's default model. +pub async fn chat( + &self, + messages: &[ChatMessage], + tools: &[ToolDefinition], +) -> Result { + let default_model = self.registry.default_provider_name().to_string(); + self.chat_completion_with_model(messages, tools, &default_model) + .await + .map(|c| c.message) +} +``` + +Remove these methods (no callers): +- `chat_with_model()` β€” unused after refactor +- `chat_completion()` β€” unused after refactor + +- [ ] **Step 5: Update soul-reflection caller in agent.rs** + +```rust +// Old (agent.rs ~line 1059): +// agent.llm.chat(&reflection_messages, &[]).await +// New: +agent.llm.chat_completion_with_model( + &reflection_messages, &[], + &*agent.current_model.read().await +).await.map(|c| c.message) +``` + +- [ ] **Step 6: Remove fetch_models() from LlmClient** (moved to OpenRouterProvider.list_models()) + +- [ ] **Step 7: Build** + +```bash +cargo check --lib +``` +Expected: Compilation succeeds (with the Agent changes still pending, but lib check covers llm.rs changes). + +- [ ] **Step 8: Commit** + +```bash +git add src/llm.rs src/agent.rs +git commit -m "refactor: LlmClient routes through ProviderRegistry, removed direct config dependency" +``` + +--- + +### Task 7: Update Agent for registry, fallback, and per-provider vision + +**Files:** +- Modify: `src/agent.rs` + +- [ ] **Step 1: Add registry field to Agent struct** + +```rust +pub struct Agent { + pub llm: LlmClient, + pub registry: Arc, + pub config: Config, + // ... existing fields unchanged +} +``` + +- [ ] **Step 2: Update Agent::new() to receive and store registry** + +```rust +pub fn new( + config: Config, + registry: Arc, + mcp: McpManager, + memory: MemoryStore, + // ... other params unchanged +) -> Self { + let llm = LlmClient::new(registry.clone()); + let initial_model = format!("{}/{}", + registry.default_provider_name(), + // get default model from default provider + registry.resolve_model(®istry.default_provider_name()).0.default_model() + ); + Self { + llm, + registry, + config, + // ... + current_model: tokio::sync::RwLock::new(initial_model), + // ... + } +} +``` + +Note: The initial model string is `/`, e.g. `"openrouter/moonshotai/kimi-k2.6"`. + +- [ ] **Step 3: Update set_model() validation** + +```rust +pub async fn set_model(&self, model_id: &str) -> anyhow::Result<()> { + if model_id.is_empty() { + anyhow::bail!("Model ID cannot be empty"); + } + + // Validate: resolve succeeds for any string, but warn on unknown prefix + let (provider, actual_model) = self.registry.resolve_model(model_id); + // Check if user explicitly specified a prefix that doesn't exist + if let Some((prefix, _)) = model_id.split_once('/') { + if self.registry.get_provider(prefix).is_none() { + tracing::warn!( + "Model '{}': prefix '{}' does not match any known provider \ + (falling through to default '{}')", + model_id, prefix, self.registry.default_provider_name() + ); + } + } + + // Persist to config.toml + let content = tokio::fs::read_to_string(&self.config_path).await?; + let mut doc: toml::value::Table = toml::from_str(&content)?; + + // Determine which TOML section to update based on the provider name + let provider_name = provider.name().to_string(); + + // For the legacy [openrouter] section: + if provider_name == "openrouter" && doc.contains_key("openrouter") { + if let Some(table) = doc.get_mut("openrouter").and_then(|v| v.as_table_mut()) { + table.insert("model".to_string(), toml::Value::String(actual_model.to_string())); + } + } else { + // For [[provider]] entries, find the right one by name in the array + if let Some(provider_array) = doc.get_mut("provider").and_then(|v| v.as_array_mut()) { + for entry in provider_array.iter_mut() { + if let Some(table) = entry.as_table_mut() { + if table.get("name").and_then(|v| v.as_str()) == Some(&provider_name) { + table.insert( + "model".to_string(), + toml::Value::String(actual_model.to_string()), + ); + } + } + } + } + } + + let new_content = toml::to_string_pretty(&doc)?; + tokio::fs::write(&self.config_path, &new_content).await?; + + let mut current = self.current_model.write().await; + *current = model_id.to_string(); + + tracing::info!(model = %model_id, provider = %provider_name, "Model changed and persisted"); + Ok(()) +} +``` + +- [ ] **Step 4: Add fallback chain to process_message main loop** + +Wrap the LLM call in the main agent loop (find the `chat_completion_with_model` call in `process_message`) with fallback logic: + +```rust +let model = self.current_model.read().await.clone(); +let fallback_chain = &self.config.fallback.chain; +let mut last_error = None; + +for attempt in 0..=fallback_chain.len() { + let current_model = if attempt == 0 { + model.clone() + } else { + fallback_chain[attempt - 1].clone() + }; + + match self.llm.chat_completion_with_model( + &prompt.messages, &all_tools, ¤t_model + ).await { + Ok(c) => { + // If we used a fallback, update current_model in memory (not disk) + if attempt > 0 { + tracing::info!( + "Fallback succeeded: switched from '{}' to '{}'", + model, current_model + ); + *self.current_model.write().await = current_model.clone(); + } + completion = c; + break; + } + Err(e) => { + tracing::warn!("Model '{}' failed (attempt {}/{}): {}", + current_model, attempt, fallback_chain.len(), e); + last_error = Some(e); + if attempt == fallback_chain.len() { + // All attempts exhausted β€” return the last error + return Err(last_error.unwrap()); + } + continue; + } + } +} +``` + +- [ ] **Step 5: Update per-provider vision check in agent.rs and file_processor** + +**agent.rs** β€” Replace `self.config.openrouter.supports_vision` references in `process_message`: + +Find the line in agent.rs that checks `supports_vision` (for deciding image vs OCR path). Replace: + +```rust +// Old: if self.config.openrouter.supports_vision { +// New: +let current = self.current_model.read().await; +let (provider, _) = self.registry.resolve_model(¤t); +if provider.supports_vision() { +``` + +**file_processor/mod.rs** β€” The `process_attachments` function currently reads `config.openrouter.supports_vision` directly (line 40). Change it to accept the flag as a parameter: + +```rust +// Old signature: +pub async fn process_attachments( + attachments: &[Attachment], + user_query: &str, + config: &Config, + memory: &MemoryStore, +) -> (String, Vec) { + +// Change the call site in agent.rs to pass the active provider's supports_vision: +let supports_vision = { + let current = self.current_model.read().await; + let (provider, _) = self.registry.resolve_model(¤t); + provider.supports_vision() +}; + +// Old: config.openrouter.supports_vision, +// New: supports_vision, +``` + +Update `process_attachments` signature to accept `supports_vision: bool` instead of reading from config: + +```rust +pub async fn process_attachments( + attachments: &[Attachment], + user_query: &str, + config: &Config, + memory: &MemoryStore, + supports_vision: bool, // <-- new parameter, from active provider +) -> (String, Vec) { +``` + +Replace line 40: +```rust +// Old: +config.openrouter.supports_vision, +// New: +supports_vision, +``` +```rust +// Old: if self.config.openrouter.supports_vision { +// New: +let current = self.current_model.read().await; +let (provider, _) = self.registry.resolve_model(¤t); +if provider.supports_vision() { +``` + +- [ ] **Step 6: Build and fix compilation errors** + +```bash +cargo check +``` +Expected: Compilation succeeds. + +- [ ] **Step 7: Run existing tests** + +```bash +cargo test --lib +``` +Expected: All existing tests pass. + +- [ ] **Step 8: Commit** + +```bash +git add src/agent.rs +git commit -m "feat: integrate ProviderRegistry into Agent with fallback chain and per-provider vision" +``` + +--- + +### Task 8: Wire providers in main.rs at startup + +**Files:** +- Modify: `src/main.rs` + +- [ ] **Step 1: Build registry from config and pass to Agent** + +After config is loaded (after `config.load()`), add: + +```rust +use crate::provider; + +// Build provider registry +let (provider_sections, default_provider, fallback_chain) = config.build_providers(); +let registry = Arc::new(provider::build_registry(&provider_sections, &default_provider) + .context("Failed to build LLM provider registry")?); +``` + +Then pass `registry.clone()` to `Agent::new()`: + +```rust +let agent = Arc::new_cyclic(|weak| { + Agent::new( + config, + registry.clone(), + // ... existing params unchanged + ) +}); +``` + +- [ ] **Step 2: Remove the old LlmClient::new creation from Agent::new** (it's now created inside Agent::new from the registry) + +- [ ] **Step 3: Build** + +```bash +cargo check +``` +Expected: Compilation succeeds. + +- [ ] **Step 4: Commit** + +```bash +git add src/main.rs +git commit -m "feat: wire ProviderRegistry startup in main.rs" +``` + +--- + +### Task 9: Redesign `/models` Telegram command with multi-step providerβ†’model selection + +**Files:** +- Modify: `src/platform/telegram.rs` + +- [ ] **Step 1: Update `/models` handler for provider selection** + +Replace the current `/models` command handler (around line 672) with: + +```rust +"models" => { + let registry = &agent.registry; + let providers = registry.provider_names(); + + if providers.len() == 1 { + // Single provider: jump straight to model search for it + let provider_name = &providers[0]; + let provider = registry.get_provider(provider_name).unwrap(); + return handle_provider_model_select( + bot, msg.chat.id, agent, provider_name, provider + ).await; + } + + // Multiple providers: show inline keyboard + let current = agent.current_model.read().await; + let reply = format!( + "Active model: `{}`\n\nSelect a provider:", + *current + ); + use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; + let mut keyboard: Vec> = providers.iter() + .map(|name| { + vec![InlineKeyboardButton::callback( + name.clone(), + format!("provider_select:{}", name), + )] + }) + .collect(); + keyboard.push(vec![InlineKeyboardButton::callback( + "❌ Cancel", "model_select:cancel", + )]); + + bot.send_message(msg.chat.id, &reply) + .reply_markup(InlineKeyboardMarkup::new(keyboard)) + .await?; + return Ok(()); +} +``` + +- [ ] **Step 2: Add provider_select callback in handle_model_callback** + +`handle_model_callback` receives `CallbackQuery`, not `Message`. The message is accessed via `q.regular_message()`. Add before the existing `model_select:` check: + +```rust +if let Some(provider_name) = data.strip_prefix("provider_select:") { + if let Some(provider) = agent.registry.get_provider(provider_name) { + let msg = q.regular_message().cloned(); + if let Some(ref m) = msg { + return handle_provider_model_select( + bot, m.chat.id, agent, provider_name, provider + ).await; + } + } + return Ok(()); +} +``` + +- [ ] **Step 3: Add handle_provider_model_select helper (uses `ChatId`, not `Message`)** + +```rust +async fn handle_provider_model_select( + bot: Bot, + chat_id: ChatId, + agent: &Arc, + provider_name: &str, + provider: &dyn Provider, +) -> ResponseResult<()> { + + if !provider.config().discover_models { + // No discovery: prompt for text search + let prompt = format!( + "Send me a model name or ID to search for on **{}**.\nExamples: {}", + provider_name, + provider.default_model() + ); + bot.send_message(chat_id, &prompt).await?; + return Ok(()); + } + + // Try to discover models + match provider.list_models(&agent.llm.client).await { + Ok(models) if models.len() <= 20 => { + use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; + let mut keyboard: Vec> = models.iter() + .map(|m| { + let qualified = format!("{}/{}", provider_name, m); + vec![InlineKeyboardButton::callback( + m.clone(), + format!("model_select:{}", qualified), + )] + }) + .collect(); + keyboard.push(vec![InlineKeyboardButton::callback( + "πŸ” Search all", "model_search_prompt", + )]); + keyboard.push(vec![InlineKeyboardButton::callback( + "❌ Cancel", "model_select:cancel", + )]); + + let reply = format!("Models on **{}** ({}):", provider_name, models.len()); + bot.send_message(chat_id, &reply) + .reply_markup(InlineKeyboardMarkup::new(keyboard)) + .await?; + } + Ok(models) => { + // Too many models: prompt for text search + let prompt = format!( + "**{}** has {} models available.\nSend me a model name or ID to search for.", + provider_name, + models.len() + ); + bot.send_message(chat_id, &prompt).await?; + } + Err(e) => { + // Discovery failed: prompt for search + let prompt = format!( + "Could not load model list from **{}**: {}\nSend a model name or ID directly.", + provider_name, e + ); + bot.send_message(chat_id, &prompt).await?; + } + } + + Ok(()) +} +``` + +- [ ] **Step 4: Update existing model_select callback to handle qualified IDs** + +The existing `model_select:{model_id}` callback now receives qualified IDs like `model_select:ollama/llama3.1:8b`. The `set_model()` call already handles the full qualified string β€” no change needed to the set_model call itself. Just ensure the callback data format is `model_select:{qualified_id}`. + +- [ ] **Step 5: Update the model search flow to be provider-scoped** + +When a user types a model name after being prompted for search, `handle_model_search` should store which provider they're searching. Use the existing `model_search_pending_{user_id}` memory key to also store the provider name. + +In the search prompt flow, after user selects a provider: +```rust +agent.memory.remember("settings", + &format!("model_search_provider_{}", user_id), + provider_name, + None, +).await.ok(); +``` + +In `handle_message`'s model search path (after the user types a search query), read the stored provider and scope the search: +```rust +// When user types a search query (not a command): +let search_provider = agent.memory.recall("settings", + &format!("model_search_provider_{}", user_id) +).await.ok().flatten(); +// Pass it to handle_model_search which filters by provider +``` + +- [ ] **Step 6: Build** + +```bash +cargo check +``` +Expected: Compilation succeeds. + +- [ ] **Step 7: Commit** + +```bash +git add src/platform/telegram.rs +git commit -m "feat: redesign /models with multi-step provider->model selection via inline keyboards" +``` + +--- + +--- + +### Task 10: Add unit tests for provider system + +**Files:** +- Modify: `src/provider.rs` (add `#[cfg(test)] mod tests`) +- Modify: `src/config.rs` (add test for new config types) + +- [ ] **Step 1: Add resolve_model tests to provider.rs** + +Append at the end of `src/provider.rs`: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + fn test_registry() -> ProviderRegistry { + let mut providers: HashMap> = HashMap::new(); + providers.insert( + "openrouter".to_string(), + Arc::new(OpenRouterProvider::new(ProviderConfig { + name: "openrouter".to_string(), + provider_type: ProviderType::OpenRouter, + base_url: "https://openrouter.ai/api/v1".to_string(), + api_key: Some("sk-test".to_string()), + default_model: "moonshotai/kimi-k2.6".to_string(), + supports_vision: false, + max_tokens: 4096, + discover_models: false, + })), + ); + providers.insert( + "ollama".to_string(), + Arc::new(OllamaProvider::new(ProviderConfig { + name: "ollama".to_string(), + provider_type: ProviderType::Ollama, + base_url: "http://localhost:11434/v1".to_string(), + api_key: None, + default_model: "llama3.1".to_string(), + supports_vision: false, + max_tokens: 4096, + discover_models: true, + })), + ); + ProviderRegistry::new(providers, "openrouter".to_string()) + } + + #[test] + fn test_resolve_model_explicit_provider() { + let registry = test_registry(); + let (provider, model) = registry.resolve_model("ollama/llama3"); + assert_eq!(provider.name(), "ollama"); + assert_eq!(model, "llama3"); + } + + #[test] + fn test_resolve_model_falls_through_to_default() { + let registry = test_registry(); + // "moonshotai" is not a known provider β†’ falls to default (openrouter) + let (provider, model) = registry.resolve_model("moonshotai/kimi-k2.6"); + assert_eq!(provider.name(), "openrouter"); + assert_eq!(model, "moonshotai/kimi-k2.6"); + } + + #[test] + fn test_resolve_model_unknown_prefix_falls_through() { + let registry = test_registry(); + let (provider, model) = registry.resolve_model("nope/llama3"); + assert_eq!(provider.name(), "openrouter"); + assert_eq!(model, "nope/llama3"); + } + + #[test] + fn test_resolve_model_no_slash_uses_default() { + let registry = test_registry(); + let (provider, model) = registry.resolve_model("llama3"); + assert_eq!(provider.name(), "openrouter"); + assert_eq!(model, "llama3"); + } + + #[test] + fn test_provider_count_and_names() { + let registry = test_registry(); + assert_eq!(registry.provider_count(), 2); + let mut names = registry.provider_names(); + names.sort(); + assert_eq!(names, vec!["ollama", "openrouter"]); + } + + #[test] + fn test_get_provider_found() { + let registry = test_registry(); + let p = registry.get_provider("ollama"); + assert!(p.is_some()); + assert_eq!(p.unwrap().name(), "ollama"); + } + + #[test] + fn test_get_provider_not_found() { + let registry = test_registry(); + assert!(registry.get_provider("nonexistent").is_none()); + } +} +``` + +- [ ] **Step 2: Run resolve_model tests** + +```bash +cargo test -p rustfox -- provider::tests --nocapture +``` +Expected: All 7 tests pass. + +- [ ] **Step 3: Add backward-compat config parsing test to config.rs** + +```rust +#[test] +fn test_provider_section_parses_ollama() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + model = "moonshotai/kimi-k2.6" + [[provider]] + name = "ollama" + type = "ollama" + base_url = "http://localhost:11434/v1" + model = "llama3.1" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert_eq!(cfg.provider.len(), 1); + assert_eq!(cfg.provider[0].name, "ollama"); + assert_eq!(cfg.provider[0].model, "llama3.1"); +} + +#[test] +fn test_legacy_openrouter_auto_creates_provider() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + model = "moonshotai/kimi-k2.6" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + let (providers, default_name, _) = cfg.build_providers(); + assert!(providers.iter().any(|p| p.name == "openrouter")); + assert_eq!(default_name, "openrouter"); +} + +#[test] +fn test_visible_provider_overrides_legacy() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "old-key" + model = "moonshotai/kimi-k2.6" + [[provider]] + name = "openrouter" + type = "openrouter" + base_url = "https://openrouter.ai/api/v1" + api_key = "new-key" + model = "anthropic/claude-sonnet-4-6" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + let (providers, _default, _) = cfg.build_providers(); + // Should not have duplicate openrouter providers + let or_count = providers.iter().filter(|p| p.name == "openrouter").count(); + assert_eq!(or_count, 1, "should not duplicate openrouter when explicit [[provider]] exists"); + let or = providers.iter().find(|p| p.name == "openrouter").unwrap(); + assert_eq!(or.api_key.as_deref(), Some("new-key"), "explicit [[provider]] should win"); +} + +#[test] +fn test_fallback_config_parses() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + [fallback] + chain = ["openrouter/model-a", "ollama/model-b"] + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert_eq!(cfg.fallback.chain.len(), 2); + assert_eq!(cfg.fallback.chain[0], "openrouter/model-a"); +} + +#[test] +fn test_fallback_defaults_empty() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert!(cfg.fallback.chain.is_empty()); +} +``` + +- [ ] **Step 4: Run config tests** + +```bash +cargo test -p rustfox -- config::tests --nocapture +``` +Expected: All new and existing config tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/provider.rs src/config.rs +git commit -m "test: add unit tests for ProviderRegistry, config parsing, backward compat" +``` + +--- + +### Task 11: Update config.example.toml with new provider sections + +**Files:** +- Modify: `config.example.toml` + +- [ ] **Step 1: Add [[provider]] and [fallback] documentation** + +Add after the existing `[openrouter]` example section in `config.example.toml`: + +```toml +# ── Multi-Provider Configuration (optional) ──────────────────────────── +# Define additional LLM providers alongside OpenRouter. +# Each [[provider]] block adds a new provider. Model strings use the +# format `provider_name/model_id` (e.g., `ollama/llama3.1`). +# The first provider becomes the default. + +# Example: Ollama (local models) +# [[provider]] +# name = "ollama" +# type = "ollama" +# base_url = "http://localhost:11434/v1" +# model = "llama3.1" +# discover_models = true + +# Example: Any OpenAI-compatible endpoint (LM Studio, vLLM, llama.cpp, etc.) +# [[provider]] +# name = "lmstudio" +# type = "openai_compatible" +# base_url = "http://localhost:1234/v1" +# model = "qwen2.5-7b-instruct" + +# ── Fallback Chain (optional) ────────────────────────────────────────── +# When the primary model fails, RustFox tries each fallback in order. +# Each entry is a full provider-prefixed model string. +# [fallback] +# chain = [ +# "openrouter/moonshotai/kimi-k2.6", +# "ollama/llama3.1", +# "lmstudio/qwen2.5-7b-instruct", +# ] +``` + +- [ ] **Step 2: Commit** + +```bash +git add config.example.toml +git commit -m "docs: add [[provider]] and [fallback] examples to config.example.toml" +``` + +--- + +### Self-Review Checklist + +- **Spec coverage:** Every section of the design spec has a corresponding task (including Β§7 config.example.toml β†’ Task 11): + - Β§3.1 Provider Trait β†’ Task 2 + - Β§3.2 Provider Types β†’ Tasks 3, 4, 5 + - Β§3.3 ProviderConfig β†’ Task 2 + - Β§3.4 ProviderRegistry β†’ Task 2 + - Β§3.5 LlmClient β†’ Task 6 + - Β§3.6 Fallback β†’ Task 7 (step 4) + - Β§4 Config β†’ Task 1 + - Β§5 Agent β†’ Task 7 + - Β§6 /models β†’ Task 9 + - Β§7 File changes β†’ covered + - Β§8 Error handling β†’ covered inline + - Β§9 Testing β†’ next task +- **Placeholder scan:** No TODOs or placeholders remain. All code blocks are complete. +- **Type consistency:** `ProviderConfig` defined in Task 2, used in Tasks 3/4/5/6/7. `ProviderRegistry::resolve_model()` returns `(&dyn Provider, &str)`, used consistently. diff --git a/docs/superpowers/specs/2026-06-22-multi-provider-design.md b/docs/superpowers/specs/2026-06-22-multi-provider-design.md new file mode 100644 index 0000000..74e7238 --- /dev/null +++ b/docs/superpowers/specs/2026-06-22-multi-provider-design.md @@ -0,0 +1,464 @@ +# Multi-Provider LLM Architecture β€” Design Spec + +**Date:** 2026-06-22 +**Status:** Draft +**Author:** RustFox Brainstorming Session + +## 1. Problem + +RustFox currently supports a single LLM provider (OpenRouter) via `[openrouter]` config. +This creates three limitations: + +1. **No fallback** β€” if OpenRouter is down, the bot is unusable +2. **No local models** β€” users cannot use Ollama, LM Studio, llama.cpp, or vLLM +3. **No provider mixing** β€” subagents always use the same endpoint as the main agent + +## 2. Goals + +- Support multiple LLM providers simultaneously (OpenRouter + local models) +- Each provider connects via an OpenAI-compatible `/chat/completions` endpoint +- Fallback chains: on provider failure, try the next in the chain +- Subagents can use any provider independently (e.g., main agent on Kimi, subagent on Ollama) +- Auto-discovery of local model lists where possible +- Backward compatible β€” existing `[openrouter]` config continues to work +- `/models` command gains multi-step providerβ†’model selection via inline keyboards + +## 3. Architecture + +### 3.1 Provider Trait + +New module `src/provider.rs` defines the core abstraction: + +```rust +#[async_trait] +pub trait Provider: Send + Sync { + fn name(&self) -> &str; + fn default_model(&self) -> &str; + fn supports_vision(&self) -> bool; + fn config(&self) -> &ProviderConfig; + + /// Send a chat completion request to this provider's endpoint. + /// `model` is already stripped of the provider prefix. + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result; + + /// Return available model IDs. Used by /models command. + async fn list_models(&self, client: &reqwest::Client) -> Result>; +} +``` + +### 3.2 Provider Types + +Three concrete implementations, all in `src/provider.rs`: + +| Provider | `list_models()` | Special behavior | +|---|---|---| +| `OpenRouterProvider` | `GET /v1/models` | Parameter sanitization, Kimi tool-call fallback | +| `OpenAICompatibleProvider` | `GET /v1/models` | Pure OpenAI protocol, `api_key` optional | +| `OllamaProvider` | `GET /api/tags` | Delegates chat to OpenAICompatibleProvider, custom discovery | + +`OpenAICompatibleProvider` handles LM Studio, llama.cpp, vLLM, Together AI, etc. β€” +any service exposing `/v1/chat/completions` with the OpenAI schema. + +`OllamaProvider` delegates its `chat_completion()` to an inner `OpenAICompatibleProvider` +(since Ollama's `/v1/chat/completions` is identical to the OpenAI protocol), but has its +own `list_models()` that calls `GET /api/tags` with a modified base URL path. +The `base_url` is stored as the full OpenAI-compatible path (e.g. `http://localhost:11434/v1`); +the Ollama provider strips a trailing `/v1`, `/v2`, or trailing slash then appends `/api/tags` +for discovery. If no versioned prefix is found, it appends `/api/tags` directly. +(e.g., `http://localhost:11434` β†’ `http://localhost:11434/api/tags`). + +### 3.3 ProviderConfig + +Deserialized from TOML, then used to construct a concrete Provider: + +```rust +#[derive(Debug, Clone)] +pub struct ProviderConfig { + pub name: String, // "ollama", "openrouter", etc. + pub provider_type: ProviderType, + pub base_url: String, + pub api_key: Option, + pub default_model: String, + pub supports_vision: bool, + pub max_tokens: u32, + pub discover_models: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum ProviderType { + OpenRouter, + OpenAICompatible, + Ollama, +} +``` + +### 3.4 ProviderRegistry + +Holds all configured providers and resolves model strings: + +```rust +pub struct ProviderRegistry { + providers: HashMap>, + default_provider: String, +} + +impl ProviderRegistry { + /// "ollama/llama3" β†’ (ollama_provider, "llama3") + /// "moonshotai/kimi-k2.6" β†’ (openrouter_provider, "moonshotai/kimi-k2.6") + /// (no provider named "moonshotai" β†’ falls through to default) + /// "llama3" (no slash) β†’ (default_provider, "llama3") + /// "nope/llama3" (unknown prefix) β†’ (default_provider, "nope/llama3") + pub fn resolve_model(&self, model: &str) -> (&dyn Provider, &str) { ... } + + pub fn get_provider(&self, name: &str) -> Option<&dyn Provider> { ... } + pub fn providers(&self) -> impl Iterator { ... } + pub fn default_provider_name(&self) -> &str { ... } +} +``` + +Model string resolution algorithm: +1. Split on first `/` +2. If first segment matches a known provider name β†’ use that provider, rest is model +3. If no match β†’ use default provider, full string is model + +This is backward compatible: `moonshotai/kimi-k2.6` with no provider named `moonshotai` +falls through to the default provider (openrouter) and sends `moonshotai/kimi-k2.6` as the +model name β€” exactly as today. + +### 3.5 LlmClient Changes + +LlmClient becomes a thin routing layer: + +```rust +pub struct LlmClient { + client: reqwest::Client, + registry: Arc, +} + +impl LlmClient { + pub async fn chat_completion_with_model( + &self, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, // "ollama/llama3" or "moonshotai/kimi-k2.6" + ) -> Result { + let (provider, actual_model) = self.registry.resolve_model(model); + let max_tokens = provider.config().max_tokens; + provider.chat_completion( + &self.client, messages, tools, actual_model, max_tokens + ).await + .map(|mut completion| { + completion.model = model.to_string(); + completion + }) + } + + pub async fn list_all_models(&self) -> Vec { + // Iterate all providers, call list_models() for discoverable ones + } +} + +// --- Migration of existing API --- +// The old `chat_completion(&self, messages, tools)` method (no model param) is **removed**. +// It was only used internally; all callers now pass the model explicitly. +// `chat(&self, messages, tools)` stays as a convenience that reads from +// `self.registry.default_provider` for the model string. +// `chat_with_model()` is replaced by `chat_completion_with_model()`. +// `fetch_models()` becomes provider-specific: call `registry.get_provider("openrouter").list_models()`. + +/// A model as shown in the /models command, prefixed with its provider name. +pub struct ProviderModel { + pub provider: String, + pub model_id: String, // e.g. "llama3.1:8b" + pub qualified_id: String, // e.g. "ollama/llama3.1:8b" + pub description: String, // optional human-readable name from the provider + // OpenRouter: from ModelInfo.name + // Ollama: from /api/tags response name + // OpenAICompatible: from /v1/models name or empty +} +``` + +### 3.6 Fallback Chain + +Fallback lives in the Agent loop, not LlmClient. The agent's main loop and subagent loop +already have retry logic β€” we extend it to try different models on failure: + +```rust +// In process_message main loop: +let fallback_chain = &self.config.fallback.chain; // Vec + +for attempt in 0..=fallback_chain.len() { + let model = if attempt == 0 { + self.current_model.read().await.clone() + } else { + fallback_chain[attempt - 1].clone() + }; + + match self.llm.chat_completion_with_model(&prompt, &all_tools, &model).await { + Ok(completion) => { /* handle success */ break; } + Err(e) => { + warn!("Model '{}' failed: {}", model, e); + if attempt == fallback_chain.len() { + return Err(e); // all exhausted + } + continue; // try next fallback + } + } +} +``` + +Each model in the fallback chain is a full provider-prefixed string like +`"ollama/llama3.1"` β€” it resolves through `resolve_model` to whatever provider +owns that model. The same logic works for all providers. + +**After successful fallback**, the agent updates `current_model` to the working +fallback model in memory (for subsequent turns in the same conversation) but does +NOT persist to config.toml. The user can explicitly persist the working model +via `/model ` if desired. + +**Fallback applies only to the main agent loop**, not to subagent loops. Subagents +have an explicit model override (e.g., `model: "ollama/llama3"`) β€” if that provider +fails, the error propagates to the main agent, which can decide to retry the subagent +with a different model. This keeps subagent behavior predictable and avoids +surprising fallback within a delegated task. + +## 4. Config Changes + +### 4.1 New Config Struct Fields + +Add to `Config` in `src/config.rs`: + +```rust +#[serde(default)] +pub provider: Vec, + +#[serde(default)] +pub fallback: FallbackConfig, +``` + +```rust +pub struct ProviderSection { + pub name: String, + #[serde(rename = "type")] + pub provider_type: String, // "openrouter", "openai_compatible", "ollama" + // Unrecognized types cause a startup error + pub base_url: String, + pub api_key: Option, + #[serde(default = "default_model")] + pub model: String, + #[serde(default)] + pub supports_vision: bool, + #[serde(default = "default_max_tokens")] + pub max_tokens: u32, + #[serde(default)] + pub discover_models: bool, +} + +pub struct FallbackConfig { + #[serde(default)] + pub chain: Vec, +} +``` + +### 4.2 Backward Compatibility + +At startup, if the legacy `[openrouter]` section exists, it auto-creates a provider +named `"openrouter"` with those settings. The default provider is `"openrouter"` if +it exists, otherwise the first configured provider. + +If both `[openrouter]` AND `[[provider]] name = "openrouter"` are present, the explicit +`[[provider]]` entry wins and the legacy section is ignored (with a warning). +`set_model()` with an openrouter-prefixed model always modifies the same section that +was used to create the provider β€” there is no split-state path. + +### 4.3 Example config.toml + +```toml +# Legacy section β€” auto-creates "openrouter" provider (backward compat) +[openrouter] +api_key = "sk-..." +model = "moonshotai/kimi-k2.6" + +# Local Ollama (auto-discover models) +[[provider]] +name = "ollama" +type = "ollama" +base_url = "http://localhost:11434/v1" +model = "llama3.1" +discover_models = true + +# LM Studio (no api_key needed for local) +[[provider]] +name = "lmstudio" +type = "openai_compatible" +base_url = "http://localhost:1234/v1" +model = "qwen2.5-7b-instruct" + +# Fallback chain +[fallback] +chain = [ + "openrouter/moonshotai/kimi-k2.6", + "ollama/llama3.1", + "lmstudio/qwen2.5-7b-instruct", +] +``` + +## 5. Agent Integration + +### 5.1 Agent Constructor + +`Agent::new()` receives `Arc` and `LlmClient`: + +```rust +let llm = LlmClient::new(registry.clone()); +Self { + llm, + registry, + current_model: RwLock::new(initial_model), // full provider-prefixed string + ... +} +``` + +### 5.2 set_model() + +`set_model()` now: +1. Resolves the model string via `registry.resolve_model()` β€” always succeeds. + Additionally checks: if the model string contains a `/` but the prefix doesn't match + any known provider, log a warning (possible typo, but could be a valid model ID + like `moonshotai/kimi-k2.6` β€” never block the change). +2. Persists the full provider-prefixed model string (e.g., `"ollama/llama3.1"`) +3. Persists to config.toml as the `model` field of the corresponding provider section + (or the `[openrouter]` model if that's the provider) + +Config persistence follows the same approach as the current `set_model()`: +read TOML via `toml::from_str`, modify the `model` field in the right section, +write back via `toml::to_string_pretty`. This does NOT preserve comments/formatting +(consistent with existing behavior). If round-trip preservation is needed later, +a separate migration to `toml_edit` would cover all config writes at once. + +Model selection via `set_model()` persists the model to disk so it survives restarts. +At runtime, `current_model` is per-Agent (the bot has one agent instance, so it +appears "global" β€” affecting all conversations in that process). This is consistent +with the current behavior where model changes affect all users until the next restart. + +### 5.3 Subagent Model Resolution + +Subagent `model` overrides are full provider-prefixed strings like `"ollama/llama3"`. +The same `resolve_model()` path handles them β€” no special subagent logic needed. + +When a subagent is invoked via `invoke_agent` with `model: "ollama/llama3"`: +- The subagent loop calls `self.llm.chat_completion_with_model(..., "ollama/llama3")` +- LlmClient resolves to the `ollama` provider with model `llama3` +- The subagent runs entirely against that provider + +### 5.4 Per-Provider Vision + +The `supports_vision` flag moves from `config.openrouter.supports_vision` to +per-provider `ProviderConfig`. The agent checks this via the active provider: + +```rust +// In process_message, when deciding whether to send images or OCR: +let current = self.current_model.read().await; +let (provider, _) = self.registry.resolve_model(¤t); +if provider.config().supports_vision { + // Send base64 images as content parts +} else { + // OCR the images +} +``` + +## 6. `/models` Command Redesign + +### 6.1 Interactive Flow + +**Step 1 β€” `/models` with no args:** +If only one provider β†’ jump directly to Step 2 for that provider. +If multiple providers β†’ show inline keyboard: + +``` +Bot: Active model: openrouter/moonshotai/kimi-k2.6 + Select a provider: + [ollama] [openrouter] [lmstudio] + [❌ Cancel] +``` + +Callback format: `provider_select:{name}` + +**Step 2 β€” Provider selected:** + +*If provider has `discover_models=true` AND <= 20 models (e.g., Ollama):* +``` +Bot: Models on ollama (http://localhost:11434/v1): + [llama3.1:8b] [mistral:7b] [qwen2.5:7b] + [πŸ” Search all] [❌ Cancel] +``` + +*If provider has no discovery OR 300+ models (e.g., OpenRouter):* +``` +Bot: Models on openrouter (300+ available). + Send me a model name or ID to search for. + Examples: kimi, claude, gpt-4, gemini +``` + +**Step 3 β€” Model selection:** +Inline keyboard tap or text search β†’ `agent.set_model("ollama/llama3.1:8b")` + +``` +Bot: βœ… Model changed to "ollama/llama3.1:8b" +``` + +### 6.2 Implementation + +- New callback prefix `provider_select:{name}` handled in `handle_model_callback()` +- Existing `model_select:{id}` stores the full provider-prefixed string +- Search in `handle_model_search()` is scoped to the selected provider +- State tracked via `model_search_pending_{user_id}` includes the provider name +- `model_select:cancel` reverts to the provider picker + +## 7. File Changes Summary + +### New Files +| File | Purpose | +|---|---| +| `src/provider.rs` | `Provider` trait, `ProviderRegistry`, three impls | + +### Modified Files +| File | Changes | +|---|---| +| `src/config.rs` | Add `ProviderSection`, `FallbackConfig`; backward compat builder | +| `src/llm.rs` | `LlmClient` takes `Arc`, routing logic | +| `src/agent.rs` | Receive registry, fallback loop, subagent model resolution | +| `src/main.rs` | Build registry from config, pass to agent | +| `src/platform/telegram.rs` | `/models` multi-step providerβ†’model selection | +| `config.example.toml` | Document `[[provider]]` and `[fallback]` sections | + +## 8. Error Handling + +- **Unknown provider in model string**: `resolve_model()` always falls through to the default provider. `set_model()` may optionally warn about unrecognized prefixes as a typo check +- **Unknown `type` in config**: Startup error β€” unrecognized provider type is a hard config error +- **Provider connection failure**: Logged, fallback chain triggered +- **All fallbacks exhausted**: Error propagated to user as today +- **Auto-discovery failure**: Logged, provider still usable with its `default_model` +- **Vision on non-vision provider**: Falls back to OCR as today (per-provider `supports_vision` flag) + +## 9. Testing + +- `ProviderRegistry::resolve_model()` β€” various model string formats +- Fallback chain iteration +- Provider chat_completion serialization (each provider type) +- Callback handler parsing for new `provider_select:` prefix +- Backward compat config parsing + +## 10. Non-Goals + +- Rate limiting per provider (out of scope) +- Cost tracking per provider (out of scope) +- Dynamic provider discovery via mDNS (out of scope) +- Provider health checks / circuit breaker (future improvement) From c88b63686066c181b85fa658eacca95b8bcb1fbf Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Thu, 25 Jun 2026 15:54:28 +0800 Subject: [PATCH 09/19] feat: add fuzzy model matching via provider list_models in /models search --- src/platform/telegram.rs | 139 ++++++++++++++++++++++++++++++++++----- 1 file changed, 124 insertions(+), 15 deletions(-) diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index 56cc247..e1b065f 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -319,9 +319,12 @@ async fn handle_provider_model_select( Ok(()) } /// Accept a text model query and attempt to set the active model. -/// If the user previously selected a provider (via the inline keyboard), -/// the bare query is prefixed with that provider's name to form a -/// qualified model ID. +/// If the query is a bare name (e.g. "deepseek v4 flash"), fetches the +/// provider's model list via `list_models()` and does fuzzy matching to +/// resolve the actual model ID (e.g. "deepseek/deepseek-v4-flash"). +/// +/// If the user previously selected a provider via the inline keyboard, +/// the search is scoped to that provider. async fn handle_model_search( bot: Bot, chat_id: ChatId, @@ -329,32 +332,139 @@ async fn handle_model_search( query: &str, user_id: &str, ) -> ResponseResult<()> { - // Check if user has a stored provider from the interactive picker + // If query already has a known provider prefix, set directly + if let Some((prefix, _)) = query.split_once('/') { + if agent.registry.get_provider(prefix).is_some() { + return set_model_and_reply(bot, chat_id, agent, query).await; + } + } + + // Determine which provider to search let stored_provider = agent .memory .recall("settings", &format!("model_search_provider_{}", user_id)) .await .unwrap_or(None); - let qualified = if let Some(ref provider_name) = stored_provider { - // Clear the stored provider so it doesn't affect future searches + let provider_name = stored_provider + .clone() + .unwrap_or_else(|| agent.registry.default_provider_name().to_string()); + + // Clear stored provider so it doesn't affect future searches + if stored_provider.is_some() { agent .memory .forget("settings", &format!("model_search_provider_{}", user_id)) .await .ok(); - if query.contains('/') { - query.to_string() - } else { - format!("{}/{}", provider_name, query) + } + + let provider = match agent.registry.get_provider(&provider_name) { + Some(p) => p, + None => { + return set_model_and_reply( + bot, + chat_id, + agent, + &format!("{}/{}", provider_name, query), + ) + .await; } - } else { - query.to_string() }; - match agent.set_model(&qualified).await { + // Fetch model list and fuzzy match + match provider.list_models(&agent.llm.client).await { + Ok(models) if !models.is_empty() => { + let q = query + .to_lowercase() + .replace(['-', '_', '.', ' '], ""); + + // Exact match (full model ID) + if let Some(exact) = models + .iter() + .find(|m| m.to_lowercase() == query.to_lowercase()) + { + return set_model_and_reply( + bot, + chat_id, + agent, + &format!("{}/{}", provider_name, exact), + ) + .await; + } + + // Fuzzy match: normalize both sides and check containment + let mut matches: Vec<&String> = models + .iter() + .filter(|m| { + m.to_lowercase() + .replace(['-', '_', '.', ' '], "") + .contains(&q) + }) + .collect(); + matches.sort(); + matches.truncate(10); + + match matches.len() { + 0 => { + // No fuzzy match β€” try direct set anyway + return set_model_and_reply( + bot, + chat_id, + agent, + &format!("{}/{}", provider_name, query), + ) + .await; + } + 1 => { + return set_model_and_reply( + bot, + chat_id, + agent, + &format!("{}/{}", provider_name, matches[0]), + ) + .await; + } + _ => { + let mut reply = format!( + "Multiple models match '{}' on **{}**:\n\n", + query, provider_name + ); + for m in &matches { + reply.push_str(&format!("`{}/{m}`\n", provider_name)); + } + reply.push_str("\nUse `/models ` to set one."); + bot.send_message(chat_id, escape_text(&reply)) + .parse_mode(ParseMode::MarkdownV2) + .await?; + } + } + } + _ => { + // API unavailable or empty list β€” try direct set + return set_model_and_reply( + bot, + chat_id, + agent, + &format!("{}/{}", provider_name, query), + ) + .await; + } + } + + Ok(()) +} + +/// Set the model and send a success/failure reply. +async fn set_model_and_reply( + bot: Bot, + chat_id: ChatId, + agent: &Arc, + model_id: &str, +) -> ResponseResult<()> { + match agent.set_model(model_id).await { Ok(()) => { - let reply = format!("βœ… Model changed to `{}`", qualified); + let reply = format!("βœ… Model changed to `{}`", model_id); bot.send_message(chat_id, escape_text(&reply)) .parse_mode(ParseMode::MarkdownV2) .await?; @@ -368,7 +478,6 @@ async fn handle_model_search( .await?; } } - Ok(()) } From 241fbb86051b4ec4a99753b1733c69c4c3abcdfe Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Thu, 25 Jun 2026 16:12:39 +0800 Subject: [PATCH 10/19] fmt: fix chained method call formatting --- src/platform/telegram.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index e1b065f..cb998ee 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -375,9 +375,7 @@ async fn handle_model_search( // Fetch model list and fuzzy match match provider.list_models(&agent.llm.client).await { Ok(models) if !models.is_empty() => { - let q = query - .to_lowercase() - .replace(['-', '_', '.', ' '], ""); + let q = query.to_lowercase().replace(['-', '_', '.', ' '], ""); // Exact match (full model ID) if let Some(exact) = models From 05996f34ff3c921464287de9d011b614f84f3e56 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 29 Jun 2026 11:27:02 +0800 Subject: [PATCH 11/19] Add context_window field to ProviderSection and ProviderConfig --- src/config.rs | 7 +++++++ src/provider.rs | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/src/config.rs b/src/config.rs index 1edd8ea..8fc02e1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -166,6 +166,8 @@ pub struct ProviderSection { pub max_tokens: u32, #[serde(default)] pub discover_models: bool, + #[serde(default = "default_context_window")] + pub context_window: usize, } #[derive(Debug, Clone, Default, Deserialize)] @@ -439,6 +441,10 @@ fn default_ocr_config() -> OcrConfig { } } +fn default_context_window() -> usize { + 512_000 +} + fn default_true() -> bool { true } @@ -611,6 +617,7 @@ impl Config { supports_vision: self.openrouter.supports_vision, max_tokens: self.openrouter.max_tokens, discover_models: false, + context_window: default_context_window(), }); } diff --git a/src/provider.rs b/src/provider.rs index 6663861..3b4fb61 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -22,6 +22,7 @@ pub struct ProviderConfig { pub supports_vision: bool, pub max_tokens: u32, pub discover_models: bool, + pub context_window: usize, } impl From<&ProviderSection> for ProviderConfig { @@ -35,6 +36,7 @@ impl From<&ProviderSection> for ProviderConfig { supports_vision: s.supports_vision, max_tokens: s.max_tokens, discover_models: s.discover_models, + context_window: s.context_window, } } } @@ -516,6 +518,7 @@ mod tests { supports_vision: false, max_tokens: 1024, discover_models: false, + context_window: 512_000, } } @@ -534,6 +537,7 @@ mod tests { assert_eq!(cfg.api_key.as_deref(), Some("test-key")); assert_eq!(cfg.default_model, "anthropic/claude-sonnet-4-6"); assert_eq!(cfg.max_tokens, 1024); + assert_eq!(cfg.context_window, 512_000); } #[test] @@ -665,6 +669,7 @@ mod tests { supports_vision: false, max_tokens: 1024, discover_models: true, + context_window: 512_000, }; let p = OllamaProvider::new(cfg); assert_eq!(p.discovery_url(), "http://localhost:11434/api/tags"); @@ -681,6 +686,7 @@ mod tests { supports_vision: false, max_tokens: 1024, discover_models: true, + context_window: 512_000, }; let p = OllamaProvider::new(cfg); assert_eq!(p.discovery_url(), "http://localhost:11434/api/tags"); @@ -697,6 +703,7 @@ mod tests { supports_vision: false, max_tokens: 1024, discover_models: true, + context_window: 512_000, }; let p = OllamaProvider::new(cfg); assert_eq!(p.discovery_url(), "http://localhost:11434/api/tags"); From 958a48e8034bc84dc357602eba1d5c9d0b7bbef9 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 29 Jun 2026 11:29:18 +0800 Subject: [PATCH 12/19] feat: add ChatMessage::has_tool_calls() helper --- src/llm.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/llm.rs b/src/llm.rs index 9177867..053186b 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -75,6 +75,14 @@ pub struct ChatMessage { pub tool_call_id: Option, } +impl ChatMessage { + pub fn has_tool_calls(&self) -> bool { + self.tool_calls + .as_ref() + .is_some_and(|calls| !calls.is_empty()) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ToolCall { pub id: String, From b3d7c0402ed41f13362abebd024acdf72ae48b14 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 29 Jun 2026 11:34:47 +0800 Subject: [PATCH 13/19] feat: rewrite agent_prompt with 4-tier compaction (Tiers 1-2 sync) --- src/agent.rs | 6 +- src/agent_prompt.rs | 1114 +++++++++++++++++++++++-------------------- src/provider.rs | 6 + 3 files changed, 612 insertions(+), 514 deletions(-) diff --git a/src/agent.rs b/src/agent.rs index f2b4e31..a75b749 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -642,7 +642,8 @@ impl Agent { let response: ChatMessage; // Prepare prompt with optional compaction (invariant across retries) - let base_prompt = prepare_messages_for_llm(&messages); + let context_window = self.registry.default_context_window(); + let base_prompt = prepare_messages_for_llm(&messages, context_window); loop { // Clone the base prompt for this retry attempt @@ -1918,7 +1919,8 @@ impl Agent { let response: ChatMessage; // Prepare prompt with optional compaction (invariant across retries) - let base_prompt = prepare_messages_for_llm(messages); + let context_window = self.registry.default_context_window(); + let base_prompt = prepare_messages_for_llm(messages, context_window); loop { let mut prompt_prepared = base_prompt.clone(); diff --git a/src/agent_prompt.rs b/src/agent_prompt.rs index 83773bd..bf60bf7 100644 --- a/src/agent_prompt.rs +++ b/src/agent_prompt.rs @@ -1,26 +1,35 @@ //! In-memory prompt preparation and compaction for LLM context management. //! -//! This module provides functionality to prepare conversation history for LLM calls, -//! including automatic compaction of tool-heavy conversations to stay within context -//! limits while preserving recent and relevant information. +//! Tiers 1-2 (sync, 0 LLM cost): +//! Tier 1: observation_mask β€” replace old tool results with placeholder, +//! neutralize old [RustFox compacted:...] markers +//! Tier 2: collapse_context β€” remove oldest tool groups entirely, +//! insert boundary marker +//! +//! Tiers 3-4 live in agent.rs (async, require LLM call). use crate::llm::{ChatMessage, MessageContent}; -const COMPACTION_MESSAGE_COUNT_THRESHOLD: usize = 10; -const COMPACTION_PROMPT_BYTE_THRESHOLD: usize = 20_000; -const TOOL_ARGUMENT_COMPACT_THRESHOLD: usize = 1_000; -const TOOL_RESULT_COMPACT_THRESHOLD: usize = 2_000; -const TOOL_RESULT_PREVIEW_CHARS: usize = 1_000; -const PRESERVED_TOOL_GROUPS: usize = 2; +/// Percentage of context_window that triggers Tier 1 (observation masking). +const OBSERVATION_MASK_PCT: f64 = 0.20; +/// Percentage that triggers Tier 2 (context collapse). +const COLLAPSE_PCT: f64 = 0.60; +/// Percentage that triggers Tier 3 (auto compact). +pub const COMPACT_PCT: f64 = 0.85; +/// Percentage that triggers Tier 4 (reactive compact). +#[allow(dead_code)] +pub(crate) const REACTIVE_PCT: f64 = 0.95; +/// Minimum turns between Tier 3/4 compactions. +pub const COMPACT_TURN_GAP: usize = 5; +/// Number of most recent tool groups to preserve verbatim. +pub const PRESERVED_TOOL_GROUPS: usize = 2; +/// Absolute hard cap safety net (applied regardless of context_window). const PROMPT_HARD_CAP_BYTES: usize = 100_000; +/// Minimum message count to consider Tier 3 compact. +const COMPACT_MIN_MESSAGE_COUNT: usize = 15; -/// Truncate a string to at most `max_chars` characters at a safe character boundary. -/// -/// Returns a new string containing up to `max_chars` characters from `value`. -/// If the string is longer, it is truncated at a character boundary (not mid-codepoint). -fn truncate_chars(value: &str, max_chars: usize) -> String { - value.chars().take(max_chars).collect() -} +/// Keep for backward compat β€” `is_compacted_regurgitation` references this. +pub const COMPACTION_MARKER_PREFIX: &str = "[RustFox compacted:"; /// Statistics about prompt preparation and compaction. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -39,10 +48,37 @@ pub struct PreparedPrompt { pub stats: PromptStats, } -/// Estimate the byte size of a prompt from its messages. +/// Per-conversation compaction metadata. /// -/// Counts message content lengths and tool call argument lengths. -/// Uses bytes (not chars) as a rough proxy for token count (~4 bytes β‰ˆ 1 token). +/// Tracked in-memory alongside the message list. The agent loop increments +/// `current_turn` each iteration and updates `last_compact_turn` after +/// Tier 3/4 fires. +#[derive(Debug, Clone)] +pub struct ConversationMeta { + pub last_compact_turn: usize, + pub has_attempted_reactive_compact: bool, + pub is_compact_agent: bool, + pub current_turn: usize, +} + +impl ConversationMeta { + pub fn new() -> Self { + Self { + last_compact_turn: 0, + has_attempted_reactive_compact: false, + is_compact_agent: false, + current_turn: 0, + } + } +} + +impl Default for ConversationMeta { + fn default() -> Self { + Self::new() + } +} + +/// Estimate the byte size of a prompt from its messages. pub fn estimate_prompt_bytes(messages: &[ChatMessage]) -> usize { messages .iter() @@ -64,9 +100,6 @@ pub fn estimate_prompt_bytes(messages: &[ChatMessage]) -> usize { } /// Create a recovery nudge message appropriate for the conversation context. -/// -/// If the previous message role is `tool`, mentions "tool result above". -/// Otherwise mentions "user's request above". pub fn recovery_nudge_for(messages: &[ChatMessage]) -> ChatMessage { let previous_is_tool = messages.last().is_some_and(|msg| msg.role == "tool"); @@ -84,150 +117,69 @@ pub fn recovery_nudge_for(messages: &[ChatMessage]) -> ChatMessage { } } -/// Prepare messages for LLM by applying compaction if needed. +/// Tier 1: Observation Masking /// -/// Applies compaction only when: -/// - Message count > 10 AND -/// - Estimated prompt size > 20,000 chars -pub fn prepare_messages_for_llm(messages: &[ChatMessage]) -> PreparedPrompt { - let original_message_count = messages.len(); - let original_prompt_bytes = estimate_prompt_bytes(messages); - - let should_compact = original_message_count > COMPACTION_MESSAGE_COUNT_THRESHOLD - && original_prompt_bytes > COMPACTION_PROMPT_BYTE_THRESHOLD; - - let prepared_messages = if should_compact { - let messages = compact_tool_heavy_history(messages); - // Second pass: if still over the hard cap, reduce preserved groups - if estimate_prompt_bytes(&messages) > PROMPT_HARD_CAP_BYTES { - compact_tool_heavy_history_with_preserved_groups(&messages, 1) - } else { - messages - } - } else { - messages.to_vec() - }; - - let prepared_message_count = prepared_messages.len(); - let prepared_prompt_bytes = estimate_prompt_bytes(&prepared_messages); - - PreparedPrompt { - messages: prepared_messages, - stats: PromptStats { - original_message_count, - prepared_message_count, - original_prompt_chars: original_prompt_bytes, - prepared_prompt_chars: prepared_prompt_bytes, - compaction_applied: should_compact, - }, - } -} - -/// Compact tool-heavy conversation history, preserving the default 2 most recent groups. -pub fn compact_tool_heavy_history(messages: &[ChatMessage]) -> Vec { - compact_tool_heavy_history_with_preserved_groups(messages, PRESERVED_TOOL_GROUPS) -} - -/// Compact tool-heavy conversation history with configurable preserved groups. +/// Replace old tool result content with a masked placeholder. The LLM +/// knows it made the call but the bulky payload is gone. Also neutralize +/// any old [RustFox compacted:...] markers in tool call arguments. /// -/// Strategy: -/// - Preserve all system and user messages unchanged -/// - Preserve the most recent `preserved_count` assistant-with-tool-calls groups and their tool results -/// - Compact older assistant tool arguments longer than 1,000 chars -/// - Compact older tool results longer than 2,000 chars -/// - Maintain message order -fn compact_tool_heavy_history_with_preserved_groups( - messages: &[ChatMessage], - preserved_count: usize, -) -> Vec { - // First, identify tool groups: assistant messages with tool calls and their following tool messages - let mut tool_groups: Vec<(usize, Vec)> = Vec::new(); - let mut current_group: Option<(usize, Vec)> = None; - - for (idx, msg) in messages.iter().enumerate() { - if msg.role == "assistant" - && msg - .tool_calls - .as_ref() - .is_some_and(|calls| !calls.is_empty()) - { - // Start a new tool group - if let Some(group) = current_group.take() { - tool_groups.push(group); - } - current_group = Some((idx, Vec::new())); - } else if msg.role == "tool" { - // Add to current group if one exists - if let Some((_, ref mut tool_indices)) = current_group { - tool_indices.push(idx); - } - } else if current_group.is_some() { - // Non-tool message encountered, close current group - if let Some(group) = current_group.take() { - tool_groups.push(group); - } - } - } - // Don't forget the last group - if let Some(group) = current_group { - tool_groups.push(group); +/// Trigger: estimated bytes > context_window * OBSERVATION_MASK_PCT +/// Applies to: tool results older than PRESERVED_TOOL_GROUPS. +pub fn observation_mask(messages: &[ChatMessage], context_window: usize) -> Vec { + let threshold = (context_window as f64 * OBSERVATION_MASK_PCT) as usize; + if estimate_prompt_bytes(messages) <= threshold { + return messages.to_vec(); } - // Determine which indices to preserve (most recent preserved_count groups) - let preserved_groups_start = tool_groups.len().saturating_sub(preserved_count); + // Identify tool groups + let tool_groups = find_tool_groups(messages); + let preserved_start = tool_groups.len().saturating_sub(PRESERVED_TOOL_GROUPS); + let mut preserved_indices = std::collections::HashSet::new(); - for group in tool_groups.iter().skip(preserved_groups_start) { - preserved_indices.insert(group.0); - for &tool_idx in &group.1 { - preserved_indices.insert(tool_idx); + for group in tool_groups.iter().skip(preserved_start) { + preserved_indices.insert(group.assistant_idx); + for &ti in &group.tool_result_indices { + preserved_indices.insert(ti); } } - // Compact messages messages .iter() .enumerate() .map(|(idx, msg)| { - // Never compact system or user messages + // System and user: pass through if msg.role == "system" || msg.role == "user" { return msg.clone(); } - // Don't compact preserved indices + // Preserved indices: pass through if preserved_indices.contains(&idx) { return msg.clone(); } - // Compact assistant tool calls - if msg.role == "assistant" - && msg - .tool_calls - .as_ref() - .is_some_and(|calls| !calls.is_empty()) - { - return compact_assistant_tool_calls(msg); - } - - // Compact assistant text-only messages with long content - if msg.role == "assistant" && msg.tool_calls.is_none() { - if let Some(ref content) = msg.content { - let text = content.as_text(); - if text.len() > TOOL_ARGUMENT_COMPACT_THRESHOLD { - let mut compacted = msg.clone(); - let preview = truncate_chars(&text, TOOL_RESULT_PREVIEW_CHARS); - compacted.content = Some(MessageContent::Text(format!( - "[RustFox compacted: previous assistant response with {} bytes]\n{}...", - text.len(), - preview - ))); - return compacted; + // Neutralize old compaction markers in tool call arguments + if msg.role == "assistant" && msg.has_tool_calls() { + let mut compacted = msg.clone(); + if let Some(calls) = &mut compacted.tool_calls { + for call in calls.iter_mut() { + if call.function.arguments.contains(COMPACTION_MARKER_PREFIX) { + call.function.arguments = call + .function + .arguments + .replace(COMPACTION_MARKER_PREFIX, "[compacted"); + } } } + return compacted; } - // Compact tool results + // Mask tool results if msg.role == "tool" { - return compact_tool_result(msg); + let mut masked = msg.clone(); + masked.content = Some(MessageContent::Text( + "[previous tool result β€” masked]".to_string(), + )); + return masked; } msg.clone() @@ -235,118 +187,329 @@ fn compact_tool_heavy_history_with_preserved_groups( .collect() } -/// Public so `agent.rs` can detect regurgitated compaction markers. -pub const COMPACTION_MARKER_PREFIX: &str = "[RustFox compacted:"; - -/// Compact assistant message with tool calls by shortening long arguments. +/// Tier 2: Context Collapse +/// +/// Remove the oldest 50% of non-preserved tool groups (assistant + tool +/// messages). Insert a structural boundary marker at the collapse point +/// so the LLM doesn't perceive a confusing jump. /// -/// Uses a plain-text marker instead of JSON to prevent the LLM from learning -/// the format and regurgitating it as its own tool call arguments β€” the root -/// cause of the "Missing skill" bug when a model copied the old JSON-shaped -/// `{"_rustfox_compacted_arguments": true, …}` object verbatim. -fn compact_assistant_tool_calls(msg: &ChatMessage) -> ChatMessage { - let mut compacted = msg.clone(); - - if let Some(tool_calls) = &mut compacted.tool_calls { - for call in tool_calls.iter_mut() { - let args_len = call.function.arguments.len(); - if args_len > TOOL_ARGUMENT_COMPACT_THRESHOLD { - call.function.arguments = format!( - "{} previous {} call with {} bytes of arguments]", - COMPACTION_MARKER_PREFIX, call.function.name, args_len, - ); +/// Trigger: still > context_window * COLLAPSE_PCT after Tier 1. +pub fn collapse_context(messages: &[ChatMessage], context_window: usize) -> Vec { + let threshold = (context_window as f64 * COLLAPSE_PCT) as usize; + if estimate_prompt_bytes(messages) <= threshold { + return messages.to_vec(); + } + + let tool_groups = find_tool_groups(messages); + let non_preserved_count = tool_groups.len().saturating_sub(PRESERVED_TOOL_GROUPS); + // Remove oldest 50% of non-preserved groups only + let keep_start = non_preserved_count / 2; + + let mut keep_indices = std::collections::HashSet::new(); + + for (i, group) in tool_groups.iter().enumerate() { + if i >= keep_start { + keep_indices.insert(group.assistant_idx); + for &ti in &group.tool_result_indices { + keep_indices.insert(ti); + } + } + } + + // Always keep system + user messages + for (idx, msg) in messages.iter().enumerate() { + if msg.role == "system" || msg.role == "user" { + keep_indices.insert(idx); + } + } + + // Keep messages after the last kept group + if let Some(last_keep) = tool_groups + .iter() + .rev() + .find(|g| keep_indices.contains(&g.assistant_idx)) + { + for idx in (last_keep.assistant_idx + 1)..messages.len() { + keep_indices.insert(idx); + } + } + + // Build result: filter to kept indices in order, inserting boundary marker + let mut result: Vec = Vec::new(); + let mut inserted_boundary = false; + + // Find the first index that is kept after the removal zone + let boundary_group_idx = tool_groups + .iter() + .position(|g| keep_indices.contains(&g.assistant_idx)); + + for (idx, msg) in messages.iter().enumerate() { + if keep_indices.contains(&idx) { + if !inserted_boundary { + if let Some(bg_idx) = boundary_group_idx { + if let Some(group) = tool_groups.iter().find(|g| g.assistant_idx == idx) { + if tool_groups + .iter() + .position(|g| g.assistant_idx == group.assistant_idx) + == Some(bg_idx) + { + result.push(ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text( + "β˜… earlier conversation collapsed β˜…".to_string(), + )), + tool_calls: None, + tool_call_id: None, + }); + inserted_boundary = true; + } + } + } else { + inserted_boundary = true; + } } + result.push(msg.clone()); } } - compacted + // If boundary still not inserted and messages were removed + if !inserted_boundary && messages.len() > result.len() { + result.insert( + 1, + ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text( + "β˜… earlier conversation collapsed β˜…".to_string(), + )), + tool_calls: None, + tool_call_id: None, + }, + ); + } + + result +} + +/// Helper: identify tool groups in a message list. +pub struct ToolGroup { + pub assistant_idx: usize, + pub tool_result_indices: Vec, +} + +pub fn find_tool_groups(messages: &[ChatMessage]) -> Vec { + let mut groups: Vec = Vec::new(); + let mut current: Option = None; + + for (idx, msg) in messages.iter().enumerate() { + if msg.role == "assistant" && msg.has_tool_calls() { + if let Some(g) = current.take() { + groups.push(g); + } + current = Some(ToolGroup { + assistant_idx: idx, + tool_result_indices: Vec::new(), + }); + } else if msg.role == "tool" { + if let Some(ref mut g) = current { + g.tool_result_indices.push(idx); + } + } else if current.is_some() { + if let Some(g) = current.take() { + groups.push(g); + } + } + } + if let Some(g) = current { + groups.push(g); + } + groups } -/// Compact tool result message by truncating long content. +/// Prepare messages for LLM by applying compaction if needed. /// -/// Preserves multi-modal structure: text parts are compacted, image parts -/// pass through unchanged. -fn compact_tool_result(msg: &ChatMessage) -> ChatMessage { - use crate::llm::ContentPart; - - let mut compacted = msg.clone(); - - if let Some(content) = &compacted.content { - match content { - MessageContent::Text(ref text) => { - let text_len = text.len(); - if text_len > TOOL_RESULT_COMPACT_THRESHOLD { - let preview = truncate_chars(text, TOOL_RESULT_PREVIEW_CHARS); - compacted.content = Some(MessageContent::Text(format!( - "[rustfox compacted tool result: {} bytes]\n{}...", - text_len, preview - ))); +/// Sync function: applies Tiers 1-2 only. +/// - Tier 1: observation masking (replace old tool results with placeholder) +/// - Tier 2: context collapse (remove oldest tool groups, insert boundary marker) +/// +/// If still over PROMPT_HARD_CAP_BYTES after both tiers, reduce preserved +/// groups to 1 as a last resort. +pub fn prepare_messages_for_llm(messages: &[ChatMessage], context_window: usize) -> PreparedPrompt { + let original_message_count = messages.len(); + let original_prompt_bytes = estimate_prompt_bytes(messages); + + let obs_threshold = (context_window as f64 * OBSERVATION_MASK_PCT) as usize; + let coll_threshold = (context_window as f64 * COLLAPSE_PCT) as usize; + + let should_compact = original_prompt_bytes > obs_threshold + && original_message_count > compact_min_message_count(context_window); + + let prepared_messages = if should_compact { + // Tier 1: observation masking + let after_tier1 = observation_mask(messages, context_window); + + // Tier 2: context collapse if still over COLLAPSE_PCT + let after_tier2 = if estimate_prompt_bytes(&after_tier1) > coll_threshold { + collapse_context(&after_tier1, context_window) + } else { + after_tier1 + }; + + // Safety net: if still over hard cap, keep all sys/user + the 2 newest messages (1 preserved pair) + if estimate_prompt_bytes(&after_tier2) > PROMPT_HARD_CAP_BYTES { + let preserved_count = after_tier2 + .iter() + .filter(|m| m.role == "system" || m.role == "user") + .count(); + let mut hard_cap_messages: Vec = Vec::with_capacity(preserved_count + 2); + for m in &after_tier2 { + if m.role == "system" || m.role == "user" { + hard_cap_messages.push(m.clone()); } } - MessageContent::Parts(ref parts) => { - let new_parts: Vec = parts - .iter() - .map(|part| match part { - ContentPart::Text { text } - if text.len() > TOOL_RESULT_COMPACT_THRESHOLD => - { - let preview = truncate_chars(text, TOOL_RESULT_PREVIEW_CHARS); - ContentPart::Text { - text: format!( - "[rustfox compacted tool result: {} bytes]\n{}...", - text.len(), - preview - ), - } - } - other => other.clone(), - }) - .collect(); - compacted.content = Some(MessageContent::Parts(new_parts)); + // Append the 2 newest non-system/user messages (preserves latest preserved pair) + let mut newest_pair: Vec = Vec::with_capacity(2); + for m in after_tier2.iter().rev() { + if m.role != "system" && m.role != "user" { + newest_pair.push(m.clone()); + if newest_pair.len() == 2 { + break; + } + } } + hard_cap_messages.extend(newest_pair.into_iter().rev()); + hard_cap_messages + } else { + after_tier2 } + } else { + messages.to_vec() + }; + + let prepared_message_count = prepared_messages.len(); + let prepared_prompt_bytes = estimate_prompt_bytes(&prepared_messages); + + PreparedPrompt { + messages: prepared_messages, + stats: PromptStats { + original_message_count, + prepared_message_count, + original_prompt_chars: original_prompt_bytes, + prepared_prompt_chars: prepared_prompt_bytes, + compaction_applied: should_compact, + }, } +} - compacted +/// Returns minimum message count to consider compaction, scaled by +/// context_window size. Larger windows need more messages to justify +/// the LLM cost of Tier 3. +fn compact_min_message_count(context_window: usize) -> usize { + // Base: 15 messages. Scale linearly: 15 for 128K, 30 for 1M, etc. + let base = COMPACT_MIN_MESSAGE_COUNT; + let window_k = context_window / 512_000; // 512K = 128K tokens + base + (base / 2) * window_k +} + +/// Check whether Tier 3 auto-compact should trigger. +/// +/// All conditions must be true: +/// - Message count > compact_min_message_count +/// - Estimated prompt bytes > context_window * COMPACT_PCT +/// - Turn gap >= COMPACT_TURN_GAP since last compact +/// - Not already in compact agent loop (recursion guard) +pub fn should_auto_compact( + messages: &[ChatMessage], + meta: &ConversationMeta, + context_window: usize, +) -> bool { + let threshold = (context_window as f64 * COMPACT_PCT) as usize; + let bytes = estimate_prompt_bytes(messages); + + bytes > threshold + && messages.len() > compact_min_message_count(context_window) + && meta.current_turn - meta.last_compact_turn >= COMPACT_TURN_GAP + && !meta.is_compact_agent +} + +/// Create the summary prompt content used for Tier 3 and Tier 4 LLM calls. +/// Returns a system-role message instructing the LLM to summarize. +pub fn build_compact_summary_prompt() -> ChatMessage { + let prompt_text = vec![ + "Your task is to create a detailed summary of the conversation so far.", + "", + "Your summary must include these sections:", + "", + "1. Primary Request and Intent: What was the user's original request?", + "2. Key Technical Concepts: Technologies, frameworks, approaches discussed", + "3. Files and Code Sections: Files read, created, or modified", + "4. Errors and Fixes: Errors encountered and how they were fixed", + "5. All User Messages: List ALL user messages verbatim (not tool results)", + "6. Pending Tasks: What tasks were explicitly requested but not completed", + "7. Current Work: Exactly what was being worked on before this summary", + "8. Next Step: The next logical action based on the most recent user request", + "", + "IMPORTANT: Do NOT call any tools. Respond with text only.", + ] + .join("\n"); + + ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text(prompt_text)), + tool_calls: None, + tool_call_id: None, + } +} + +/// Build the compact boundary marker message (pre-compact stats). +pub fn build_compact_boundary_marker(original_count: usize, compacted_count: usize) -> ChatMessage { + ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text(format!( + "β˜… COMPACT SUMMARY β€” previous {} messages compressed into this summary (now {} messages) β˜…", + original_count, compacted_count, + ))), + tool_calls: None, + tool_call_id: None, + } } #[cfg(test)] mod tests { use super::*; - use crate::llm::{FunctionCall, MessageContent, ToolCall}; + use crate::llm::{FunctionCall, ToolCall}; #[test] fn estimate_prompt_bytes_counts_content_and_tool_arguments() { let messages = vec![ ChatMessage { role: "user".to_string(), - content: Some(MessageContent::Text("Hello".to_string())), // 5 chars + content: Some(MessageContent::Text("Hello".to_string())), tool_calls: None, tool_call_id: None, }, ChatMessage { role: "assistant".to_string(), - content: Some(MessageContent::Text("Hi".to_string())), // 2 chars + content: Some(MessageContent::Text("Hi".to_string())), tool_calls: Some(vec![ToolCall { id: "call_1".to_string(), call_type: "function".to_string(), function: FunctionCall { name: "test_tool".to_string(), - arguments: r#"{"arg":"value"}"#.to_string(), // 15 chars + arguments: r#"{"arg":"value"}"#.to_string(), }, }]), tool_call_id: None, }, ChatMessage { role: "tool".to_string(), - content: Some(MessageContent::Text("result".to_string())), // 6 chars + content: Some(MessageContent::Text("result".to_string())), tool_calls: None, tool_call_id: Some("call_1".to_string()), }, ]; let total = estimate_prompt_bytes(&messages); - assert_eq!(total, 5 + 2 + 15 + 6); // 28 bytes + assert_eq!(total, 5 + 2 + 15 + 6); } #[test] @@ -397,7 +560,6 @@ mod tests { #[test] fn prepare_messages_skips_compaction_for_short_prompts() { - // Less than 10 messages let messages: Vec = (0..5) .map(|i| ChatMessage { role: "user".to_string(), @@ -407,445 +569,373 @@ mod tests { }) .collect(); - let result = prepare_messages_for_llm(&messages); + let result = prepare_messages_for_llm(&messages, 512_000); assert!(!result.stats.compaction_applied); assert_eq!(result.messages.len(), messages.len()); - - // More than 10 messages but under char threshold - let short_messages: Vec = (0..15) - .map(|i| ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text(format!("msg{}", i))), // Very short - tool_calls: None, - tool_call_id: None, - }) - .collect(); - - let result2 = prepare_messages_for_llm(&short_messages); - assert!(!result2.stats.compaction_applied); } #[test] - fn compaction_preserves_newest_two_tool_groups_and_compacts_older_group() { - // Build a conversation with 3 tool groups + fn observation_mask_replaces_old_tool_results_and_keeps_recent() { + let ctx = 2000; let mut messages = Vec::new(); - // System message messages.push(ChatMessage { role: "system".to_string(), - content: Some(MessageContent::Text( - "You are a helpful assistant.".to_string(), - )), + content: Some(MessageContent::Text("sys".to_string())), tool_calls: None, tool_call_id: None, }); - - // User message messages.push(ChatMessage { role: "user".to_string(), - content: Some(MessageContent::Text("Do task 1".to_string())), + content: Some(MessageContent::Text("hello".to_string())), tool_calls: None, tool_call_id: None, }); - // Old tool group 1 (should be compacted) - let long_args = "x".repeat(1500); + let old_args = "x".repeat(500); messages.push(ChatMessage { role: "assistant".to_string(), content: None, tool_calls: Some(vec![ToolCall { - id: "old_call_1".to_string(), + id: "call_1".to_string(), call_type: "function".to_string(), function: FunctionCall { name: "old_tool".to_string(), - arguments: long_args.clone(), + arguments: old_args, }, }]), tool_call_id: None, }); - - let long_result = "y".repeat(2500); messages.push(ChatMessage { role: "tool".to_string(), - content: Some(MessageContent::Text(long_result.clone())), + content: Some(MessageContent::Text("y".repeat(800))), tool_calls: None, - tool_call_id: Some("old_call_1".to_string()), + tool_call_id: Some("call_1".to_string()), }); - // Recent tool group 1 (should be preserved) messages.push(ChatMessage { role: "assistant".to_string(), content: None, tool_calls: Some(vec![ToolCall { - id: "recent_call_1".to_string(), + id: "call_2".to_string(), call_type: "function".to_string(), function: FunctionCall { - name: "recent_tool_1".to_string(), - arguments: "x".repeat(1500), + name: "recent_tool".to_string(), + arguments: "test".to_string(), }, }]), tool_call_id: None, }); - messages.push(ChatMessage { role: "tool".to_string(), - content: Some(MessageContent::Text("y".repeat(2500))), + content: Some(MessageContent::Text("recent result".to_string())), tool_calls: None, - tool_call_id: Some("recent_call_1".to_string()), + tool_call_id: Some("call_2".to_string()), }); - // Recent tool group 2 (should be preserved) messages.push(ChatMessage { role: "assistant".to_string(), content: None, tool_calls: Some(vec![ToolCall { - id: "recent_call_2".to_string(), + id: "call_3".to_string(), call_type: "function".to_string(), function: FunctionCall { - name: "recent_tool_2".to_string(), - arguments: "z".repeat(1500), + name: "latest".to_string(), + arguments: "tiny".to_string(), }, }]), tool_call_id: None, }); - messages.push(ChatMessage { role: "tool".to_string(), - content: Some(MessageContent::Text("w".repeat(2500))), + content: Some(MessageContent::Text("latest result".to_string())), tool_calls: None, - tool_call_id: Some("recent_call_2".to_string()), + tool_call_id: Some("call_3".to_string()), }); - let compacted = compact_tool_heavy_history(&messages); - - // System and user messages should be unchanged - assert_eq!(compacted[0].role, "system"); - assert_eq!(compacted[1].role, "user"); - - // Old tool group should be compacted - let old_assistant = &compacted[2]; - assert_eq!(old_assistant.role, "assistant"); - let old_args = &old_assistant.tool_calls.as_ref().unwrap()[0] - .function - .arguments; - assert!(old_args.contains("[RustFox compacted:")); - assert!(old_args.len() < long_args.len()); - - let old_tool = &compacted[3]; - assert_eq!(old_tool.role, "tool"); - let old_content = old_tool.content.as_ref().unwrap(); - assert!(old_content - .as_text() - .contains("rustfox compacted tool result")); - assert!(old_content.as_text().len() < long_result.len()); - - // Recent tool groups should be preserved unchanged - let recent1_assistant = &compacted[4]; - assert_eq!( - recent1_assistant.tool_calls.as_ref().unwrap()[0] - .function - .arguments - .len(), - 1500 - ); + let result = observation_mask(&messages, ctx); - let recent1_tool = &compacted[5]; - assert_eq!(recent1_tool.content.as_ref().unwrap().as_text().len(), 2500); + assert_eq!(result[0].role, "system"); + assert_eq!(result[1].role, "user"); - let recent2_assistant = &compacted[6]; - assert_eq!( - recent2_assistant.tool_calls.as_ref().unwrap()[0] - .function - .arguments - .len(), - 1500 + let old_content = result[3].content.as_ref().unwrap().as_text(); + assert!( + old_content.contains("masked"), + "old tool result should be masked: {}", + old_content ); - let recent2_tool = &compacted[7]; - assert_eq!(recent2_tool.content.as_ref().unwrap().as_text().len(), 2500); + let recent_content = result[5].content.as_ref().unwrap().as_text(); + assert_eq!(recent_content, "recent result"); + + let latest_content = result[7].content.as_ref().unwrap().as_text(); + assert_eq!(latest_content, "latest result"); + + assert_eq!(result.len(), messages.len()); } #[test] - fn compacted_message_order_is_unchanged() { - let messages = vec![ - ChatMessage { - role: "system".to_string(), - content: Some(MessageContent::Text("System".to_string())), - tool_calls: None, - tool_call_id: None, - }, - ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text("User 1".to_string())), - tool_calls: None, - tool_call_id: None, - }, - ChatMessage { + fn collapse_context_removes_oldest_tool_groups_and_inserts_boundary_marker() { + let ctx = 5000; + let mut messages = Vec::new(); + + messages.push(ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("sys".to_string())), + tool_calls: None, + tool_call_id: None, + }); + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("user msg".to_string())), + tool_calls: None, + tool_call_id: None, + }); + + for i in 0..4 { + messages.push(ChatMessage { role: "assistant".to_string(), content: None, tool_calls: Some(vec![ToolCall { - id: "call_1".to_string(), + id: format!("c{}", i), call_type: "function".to_string(), function: FunctionCall { - name: "tool".to_string(), - arguments: "x".repeat(1500), + name: format!("t{}", i), + arguments: "x".repeat(500), }, }]), tool_call_id: None, - }, - ChatMessage { + }); + messages.push(ChatMessage { role: "tool".to_string(), - content: Some(MessageContent::Text("y".repeat(2500))), - tool_calls: None, - tool_call_id: Some("call_1".to_string()), - }, - ChatMessage { - role: "assistant".to_string(), - content: Some(MessageContent::Text("Final response".to_string())), + content: Some(MessageContent::Text("y".repeat(800))), tool_calls: None, - tool_call_id: None, - }, - ]; - - let compacted = compact_tool_heavy_history(&messages); + tool_call_id: Some(format!("c{}", i)), + }); + } - // Check that roles are in the same order - let original_roles: Vec<_> = messages.iter().map(|m| m.role.as_str()).collect(); - let compacted_roles: Vec<_> = compacted.iter().map(|m| m.role.as_str()).collect(); - assert_eq!(original_roles, compacted_roles); + messages.push(ChatMessage { + role: "assistant".to_string(), + content: Some(MessageContent::Text("done".to_string())), + tool_calls: None, + tool_call_id: None, + }); - // Check message count is unchanged - assert_eq!(messages.len(), compacted.len()); - } + let result = collapse_context(&messages, ctx); - #[test] - fn truncate_chars_handles_unicode_safely() { - // ASCII text - assert_eq!(truncate_chars("hello world", 5), "hello"); - assert_eq!(truncate_chars("hello", 10), "hello"); - - // Emoji (multi-byte UTF-8) - let emoji_text = "Hello πŸ‘‹ World 🌍"; - let truncated = truncate_chars(emoji_text, 8); - assert_eq!(truncated, "Hello πŸ‘‹ "); - assert_eq!(truncated.chars().count(), 8); - - // CJK characters - let cjk_text = "δ½ ε₯½δΈ–η•Œ"; - let truncated = truncate_chars(cjk_text, 2); - assert_eq!(truncated, "δ½ ε₯½"); - assert_eq!(truncated.chars().count(), 2); - - // Mixed: emoji + CJK + ASCII - let mixed = "Test ζ΅‹θ―• πŸŽ‰ emoji 葨情"; - let truncated = truncate_chars(mixed, 11); - assert_eq!(truncated, "Test ζ΅‹θ―• πŸŽ‰ e"); - assert_eq!(truncated.chars().count(), 11); - } + assert_eq!(result[0].role, "system"); + assert_eq!(result[1].role, "user"); - #[test] - fn compaction_handles_unicode_tool_arguments_safely() { - // Create a long tool argument with emoji and CJK that would panic with byte slicing - // This string has multi-byte UTF-8 characters; byte index 200 could fall mid-character - let long_args_with_unicode = format!( - "{{\"message\": \"{}ζ΅‹θ―•πŸŽ‰{}\"}}", - "x".repeat(900), - "y".repeat(200) + let collapse_idx = result.iter().position(|m| { + m.role == "system" && m.content.as_ref().unwrap().as_text().contains("collapsed") + }); + assert!( + collapse_idx.is_some(), + "should have collapse boundary marker" ); - assert!(long_args_with_unicode.len() > TOOL_ARGUMENT_COMPACT_THRESHOLD); - let message = ChatMessage { - role: "assistant".to_string(), - content: None, - tool_calls: Some(vec![ToolCall { - id: "call_1".to_string(), - call_type: "function".to_string(), - function: FunctionCall { - name: "unicode_tool".to_string(), - arguments: long_args_with_unicode.clone(), - }, - }]), - tool_call_id: None, - }; - - // This should not panic - let compacted = compact_assistant_tool_calls(&message); - let compacted_args = &compacted.tool_calls.as_ref().unwrap()[0].function.arguments; - - // Verify it's been compacted β€” plain-text marker, no JSON - assert!(compacted_args.contains("[RustFox compacted:")); - assert!(compacted_args.contains("unicode_tool")); - assert!(compacted_args.contains("bytes of arguments")); + let preserved_start = collapse_idx.unwrap() + 1; + assert!( + preserved_start < result.len(), + "should have messages after boundary" + ); } #[test] - fn compaction_handles_unicode_tool_results_safely() { - // Create a long tool result with emoji and CJK that would panic with byte slicing - let long_result_with_unicode = - format!("Result: {}πŸŒŸζ΅‹θ―•{}πŸ’―", "a".repeat(1000), "b".repeat(1500)); - assert!(long_result_with_unicode.len() > TOOL_RESULT_COMPACT_THRESHOLD); - - let message = ChatMessage { - role: "tool".to_string(), - content: Some(MessageContent::Text(long_result_with_unicode.clone())), - tool_calls: None, - tool_call_id: Some("call_1".to_string()), + fn should_auto_compact_checks_bytes_turns_and_recursion_guard() { + let mut meta = ConversationMeta { + last_compact_turn: 0, + has_attempted_reactive_compact: false, + is_compact_agent: false, + current_turn: 10, }; - // This should not panic - let compacted = compact_tool_result(&message); - let compacted_content = compacted.content.as_ref().unwrap(); - - // Verify it's been compacted - assert!(compacted_content - .as_text() - .contains("rustfox compacted tool result")); - assert!(compacted_content.as_text().len() < long_result_with_unicode.len()); - - // Verify the preview portion is valid UTF-8 and respects character boundaries - // The format is: "[rustfox compacted tool result: N chars]\n{preview}..." - let text = compacted_content.as_text(); - let lines: Vec<&str> = text.lines().collect(); - assert_eq!(lines.len(), 2); - let preview_line = lines[1]; - assert!(preview_line.ends_with("...")); - - // Verify we can count characters without panicking (proves valid UTF-8) - let preview_without_ellipsis = preview_line.trim_end_matches("..."); - let char_count = preview_without_ellipsis.chars().count(); - assert!(char_count <= TOOL_RESULT_PREVIEW_CHARS); - } + let small: Vec = (0..3) + .map(|_| ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("hi".to_string())), + tool_calls: None, + tool_call_id: None, + }) + .collect(); + assert!(!should_auto_compact(&small, &meta, 512_000)); - #[test] - fn compaction_with_unicode_preserves_structure() { - // Test full compaction flow with Unicode content - let mut messages = vec![ - ChatMessage { - role: "system".to_string(), - content: Some(MessageContent::Text("System prompt 系统提瀺".to_string())), + let few_but_big: Vec = (0..5) + .map(|_| ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("x".repeat(100_000))), tool_calls: None, tool_call_id: None, - }, - ChatMessage { + }) + .collect(); + assert!(!should_auto_compact(&few_but_big, &meta, 512_000)); + + let many_big: Vec = (0..23) + .map(|_| ChatMessage { role: "user".to_string(), - content: Some(MessageContent::Text( - "User request with emoji πŸš€".to_string(), - )), + content: Some(MessageContent::Text("x".repeat(50_000))), tool_calls: None, tool_call_id: None, - }, - ]; + }) + .collect(); - // Add an OLD tool group with Unicode content (this should be compacted) - let unicode_args = format!( - "{{\"data\": \"{}δΈ­ζ–‡{}🎨{}\"}}", - "x".repeat(800), - "y".repeat(200), - "z".repeat(100) - ); - messages.push(ChatMessage { - role: "assistant".to_string(), - content: None, - tool_calls: Some(vec![ToolCall { - id: "old_call".to_string(), - call_type: "function".to_string(), - function: FunctionCall { - name: "unicode_tool".to_string(), - arguments: unicode_args, - }, - }]), - tool_call_id: None, - }); + meta.is_compact_agent = true; + assert!(!should_auto_compact(&many_big, &meta, 512_000)); + meta.is_compact_agent = false; + + meta.last_compact_turn = 8; + assert!(!should_auto_compact(&many_big, &meta, 512_000)); + meta.last_compact_turn = 0; + + assert!(should_auto_compact(&many_big, &meta, 512_000)); + } - let unicode_result = format!("Result η»“ζžœ: {}🌈{}", "a".repeat(1500), "b".repeat(1200)); + #[test] + fn find_tool_groups_detects_consecutive_tool_calls() { + let mut messages = Vec::new(); messages.push(ChatMessage { - role: "tool".to_string(), - content: Some(MessageContent::Text(unicode_result)), + role: "system".to_string(), + content: Some(MessageContent::Text("sys".to_string())), tool_calls: None, - tool_call_id: Some("old_call".to_string()), + tool_call_id: None, }); - - // Add recent tool group 1 (should be preserved) + // Group 1 messages.push(ChatMessage { role: "assistant".to_string(), content: None, tool_calls: Some(vec![ToolCall { - id: "recent_call_1".to_string(), + id: "c1".to_string(), call_type: "function".to_string(), function: FunctionCall { - name: "recent_tool_1".to_string(), - arguments: "x".repeat(1500), + name: "t1".to_string(), + arguments: "{}".to_string(), }, }]), tool_call_id: None, }); - messages.push(ChatMessage { role: "tool".to_string(), - content: Some(MessageContent::Text("y".repeat(2500))), + content: Some(MessageContent::Text("r1".to_string())), tool_calls: None, - tool_call_id: Some("recent_call_1".to_string()), + tool_call_id: Some("c1".to_string()), }); - - // Add recent tool group 2 (should be preserved) + // Group 2 messages.push(ChatMessage { role: "assistant".to_string(), content: None, tool_calls: Some(vec![ToolCall { - id: "recent_call_2".to_string(), + id: "c2".to_string(), call_type: "function".to_string(), function: FunctionCall { - name: "recent_tool_2".to_string(), - arguments: "z".repeat(1500), + name: "t2".to_string(), + arguments: "{}".to_string(), }, }]), tool_call_id: None, }); - messages.push(ChatMessage { role: "tool".to_string(), - content: Some(MessageContent::Text("w".repeat(2500))), + content: Some(MessageContent::Text("r2".to_string())), tool_calls: None, - tool_call_id: Some("recent_call_2".to_string()), + tool_call_id: Some("c2".to_string()), }); - // Compact and verify no panics - let compacted = compact_tool_heavy_history(&messages); + let groups = find_tool_groups(&messages); + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].assistant_idx, 1); + assert_eq!(groups[0].tool_result_indices, vec![2]); + assert_eq!(groups[1].assistant_idx, 3); + assert_eq!(groups[1].tool_result_indices, vec![4]); + } + + #[test] + fn should_auto_compact_needs_minimum_message_count() { + let meta = ConversationMeta::new(); + let few: Vec = (0..5) + .map(|_| ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("x".repeat(100_000))), + tool_calls: None, + tool_call_id: None, + }) + .collect(); + assert!(!should_auto_compact(&few, &meta, 512_000)); + } - // Verify structure is preserved - assert_eq!(compacted.len(), messages.len()); - assert_eq!(compacted[0].role, "system"); - assert_eq!(compacted[1].role, "user"); - assert_eq!(compacted[2].role, "assistant"); - assert_eq!(compacted[3].role, "tool"); + #[test] + fn conversation_meta_defaults_to_zero() { + let meta = ConversationMeta::new(); + assert_eq!(meta.last_compact_turn, 0); + assert!(!meta.has_attempted_reactive_compact); + assert!(!meta.is_compact_agent); + assert_eq!(meta.current_turn, 0); + } - // Verify OLD tool group was compacted - let compacted_args = &compacted[2].tool_calls.as_ref().unwrap()[0] - .function - .arguments; - assert!(compacted_args.contains("[RustFox compacted:")); + #[test] + fn prepare_messages_applies_tier2_when_tier1_not_enough() { + let ctx = 50000; + let mut messages = Vec::new(); + messages.push(ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text("sys".to_string())), + tool_calls: None, + tool_call_id: None, + }); + messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("do things".to_string())), + tool_calls: None, + tool_call_id: None, + }); + for i in 0..6 { + messages.push(ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![ToolCall { + id: format!("c{}", i), + call_type: "function".to_string(), + function: FunctionCall { + name: "tool".to_string(), + arguments: "x".repeat(8000), + }, + }]), + tool_call_id: None, + }); + messages.push(ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text("y".repeat(8000))), + tool_calls: None, + tool_call_id: Some(format!("c{}", i)), + }); + } + for i in 6..8 { + messages.push(ChatMessage { + role: "assistant".to_string(), + content: None, + tool_calls: Some(vec![ToolCall { + id: format!("c{}", i), + call_type: "function".to_string(), + function: FunctionCall { + name: "tool".to_string(), + arguments: "short".to_string(), + }, + }]), + tool_call_id: None, + }); + messages.push(ChatMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text("preserved".to_string())), + tool_calls: None, + tool_call_id: Some(format!("c{}", i)), + }); + } - let compacted_result = compacted[3].content.as_ref().unwrap(); - assert!(compacted_result - .as_text() - .contains("rustfox compacted tool result")); - - // Verify recent groups are preserved unchanged - assert_eq!( - compacted[4].tool_calls.as_ref().unwrap()[0] - .function - .arguments - .len(), - 1500 - ); - assert_eq!(compacted[5].content.as_ref().unwrap().as_text().len(), 2500); - assert_eq!( - compacted[6].tool_calls.as_ref().unwrap()[0] - .function - .arguments - .len(), - 1500 - ); - assert_eq!(compacted[7].content.as_ref().unwrap().as_text().len(), 2500); + let result = prepare_messages_for_llm(&messages, ctx); + assert!(result.stats.compaction_applied); + assert!(result.messages.len() < messages.len()); + let has_boundary = result.messages.iter().any(|m| { + m.role == "system" && m.content.as_ref().unwrap().as_text().contains("collapsed") + }); + assert!(has_boundary); } } diff --git a/src/provider.rs b/src/provider.rs index 3b4fb61..193baf0 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -127,6 +127,12 @@ impl ProviderRegistry { let provider = &self.providers[&self.default_provider]; format!("{}/{}", self.default_provider, provider.default_model()) } + + /// Return the context window size of the default provider. + pub fn default_context_window(&self) -> usize { + let provider = &self.providers[&self.default_provider]; + provider.config().context_window + } } /// Build a ProviderRegistry from config sections. From 451fc0c64a7e5941fa206de90241f117fe30008a Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 29 Jun 2026 11:44:31 +0800 Subject: [PATCH 14/19] feat: wire Tier 3 auto-compact and Tier 4 reactive compact into agent loops --- src/agent.rs | 299 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 283 insertions(+), 16 deletions(-) diff --git a/src/agent.rs b/src/agent.rs index a75b749..d9db683 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -10,7 +10,9 @@ use teloxide::types::{ChatId, InputFile}; use teloxide::Bot; use crate::agent_prompt::{ - estimate_prompt_bytes, prepare_messages_for_llm, recovery_nudge_for, PreparedPrompt, + estimate_prompt_bytes, prepare_messages_for_llm, recovery_nudge_for, should_auto_compact, + build_compact_summary_prompt, build_compact_boundary_marker, ConversationMeta, + PreparedPrompt, PRESERVED_TOOL_GROUPS, }; use crate::config::Config; use crate::langsmith::LangSmithClient; @@ -593,6 +595,9 @@ impl Agent { }; messages.push(user_msg); + // Compaction state for this conversation session (persists across iterations) + let mut conv_meta = ConversationMeta::new(); + // Gather all tool definitions let mut all_tools: Vec = tools::builtin_tool_definitions(); all_tools.extend(self.mcp.tool_definitions()); @@ -630,6 +635,13 @@ impl Agent { self.soul_updated .store(false, std::sync::atomic::Ordering::Relaxed); + // Resolve context_window once before the loop (can't .await inside the loop) + let context_window = { + let model = self.current_model.read().await; + let (provider, _) = self.registry.resolve_model(&model); + provider.config().context_window + }; + for iteration in 0..max_iterations { debug!( "Trying iteration {}: messages length: {}", @@ -641,7 +653,55 @@ impl Agent { let mut retry_count = 0u32; let response: ChatMessage; - // Prepare prompt with optional compaction (invariant across retries) + // Tier 3: auto-compact (async, LLM call) β€” before Tiers 1-2 + conv_meta.current_turn = iteration as usize; + if should_auto_compact(&messages, &conv_meta, context_window) { + // Capture pre-compact metrics + let messages_before = messages.clone(); + let messages_before_bytes = estimate_prompt_bytes(&messages_before); + + if let Ok(compacted) = Self::auto_compact_conversation( + &self.llm, &messages, context_window, + ).await { + conv_meta.last_compact_turn = iteration as usize; + messages = compacted; + info!( + "Tier 3 auto-compact applied: {} messages reduced", + messages.len(), + ); + + // LangSmith logging for Tier 3 + let compacted_bytes = estimate_prompt_bytes(&messages); + let compact_run_id = uuid::Uuid::new_v4().to_string(); + self.langsmith.start_run(crate::langsmith::RunParams { + id: compact_run_id.clone(), + name: "auto_compact".to_string(), + run_type: crate::langsmith::RunType::Chain, + parent_run_id: Some(chain_run_id.clone()), + inputs: serde_json::json!({ + "tier": 3, + "pre_bytes": messages_before_bytes, + "post_bytes": compacted_bytes, + "pre_count": messages_before.len(), + "post_count": messages.len(), + }), + session_name: ls_project.clone(), + start_time: Self::now_iso8601_static(), + }); + self.langsmith.end_run(crate::langsmith::EndRunParams { + id: compact_run_id, + outputs: Some(serde_json::json!({ + "tier": 3, + "delta_bytes": (messages_before_bytes as i64 - compacted_bytes as i64).max(0), + "delta_messages": messages_before.len() as i64 - messages.len() as i64, + })), + error: None, + end_time: Self::now_iso8601_static(), + }); + } + } + + // Tiers 1-2: sync compaction let context_window = self.registry.default_context_window(); let base_prompt = prepare_messages_for_llm(&messages, context_window); @@ -742,25 +802,90 @@ impl Agent { }; // Handle LLM transport/API errors + let mut recovered_from_413 = false; + let completion = match completion_result { Ok(c) => c, Err(e) => { - self.langsmith.end_run(crate::langsmith::EndRunParams { - id: llm_run_id, - outputs: None, - error: Some(format!("{:#}", e)), - end_time: Self::now_iso8601_static(), - }); - self.langsmith.end_run(crate::langsmith::EndRunParams { - id: chain_run_id, - outputs: None, - error: Some(format!("{:#}", e)), - end_time: Self::now_iso8601_static(), - }); - return Err(e); + let err_str = format!("{:#}", e); + let is_413 = err_str.contains("413") + || err_str.to_lowercase().contains("prompt too long") + || err_str.to_lowercase().contains("context length") + || err_str.to_lowercase().contains("maximum context"); + if is_413 && !conv_meta.has_attempted_reactive_compact { + conv_meta.has_attempted_reactive_compact = true; + let messages_before_compact = messages.len(); + match Self::reactive_compact(&self.llm, &messages, context_window).await { + Ok(compacted) => { + let compacted_len = compacted.len(); + messages = compacted; + recovered_from_413 = true; + + // LangSmith logging for Tier 4 + let compact_run_id = uuid::Uuid::new_v4().to_string(); + self.langsmith.start_run(crate::langsmith::RunParams { + id: compact_run_id.clone(), + name: "reactive_compact".to_string(), + run_type: crate::langsmith::RunType::Chain, + parent_run_id: Some(chain_run_id.clone()), + inputs: serde_json::json!({ + "tier": 4, + "reason": err_str, + "pre_count": messages_before_compact, + }), + session_name: ls_project.clone(), + start_time: Self::now_iso8601_static(), + }); + self.langsmith.end_run(crate::langsmith::EndRunParams { + id: compact_run_id, + outputs: Some(serde_json::json!({ + "tier": 4, + "post_count": compacted_len, + })), + error: None, + end_time: Self::now_iso8601_static(), + }); + } + Err(compact_err) => { + warn!("Reactive compact failed: {}", compact_err); + } + } + } + if !recovered_from_413 { + self.langsmith.end_run(crate::langsmith::EndRunParams { + id: llm_run_id, + outputs: None, + error: Some(err_str.clone()), + end_time: Self::now_iso8601_static(), + }); + self.langsmith.end_run(crate::langsmith::EndRunParams { + id: chain_run_id, + outputs: None, + error: Some(err_str), + end_time: Self::now_iso8601_static(), + }); + return Err(e); + } + // recovered_from_413 is true but the compiler can't see this; + // return a dummy completion (never used because of continue below) + crate::llm::ChatCompletion { + message: ChatMessage { + role: String::new(), + content: None, + tool_calls: None, + tool_call_id: None, + }, + finish_reason: None, + model: String::new(), + } } }; + if recovered_from_413 { + // Skip retry loop, restart outer for iteration + continue; + } + // Check if response is empty (no content and no tool calls) if is_empty_assistant_response(&completion.message) { warn!( @@ -1189,6 +1314,144 @@ impl Agent { Ok("I've reached the maximum number of tool call iterations. Please try rephrasing your request.".to_string()) } + /// Tier 3: Auto-compact via LLM summarization. + async fn auto_compact_conversation( + llm: &LlmClient, + messages: &[ChatMessage], + _context_window: usize, + ) -> Result> { + let tool_groups = crate::agent_prompt::find_tool_groups(messages); + + let preserve_count = PRESERVED_TOOL_GROUPS.min(tool_groups.len()); + let preserved_groups_start = tool_groups.len().saturating_sub(preserve_count); + + let summary_end = if preserved_groups_start > 0 { + let last_summary = &tool_groups[preserved_groups_start - 1]; + *last_summary.tool_result_indices.last().unwrap_or(&last_summary.assistant_idx) + 1 + } else { + return Ok(messages.to_vec()); + }; + + let to_summarize = &messages[..summary_end]; + let preserved = &messages[summary_end..]; + + let summary_prompt = build_compact_summary_prompt(); + let mut compact_messages = Vec::with_capacity(2 + preserved.len() + 1); + compact_messages.push(summary_prompt); + compact_messages.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "Summarize the following conversation portion ({} messages):", + to_summarize.len(), + ))), + tool_calls: None, + tool_call_id: None, + }); + compact_messages.extend(to_summarize.iter().cloned()); + + let summary_response = match llm.chat(&compact_messages, &[]).await { + Ok(c) => c, + Err(e) => anyhow::bail!("Auto-compact LLM call failed: {}", e), + }; + + let summary_text = summary_response + .content + .as_ref() + .map(|c| c.as_text()) + .unwrap_or_default(); + + if summary_text.is_empty() { + anyhow::bail!("Auto-compact returned empty summary"); + } + + let mut result: Vec = Vec::new(); + let boundary = build_compact_boundary_marker(to_summarize.len(), 1); + result.push(boundary); + + let summary_msg = ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text(format!( + "β˜… COMPACT SUMMARY β˜…\n\n{}", + summary_text + ))), + tool_calls: None, + tool_call_id: None, + }; + result.push(summary_msg); + + result.extend(preserved.iter().cloned()); + + let nudge = recovery_nudge_for(&result); + result.push(nudge); + + Ok(result) + } + + /// Tier 4: Reactive compact β€” emergency 413 recovery. + async fn reactive_compact( + llm: &LlmClient, + messages: &[ChatMessage], + _context_window: usize, + ) -> Result> { + if messages.len() <= 4 { + anyhow::bail!("Too few messages for reactive compact"); + } + + let split = messages.len().saturating_sub(4); + let to_summarize = &messages[..split]; + let preserved = &messages[split..]; + + let summary_prompt = build_compact_summary_prompt(); + let mut compact_msgs = Vec::with_capacity(to_summarize.len() + 2); + compact_msgs.push(summary_prompt); + compact_msgs.push(ChatMessage { + role: "user".to_string(), + content: Some(MessageContent::Text(format!( + "Summarize the following conversation portion ({} messages) β€” emergency compact:", + to_summarize.len(), + ))), + tool_calls: None, + tool_call_id: None, + }); + compact_msgs.extend(to_summarize.iter().cloned()); + + let summary_response = match llm.chat(&compact_msgs, &[]).await { + Ok(c) => c, + Err(e) => anyhow::bail!("Reactive compact LLM call failed: {}", e), + }; + + let summary_text = summary_response + .content + .as_ref() + .map(|c| c.as_text()) + .unwrap_or_default(); + + if summary_text.is_empty() { + anyhow::bail!("Reactive compact returned empty summary"); + } + + let boundary = build_compact_boundary_marker(to_summarize.len(), 1); + let summary_msg = ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text(format!( + "β˜… COMPACT SUMMARY (EMERGENCY) β˜…\n\n{}", + summary_text + ))), + tool_calls: None, + tool_call_id: None, + }; + + let mut result = Vec::with_capacity(3 + preserved.len()); + result.push(boundary); + result.push(summary_msg); + result.extend(preserved.iter().cloned()); + + let nudge = recovery_nudge_for(&result); + result.push(nudge); + + Ok(result) + } + /// Re-register all active scheduled tasks from the DB into the scheduler. /// Called once at startup after the agent is constructed. pub async fn restore_scheduled_tasks(&self) { @@ -1919,7 +2182,11 @@ impl Agent { let response: ChatMessage; // Prepare prompt with optional compaction (invariant across retries) - let context_window = self.registry.default_context_window(); + let context_window = { + let (provider, _) = self.registry.resolve_model(model); + provider.config().context_window + }; + // TODO: consider adding Tier 3/4 to subagent loops in a follow-up let base_prompt = prepare_messages_for_llm(messages, context_window); loop { From 2da3e206f89cfd89d9f38df6afca78fdbcf23953 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 29 Jun 2026 11:48:01 +0800 Subject: [PATCH 15/19] fix: use model-specific context_window in Tiers 1-2, not default --- src/agent.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/agent.rs b/src/agent.rs index d9db683..319f79a 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -702,7 +702,6 @@ impl Agent { } // Tiers 1-2: sync compaction - let context_window = self.registry.default_context_window(); let base_prompt = prepare_messages_for_llm(&messages, context_window); loop { From 55344805ad0abf81e707924e84a139ac217a215d Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 29 Jun 2026 14:01:03 +0800 Subject: [PATCH 16/19] fix: correct buffer capacity in auto_compact_conversation, add named constant for preserve count --- src/agent.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/agent.rs b/src/agent.rs index 319f79a..11605ee 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -10,8 +10,8 @@ use teloxide::types::{ChatId, InputFile}; use teloxide::Bot; use crate::agent_prompt::{ - estimate_prompt_bytes, prepare_messages_for_llm, recovery_nudge_for, should_auto_compact, - build_compact_summary_prompt, build_compact_boundary_marker, ConversationMeta, + build_compact_boundary_marker, build_compact_summary_prompt, estimate_prompt_bytes, + prepare_messages_for_llm, recovery_nudge_for, should_auto_compact, ConversationMeta, PreparedPrompt, PRESERVED_TOOL_GROUPS, }; use crate::config::Config; @@ -660,9 +660,9 @@ impl Agent { let messages_before = messages.clone(); let messages_before_bytes = estimate_prompt_bytes(&messages_before); - if let Ok(compacted) = Self::auto_compact_conversation( - &self.llm, &messages, context_window, - ).await { + if let Ok(compacted) = + Self::auto_compact_conversation(&self.llm, &messages, context_window).await + { conv_meta.last_compact_turn = iteration as usize; messages = compacted; info!( @@ -814,7 +814,8 @@ impl Agent { if is_413 && !conv_meta.has_attempted_reactive_compact { conv_meta.has_attempted_reactive_compact = true; let messages_before_compact = messages.len(); - match Self::reactive_compact(&self.llm, &messages, context_window).await { + match Self::reactive_compact(&self.llm, &messages, context_window).await + { Ok(compacted) => { let compacted_len = compacted.len(); messages = compacted; @@ -1326,7 +1327,11 @@ impl Agent { let summary_end = if preserved_groups_start > 0 { let last_summary = &tool_groups[preserved_groups_start - 1]; - *last_summary.tool_result_indices.last().unwrap_or(&last_summary.assistant_idx) + 1 + *last_summary + .tool_result_indices + .last() + .unwrap_or(&last_summary.assistant_idx) + + 1 } else { return Ok(messages.to_vec()); }; @@ -1335,7 +1340,7 @@ impl Agent { let preserved = &messages[summary_end..]; let summary_prompt = build_compact_summary_prompt(); - let mut compact_messages = Vec::with_capacity(2 + preserved.len() + 1); + let mut compact_messages = Vec::with_capacity(to_summarize.len() + 2); compact_messages.push(summary_prompt); compact_messages.push(ChatMessage { role: "user".to_string(), @@ -1392,11 +1397,12 @@ impl Agent { messages: &[ChatMessage], _context_window: usize, ) -> Result> { - if messages.len() <= 4 { + const PRESERVE_COUNT: usize = 4; + if messages.len() <= PRESERVE_COUNT { anyhow::bail!("Too few messages for reactive compact"); } - let split = messages.len().saturating_sub(4); + let split = messages.len().saturating_sub(PRESERVE_COUNT); let to_summarize = &messages[..split]; let preserved = &messages[split..]; From 3b98c045c7855d442902ef98a8f5c70b869a0b73 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 29 Jun 2026 14:09:17 +0800 Subject: [PATCH 17/19] fix: end orphaned LangSmith llm_run on 413 recovery --- src/agent.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/agent.rs b/src/agent.rs index 11605ee..36b993e 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -882,7 +882,15 @@ impl Agent { }; if recovered_from_413 { - // Skip retry loop, restart outer for iteration + // End the leaked llm_run before continuing + self.langsmith.end_run(crate::langsmith::EndRunParams { + id: llm_run_id, + outputs: None, + error: Some( + "413 context exceeded β€” recovered via Tier 4 compact".to_string(), + ), + end_time: Self::now_iso8601_static(), + }); continue; } From d6be08257f5692dc3f14a27c844a1b13f2341fa3 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 29 Jun 2026 15:37:10 +0800 Subject: [PATCH 18/19] refactor: extract summarize_and_replace helper, document REACTIVE_PCT, add defensive fallback check --- src/agent.rs | 97 +++++++++++++++++++-------------------------- src/agent_prompt.rs | 28 +++++++------ 2 files changed, 57 insertions(+), 68 deletions(-) diff --git a/src/agent.rs b/src/agent.rs index 36b993e..fe6d65f 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -1323,6 +1323,10 @@ impl Agent { } /// Tier 3: Auto-compact via LLM summarization. + /// + /// `_context_window` is reserved for future threshold-based + /// split-point calculation β€” the summarization LLM handles + /// split decisions internally. async fn auto_compact_conversation( llm: &LlmClient, messages: &[ChatMessage], @@ -1347,59 +1351,21 @@ impl Agent { let to_summarize = &messages[..summary_end]; let preserved = &messages[summary_end..]; - let summary_prompt = build_compact_summary_prompt(); - let mut compact_messages = Vec::with_capacity(to_summarize.len() + 2); - compact_messages.push(summary_prompt); - compact_messages.push(ChatMessage { - role: "user".to_string(), - content: Some(MessageContent::Text(format!( - "Summarize the following conversation portion ({} messages):", - to_summarize.len(), - ))), - tool_calls: None, - tool_call_id: None, - }); - compact_messages.extend(to_summarize.iter().cloned()); - - let summary_response = match llm.chat(&compact_messages, &[]).await { - Ok(c) => c, - Err(e) => anyhow::bail!("Auto-compact LLM call failed: {}", e), - }; - - let summary_text = summary_response - .content - .as_ref() - .map(|c| c.as_text()) - .unwrap_or_default(); - - if summary_text.is_empty() { - anyhow::bail!("Auto-compact returned empty summary"); - } - - let mut result: Vec = Vec::new(); - let boundary = build_compact_boundary_marker(to_summarize.len(), 1); - result.push(boundary); - - let summary_msg = ChatMessage { - role: "system".to_string(), - content: Some(MessageContent::Text(format!( - "β˜… COMPACT SUMMARY β˜…\n\n{}", - summary_text - ))), - tool_calls: None, - tool_call_id: None, - }; - result.push(summary_msg); - - result.extend(preserved.iter().cloned()); - - let nudge = recovery_nudge_for(&result); - result.push(nudge); - - Ok(result) + Self::summarize_and_replace( + llm, + to_summarize, + preserved, + "Auto-compact", + "β˜… COMPACT SUMMARY β˜…", + ) + .await } /// Tier 4: Reactive compact β€” emergency 413 recovery. + /// + /// `_context_window` is reserved for future threshold-based + /// split-point calculation β€” the summarization LLM handles + /// split decisions internally. async fn reactive_compact( llm: &LlmClient, messages: &[ChatMessage], @@ -1414,13 +1380,32 @@ impl Agent { let to_summarize = &messages[..split]; let preserved = &messages[split..]; + Self::summarize_and_replace( + llm, + to_summarize, + preserved, + "Reactive compact", + "β˜… COMPACT SUMMARY (EMERGENCY) β˜…", + ) + .await + } + + /// Shared helper for Tiers 3 and 4: send messages to LLM for + /// summarization, then assemble the compacted result. + async fn summarize_and_replace( + llm: &LlmClient, + to_summarize: &[ChatMessage], + preserved: &[ChatMessage], + error_label: &str, + summary_label: &str, + ) -> Result> { let summary_prompt = build_compact_summary_prompt(); let mut compact_msgs = Vec::with_capacity(to_summarize.len() + 2); compact_msgs.push(summary_prompt); compact_msgs.push(ChatMessage { role: "user".to_string(), content: Some(MessageContent::Text(format!( - "Summarize the following conversation portion ({} messages) β€” emergency compact:", + "Summarize the following conversation portion ({} messages):", to_summarize.len(), ))), tool_calls: None, @@ -1430,7 +1415,7 @@ impl Agent { let summary_response = match llm.chat(&compact_msgs, &[]).await { Ok(c) => c, - Err(e) => anyhow::bail!("Reactive compact LLM call failed: {}", e), + Err(e) => anyhow::bail!("{} LLM call failed: {}", error_label, e), }; let summary_text = summary_response @@ -1440,21 +1425,21 @@ impl Agent { .unwrap_or_default(); if summary_text.is_empty() { - anyhow::bail!("Reactive compact returned empty summary"); + anyhow::bail!("{} returned empty summary", error_label); } let boundary = build_compact_boundary_marker(to_summarize.len(), 1); let summary_msg = ChatMessage { role: "system".to_string(), content: Some(MessageContent::Text(format!( - "β˜… COMPACT SUMMARY (EMERGENCY) β˜…\n\n{}", - summary_text + "{}\n\n{}", + summary_label, summary_text ))), tool_calls: None, tool_call_id: None, }; - let mut result = Vec::with_capacity(3 + preserved.len()); + let mut result: Vec = Vec::with_capacity(3 + preserved.len()); result.push(boundary); result.push(summary_msg); result.extend(preserved.iter().cloned()); diff --git a/src/agent_prompt.rs b/src/agent_prompt.rs index bf60bf7..57267ec 100644 --- a/src/agent_prompt.rs +++ b/src/agent_prompt.rs @@ -16,7 +16,9 @@ const OBSERVATION_MASK_PCT: f64 = 0.20; const COLLAPSE_PCT: f64 = 0.60; /// Percentage that triggers Tier 3 (auto compact). pub const COMPACT_PCT: f64 = 0.85; -/// Percentage that triggers Tier 4 (reactive compact). +/// Documentary threshold β€” Tier 4 is triggered by HTTP 413 errors, +/// not by a percentage, but this documents the utilization level at +/// which a 413 would typically occur. #[allow(dead_code)] pub(crate) const REACTIVE_PCT: f64 = 0.95; /// Minimum turns between Tier 3/4 compactions. @@ -274,17 +276,19 @@ pub fn collapse_context(messages: &[ChatMessage], context_window: usize) -> Vec< // If boundary still not inserted and messages were removed if !inserted_boundary && messages.len() > result.len() { - result.insert( - 1, - ChatMessage { - role: "system".to_string(), - content: Some(MessageContent::Text( - "β˜… earlier conversation collapsed β˜…".to_string(), - )), - tool_calls: None, - tool_call_id: None, - }, - ); + let boundary_msg = ChatMessage { + role: "system".to_string(), + content: Some(MessageContent::Text( + "β˜… earlier conversation collapsed β˜…".to_string(), + )), + tool_calls: None, + tool_call_id: None, + }; + if result.is_empty() { + result.push(boundary_msg); + } else { + result.insert(1, boundary_msg); + } } result From 6128af767218bdc5ec1024a4bea90f8ae014ad53 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Tue, 30 Jun 2026 09:01:48 +0800 Subject: [PATCH 19/19] fmt: remove extra blank line before default_context_window --- src/provider.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/provider.rs b/src/provider.rs index 7473674..193baf0 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -128,7 +128,6 @@ impl ProviderRegistry { format!("{}/{}", self.default_provider, provider.default_model()) } - /// Return the context window size of the default provider. pub fn default_context_window(&self) -> usize { let provider = &self.providers[&self.default_provider];