From 4bad1def54ea1df049e0347583638ecb8e970605 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Thu, 18 Jun 2026 00:19:43 +0800 Subject: [PATCH 01/10] docs: improve self-upgrade/model-switching sections + multi-session brainstorming roadmap --- README.md | 33 ++++++++- docs/roadmap/multi-session-multi-model.md | 86 +++++++++++++++++++++++ 2 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 docs/roadmap/multi-session-multi-model.md diff --git a/README.md b/README.md index 4505d90..d87a01c 100644 --- a/README.md +++ b/README.md @@ -40,10 +40,37 @@ Star the repo ⭐, fork to contribute, or open an issue for feedback. β†’ Full feature reference: [docs/GUIDE.md](docs/GUIDE.md#advanced-features) +### Agent Lifecycle & Runtime + +| Capability | Description | +|------------|-------------| +| **Self-Upgrade** | Trigger an in-place upgrade: pulls from git source or downloads the latest GitHub release binary. Auto-restarts after upgrade β€” no SSH, no manual steps. | +| **Model Switching** | Switch OpenRouter models at runtime via `/models`. Interactive picker lets you choose the best model per task: fast/cheap for simple queries, powerful for complex reasoning. | +| **Soul Files** | SOUL.md (persona), AGENTS.md (behaviour), USER.md (preferences) β€” persistent identity files auto-injected into every system prompt. Session-end self-reflection with `.bak` backups. | + --- -| πŸš€ **Self-Upgrade** | `/self-upgrade` slash command + `self_upgrade` tool, auto-restart after upgrade, git source or GitHub release binary | -| πŸ”€ **Model Switching** | `/models` slash command to switch OpenRouter LLM models at runtime, with interactive model picker | -| πŸ“ **Soul Files** | SOUL.md / AGENTS.md / USER.md persistent identity files, auto-injected into system prompt, session-end reflection with `.bak` backups | + +### πŸ’­ Multi-Session & Multi-Model (Brainstorming) + +> ⚠️ **Planning phase** β€” not yet implemented. This section captures ideas explored in the `feat/readme-improve-multi-session-brainstorm` branch. + +The vision: run multiple concurrent chat sessions, each with its own model and isolated context. + +| Use Case | Description | +|----------|-------------| +| **Parallel execution** | Run a cheap model for quick tasks while a powerful model tackles deep analysis β€” concurrently, not sequentially | +| **Per-user isolation** | Each Telegram user gets their own session with independent conversation context and model preference | +| **Sub-agent delegation** | Spawn sub-agents with different models (e.g., GPT-4o for code review, Claude for writing) without polluting the main session | + +**Topics to explore:** + +- Session lifecycle β€” create, switch, merge, archive +- Per-session model binding vs global default model +- Context isolation between sessions (independent or shared RAG?) +- Telegram UX for multi-session management (inline buttons? slash commands?) +- Persistence and RAG across session boundaries + +See [docs/roadmap/multi-session.md](docs/roadmap/multi-session.md) for detailed design notes. ## Quick Start diff --git a/docs/roadmap/multi-session-multi-model.md b/docs/roadmap/multi-session-multi-model.md new file mode 100644 index 0000000..376982c --- /dev/null +++ b/docs/roadmap/multi-session-multi-model.md @@ -0,0 +1,86 @@ +# Multi-Session & Multi-Model β€” Brainstorming Roadmap + +> ⚠️ **Planning phase** β€” not yet implemented. This document captures the vision, design constraints, and open questions. + +## Motivation + +Today RustFox runs a **single chat session** with **one model at a time**. You `/models` to switch, and the whole bot changes personality. + +But power users want: + +1. **Multiple concurrent sessions** β€” work on 3 different projects in 3 different chats, each with independent context. +2. **Different models per session** β€” code review chat uses Claude Opus, casual chat uses Haiku. +3. **Agent-to-agent delegation** (already partially solved) β€” one session can ask another session's model to do something. + +## Core Concepts + +### Session = Chat Thread + +Each Telegram chat (DM or group) is a **session**. Sessions are isolated: +- Each has its own conversation history (context window) +- Each can be assigned a different model +- Each can have different system prompts / soul files + +### Session Model Binding + +| Concept | Description | +|---------|-------------| +| **Default model** | The model used for new sessions (from config) | +| **Session model** | Override for a specific session via `/models` | +| **Session stickiness** | Model choice persists across bot restarts | + +### Context Isolation + +| Aspect | Shared | Per-Session | +|--------|--------|-------------| +| Soul files (SOUL.md, AGENTS.md, USER.md) | βœ… | ❌ | +| Conversation history | ❌ | βœ… | +| Scheduled tasks | βœ… | ❌ (scheduler is global) | +| Active model | ❌ | βœ… | +| Plans | ❌ | βœ… (per-session plan tracking) | +| Skills | βœ… | ❌ | + +## Potential Approach + +### Option A: Stateless Sessions + +- Session state lives entirely in Telegram (message history replayed on context window overflow) +- No per-session persistence β€” just model binding +- **Pros**: Simple, no new infrastructure +- **Cons**: Context window limited to recent messages, slow startup replay + +### Option B: Lightweight Session Store + +- SQLite table `sessions (chat_id, model_name, context_summary, last_active)` +- On each new message, load session state, inject context summary, respond +- **Pros**: Context summaries survive restarts, faster than full replay +- **Cons**: New persistence complexity, summary drift + +### Option C: Full Context Persistence + +- Store full conversation vector embeddings + message history in SQLite +- RAG-style retrieval at the start of each turn +- **Pros**: Maximum context recall +- **Cons**: Heavy infrastructure, token cost, complexity + +## Open Questions + +1. **Session lifecycle**: When does a session "expire"? After inactivity? Never? +2. **Cross-session memory**: Should sessions share learned facts (RLHF-style)? +3. **SQLite schema**: How does session state compose with the existing learning/scheduler tables? +4. **Cost tracking**: Per-session token usage & cost reporting? +5. **Billing model**: Some users may want per-session billing (e.g., `gpt-4` costs more than `haiku`). +6. **Migration path**: Users upgrading from single-model shouldn't lose their config. + +## Related Work + +- `invoke_agent` / `spawn_agents` β€” already supports subagent isolation +- `/models` command β€” already supports runtime model switching +- Soul files β€” already support persistent identity + +## Next Steps + +1. Choose an approach (A, B, or hybrid) +2. Design SQLite schema if needed +3. Scope implementation into trackable issues +4. Plan migration path for existing users \ No newline at end of file From 0c51d9cb010a8d9a989b7f02646a4fa667f0f94a Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 16:32:22 +0800 Subject: [PATCH 02/10] feat: add provider and fallback config types --- src/config.rs | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/config.rs b/src/config.rs index 424e295..34f5fe0 100644 --- a/src/config.rs +++ b/src/config.rs @@ -31,6 +31,15 @@ pub struct Config { pub supervisor: SupervisorConfig, #[serde(default)] pub subagents: SubagentsConfig, + /// Explicit provider sections (multi-provider mode). Optional β€” + /// when empty, `build_providers()` synthesizes a single OpenRouter + /// provider from the legacy `[openrouter]` section. + #[serde(default)] + pub provider: Vec, + /// Fallback chain β€” additional provider/model names tried when + /// the primary call fails. + #[serde(default)] + pub fallback: FallbackConfig, /// Absolute home root resolved at load time (not read from TOML). #[serde(skip)] pub resolved_home: Option, @@ -125,6 +134,46 @@ pub struct OpenRouterConfig { pub supports_vision: bool, } +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub enum ProviderType { + #[serde(rename = "openrouter")] + OpenRouter, + #[serde(rename = "openai_compatible")] + OpenAICompatible, + #[serde(rename = "ollama")] + Ollama, +} + +#[allow(clippy::derivable_impls)] +impl Default for ProviderType { + fn default() -> Self { + Self::OpenRouter + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ProviderSection { + pub name: String, + #[serde(rename = "type")] + pub provider_type: ProviderType, + pub base_url: String, + pub api_key: Option, + #[serde(default = "default_model")] + pub model: String, + #[serde(default)] + pub supports_vision: bool, + #[serde(default = "default_max_tokens")] + pub max_tokens: u32, + #[serde(default)] + pub discover_models: bool, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct FallbackConfig { + #[serde(default)] + pub chain: Vec, +} + #[derive(Debug, Deserialize, Clone)] pub struct OcrConfig { /// Directory where OCR model files are cached. @@ -540,6 +589,40 @@ impl Config { Ok(config) } + + /// Build the provider list from config, handling legacy [openrouter] backward compat. + /// Returns (providers, default_provider_name, fallback_chain). + pub fn build_providers(&self) -> (Vec, String, Vec) { + let mut providers: Vec = self.provider.clone(); + + // Backward compat: if [openrouter] section exists and no explicit provider named "openrouter" + let has_openrouter = providers.iter().any(|p| p.name == "openrouter"); + if has_openrouter { + tracing::warn!( + "Both [[provider]] name=\"openrouter\" and [openrouter] configured β€” explicit [[provider]] entry takes precedence" + ); + } else { + providers.push(ProviderSection { + name: "openrouter".to_string(), + provider_type: ProviderType::OpenRouter, + base_url: self.openrouter.base_url.clone(), + api_key: Some(self.openrouter.api_key.clone()), + model: self.openrouter.model.clone(), + supports_vision: self.openrouter.supports_vision, + max_tokens: self.openrouter.max_tokens, + discover_models: false, + }); + } + + let default = if providers.is_empty() { + "openrouter".to_string() + } else { + providers[0].name.clone() + }; + + let fallback = self.fallback.chain.clone(); + (providers, default, fallback) + } } #[cfg(test)] From e6276dc748d1c381773601fc9f1d75cb42f5a38f Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 16:41:18 +0800 Subject: [PATCH 03/10] feat: add Provider trait, ProviderRegistry, OpenRouter/OpenAICompatible/Ollama providers --- src/lib.rs | 1 + src/llm.rs | 31 ++- src/provider.rs | 694 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 713 insertions(+), 13 deletions(-) create mode 100644 src/provider.rs diff --git a/src/lib.rs b/src/lib.rs index 8d87714..8c1eac2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ pub mod llm; pub mod mcp; pub mod memory; pub mod platform; +pub mod provider; pub mod scheduler; pub mod setup; pub mod skills; diff --git a/src/llm.rs b/src/llm.rs index 5d55327..4651210 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -130,26 +130,31 @@ pub fn is_empty_assistant_response(message: &ChatMessage) -> bool { } #[derive(Debug, Serialize)] -struct ChatRequest { - model: String, - messages: Vec, +pub struct ChatRequest { + pub model: String, + pub messages: Vec, #[serde(skip_serializing_if = "Option::is_none")] - tools: Option>, + pub tools: Option>, #[serde(skip_serializing_if = "Option::is_none")] - tool_choice: Option, - max_tokens: u32, + pub tool_choice: Option, + pub max_tokens: u32, } #[derive(Debug, Deserialize)] -struct ChatResponse { - choices: Vec, +pub struct ChatResponse { + pub choices: Vec, } #[derive(Debug, Deserialize)] -struct Choice { - message: ChatMessage, +pub struct Choice { + pub message: ChatMessage, #[serde(default)] - finish_reason: Option, + pub finish_reason: Option, +} + +#[doc(hidden)] +pub mod internal { + pub use super::{ChatRequest, ChatResponse, Choice}; } /// Sanitize a JSON Schema parameter object so it is accepted by strict providers @@ -170,7 +175,7 @@ struct Choice { /// /// This function mutates the schema in-place and recurses into `properties`, /// `items`, `anyOf`, `oneOf`, and `allOf` sub-schemas. -fn sanitize_parameters(schema: &mut serde_json::Value) { +pub fn sanitize_parameters(schema: &mut serde_json::Value) { let obj = match schema.as_object_mut() { Some(o) => o, None => return, @@ -276,7 +281,7 @@ fn sanitize_parameters(schema: &mut serde_json::Value) { /// /// Returns `Some(Vec)` with at least one entry when the format is /// detected, or `None` if the content does not contain the Kimi markers. -fn parse_kimi_tool_calls(content: &str) -> Option> { +pub fn parse_kimi_tool_calls(content: &str) -> Option> { if !content.contains("<|tool_calls_section_begin|>") { return None; } diff --git a/src/provider.rs b/src/provider.rs new file mode 100644 index 0000000..cf53e18 --- /dev/null +++ b/src/provider.rs @@ -0,0 +1,694 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use async_trait::async_trait; + +use crate::config::{ProviderSection, ProviderType}; +use crate::llm::{ChatCompletion, ChatMessage, ToolDefinition}; + +/// Runtime configuration for a single LLM provider. +/// +/// This is the "live" view of a provider built from a [`ProviderSection`] in +/// the config file. The `From<&ProviderSection>` conversion is the canonical +/// way to construct one. +#[derive(Debug, Clone)] +pub struct ProviderConfig { + pub name: String, + pub provider_type: ProviderType, + pub base_url: String, + pub api_key: Option, + pub default_model: String, + pub supports_vision: bool, + pub max_tokens: u32, + pub discover_models: bool, +} + +impl From<&ProviderSection> for ProviderConfig { + fn from(s: &ProviderSection) -> Self { + Self { + name: s.name.clone(), + provider_type: s.provider_type.clone(), + base_url: s.base_url.clone(), + api_key: s.api_key.clone(), + default_model: s.model.clone(), + supports_vision: s.supports_vision, + max_tokens: s.max_tokens, + discover_models: s.discover_models, + } + } +} + +/// Unified LLM provider abstraction. +/// +/// Each implementation owns its own HTTP shaping (URL, headers, model list +/// endpoint) and delegates the request/response body shapes to the shared +/// types in [`crate::llm`]. +#[async_trait] +pub trait Provider: Send + Sync { + fn name(&self) -> &str; + fn default_model(&self) -> &str; + fn supports_vision(&self) -> bool; + fn config(&self) -> &ProviderConfig; + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result; + + async fn list_models(&self, client: &reqwest::Client) -> Result>; +} + +/// Holds the set of providers available to the agent and the default provider +/// name used when a model string does not carry an explicit `provider/` prefix. +pub struct ProviderRegistry { + providers: HashMap>, + default_provider: String, +} + +impl std::fmt::Debug for ProviderRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProviderRegistry") + .field("providers", &self.providers.keys().collect::>()) + .field("default_provider", &self.default_provider) + .finish() + } +} + +impl ProviderRegistry { + pub fn new(providers: HashMap>, default_provider: String) -> Self { + Self { + providers, + default_provider, + } + } + + /// Resolve a model string to (provider, stripped_model). + /// Never fails β€” unknown prefixes fall through to default provider. + pub fn resolve_model<'a>(&'a self, model: &'a str) -> (&'a dyn Provider, &'a str) { + if let Some((prefix, rest)) = model.split_once('/') { + if let Some(provider) = self.providers.get(prefix) { + return (provider.as_ref(), rest); + } + } + // Fall through to default provider with full string + let default = &self.providers[&self.default_provider]; + (default.as_ref(), model) + } + + pub fn get_provider(&self, name: &str) -> Option<&dyn Provider> { + self.providers.get(name).map(|p| p.as_ref()) + } + + pub fn providers(&self) -> impl Iterator { + self.providers.values().map(|p| p.as_ref()) + } + + pub fn default_provider_name(&self) -> &str { + &self.default_provider + } + + pub fn provider_count(&self) -> usize { + self.providers.len() + } + + pub fn provider_names(&self) -> Vec { + self.providers.keys().cloned().collect() + } +} + +/// Build a ProviderRegistry from config sections. +pub fn build_registry( + sections: &[ProviderSection], + default_name: &str, +) -> Result { + let mut providers: HashMap> = HashMap::new(); + + for section in sections { + let cfg: ProviderConfig = ProviderConfig::from(section); + let provider: Arc = match section.provider_type { + ProviderType::OpenRouter => Arc::new(OpenRouterProvider::new(cfg)), + ProviderType::OpenAICompatible => Arc::new(OpenAICompatibleProvider::new(cfg)), + ProviderType::Ollama => Arc::new(OllamaProvider::new(cfg)), + }; + providers.insert(section.name.clone(), provider); + } + + if providers.is_empty() { + anyhow::bail!("No LLM providers configured"); + } + + if !providers.contains_key(default_name) { + anyhow::bail!( + "Default provider '{}' not found in configured providers", + default_name + ); + } + + Ok(ProviderRegistry::new(providers, default_name.to_string())) +} + +// === OpenRouterProvider === + +pub struct OpenRouterProvider { + config: ProviderConfig, +} + +impl OpenRouterProvider { + pub fn new(config: ProviderConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl Provider for OpenRouterProvider { + fn name(&self) -> &str { + &self.config.name + } + fn default_model(&self) -> &str { + &self.config.default_model + } + fn supports_vision(&self) -> bool { + self.config.supports_vision + } + fn config(&self) -> &ProviderConfig { + &self.config + } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + let tools_param = if tools.is_empty() { + None + } else { + let sanitized: Vec = tools + .iter() + .map(|t| { + let mut t = t.clone(); + crate::llm::sanitize_parameters(&mut t.function.parameters); + t + }) + .collect(); + Some(sanitized) + }; + + let request = crate::llm::internal::ChatRequest { + model: model.to_string(), + messages: messages.to_vec(), + tools: tools_param, + tool_choice: None, + max_tokens, + }; + + let url = format!("{}/chat/completions", self.config.base_url); + let mut req = client.post(&url).json(&request); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + + let response = req.send().await.context("Failed to send request to OpenRouter")?; + + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("OpenRouter API error ({}): {}", status, body); + } + + let chat_response: crate::llm::internal::ChatResponse = response + .json() + .await + .context("Failed to parse OpenRouter response")?; + + let mut choice = chat_response + .choices + .into_iter() + .next() + .context("No response from OpenRouter")?; + + // Kimi tool-call fallback + let has_tool_calls = choice + .message + .tool_calls + .as_ref() + .is_some_and(|t| !t.is_empty()); + if !has_tool_calls { + if let Some(ref content) = choice.message.content { + if let Some(parsed) = crate::llm::parse_kimi_tool_calls(&content.as_text()) { + choice.message.tool_calls = Some(parsed); + choice.message.content = None; + } + } + } + + Ok(ChatCompletion { + message: choice.message, + finish_reason: choice.finish_reason, + model: model.to_string(), + }) + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = format!("{}/models", self.config.base_url); + let mut req = client.get(&url); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + let response = req.send().await.context("Failed to fetch models from OpenRouter")?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("OpenRouter models API error ({}): {}", status, body); + } + let list: serde_json::Value = response.json().await?; + let models = list["data"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|m| m["id"].as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + Ok(models) + } +} + +// === OpenAICompatibleProvider === + +pub struct OpenAICompatibleProvider { + config: ProviderConfig, +} + +impl OpenAICompatibleProvider { + pub fn new(config: ProviderConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl Provider for OpenAICompatibleProvider { + fn name(&self) -> &str { + &self.config.name + } + fn default_model(&self) -> &str { + &self.config.default_model + } + fn supports_vision(&self) -> bool { + self.config.supports_vision + } + fn config(&self) -> &ProviderConfig { + &self.config + } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + let tools_param = if tools.is_empty() { + None + } else { + Some(tools.to_vec()) + }; + + let request = crate::llm::internal::ChatRequest { + model: model.to_string(), + messages: messages.to_vec(), + tools: tools_param, + tool_choice: None, + max_tokens, + }; + + let url = format!("{}/chat/completions", self.config.base_url); + let mut req = client.post(&url).json(&request); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + + let response = req + .send() + .await + .context("Failed to send request to OpenAI-compatible provider")?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!( + "Provider '{}' error ({}): {}", + self.config.name, + status, + body + ); + } + + let chat_response: crate::llm::internal::ChatResponse = response + .json() + .await + .context("Failed to parse OpenAI-compatible provider response")?; + let choice = + chat_response.choices.into_iter().next().ok_or_else(|| { + anyhow::anyhow!("No response from provider '{}'", self.config.name) + })?; + + Ok(ChatCompletion { + message: choice.message, + finish_reason: choice.finish_reason, + model: model.to_string(), + }) + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = format!("{}/models", self.config.base_url); + let mut req = client.get(&url); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + let response = req + .send() + .await + .context("Failed to fetch models from OpenAI-compatible provider")?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!( + "Provider '{}' models API error ({}): {}", + self.config.name, + status, + body + ); + } + let list: serde_json::Value = response.json().await?; + let models = list["data"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|m| m["id"].as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + Ok(models) + } +} + +// === OllamaProvider === + +pub struct OllamaProvider { + config: ProviderConfig, + inner: OpenAICompatibleProvider, +} + +impl OllamaProvider { + pub fn new(config: ProviderConfig) -> Self { + let inner_cfg = config.clone(); + let inner = OpenAICompatibleProvider::new(inner_cfg); + Self { config, inner } + } + + /// Compute the native Ollama `/api/tags` model-discovery URL. + /// + /// Ollama's native endpoint lives at `/api/tags` on the server root + /// (not under `/v1`). Strip any common version suffix from the configured + /// base URL before appending the tags path. + fn discovery_url(&self) -> String { + let base = self.config.base_url.trim_end_matches('/'); + let base = base + .strip_suffix("/v1") + .or_else(|| base.strip_suffix("/v2")) + .or_else(|| base.strip_suffix("/v3")) + .unwrap_or(base); + format!("{}/api/tags", base.trim_end_matches('/')) + } +} + +#[async_trait] +impl Provider for OllamaProvider { + fn name(&self) -> &str { + &self.config.name + } + fn default_model(&self) -> &str { + &self.config.default_model + } + fn supports_vision(&self) -> bool { + self.config.supports_vision + } + fn config(&self) -> &ProviderConfig { + &self.config + } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + self.inner + .chat_completion(client, messages, tools, model, max_tokens) + .await + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = self.discovery_url(); + let mut req = client.get(&url); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + let response = req + .send() + .await + .context("Failed to fetch models from Ollama")?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("Ollama models API error ({}): {}", status, body); + } + let body: serde_json::Value = response.json().await?; + let models = body["models"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|m| m["name"].as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + Ok(models) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::ProviderSection; + + fn make_section( + name: &str, + provider_type: ProviderType, + base_url: &str, + model: &str, + ) -> ProviderSection { + ProviderSection { + name: name.to_string(), + provider_type, + base_url: base_url.to_string(), + api_key: Some("test-key".to_string()), + model: model.to_string(), + supports_vision: false, + max_tokens: 1024, + discover_models: false, + } + } + + #[test] + fn provider_config_from_section_copies_all_fields() { + let section = make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + ); + let cfg = ProviderConfig::from(§ion); + assert_eq!(cfg.name, "alpha"); + assert_eq!(cfg.provider_type, ProviderType::OpenRouter); + assert_eq!(cfg.base_url, "https://openrouter.ai/api/v1"); + assert_eq!(cfg.api_key.as_deref(), Some("test-key")); + assert_eq!(cfg.default_model, "anthropic/claude-sonnet-4-6"); + assert_eq!(cfg.max_tokens, 1024); + } + + #[test] + fn build_registry_creates_one_provider_per_section() { + let sections = vec![ + make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + ), + make_section( + "beta", + ProviderType::Ollama, + "http://localhost:11434/v1", + "llama3.1", + ), + ]; + let reg = build_registry(§ions, "alpha").unwrap(); + assert_eq!(reg.provider_count(), 2); + assert!(reg.get_provider("alpha").is_some()); + assert!(reg.get_provider("beta").is_some()); + assert_eq!(reg.default_provider_name(), "alpha"); + } + + #[test] + fn build_registry_fails_with_no_providers() { + let result = build_registry(&[], "alpha"); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("No LLM providers")); + } + + #[test] + fn build_registry_fails_when_default_missing() { + let sections = vec![make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + )]; + let result = build_registry(§ions, "missing"); + assert!(result.is_err()); + assert!(result + .unwrap_err() + .to_string() + .contains("Default provider 'missing'")); + } + + #[test] + fn resolve_model_uses_prefix_when_provider_known() { + let sections = vec![ + make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + ), + make_section( + "beta", + ProviderType::Ollama, + "http://localhost:11434/v1", + "llama3.1", + ), + ]; + let reg = build_registry(§ions, "alpha").unwrap(); + let (provider, model) = reg.resolve_model("beta/llama3.1:8b"); + assert_eq!(provider.name(), "beta"); + assert_eq!(model, "llama3.1:8b"); + } + + #[test] + fn resolve_model_falls_back_to_default_on_unknown_prefix() { + let sections = vec![make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + )]; + let reg = build_registry(§ions, "alpha").unwrap(); + let (provider, model) = reg.resolve_model("moonshotai/kimi-k2.5"); + assert_eq!(provider.name(), "alpha"); + assert_eq!(model, "moonshotai/kimi-k2.5"); + } + + #[test] + fn resolve_model_uses_default_for_bare_model() { + let sections = vec![make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + )]; + let reg = build_registry(§ions, "alpha").unwrap(); + let (provider, model) = reg.resolve_model("claude-sonnet-4-6"); + assert_eq!(provider.name(), "alpha"); + assert_eq!(model, "claude-sonnet-4-6"); + } + + #[test] + fn provider_names_returns_all_registered_names() { + let sections = vec![ + make_section( + "alpha", + ProviderType::OpenRouter, + "https://openrouter.ai/api/v1", + "anthropic/claude-sonnet-4-6", + ), + make_section( + "beta", + ProviderType::Ollama, + "http://localhost:11434/v1", + "llama3.1", + ), + ]; + let reg = build_registry(§ions, "alpha").unwrap(); + let mut names = reg.provider_names(); + names.sort(); + assert_eq!(names, vec!["alpha".to_string(), "beta".to_string()]); + } + + #[test] + fn ollama_discovery_url_strips_v1_suffix() { + let cfg = ProviderConfig { + name: "local".to_string(), + provider_type: ProviderType::Ollama, + base_url: "http://localhost:11434/v1".to_string(), + api_key: None, + default_model: "llama3.1".to_string(), + supports_vision: false, + max_tokens: 1024, + discover_models: true, + }; + let p = OllamaProvider::new(cfg); + assert_eq!(p.discovery_url(), "http://localhost:11434/api/tags"); + } + + #[test] + fn ollama_discovery_url_strips_trailing_slash() { + let cfg = ProviderConfig { + name: "local".to_string(), + provider_type: ProviderType::Ollama, + base_url: "http://localhost:11434/".to_string(), + api_key: None, + default_model: "llama3.1".to_string(), + supports_vision: false, + max_tokens: 1024, + discover_models: true, + }; + let p = OllamaProvider::new(cfg); + assert_eq!(p.discovery_url(), "http://localhost:11434/api/tags"); + } + + #[test] + fn ollama_discovery_url_appends_to_bare_host() { + let cfg = ProviderConfig { + name: "local".to_string(), + provider_type: ProviderType::Ollama, + base_url: "http://localhost:11434".to_string(), + api_key: None, + default_model: "llama3.1".to_string(), + supports_vision: false, + max_tokens: 1024, + discover_models: true, + }; + let p = OllamaProvider::new(cfg); + assert_eq!(p.discovery_url(), "http://localhost:11434/api/tags"); + } +} From 94c5935f217ef6c007bb7844554469c6e16095be Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 16:58:11 +0800 Subject: [PATCH 04/10] refactor: wire ProviderRegistry into LlmClient, Agent, and main.rs with fallback chain --- src/agent.rs | 137 +++++++++++++++++----- src/file_processor/mod.rs | 3 +- src/llm.rs | 236 ++++---------------------------------- src/main.rs | 17 ++- src/platform/telegram.rs | 113 ++---------------- src/provider.rs | 12 +- 6 files changed, 170 insertions(+), 348 deletions(-) diff --git a/src/agent.rs b/src/agent.rs index 483bf28..f1f1e63 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -39,6 +39,7 @@ pub struct ScheduledJobRequest { /// Platform-agnostic β€” receives IncomingMessage, returns response text. pub struct Agent { pub llm: LlmClient, + pub registry: Arc, pub config: Config, pub mcp: McpManager, pub memory: MemoryStore, @@ -100,6 +101,7 @@ impl Agent { #[allow(clippy::too_many_arguments)] pub fn new( config: Config, + registry: Arc, mcp: McpManager, memory: MemoryStore, skills: SkillRegistry, @@ -112,10 +114,18 @@ impl Agent { langsmith: Arc, config_path: PathBuf, ) -> Self { - let llm = LlmClient::new(config.openrouter.clone()); - let initial_model = config.openrouter.model.clone(); + let llm = LlmClient::new(registry.clone()); + let initial_model = format!( + "{}/{}", + registry.default_provider_name(), + registry + .resolve_model(registry.default_provider_name()) + .0 + .default_model() + ); Self { llm, + registry, config, mcp, memory, @@ -325,32 +335,53 @@ impl Agent { anyhow::bail!("Model ID cannot be empty"); } - let content = tokio::fs::read_to_string(&self.config_path) - .await - .context("Failed to read config.toml")?; - let mut doc: toml::value::Table = - toml::from_str(&content).context("Failed to parse config.toml")?; - - let openrouter_entry = doc - .entry("openrouter") - .or_insert_with(|| toml::Value::Table(toml::value::Table::new())); - if let Some(table) = openrouter_entry.as_table_mut() { - table.insert( - "model".to_string(), - toml::Value::String(model_id.to_string()), - ); + // Validate: resolve succeeds for any string + let (provider, actual_model) = self.registry.resolve_model(model_id); + if let Some((prefix, _)) = model_id.split_once('/') { + if self.registry.get_provider(prefix).is_none() { + tracing::warn!( + "Model '{}': prefix '{}' does not match any known provider \ + (falling through to default '{}')", + model_id, + prefix, + self.registry.default_provider_name() + ); + } } - let new_content = - toml::to_string_pretty(&doc).context("Failed to serialize config.toml")?; - tokio::fs::write(&self.config_path, &new_content) - .await - .with_context(|| format!("Failed to write {}", self.config_path.display()))?; + let content = tokio::fs::read_to_string(&self.config_path).await?; + let mut doc: toml::value::Table = toml::from_str(&content)?; + + let provider_name = provider.name().to_string(); + + if provider_name == "openrouter" && doc.contains_key("openrouter") { + if let Some(table) = doc.get_mut("openrouter").and_then(|v| v.as_table_mut()) { + table.insert( + "model".to_string(), + toml::Value::String(actual_model.to_string()), + ); + } + } else if let Some(provider_array) = doc.get_mut("provider").and_then(|v| v.as_array_mut()) + { + for entry in provider_array.iter_mut() { + if let Some(table) = entry.as_table_mut() { + if table.get("name").and_then(|v| v.as_str()) == Some(&provider_name) { + table.insert( + "model".to_string(), + toml::Value::String(actual_model.to_string()), + ); + } + } + } + } + + let new_content = toml::to_string_pretty(&doc)?; + tokio::fs::write(&self.config_path, &new_content).await?; let mut current = self.current_model.write().await; *current = model_id.to_string(); - tracing::info!(model = %model_id, "Model changed and persisted"); + tracing::info!(model = %model_id, provider = %provider_name, "Model changed and persisted"); Ok(()) } @@ -494,12 +525,19 @@ impl Agent { } // Process attachments (images β†’ vision parts or OCR text; PDFs/DOCXs β†’ extracted text) + let supports_vision = { + let current = self.current_model.read().await; + let (provider, _) = self.registry.resolve_model(¤t); + provider.supports_vision() + }; + let (attachment_text, image_parts) = if !incoming.attachments.is_empty() { crate::file_processor::process_attachments( &incoming.attachments, &incoming.text, &self.config, &self.memory, + supports_vision, // NEW parameter ) .await } else { @@ -652,12 +690,57 @@ impl Agent { start_time: llm_start, }); - // Call LLM with prepared prompt + // Call LLM with prepared prompt (with fallback chain) let model = self.current_model.read().await.clone(); - let completion_result = self - .llm - .chat_completion_with_model(&prompt.messages, &all_tools, &model) - .await; + let fallback_chain = &self.config.fallback.chain; + let mut last_error = None; + + let completion_result = 'fallback: { + for attempt in 0..=fallback_chain.len() { + let current_model = if attempt == 0 { + model.clone() + } else { + fallback_chain[attempt - 1].clone() + }; + + match self + .llm + .chat_completion_with_model( + &prompt.messages, + &all_tools, + ¤t_model, + ) + .await + { + Ok(c) => { + if attempt > 0 { + tracing::info!( + "Fallback succeeded: switched from '{}' to '{}'", + model, + current_model + ); + *self.current_model.write().await = current_model.clone(); + } + break 'fallback Ok(c); + } + Err(e) => { + tracing::warn!( + "Model '{}' failed (attempt {}/{}): {}", + current_model, + attempt, + fallback_chain.len(), + e + ); + last_error = Some(e); + if attempt == fallback_chain.len() { + break 'fallback Err(last_error.unwrap()); + } + continue; + } + } + } + Err(last_error.unwrap()) + }; // Handle LLM transport/API errors let completion = match completion_result { diff --git a/src/file_processor/mod.rs b/src/file_processor/mod.rs index 1edcd9d..a34bcb8 100644 --- a/src/file_processor/mod.rs +++ b/src/file_processor/mod.rs @@ -27,6 +27,7 @@ pub async fn process_attachments( user_query: &str, config: &Config, memory: &MemoryStore, + supports_vision: bool, ) -> (String, Vec) { let mut text_parts: Vec = Vec::new(); let mut image_parts: Vec = Vec::new(); @@ -37,7 +38,7 @@ pub async fn process_attachments( match process_image( &attachment.path, &attachment.mime_type, - config.openrouter.supports_vision, + supports_vision, &config.ocr.model_dir, ) .await diff --git a/src/llm.rs b/src/llm.rs index 4651210..9177867 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -1,8 +1,8 @@ -use anyhow::{Context, Result}; +use anyhow::Result; use serde::{Deserialize, Serialize}; -use tracing::{debug, warn}; +use tracing::debug; -use crate::config::OpenRouterConfig; +use std::sync::Arc; /// A single part in a multi-modal message #[derive(Debug, Clone, Serialize, Deserialize)] @@ -135,8 +135,6 @@ pub struct ChatRequest { pub messages: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub tools: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub tool_choice: Option, pub max_tokens: u32, } @@ -353,174 +351,49 @@ pub fn parse_kimi_tool_calls(content: &str) -> Option> { #[derive(Clone)] pub struct LlmClient { - client: reqwest::Client, - config: OpenRouterConfig, + pub client: reqwest::Client, + pub registry: Arc, } impl LlmClient { - pub fn new(config: OpenRouterConfig) -> Self { + pub fn new(registry: Arc) -> Self { Self { client: reqwest::Client::new(), - config, + registry, } } /// Core chat method returning full completion metadata (message, finish_reason, model). /// - /// This is the lowest-level method that performs the HTTP request and returns - /// all available metadata from the LLM response. All other chat methods delegate - /// to this one. + /// Resolves the model string through the registry to pick the right provider, + /// then delegates the actual HTTP call to that provider. pub async fn chat_completion_with_model( &self, messages: &[ChatMessage], tools: &[ToolDefinition], model: &str, ) -> Result { - let tools_param = if tools.is_empty() { - None - } else { - let sanitized = tools - .iter() - .map(|t| { - let mut t = t.clone(); - sanitize_parameters(&mut t.function.parameters); - t - }) - .collect(); - Some(sanitized) - }; - - let tool_choice = if tools_param.is_some() { - Some("auto".to_string()) - } else { - None - }; - - let request = ChatRequest { - model: model.to_string(), - messages: messages.to_vec(), - tools: tools_param, - tool_choice, - max_tokens: self.config.max_tokens, - }; - - let url = format!("{}/chat/completions", self.config.base_url); - - debug!( - url = %url, - model = %model, - message_count = messages.len(), - tool_count = tools.len(), - "Sending request to OpenRouter" - ); - - let response = self - .client - .post(&url) - .header("Authorization", format!("Bearer {}", self.config.api_key)) - .header("Content-Type", "application/json") - .json(&request) - .send() - .await - .context("Failed to send request to OpenRouter")?; - - let status = response.status(); - if !status.is_success() { - let error_body = response.text().await.unwrap_or_default(); - anyhow::bail!("OpenRouter API error ({}): {}", status, error_body); - } - - let chat_response: ChatResponse = response - .json() - .await - .context("Failed to parse OpenRouter response")?; - - if let Some(choice) = chat_response.choices.first() { - debug!( - finish_reason = ?choice.finish_reason, - has_content = choice.message.content.is_some(), - tool_call_count = choice.message.tool_calls.as_ref().map_or(0, |t| t.len()), - "Received LLM response" - ); - if choice - .message - .content - .as_ref() - .is_none_or(MessageContent::is_empty) - && choice.message.tool_calls.as_ref().is_none_or(Vec::is_empty) - { - warn!( - finish_reason = ?choice.finish_reason, - "LLM returned no content and no tool calls" - ); - } - } - - let mut choice = chat_response - .choices - .into_iter() - .next() - .context("No response from OpenRouter")?; - - // Kimi-family models occasionally leak their native tool-call syntax into - // the `content` field instead of populating `tool_calls`. Detect and fix. - let has_tool_calls = choice - .message - .tool_calls - .as_ref() - .is_some_and(|t| !t.is_empty()); - if !has_tool_calls { - if let Some(ref content) = choice.message.content.clone() { - if let Some(parsed) = parse_kimi_tool_calls(&content.as_text()) { - warn!( - tool_count = parsed.len(), - "Kimi native tool-call format detected in content β€” \ - extracting tool calls and clearing content" - ); - choice.message.tool_calls = Some(parsed); - choice.message.content = None; - choice.finish_reason = Some("tool_calls".to_string()); - } - } - } - - Ok(ChatCompletion { - message: choice.message, - finish_reason: choice.finish_reason, - model: model.to_string(), - }) - } + let (provider, actual_model) = self.registry.resolve_model(model); + let max_tokens = provider.config().max_tokens; - /// Compatibility wrapper returning only the message (delegates to chat_completion_with_model). - pub async fn chat_with_model( - &self, - messages: &[ChatMessage], - tools: &[ToolDefinition], - model: &str, - ) -> Result { - Ok(self - .chat_completion_with_model(messages, tools, model) - .await? - .message) - } + let mut completion = provider + .chat_completion(&self.client, messages, tools, actual_model, max_tokens) + .await?; - /// Chat using an explicit model, returning full completion metadata. - pub async fn chat_completion( - &self, - messages: &[ChatMessage], - tools: &[ToolDefinition], - ) -> Result { - self.chat_completion_with_model(messages, tools, &self.config.model) - .await + completion.model = model.to_string(); + Ok(completion) } - /// Chat using the model configured in config.toml (delegates to chat_completion). + /// Convenience wrapper that uses the default provider's model. pub async fn chat( &self, messages: &[ChatMessage], tools: &[ToolDefinition], ) -> Result { - Ok(self.chat_completion(messages, tools).await?.message) + let default_model = self.registry.default_provider_name().to_string(); + self.chat_completion_with_model(messages, tools, &default_model) + .await + .map(|c| c.message) } /// Stream an already-complete `content` string through a token channel in small chunks. @@ -557,68 +430,6 @@ impl LlmClient { } /// Information about an OpenRouter model, deserialized from GET /api/v1/models. -#[derive(Debug, Clone, serde::Deserialize)] -pub struct ModelInfo { - pub id: String, - pub name: String, - #[serde(default)] - pub description: String, - #[serde(default)] - pub context_length: u64, - #[serde(default)] - pub pricing: ModelPricing, - #[serde(default)] - pub architecture: ModelArchitecture, -} - -#[derive(Debug, Clone, serde::Deserialize, Default)] -pub struct ModelPricing { - #[serde(default)] - pub prompt: String, - #[serde(default)] - pub completion: String, -} - -#[derive(Debug, Clone, serde::Deserialize, Default)] -pub struct ModelArchitecture { - #[serde(default)] - pub modality: String, -} - -#[derive(Debug, serde::Deserialize)] -struct ModelsListResponse { - data: Vec, -} - -impl LlmClient { - /// Fetch the list of available models from OpenRouter. - /// The endpoint is public (no auth required for GET /api/v1/models), - /// but we send the API key in case of rate-limiting. - pub async fn fetch_models(&self) -> anyhow::Result> { - let url = format!("{}/models", self.config.base_url); - let response = self - .client - .get(&url) - .header("Authorization", format!("Bearer {}", self.config.api_key)) - .send() - .await - .context("Failed to fetch model list from OpenRouter")?; - - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("OpenRouter models API error ({}): {}", status, body); - } - - let list: ModelsListResponse = response - .json() - .await - .context("Failed to parse OpenRouter model list response")?; - - Ok(list.data) - } -} - #[cfg(test)] mod tests { use super::*; @@ -663,7 +474,6 @@ mod tests { model: "anthropic/claude-sonnet-4-6".to_string(), messages: vec![], tools: None, - tool_choice: None, max_tokens: 100, }; let json = serde_json::to_value(&req).unwrap(); @@ -672,19 +482,17 @@ mod tests { #[test] fn test_chat_request_default_model_is_different_from_override() { - // Ensures chat_with_model can use a different model than the config default + // Ensures chat_completion_with_model can use a different model than the config default let default_req = ChatRequest { model: "moonshotai/kimi-k2.5".to_string(), messages: vec![], tools: None, - tool_choice: None, max_tokens: 100, }; let override_req = ChatRequest { model: "anthropic/claude-sonnet-4-6".to_string(), messages: vec![], tools: None, - tool_choice: None, max_tokens: 100, }; let json_default = serde_json::to_value(&default_req).unwrap(); diff --git a/src/main.rs b/src/main.rs index ee3eedd..e7c95da 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ use rustfox::config::Config; use rustfox::mcp::McpManager; use rustfox::memory::MemoryStore; use rustfox::platform; +use rustfox::provider; use rustfox::scheduler::tasks::register_builtin_tasks; use rustfox::scheduler::Scheduler; use rustfox::setup; @@ -61,6 +62,19 @@ async fn main() -> Result<()> { let config = Config::load(&config_path) .with_context(|| format!("Failed to load config from {}", config_path.display()))?; + // Build provider registry from config + let (provider_sections, default_provider, fallback_chain) = config.build_providers(); + let registry = Arc::new( + provider::build_registry(&provider_sections, &default_provider) + .context("Failed to build LLM provider registry")?, + ); + info!( + " Providers: {} (default: {}, fallback: {} model(s))", + registry.provider_count(), + registry.default_provider_name(), + fallback_chain.len() + ); + info!("Configuration loaded successfully"); info!(" Model: {}", config.openrouter.model); info!(" Sandbox: {}", config.sandbox.allowed_directory.display()); @@ -164,6 +178,7 @@ async fn main() -> Result<()> { let agent = Arc::new_cyclic(|weak| { Agent::new( config.clone(), + registry.clone(), mcp_manager, memory.clone(), skills, @@ -257,7 +272,7 @@ async fn main() -> Result<()> { register_builtin_tasks( &scheduler, memory.clone(), - rustfox::llm::LlmClient::new(config.openrouter.clone()), + rustfox::llm::LlmClient::new(registry.clone()), config.memory.summarize_cron.clone(), config.memory.summarize_threshold, config.learning.user_model_cron.clone(), diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index f597194..e1cfd3f 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -8,7 +8,6 @@ use teloxide::types::ParseMode; use tracing::{error, info, warn}; use crate::agent::Agent; -use crate::llm::ModelInfo; use crate::platform::{Attachment, AttachmentKind, IncomingMessage}; use crate::utils::markdown_entities::{markdown_to_entities, split_entities}; use crate::utils::telegram_markdown::escape_text; @@ -228,114 +227,26 @@ async fn handle_model_search( agent: &Arc, query: &str, ) -> ResponseResult<()> { - let models = match agent.llm.fetch_models().await { - Ok(list) => list, + // TODO(Task 9): /models interactive picker will be reworked against the new + // ProviderRegistry (per-provider list_models, fuzzy match across providers). + // For now, accept the model string verbatim via set_model(). + let _ = agent; + match agent.set_model(query).await { + Ok(()) => { + let reply = format!("βœ… Model changed to `{}`", query); + bot.send_message(chat_id, escape_text(&reply)) + .parse_mode(ParseMode::MarkdownV2) + .await?; + } Err(e) => { bot.send_message( chat_id, - escape_text(&format!("Failed to fetch model list: {:#}", e)), + escape_text(&format!("Failed to save model: {:#}", e)), ) .parse_mode(ParseMode::MarkdownV2) .await?; - return Ok(()); } - }; - - // Try exact match first. - if let Some(model) = models.iter().find(|m| m.id == query) { - match agent.set_model(&model.id).await { - Ok(()) => { - let reply = format!("βœ… Model changed to `{}` ({})", model.id, model.name); - bot.send_message(chat_id, escape_text(&reply)) - .parse_mode(ParseMode::MarkdownV2) - .await?; - } - Err(e) => { - bot.send_message( - chat_id, - escape_text(&format!("Failed to save model: {:#}", e)), - ) - .parse_mode(ParseMode::MarkdownV2) - .await?; - } - } - return Ok(()); } - - // Fuzzy search: case-insensitive match on id or name. - let q = query.to_lowercase(); - let mut matches: Vec<&ModelInfo> = models - .iter() - .filter(|m| m.id.to_lowercase().contains(&q) || m.name.to_lowercase().contains(&q)) - .collect(); - matches.sort_by(|a, b| { - let a_name = a.name.to_lowercase().contains(&q); - let b_name = b.name.to_lowercase().contains(&q); - b_name.cmp(&a_name).then(a.id.cmp(&b.id)) - }); - matches.truncate(10); - - if matches.is_empty() { - bot.send_message( - chat_id, - escape_text(&format!( - "No models found matching '{}'. Try a different search term.", - query - )), - ) - .parse_mode(ParseMode::MarkdownV2) - .await?; - return Ok(()); - } - - if matches.len() == 1 { - let model = &matches[0]; - match agent.set_model(&model.id).await { - Ok(()) => { - let reply = format!("βœ… Model changed to `{}` ({})", model.id, model.name); - bot.send_message(chat_id, escape_text(&reply)) - .parse_mode(ParseMode::MarkdownV2) - .await?; - } - Err(e) => { - bot.send_message( - chat_id, - escape_text(&format!("Failed to save model: {:#}", e)), - ) - .parse_mode(ParseMode::MarkdownV2) - .await?; - } - } - return Ok(()); - } - - // Show top 5 results with inline keyboard buttons. - use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; - let mut keyboard: Vec> = Vec::new(); - for model in matches.iter().take(5) { - let label = if model.name.len() > 35 { - format!("{}…", &model.name[..35]) - } else { - model.name.clone() - }; - keyboard.push(vec![InlineKeyboardButton::callback( - label, - format!("model_select:{}", model.id), - )]); - } - keyboard.push(vec![InlineKeyboardButton::callback( - "❌ Cancel", - "model_select:cancel", - )]); - - let reply = format!( - "Found {} models matching '{}'. Select one:", - matches.len(), - query - ); - bot.send_message(chat_id, &reply) - .reply_markup(InlineKeyboardMarkup::new(keyboard)) - .await?; Ok(()) } diff --git a/src/provider.rs b/src/provider.rs index cf53e18..7e73131 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -205,7 +205,6 @@ impl Provider for OpenRouterProvider { model: model.to_string(), messages: messages.to_vec(), tools: tools_param, - tool_choice: None, max_tokens, }; @@ -215,7 +214,10 @@ impl Provider for OpenRouterProvider { req = req.header("Authorization", format!("Bearer {}", key)); } - let response = req.send().await.context("Failed to send request to OpenRouter")?; + let response = req + .send() + .await + .context("Failed to send request to OpenRouter")?; let status = response.status(); if !status.is_success() { @@ -262,7 +264,10 @@ impl Provider for OpenRouterProvider { if let Some(ref key) = self.config.api_key { req = req.header("Authorization", format!("Bearer {}", key)); } - let response = req.send().await.context("Failed to fetch models from OpenRouter")?; + let response = req + .send() + .await + .context("Failed to fetch models from OpenRouter")?; let status = response.status(); if !status.is_success() { let body = response.text().await.unwrap_or_default(); @@ -326,7 +331,6 @@ impl Provider for OpenAICompatibleProvider { model: model.to_string(), messages: messages.to_vec(), tools: tools_param, - tool_choice: None, max_tokens, }; From 8c268d8b5cfbe3017fe2f9fd4f0a9e12096a7a46 Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 17:04:46 +0800 Subject: [PATCH 05/10] feat: redesign /models with multi-step provider selection and model discovery --- src/platform/telegram.rs | 195 +++++++++++++++++++++++++++++++++------ 1 file changed, 169 insertions(+), 26 deletions(-) diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index e1cfd3f..14ea9e8 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -9,6 +9,7 @@ use tracing::{error, info, warn}; use crate::agent::Agent; use crate::platform::{Attachment, AttachmentKind, IncomingMessage}; +use crate::provider::Provider; use crate::utils::markdown_entities::{markdown_to_entities, split_entities}; use crate::utils::telegram_markdown::escape_text; @@ -219,6 +220,98 @@ fn is_verbose_enabled(value: Option<&str>) -> bool { value.map(|v| v == "true").unwrap_or(false) } +/// Show models for a selected provider, or prompt for text search. +/// When prompting for text search, stores pending state in memory so the next +/// user message from this user routes to `handle_model_search` scoped to this provider. +async fn handle_provider_model_select( + bot: Bot, + chat_id: ChatId, + agent: &Arc, + provider_name: &str, + provider: &dyn Provider, + user_id: &str, +) -> ResponseResult<()> { + + let set_pending = |agent: &Arc, user_id: &str| { + let agent = agent.clone(); + let user_id = user_id.to_string(); + let provider_name = provider_name.to_string(); + Box::pin(async move { + agent.memory.remember( + "settings", + &format!("model_search_pending_{}", user_id), + "true", + None, + ).await.ok(); + agent.memory.remember( + "settings", + &format!("model_search_provider_{}", user_id), + &provider_name, + None, + ).await.ok(); + }) + }; + + if !provider.config().discover_models { + let prompt = format!( + "Send me a model name or ID to search for on **{provider_name}**.\n\ + Example: `{}`", + provider.default_model() + ); + bot.send_message(chat_id, &prompt).await?; + set_pending(agent, user_id).await; + return Ok(()); + } + + match provider.list_models(&agent.llm.client).await { + Ok(models) if models.len() <= 20 => { + use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; + let mut keyboard: Vec> = models + .iter() + .map(|m| { + let qualified = format!("{}/{}", provider_name, m); + vec![InlineKeyboardButton::callback( + m.clone(), + format!("model_select:{}", qualified), + )] + }) + .collect(); + keyboard.push(vec![InlineKeyboardButton::callback( + "\u{1F50D} Search all", + "model_search_prompt", + )]); + keyboard.push(vec![InlineKeyboardButton::callback( + "\u{274C} Cancel", + "model_select:cancel", + )]); + + let reply = format!("Models on **{provider_name}** ({}):", models.len()); + bot.send_message(chat_id, &reply) + .reply_markup(InlineKeyboardMarkup::new(keyboard)) + .await?; + } + Ok(models) => { + let prompt = format!( + "**{provider_name}** has {} models available.\n\ + Send me a model name or ID to search for.", + models.len() + ); + bot.send_message(chat_id, &prompt).await?; + set_pending(agent, user_id).await; + } + Err(e) => { + let prompt = format!( + "Could not load model list from **{provider_name}**: {e}\n\ + Send a model name or ID directly." + ); + bot.send_message(chat_id, &prompt).await?; + set_pending(agent, user_id).await; + } + } + + Ok(()) +} + /// Shared model search logic: fetch models, try exact match, fall back to fuzzy search, /// show top 5 results as inline keyboard buttons. async fn handle_model_search( @@ -227,10 +320,11 @@ async fn handle_model_search( agent: &Arc, query: &str, ) -> ResponseResult<()> { - // TODO(Task 9): /models interactive picker will be reworked against the new - // ProviderRegistry (per-provider list_models, fuzzy match across providers). - // For now, accept the model string verbatim via set_model(). - let _ = agent; + // Direct model set via qualified ID (e.g. "ollama/llama3" or + // "openrouter/moonshotai/kimi-k2.6"). The agent.set_model handles validation + // through the registry; the multi-step providerβ†’model picker is in + // `handle_provider_model_select` and is invoked from the `/models` command + // and the `provider_select:` callback. match agent.set_model(query).await { Ok(()) => { let reply = format!("βœ… Model changed to `{}`", query); @@ -581,30 +675,61 @@ async fn handle_message(bot: Bot, msg: Message, agent: Arc) -> ResponseRe return Ok(()); } "models" => { - if arg.is_empty() { - let current = agent.current_model.read().await; - let reply = format!( - "Current model: `{}`\n\nTap the button below to search for a model.", - *current - ); - use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; - let keyboard = InlineKeyboardMarkup::new(vec![ - vec![InlineKeyboardButton::callback( - "πŸ” Search models", - "model_search_prompt", - )], - vec![InlineKeyboardButton::callback( - "❌ Cancel", - "model_select:cancel", - )], - ]); - bot.send_message(msg.chat.id, &reply) - .reply_markup(keyboard) - .await?; - return Ok(()); + if !arg.is_empty() { + return handle_model_search(bot, msg.chat.id, &agent, &arg).await; + } + + let providers = agent.registry.provider_names(); + + if providers.len() == 1 { + // Single provider: jump straight to model search + let provider_name = providers[0].clone(); + let provider = match agent.registry.get_provider(&provider_name) { + Some(p) => p, + None => { + bot.send_message( + msg.chat.id, + escape_text(&format!("Provider '{}' not found.", provider_name)), + ) + .parse_mode(ParseMode::MarkdownV2) + .await?; + return Ok(()); + } + }; + let user_id = user_id.to_string(); + return handle_provider_model_select( + bot, + msg.chat.id, + &agent, + &provider_name, + provider, + &user_id, + ) + .await; } - return handle_model_search(bot, msg.chat.id, &agent, &arg).await; + // Multiple providers: show inline keyboard + let current = agent.current_model.read().await; + let reply = format!("Active model: `{}`\n\nSelect a provider:", *current); + use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; + let mut keyboard: Vec> = providers + .iter() + .map(|name| { + vec![InlineKeyboardButton::callback( + name.clone(), + format!("provider_select:{}", name), + )] + }) + .collect(); + keyboard.push(vec![InlineKeyboardButton::callback( + "❌ Cancel", + "model_select:cancel", + )]); + + bot.send_message(msg.chat.id, &reply) + .reply_markup(InlineKeyboardMarkup::new(keyboard)) + .await?; + return Ok(()); } _ => {} // ignore unknown commands for now } @@ -885,6 +1010,24 @@ async fn handle_model_callback( bot.answer_callback_query(callback_id).await?; + if let Some(provider_name) = data.strip_prefix("provider_select:") { + if let Some(provider) = agent.registry.get_provider(provider_name) { + if let Some(ref m) = msg { + let user_id = q.from.id.0.to_string(); + return handle_provider_model_select( + bot, + m.chat.id, + &agent, + provider_name, + provider, + &user_id, + ) + .await; + } + } + return Ok(()); + } + if data == "model_search_prompt" { if let Some(m) = msg { let prompt = "Send me a model name or ID to search for. Examples: claude, kimi, gpt, or a full model ID like openrouter/o3-mini."; From 2f7f4cc9f5f2e9adee6afeaab689d25e9706a46f Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 17:07:12 +0800 Subject: [PATCH 06/10] docs: add [[provider]] and [fallback] examples to config.example.toml --- config.example.toml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/config.example.toml b/config.example.toml index dc90b6f..9ef3b1c 100644 --- a/config.example.toml +++ b/config.example.toml @@ -25,6 +25,37 @@ Use the available tools to help the user with their tasks. \ When using file or terminal tools, operate only within the allowed sandbox directory. \ Be concise and helpful.""" +# ── Multi-Provider Configuration (optional) ──────────────────────────── +# Define additional LLM providers alongside OpenRouter. +# Each [[provider]] block adds a new provider. Model strings use the +# format `provider_name/model_id` (e.g., `ollama/llama3.1`). +# The first provider becomes the default. + +# Example: Ollama (local models) +# [[provider]] +# name = "ollama" +# type = "ollama" +# base_url = "http://localhost:11434/v1" +# model = "llama3.1" +# discover_models = true + +# Example: Any OpenAI-compatible endpoint (LM Studio, vLLM, llama.cpp, etc.) +# [[provider]] +# name = "lmstudio" +# type = "openai_compatible" +# base_url = "http://localhost:1234/v1" +# model = "qwen2.5-7b-instruct" + +# ── Fallback Chain (optional) ────────────────────────────────────────── +# When the primary model fails, RustFox tries each fallback in order. +# Each entry is a full provider-prefixed model string. +# [fallback] +# chain = [ +# "openrouter/moonshotai/kimi-k2.6", +# "ollama/llama3.1", +# "lmstudio/qwen2.5-7b-instruct", +# ] + # ── Home directory & paths ────────────────────────────────────────────── # By default RustFox stores everything under ~/.rustfox: # ~/.rustfox/config.toml, rustfox.db, skills/, agents/, workspace/, artifacts/ From 42aba599cc291d86fe9b404fbdb072056a7a2d6d Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Mon, 22 Jun 2026 17:10:27 +0800 Subject: [PATCH 07/10] feat: redesign /models with multi-step provider selection and model discovery --- src/agent.rs | 32 ++++++------- src/config.rs | 98 ++++++++++++++++++++++++++++++++++++++++ src/main.rs | 9 +++- src/platform/telegram.rs | 85 ++++++++++++++++++++++++---------- src/provider.rs | 6 +++ 5 files changed, 187 insertions(+), 43 deletions(-) diff --git a/src/agent.rs b/src/agent.rs index f1f1e63..f2b4e31 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -115,14 +115,7 @@ impl Agent { config_path: PathBuf, ) -> Self { let llm = LlmClient::new(registry.clone()); - let initial_model = format!( - "{}/{}", - registry.default_provider_name(), - registry - .resolve_model(registry.default_provider_name()) - .0 - .default_model() - ); + let initial_model = registry.default_qualified_model(); Self { llm, registry, @@ -354,15 +347,9 @@ impl Agent { let provider_name = provider.name().to_string(); - if provider_name == "openrouter" && doc.contains_key("openrouter") { - if let Some(table) = doc.get_mut("openrouter").and_then(|v| v.as_table_mut()) { - table.insert( - "model".to_string(), - toml::Value::String(actual_model.to_string()), - ); - } - } else if let Some(provider_array) = doc.get_mut("provider").and_then(|v| v.as_array_mut()) - { + // Try explicit [[provider]] array first + let mut found_in_array = false; + if let Some(provider_array) = doc.get_mut("provider").and_then(|v| v.as_array_mut()) { for entry in provider_array.iter_mut() { if let Some(table) = entry.as_table_mut() { if table.get("name").and_then(|v| v.as_str()) == Some(&provider_name) { @@ -370,11 +357,22 @@ impl Agent { "model".to_string(), toml::Value::String(actual_model.to_string()), ); + found_in_array = true; } } } } + // Fall back to legacy [openrouter] section if not found in [[provider]] array + if !found_in_array && provider_name == "openrouter" && doc.contains_key("openrouter") { + if let Some(table) = doc.get_mut("openrouter").and_then(|v| v.as_table_mut()) { + table.insert( + "model".to_string(), + toml::Value::String(actual_model.to_string()), + ); + } + } + let new_content = toml::to_string_pretty(&doc)?; tokio::fs::write(&self.config_path, &new_content).await?; diff --git a/src/config.rs b/src/config.rs index 34f5fe0..1edd8ea 100644 --- a/src/config.rs +++ b/src/config.rs @@ -994,4 +994,102 @@ mod tests { assert_eq!(cfg.agent.empty_response_retry_limit, 0); assert_eq!(cfg.empty_response_retry_limit(), 0); } + + #[test] + fn test_provider_section_parses_ollama() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + model = "moonshotai/kimi-k2.6" + [[provider]] + name = "ollama" + type = "ollama" + base_url = "http://localhost:11434/v1" + model = "llama3.1" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert_eq!(cfg.provider.len(), 1); + assert_eq!(cfg.provider[0].name, "ollama"); + assert_eq!(cfg.provider[0].model, "llama3.1"); + } + + #[test] + fn test_legacy_openrouter_auto_creates_provider() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + model = "moonshotai/kimi-k2.6" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + let (providers, default_name, _) = cfg.build_providers(); + assert!(providers.iter().any(|p| p.name == "openrouter")); + assert_eq!(default_name, "openrouter"); + } + + #[test] + fn test_visible_provider_overrides_legacy() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "old-key" + model = "moonshotai/kimi-k2.6" + [[provider]] + name = "openrouter" + type = "openrouter" + base_url = "https://openrouter.ai/api/v1" + api_key = "new-key" + model = "anthropic/claude-sonnet-4-6" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + let (providers, _default, _) = cfg.build_providers(); + // Should not have duplicate openrouter providers + let or_count = providers.iter().filter(|p| p.name == "openrouter").count(); + assert_eq!( + or_count, 1, + "should not duplicate openrouter when explicit [[provider]] exists" + ); + let or = providers.iter().find(|p| p.name == "openrouter").unwrap(); + assert_eq!( + or.api_key.as_deref(), + Some("new-key"), + "explicit [[provider]] should win" + ); + } + + #[test] + fn test_fallback_config_parses() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + [fallback] + chain = ["openrouter/model-a", "ollama/model-b"] + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert_eq!(cfg.fallback.chain.len(), 2); + assert_eq!(cfg.fallback.chain[0], "openrouter/model-a"); + } + + #[test] + fn test_fallback_defaults_empty() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert!(cfg.fallback.chain.is_empty()); + } } diff --git a/src/main.rs b/src/main.rs index e7c95da..94a4dcf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -76,7 +76,14 @@ async fn main() -> Result<()> { ); info!("Configuration loaded successfully"); - info!(" Model: {}", config.openrouter.model); + let default_provider_obj = registry + .get_provider(registry.default_provider_name()) + .expect("default provider must exist"); + info!( + " Model: {}/{}", + registry.default_provider_name(), + default_provider_obj.default_model() + ); info!(" Sandbox: {}", config.sandbox.allowed_directory.display()); if let Some(home) = &config.resolved_home { info!(" Home: {}", home.display()); diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index 14ea9e8..56cc247 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -231,24 +231,31 @@ async fn handle_provider_model_select( provider: &dyn Provider, user_id: &str, ) -> ResponseResult<()> { - let set_pending = |agent: &Arc, user_id: &str| { let agent = agent.clone(); let user_id = user_id.to_string(); let provider_name = provider_name.to_string(); Box::pin(async move { - agent.memory.remember( - "settings", - &format!("model_search_pending_{}", user_id), - "true", - None, - ).await.ok(); - agent.memory.remember( - "settings", - &format!("model_search_provider_{}", user_id), - &provider_name, - None, - ).await.ok(); + agent + .memory + .remember( + "settings", + &format!("model_search_pending_{}", user_id), + "true", + None, + ) + .await + .ok(); + agent + .memory + .remember( + "settings", + &format!("model_search_provider_{}", user_id), + &provider_name, + None, + ) + .await + .ok(); }) }; @@ -311,23 +318,43 @@ async fn handle_provider_model_select( Ok(()) } - -/// Shared model search logic: fetch models, try exact match, fall back to fuzzy search, -/// show top 5 results as inline keyboard buttons. +/// Accept a text model query and attempt to set the active model. +/// If the user previously selected a provider (via the inline keyboard), +/// the bare query is prefixed with that provider's name to form a +/// qualified model ID. async fn handle_model_search( bot: Bot, chat_id: ChatId, agent: &Arc, query: &str, + user_id: &str, ) -> ResponseResult<()> { - // Direct model set via qualified ID (e.g. "ollama/llama3" or - // "openrouter/moonshotai/kimi-k2.6"). The agent.set_model handles validation - // through the registry; the multi-step providerβ†’model picker is in - // `handle_provider_model_select` and is invoked from the `/models` command - // and the `provider_select:` callback. - match agent.set_model(query).await { + // Check if user has a stored provider from the interactive picker + let stored_provider = agent + .memory + .recall("settings", &format!("model_search_provider_{}", user_id)) + .await + .unwrap_or(None); + + let qualified = if let Some(ref provider_name) = stored_provider { + // Clear the stored provider so it doesn't affect future searches + agent + .memory + .forget("settings", &format!("model_search_provider_{}", user_id)) + .await + .ok(); + if query.contains('/') { + query.to_string() + } else { + format!("{}/{}", provider_name, query) + } + } else { + query.to_string() + }; + + match agent.set_model(&qualified).await { Ok(()) => { - let reply = format!("βœ… Model changed to `{}`", query); + let reply = format!("βœ… Model changed to `{}`", qualified); bot.send_message(chat_id, escape_text(&reply)) .parse_mode(ParseMode::MarkdownV2) .await?; @@ -341,6 +368,7 @@ async fn handle_model_search( .await?; } } + Ok(()) } @@ -426,7 +454,7 @@ async fn handle_message(bot: Bot, msg: Message, agent: Arc) -> ResponseRe .await .ok(); // Treat message as a model search query β€” dispatch to shared model search logic. - return handle_model_search(bot, msg.chat.id, &agent, &text).await; + return handle_model_search(bot, msg.chat.id, &agent, &text, &user_id.to_string()).await; } info!( @@ -676,7 +704,14 @@ async fn handle_message(bot: Bot, msg: Message, agent: Arc) -> ResponseRe } "models" => { if !arg.is_empty() { - return handle_model_search(bot, msg.chat.id, &agent, &arg).await; + return handle_model_search( + bot, + msg.chat.id, + &agent, + &arg, + &user_id.to_string(), + ) + .await; } let providers = agent.registry.provider_names(); diff --git a/src/provider.rs b/src/provider.rs index 7e73131..6663861 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -119,6 +119,12 @@ impl ProviderRegistry { pub fn provider_names(&self) -> Vec { self.providers.keys().cloned().collect() } + + /// Return the fully-qualified default model string, e.g. `"openrouter/moonshotai/kimi-k2.6"`. + pub fn default_qualified_model(&self) -> String { + let provider = &self.providers[&self.default_provider]; + format!("{}/{}", self.default_provider, provider.default_model()) + } } /// Build a ProviderRegistry from config sections. From e33419de3049b5b55bb10ab25b7e05732940f0ad Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Tue, 23 Jun 2026 11:44:04 +0800 Subject: [PATCH 08/10] docs: add multi-provider design spec and implementation plan --- .../plans/2026-06-22-multi-provider-plan.md | 1502 +++++++++++++++++ .../specs/2026-06-22-multi-provider-design.md | 464 +++++ 2 files changed, 1966 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-22-multi-provider-plan.md create mode 100644 docs/superpowers/specs/2026-06-22-multi-provider-design.md diff --git a/docs/superpowers/plans/2026-06-22-multi-provider-plan.md b/docs/superpowers/plans/2026-06-22-multi-provider-plan.md new file mode 100644 index 0000000..136b8d8 --- /dev/null +++ b/docs/superpowers/plans/2026-06-22-multi-provider-plan.md @@ -0,0 +1,1502 @@ +# Multi-Provider LLM Architecture β€” Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add multiple LLM provider support (Ollama, LM Studio, etc.), fallback chains, and per-subagent provider mixing. + +**Architecture:** Define a `Provider` trait with concrete impls (OpenRouter, OpenAICompatible, Ollama). `ProviderRegistry` resolves model strings (`ollama/llama3` β†’ Ollama provider + model `llama3`). `LlmClient` routes calls through the registry. Fallback chains in the agent loop try alternative models on failure. + +**Tech Stack:** Rust, reqwest, serde, async_trait, teloxide (inline keyboards) + +--- + +### Task 1: Add config types for providers and fallback + +**Files:** +- Modify: `src/config.rs` +- Read: `docs/superpowers/specs/2026-06-22-multi-provider-design.md` (for reference) + +- [ ] **Step 1: Add ProviderType enum to config.rs** + +Add this after the existing config structs (before `OcrConfig` or similar): + +```rust +#[derive(Debug, Clone, Deserialize, PartialEq)] +pub enum ProviderType { + #[serde(rename = "openrouter")] + OpenRouter, + #[serde(rename = "openai_compatible")] + OpenAICompatible, + #[serde(rename = "ollama")] + Ollama, +} + +impl Default for ProviderType { + fn default() -> Self { + Self::OpenRouter + } +} +``` + +- [ ] **Step 2: Add ProviderSection and FallbackConfig structs** + +```rust +#[derive(Debug, Clone, Deserialize)] +pub struct ProviderSection { + pub name: String, + #[serde(rename = "type")] + pub provider_type: ProviderType, + pub base_url: String, + pub api_key: Option, + #[serde(default = "default_model")] + pub model: String, + #[serde(default)] + pub supports_vision: bool, + #[serde(default = "default_max_tokens")] + pub max_tokens: u32, + #[serde(default)] + pub discover_models: bool, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct FallbackConfig { + #[serde(default)] + pub chain: Vec, +} +``` + +- [ ] **Step 3: Add fields to Config struct** + +Add inside `pub struct Config`: +```rust + #[serde(default)] + pub provider: Vec, + #[serde(default)] + pub fallback: FallbackConfig, +``` + +- [ ] **Step 4: Add backward compat builder method to Config** + +```rust +/// Build the provider list from config, handling legacy [openrouter] backward compat. +/// Returns (providers, default_provider_name, fallback_chain). +pub fn build_providers(&self) -> (Vec, String, Vec) { + let mut providers: Vec = self.provider.clone(); + + // Backward compat: if [openrouter] section exists and no explicit provider named "openrouter" + let has_openrouter = providers.iter().any(|p| p.name == "openrouter"); + if has_openrouter { + tracing::warn!( + "Explicit [[provider]] name=\"openrouter\" found β€” legacy [openrouter] section will be ignored" + ); + } else { + providers.push(ProviderSection { + name: "openrouter".to_string(), + provider_type: ProviderType::OpenRouter, + base_url: self.openrouter.base_url.clone(), + api_key: Some(self.openrouter.api_key.clone()), + model: self.openrouter.model.clone(), + supports_vision: self.openrouter.supports_vision, + max_tokens: self.openrouter.max_tokens, + discover_models: false, + }); + } + + let default = if providers.is_empty() { + "openrouter".to_string() + } else { + providers[0].name.clone() + }; + + let fallback = self.fallback.chain.clone(); + (providers, default, fallback) +} +``` + +- [ ] **Step 5: Run tests to verify existing config parsing still works** + +```bash +cargo test -p rustfox -- config::tests --nocapture +``` +Expected: Existing config tests pass (backward compat). + +- [ ] **Step 6: Commit** + +```bash +git add src/config.rs +git commit -m "feat: add provider and fallback config types" +``` + +--- + +### Task 2: Create `src/provider.rs` β€” Provider trait, types, and registry + +**Files:** +- Create: `src/provider.rs` +- Modify: `src/lib.rs` + +- [ ] **Step 1: Add module declaration to lib.rs** + +```rust +pub mod provider; +``` + +- [ ] **Step 2: Create src/provider.rs with the core types** + +```rust +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::Result; +use async_trait::async_trait; + +use crate::config::{ProviderSection, ProviderType}; +use crate::llm::{ChatCompletion, ChatMessage, ToolDefinition}; + +#[derive(Debug, Clone)] +pub struct ProviderConfig { + pub name: String, + pub provider_type: ProviderType, + pub base_url: String, + pub api_key: Option, + pub default_model: String, + pub supports_vision: bool, + pub max_tokens: u32, + pub discover_models: bool, +} + +impl From<&ProviderSection> for ProviderConfig { + fn from(s: &ProviderSection) -> Self { + Self { + name: s.name.clone(), + provider_type: s.provider_type.clone(), + base_url: s.base_url.clone(), + api_key: s.api_key.clone(), + default_model: s.model.clone(), + supports_vision: s.supports_vision, + max_tokens: s.max_tokens, + discover_models: s.discover_models, + } + } +} + +#[async_trait] +pub trait Provider: Send + Sync { + fn name(&self) -> &str; + fn default_model(&self) -> &str; + fn supports_vision(&self) -> bool; + fn config(&self) -> &ProviderConfig; + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result; + + async fn list_models(&self, client: &reqwest::Client) -> Result>; +} +``` + +- [ ] **Step 3: Add ProviderRegistry** + +```rust +pub struct ProviderRegistry { + providers: HashMap>, + default_provider: String, +} + +impl ProviderRegistry { + pub fn new( + providers: HashMap>, + default_provider: String, + ) -> Self { + Self { + providers, + default_provider, + } + } + + /// Resolve a model string to (provider, stripped_model). + /// Never fails β€” unknown prefixes fall through to default provider. + pub fn resolve_model(&self, model: &str) -> (&dyn Provider, &str) { + if let Some((prefix, rest)) = model.split_once('/') { + if let Some(provider) = self.providers.get(prefix) { + return (provider.as_ref(), rest); + } + } + // Fall through to default provider with full string + let default = &self.providers[&self.default_provider]; + (default.as_ref(), model) + } + + pub fn get_provider(&self, name: &str) -> Option<&dyn Provider> { + self.providers.get(name).map(|p| p.as_ref()) + } + + pub fn providers(&self) -> impl Iterator { + self.providers.values().map(|p| p.as_ref()) + } + + pub fn default_provider_name(&self) -> &str { + &self.default_provider + } + + pub fn provider_count(&self) -> usize { + self.providers.len() + } + + pub fn provider_names(&self) -> Vec { + self.providers.keys().cloned().collect() + } +} +``` + +- [ ] **Step 4: Add construction helper** + +```rust +use crate::config::ProviderSection; + +/// Build a ProviderRegistry from config sections. +pub fn build_registry( + sections: &[ProviderSection], + default_name: &str, +) -> Result { + let mut providers: HashMap> = HashMap::new(); + + for section in sections { + let cfg: ProviderConfig = ProviderConfig::from(section); + let provider: Arc = match section.provider_type { + ProviderType::OpenRouter => Arc::new(OpenRouterProvider::new(cfg)), + ProviderType::OpenAICompatible => Arc::new(OpenAICompatibleProvider::new(cfg)), + ProviderType::Ollama => Arc::new(OllamaProvider::new(cfg)), + }; + providers.insert(section.name.clone(), provider); + } + + if providers.is_empty() { + anyhow::bail!("No LLM providers configured"); + } + + // Ensure default exists + if !providers.contains_key(default_name) { + anyhow::bail!("Default provider '{}' not found in configured providers", default_name); + } + + Ok(ProviderRegistry::new(providers, default_name.to_string())) +} +``` + +- [ ] **Step 6: Commit** + +```bash +git add src/provider.rs src/lib.rs +git commit -m "feat: add Provider trait, ProviderRegistry, ProviderConfig types" +``` + +--- + +### Task 3: Implement OpenRouterProvider + +**Files:** +- Modify: `src/provider.rs` +- Modify: `src/llm.rs` + +- [ ] **Step 0: Export ChatRequest/ChatResponse from llm.rs for reuse by all providers** + +Add to `llm.rs`: +```rust +// These are shared with provider implementations in src/provider.rs +#[doc(hidden)] +pub mod internal { + pub use super::{ChatRequest, ChatResponse}; +} +``` + +Change `ChatRequest` and `ChatResponse` from `struct ChatRequest` / `struct ChatResponse` to `pub struct ChatRequest` / `pub struct ChatResponse`. + +- [ ] **Step 1: Add OpenRouterProvider struct and impl** + +Append to `src/provider.rs`: + +```rust +// === OpenRouterProvider === + +pub struct OpenRouterProvider { + config: ProviderConfig, +} + +impl OpenRouterProvider { + pub fn new(config: ProviderConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl Provider for OpenRouterProvider { + fn name(&self) -> &str { &self.config.name } + fn default_model(&self) -> &str { &self.config.default_model } + fn supports_vision(&self) -> bool { self.config.supports_vision } + fn config(&self) -> &ProviderConfig { &self.config } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + // Same logic as current LlmClient::chat_completion_with_model: + let tools_param = if tools.is_empty() { + None + } else { + let sanitized: Vec = tools + .iter() + .map(|t| { + let mut t = t.clone(); + sanitize_parameters(&mut t.function.parameters); + t + }) + .collect(); + Some(sanitized) + }; + + let request = crate::llm::internal::ChatRequest { + model: model.to_string(), + messages: messages.to_vec(), + tools: tools_param, + max_tokens, + }; + + let url = format!("{}/chat/completions", self.config.base_url); + let response = client + .post(&url) + .header("Authorization", format!("Bearer {}", + self.config.api_key.as_deref().unwrap_or(""))) + .json(&request) + .send() + .await + .context("Failed to send request to OpenRouter")?; + + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("OpenRouter API error ({}): {}", status, body); + } + + let chat_response: crate::llm::internal::ChatResponse = response + .json() + .await + .context("Failed to parse OpenRouter response")?; + + let mut choice = chat_response + .choices + .into_iter() + .next() + .context("No response from OpenRouter")?; + + // Kimi tool-call fallback + let has_tool_calls = choice + .message + .tool_calls + .as_ref() + .is_some_and(|t| !t.is_empty()); + if !has_tool_calls { + if let Some(ref content) = choice.message.content { + if let Some(parsed) = parse_kimi_tool_calls(&content.as_text()) { + choice.message.tool_calls = Some(parsed); + choice.message.content = None; + } + } + } + + Ok(ChatCompletion { + message: choice.message, + finish_reason: choice.finish_reason, + model: model.to_string(), + }) + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = format!("{}/models", self.config.base_url); + let response = client + .get(&url) + .header("Authorization", format!("Bearer {}", self.config.api_key.as_deref().unwrap_or(""))) + .send() + .await?; + let list: serde_json::Value = response.json().await?; + let models = list["data"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|m| m["id"].as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + Ok(models) + } +} +``` + +The `sanitize_parameters` and `parse_kimi_tool_calls` functions need to remain in `llm.rs` (they are referenced from within the OpenRouterProvider impl above via `use crate::llm::{sanitize_parameters, parse_kimi_tool_calls}`). + +- [ ] **Step 2: Add the import for sanitize_parameters at top of provider.rs** + +```rust +use crate::llm::{sanitize_parameters, parse_kimi_tool_calls}; +``` + +Make `sanitize_parameters` and `parse_kimi_tool_calls` `pub` in `llm.rs`. + +- [ ] **Step 3: Commit** + +```bash +git add src/provider.rs src/llm.rs +git commit -m "feat: add OpenRouterProvider with param sanitization and Kimi fallback" +``` + +--- + +### Task 4: Implement OpenAICompatibleProvider + +**Files:** +- Modify: `src/provider.rs` + +- [ ] **Step 1: Add OpenAICompatibleProvider** + +```rust +// === OpenAICompatibleProvider === + +pub struct OpenAICompatibleProvider { + config: ProviderConfig, +} + +impl OpenAICompatibleProvider { + pub fn new(config: ProviderConfig) -> Self { + Self { config } + } +} + +#[async_trait] +impl Provider for OpenAICompatibleProvider { + fn name(&self) -> &str { &self.config.name } + fn default_model(&self) -> &str { &self.config.default_model } + fn supports_vision(&self) -> bool { self.config.supports_vision } + fn config(&self) -> &ProviderConfig { &self.config } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + // Pure OpenAI-compatible endpoint: no param sanitization, no Kimi fallback. + let tools_param = if tools.is_empty() { None } else { Some(tools.to_vec()) }; + + let request = crate::llm::internal::ChatRequest { + model: model.to_string(), + messages: messages.to_vec(), + tools: tools_param, + max_tokens, + }; + + let url = format!("{}/chat/completions", self.config.base_url); + let mut req = client.post(&url).json(&request); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + + let response = req.send().await?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("Provider '{}' error ({}): {}", self.config.name, status, body); + } + + let chat_response = response.json::().await?; + let choice = chat_response.choices.into_iter().next() + .ok_or_else(|| anyhow::anyhow!("No response from provider '{}'", self.config.name))?; + + Ok(ChatCompletion { + message: choice.message, + finish_reason: choice.finish_reason, + model: model.to_string(), + }) + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = format!("{}/models", self.config.base_url); + let mut req = client.get(&url); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + let response = req.send().await?; + let list: serde_json::Value = response.json().await?; + let models = list["data"] + .as_array() + .map(|arr| arr.iter().filter_map(|m| m["id"].as_str().map(String::from)).collect()) + .unwrap_or_default(); + Ok(models) + } +} +``` + +(The `pub mod internal` export was already added in Task 3 Step 0. No additional llm.rs changes needed here.) + +- [ ] **Step 2: Commit** + +```bash +git add src/provider.rs src/llm.rs +git commit -m "feat: add OpenAICompatibleProvider for any OpenAI-compatible endpoint" +``` + +--- + +### Task 5: Implement OllamaProvider + +**Files:** +- Modify: `src/provider.rs` + +- [ ] **Step 1: Add OllamaProvider (delegates chat to OpenAICompatibleProvider)** + +```rust +// === OllamaProvider === + +pub struct OllamaProvider { + config: ProviderConfig, + inner: OpenAICompatibleProvider, +} + +impl OllamaProvider { + pub fn new(config: ProviderConfig) -> Self { + let inner_cfg = config.clone(); + let inner = OpenAICompatibleProvider::new(inner_cfg); + Self { config, inner } + } + + /// Build the discovery URL from the OpenAI-compatible base_url. + /// Trailing `/v1` or `/vN` is stripped and replaced with `/api/tags`. + /// e.g. "http://localhost:11434/v1" β†’ "http://localhost:11434/api/tags" + /// "http://localhost:11434" β†’ "http://localhost:11434/api/tags" + fn discovery_url(&self) -> String { + let base = self.config.base_url.trim_end_matches('/'); + // Strip versioned path suffix (/v1, /v2, etc.) + let base = base.strip_suffix("/v1") + .or_else(|| base.strip_suffix("/v2")) + .or_else(|| base.strip_suffix("/v3")) + .unwrap_or(base); + format!("{}/api/tags", base.trim_end_matches('/')) + } +} + +#[async_trait] +impl Provider for OllamaProvider { + fn name(&self) -> &str { &self.config.name } + fn default_model(&self) -> &str { &self.config.default_model } + fn supports_vision(&self) -> bool { self.config.supports_vision } + fn config(&self) -> &ProviderConfig { &self.config } + + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result { + // Delegate to OpenAICompatibleProvider (same API format) + self.inner.chat_completion(client, messages, tools, model, max_tokens).await + } + + async fn list_models(&self, client: &reqwest::Client) -> Result> { + let url = self.discovery_url(); + let mut req = client.get(&url); + if let Some(ref key) = self.config.api_key { + req = req.header("Authorization", format!("Bearer {}", key)); + } + let response = req.send().await?; + let body: serde_json::Value = response.json().await?; + // Ollama /api/tags returns {"models": [{"name": "llama3.1:8b", ...}, ...]} + let models = body["models"] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|m| m["name"].as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + Ok(models) + } +} +``` + +- [ ] **Step 2: Commit** + +```bash +git add src/provider.rs +git commit -m "feat: add OllamaProvider with /api/tags discovery" +``` + +--- + +### Task 6: Refactor LlmClient to use ProviderRegistry + +**Files:** +- Modify: `src/llm.rs` + +- [ ] **Step 1: Change LlmClient struct** + +```rust +pub struct LlmClient { + pub client: reqwest::Client, + pub registry: Arc, +} +``` + +- [ ] **Step 2: Update constructor** + +```rust +impl LlmClient { + pub fn new(registry: Arc) -> Self { + Self { + client: reqwest::Client::new(), + registry, + } + } +} +``` + +- [ ] **Step 3: Rewrite chat_completion_with_model as a routing method** + +```rust + pub async fn chat_completion_with_model( + &self, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + ) -> Result { + let (provider, actual_model) = self.registry.resolve_model(model); + let max_tokens = provider.config().max_tokens; + + let mut completion = provider + .chat_completion(&self.client, messages, tools, actual_model, max_tokens) + .await?; + + // Preserve the original model string in metadata + completion.model = model.to_string(); + Ok(completion) + } +``` + +- [ ] **Step 4: Keep `chat()` as a convenience, remove `chat_with_model()` and `chat_completion()`** + +Keep `chat()` as a thin wrapper for backward compat (e.g., soul reflection in agent.rs): +```rust +/// Convenience wrapper that uses the default provider's default model. +pub async fn chat( + &self, + messages: &[ChatMessage], + tools: &[ToolDefinition], +) -> Result { + let default_model = self.registry.default_provider_name().to_string(); + self.chat_completion_with_model(messages, tools, &default_model) + .await + .map(|c| c.message) +} +``` + +Remove these methods (no callers): +- `chat_with_model()` β€” unused after refactor +- `chat_completion()` β€” unused after refactor + +- [ ] **Step 5: Update soul-reflection caller in agent.rs** + +```rust +// Old (agent.rs ~line 1059): +// agent.llm.chat(&reflection_messages, &[]).await +// New: +agent.llm.chat_completion_with_model( + &reflection_messages, &[], + &*agent.current_model.read().await +).await.map(|c| c.message) +``` + +- [ ] **Step 6: Remove fetch_models() from LlmClient** (moved to OpenRouterProvider.list_models()) + +- [ ] **Step 7: Build** + +```bash +cargo check --lib +``` +Expected: Compilation succeeds (with the Agent changes still pending, but lib check covers llm.rs changes). + +- [ ] **Step 8: Commit** + +```bash +git add src/llm.rs src/agent.rs +git commit -m "refactor: LlmClient routes through ProviderRegistry, removed direct config dependency" +``` + +--- + +### Task 7: Update Agent for registry, fallback, and per-provider vision + +**Files:** +- Modify: `src/agent.rs` + +- [ ] **Step 1: Add registry field to Agent struct** + +```rust +pub struct Agent { + pub llm: LlmClient, + pub registry: Arc, + pub config: Config, + // ... existing fields unchanged +} +``` + +- [ ] **Step 2: Update Agent::new() to receive and store registry** + +```rust +pub fn new( + config: Config, + registry: Arc, + mcp: McpManager, + memory: MemoryStore, + // ... other params unchanged +) -> Self { + let llm = LlmClient::new(registry.clone()); + let initial_model = format!("{}/{}", + registry.default_provider_name(), + // get default model from default provider + registry.resolve_model(®istry.default_provider_name()).0.default_model() + ); + Self { + llm, + registry, + config, + // ... + current_model: tokio::sync::RwLock::new(initial_model), + // ... + } +} +``` + +Note: The initial model string is `/`, e.g. `"openrouter/moonshotai/kimi-k2.6"`. + +- [ ] **Step 3: Update set_model() validation** + +```rust +pub async fn set_model(&self, model_id: &str) -> anyhow::Result<()> { + if model_id.is_empty() { + anyhow::bail!("Model ID cannot be empty"); + } + + // Validate: resolve succeeds for any string, but warn on unknown prefix + let (provider, actual_model) = self.registry.resolve_model(model_id); + // Check if user explicitly specified a prefix that doesn't exist + if let Some((prefix, _)) = model_id.split_once('/') { + if self.registry.get_provider(prefix).is_none() { + tracing::warn!( + "Model '{}': prefix '{}' does not match any known provider \ + (falling through to default '{}')", + model_id, prefix, self.registry.default_provider_name() + ); + } + } + + // Persist to config.toml + let content = tokio::fs::read_to_string(&self.config_path).await?; + let mut doc: toml::value::Table = toml::from_str(&content)?; + + // Determine which TOML section to update based on the provider name + let provider_name = provider.name().to_string(); + + // For the legacy [openrouter] section: + if provider_name == "openrouter" && doc.contains_key("openrouter") { + if let Some(table) = doc.get_mut("openrouter").and_then(|v| v.as_table_mut()) { + table.insert("model".to_string(), toml::Value::String(actual_model.to_string())); + } + } else { + // For [[provider]] entries, find the right one by name in the array + if let Some(provider_array) = doc.get_mut("provider").and_then(|v| v.as_array_mut()) { + for entry in provider_array.iter_mut() { + if let Some(table) = entry.as_table_mut() { + if table.get("name").and_then(|v| v.as_str()) == Some(&provider_name) { + table.insert( + "model".to_string(), + toml::Value::String(actual_model.to_string()), + ); + } + } + } + } + } + + let new_content = toml::to_string_pretty(&doc)?; + tokio::fs::write(&self.config_path, &new_content).await?; + + let mut current = self.current_model.write().await; + *current = model_id.to_string(); + + tracing::info!(model = %model_id, provider = %provider_name, "Model changed and persisted"); + Ok(()) +} +``` + +- [ ] **Step 4: Add fallback chain to process_message main loop** + +Wrap the LLM call in the main agent loop (find the `chat_completion_with_model` call in `process_message`) with fallback logic: + +```rust +let model = self.current_model.read().await.clone(); +let fallback_chain = &self.config.fallback.chain; +let mut last_error = None; + +for attempt in 0..=fallback_chain.len() { + let current_model = if attempt == 0 { + model.clone() + } else { + fallback_chain[attempt - 1].clone() + }; + + match self.llm.chat_completion_with_model( + &prompt.messages, &all_tools, ¤t_model + ).await { + Ok(c) => { + // If we used a fallback, update current_model in memory (not disk) + if attempt > 0 { + tracing::info!( + "Fallback succeeded: switched from '{}' to '{}'", + model, current_model + ); + *self.current_model.write().await = current_model.clone(); + } + completion = c; + break; + } + Err(e) => { + tracing::warn!("Model '{}' failed (attempt {}/{}): {}", + current_model, attempt, fallback_chain.len(), e); + last_error = Some(e); + if attempt == fallback_chain.len() { + // All attempts exhausted β€” return the last error + return Err(last_error.unwrap()); + } + continue; + } + } +} +``` + +- [ ] **Step 5: Update per-provider vision check in agent.rs and file_processor** + +**agent.rs** β€” Replace `self.config.openrouter.supports_vision` references in `process_message`: + +Find the line in agent.rs that checks `supports_vision` (for deciding image vs OCR path). Replace: + +```rust +// Old: if self.config.openrouter.supports_vision { +// New: +let current = self.current_model.read().await; +let (provider, _) = self.registry.resolve_model(¤t); +if provider.supports_vision() { +``` + +**file_processor/mod.rs** β€” The `process_attachments` function currently reads `config.openrouter.supports_vision` directly (line 40). Change it to accept the flag as a parameter: + +```rust +// Old signature: +pub async fn process_attachments( + attachments: &[Attachment], + user_query: &str, + config: &Config, + memory: &MemoryStore, +) -> (String, Vec) { + +// Change the call site in agent.rs to pass the active provider's supports_vision: +let supports_vision = { + let current = self.current_model.read().await; + let (provider, _) = self.registry.resolve_model(¤t); + provider.supports_vision() +}; + +// Old: config.openrouter.supports_vision, +// New: supports_vision, +``` + +Update `process_attachments` signature to accept `supports_vision: bool` instead of reading from config: + +```rust +pub async fn process_attachments( + attachments: &[Attachment], + user_query: &str, + config: &Config, + memory: &MemoryStore, + supports_vision: bool, // <-- new parameter, from active provider +) -> (String, Vec) { +``` + +Replace line 40: +```rust +// Old: +config.openrouter.supports_vision, +// New: +supports_vision, +``` +```rust +// Old: if self.config.openrouter.supports_vision { +// New: +let current = self.current_model.read().await; +let (provider, _) = self.registry.resolve_model(¤t); +if provider.supports_vision() { +``` + +- [ ] **Step 6: Build and fix compilation errors** + +```bash +cargo check +``` +Expected: Compilation succeeds. + +- [ ] **Step 7: Run existing tests** + +```bash +cargo test --lib +``` +Expected: All existing tests pass. + +- [ ] **Step 8: Commit** + +```bash +git add src/agent.rs +git commit -m "feat: integrate ProviderRegistry into Agent with fallback chain and per-provider vision" +``` + +--- + +### Task 8: Wire providers in main.rs at startup + +**Files:** +- Modify: `src/main.rs` + +- [ ] **Step 1: Build registry from config and pass to Agent** + +After config is loaded (after `config.load()`), add: + +```rust +use crate::provider; + +// Build provider registry +let (provider_sections, default_provider, fallback_chain) = config.build_providers(); +let registry = Arc::new(provider::build_registry(&provider_sections, &default_provider) + .context("Failed to build LLM provider registry")?); +``` + +Then pass `registry.clone()` to `Agent::new()`: + +```rust +let agent = Arc::new_cyclic(|weak| { + Agent::new( + config, + registry.clone(), + // ... existing params unchanged + ) +}); +``` + +- [ ] **Step 2: Remove the old LlmClient::new creation from Agent::new** (it's now created inside Agent::new from the registry) + +- [ ] **Step 3: Build** + +```bash +cargo check +``` +Expected: Compilation succeeds. + +- [ ] **Step 4: Commit** + +```bash +git add src/main.rs +git commit -m "feat: wire ProviderRegistry startup in main.rs" +``` + +--- + +### Task 9: Redesign `/models` Telegram command with multi-step providerβ†’model selection + +**Files:** +- Modify: `src/platform/telegram.rs` + +- [ ] **Step 1: Update `/models` handler for provider selection** + +Replace the current `/models` command handler (around line 672) with: + +```rust +"models" => { + let registry = &agent.registry; + let providers = registry.provider_names(); + + if providers.len() == 1 { + // Single provider: jump straight to model search for it + let provider_name = &providers[0]; + let provider = registry.get_provider(provider_name).unwrap(); + return handle_provider_model_select( + bot, msg.chat.id, agent, provider_name, provider + ).await; + } + + // Multiple providers: show inline keyboard + let current = agent.current_model.read().await; + let reply = format!( + "Active model: `{}`\n\nSelect a provider:", + *current + ); + use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; + let mut keyboard: Vec> = providers.iter() + .map(|name| { + vec![InlineKeyboardButton::callback( + name.clone(), + format!("provider_select:{}", name), + )] + }) + .collect(); + keyboard.push(vec![InlineKeyboardButton::callback( + "❌ Cancel", "model_select:cancel", + )]); + + bot.send_message(msg.chat.id, &reply) + .reply_markup(InlineKeyboardMarkup::new(keyboard)) + .await?; + return Ok(()); +} +``` + +- [ ] **Step 2: Add provider_select callback in handle_model_callback** + +`handle_model_callback` receives `CallbackQuery`, not `Message`. The message is accessed via `q.regular_message()`. Add before the existing `model_select:` check: + +```rust +if let Some(provider_name) = data.strip_prefix("provider_select:") { + if let Some(provider) = agent.registry.get_provider(provider_name) { + let msg = q.regular_message().cloned(); + if let Some(ref m) = msg { + return handle_provider_model_select( + bot, m.chat.id, agent, provider_name, provider + ).await; + } + } + return Ok(()); +} +``` + +- [ ] **Step 3: Add handle_provider_model_select helper (uses `ChatId`, not `Message`)** + +```rust +async fn handle_provider_model_select( + bot: Bot, + chat_id: ChatId, + agent: &Arc, + provider_name: &str, + provider: &dyn Provider, +) -> ResponseResult<()> { + + if !provider.config().discover_models { + // No discovery: prompt for text search + let prompt = format!( + "Send me a model name or ID to search for on **{}**.\nExamples: {}", + provider_name, + provider.default_model() + ); + bot.send_message(chat_id, &prompt).await?; + return Ok(()); + } + + // Try to discover models + match provider.list_models(&agent.llm.client).await { + Ok(models) if models.len() <= 20 => { + use teloxide::types::{InlineKeyboardButton, InlineKeyboardMarkup}; + let mut keyboard: Vec> = models.iter() + .map(|m| { + let qualified = format!("{}/{}", provider_name, m); + vec![InlineKeyboardButton::callback( + m.clone(), + format!("model_select:{}", qualified), + )] + }) + .collect(); + keyboard.push(vec![InlineKeyboardButton::callback( + "πŸ” Search all", "model_search_prompt", + )]); + keyboard.push(vec![InlineKeyboardButton::callback( + "❌ Cancel", "model_select:cancel", + )]); + + let reply = format!("Models on **{}** ({}):", provider_name, models.len()); + bot.send_message(chat_id, &reply) + .reply_markup(InlineKeyboardMarkup::new(keyboard)) + .await?; + } + Ok(models) => { + // Too many models: prompt for text search + let prompt = format!( + "**{}** has {} models available.\nSend me a model name or ID to search for.", + provider_name, + models.len() + ); + bot.send_message(chat_id, &prompt).await?; + } + Err(e) => { + // Discovery failed: prompt for search + let prompt = format!( + "Could not load model list from **{}**: {}\nSend a model name or ID directly.", + provider_name, e + ); + bot.send_message(chat_id, &prompt).await?; + } + } + + Ok(()) +} +``` + +- [ ] **Step 4: Update existing model_select callback to handle qualified IDs** + +The existing `model_select:{model_id}` callback now receives qualified IDs like `model_select:ollama/llama3.1:8b`. The `set_model()` call already handles the full qualified string β€” no change needed to the set_model call itself. Just ensure the callback data format is `model_select:{qualified_id}`. + +- [ ] **Step 5: Update the model search flow to be provider-scoped** + +When a user types a model name after being prompted for search, `handle_model_search` should store which provider they're searching. Use the existing `model_search_pending_{user_id}` memory key to also store the provider name. + +In the search prompt flow, after user selects a provider: +```rust +agent.memory.remember("settings", + &format!("model_search_provider_{}", user_id), + provider_name, + None, +).await.ok(); +``` + +In `handle_message`'s model search path (after the user types a search query), read the stored provider and scope the search: +```rust +// When user types a search query (not a command): +let search_provider = agent.memory.recall("settings", + &format!("model_search_provider_{}", user_id) +).await.ok().flatten(); +// Pass it to handle_model_search which filters by provider +``` + +- [ ] **Step 6: Build** + +```bash +cargo check +``` +Expected: Compilation succeeds. + +- [ ] **Step 7: Commit** + +```bash +git add src/platform/telegram.rs +git commit -m "feat: redesign /models with multi-step provider->model selection via inline keyboards" +``` + +--- + +--- + +### Task 10: Add unit tests for provider system + +**Files:** +- Modify: `src/provider.rs` (add `#[cfg(test)] mod tests`) +- Modify: `src/config.rs` (add test for new config types) + +- [ ] **Step 1: Add resolve_model tests to provider.rs** + +Append at the end of `src/provider.rs`: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + fn test_registry() -> ProviderRegistry { + let mut providers: HashMap> = HashMap::new(); + providers.insert( + "openrouter".to_string(), + Arc::new(OpenRouterProvider::new(ProviderConfig { + name: "openrouter".to_string(), + provider_type: ProviderType::OpenRouter, + base_url: "https://openrouter.ai/api/v1".to_string(), + api_key: Some("sk-test".to_string()), + default_model: "moonshotai/kimi-k2.6".to_string(), + supports_vision: false, + max_tokens: 4096, + discover_models: false, + })), + ); + providers.insert( + "ollama".to_string(), + Arc::new(OllamaProvider::new(ProviderConfig { + name: "ollama".to_string(), + provider_type: ProviderType::Ollama, + base_url: "http://localhost:11434/v1".to_string(), + api_key: None, + default_model: "llama3.1".to_string(), + supports_vision: false, + max_tokens: 4096, + discover_models: true, + })), + ); + ProviderRegistry::new(providers, "openrouter".to_string()) + } + + #[test] + fn test_resolve_model_explicit_provider() { + let registry = test_registry(); + let (provider, model) = registry.resolve_model("ollama/llama3"); + assert_eq!(provider.name(), "ollama"); + assert_eq!(model, "llama3"); + } + + #[test] + fn test_resolve_model_falls_through_to_default() { + let registry = test_registry(); + // "moonshotai" is not a known provider β†’ falls to default (openrouter) + let (provider, model) = registry.resolve_model("moonshotai/kimi-k2.6"); + assert_eq!(provider.name(), "openrouter"); + assert_eq!(model, "moonshotai/kimi-k2.6"); + } + + #[test] + fn test_resolve_model_unknown_prefix_falls_through() { + let registry = test_registry(); + let (provider, model) = registry.resolve_model("nope/llama3"); + assert_eq!(provider.name(), "openrouter"); + assert_eq!(model, "nope/llama3"); + } + + #[test] + fn test_resolve_model_no_slash_uses_default() { + let registry = test_registry(); + let (provider, model) = registry.resolve_model("llama3"); + assert_eq!(provider.name(), "openrouter"); + assert_eq!(model, "llama3"); + } + + #[test] + fn test_provider_count_and_names() { + let registry = test_registry(); + assert_eq!(registry.provider_count(), 2); + let mut names = registry.provider_names(); + names.sort(); + assert_eq!(names, vec!["ollama", "openrouter"]); + } + + #[test] + fn test_get_provider_found() { + let registry = test_registry(); + let p = registry.get_provider("ollama"); + assert!(p.is_some()); + assert_eq!(p.unwrap().name(), "ollama"); + } + + #[test] + fn test_get_provider_not_found() { + let registry = test_registry(); + assert!(registry.get_provider("nonexistent").is_none()); + } +} +``` + +- [ ] **Step 2: Run resolve_model tests** + +```bash +cargo test -p rustfox -- provider::tests --nocapture +``` +Expected: All 7 tests pass. + +- [ ] **Step 3: Add backward-compat config parsing test to config.rs** + +```rust +#[test] +fn test_provider_section_parses_ollama() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + model = "moonshotai/kimi-k2.6" + [[provider]] + name = "ollama" + type = "ollama" + base_url = "http://localhost:11434/v1" + model = "llama3.1" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert_eq!(cfg.provider.len(), 1); + assert_eq!(cfg.provider[0].name, "ollama"); + assert_eq!(cfg.provider[0].model, "llama3.1"); +} + +#[test] +fn test_legacy_openrouter_auto_creates_provider() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + model = "moonshotai/kimi-k2.6" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + let (providers, default_name, _) = cfg.build_providers(); + assert!(providers.iter().any(|p| p.name == "openrouter")); + assert_eq!(default_name, "openrouter"); +} + +#[test] +fn test_visible_provider_overrides_legacy() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "old-key" + model = "moonshotai/kimi-k2.6" + [[provider]] + name = "openrouter" + type = "openrouter" + base_url = "https://openrouter.ai/api/v1" + api_key = "new-key" + model = "anthropic/claude-sonnet-4-6" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + let (providers, _default, _) = cfg.build_providers(); + // Should not have duplicate openrouter providers + let or_count = providers.iter().filter(|p| p.name == "openrouter").count(); + assert_eq!(or_count, 1, "should not duplicate openrouter when explicit [[provider]] exists"); + let or = providers.iter().find(|p| p.name == "openrouter").unwrap(); + assert_eq!(or.api_key.as_deref(), Some("new-key"), "explicit [[provider]] should win"); +} + +#[test] +fn test_fallback_config_parses() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + [fallback] + chain = ["openrouter/model-a", "ollama/model-b"] + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert_eq!(cfg.fallback.chain.len(), 2); + assert_eq!(cfg.fallback.chain[0], "openrouter/model-a"); +} + +#[test] +fn test_fallback_defaults_empty() { + let toml = r#" + [telegram] + bot_token = "tok" + allowed_user_ids = [1] + [openrouter] + api_key = "key" + "#; + let cfg: Config = toml::from_str(toml).unwrap(); + assert!(cfg.fallback.chain.is_empty()); +} +``` + +- [ ] **Step 4: Run config tests** + +```bash +cargo test -p rustfox -- config::tests --nocapture +``` +Expected: All new and existing config tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/provider.rs src/config.rs +git commit -m "test: add unit tests for ProviderRegistry, config parsing, backward compat" +``` + +--- + +### Task 11: Update config.example.toml with new provider sections + +**Files:** +- Modify: `config.example.toml` + +- [ ] **Step 1: Add [[provider]] and [fallback] documentation** + +Add after the existing `[openrouter]` example section in `config.example.toml`: + +```toml +# ── Multi-Provider Configuration (optional) ──────────────────────────── +# Define additional LLM providers alongside OpenRouter. +# Each [[provider]] block adds a new provider. Model strings use the +# format `provider_name/model_id` (e.g., `ollama/llama3.1`). +# The first provider becomes the default. + +# Example: Ollama (local models) +# [[provider]] +# name = "ollama" +# type = "ollama" +# base_url = "http://localhost:11434/v1" +# model = "llama3.1" +# discover_models = true + +# Example: Any OpenAI-compatible endpoint (LM Studio, vLLM, llama.cpp, etc.) +# [[provider]] +# name = "lmstudio" +# type = "openai_compatible" +# base_url = "http://localhost:1234/v1" +# model = "qwen2.5-7b-instruct" + +# ── Fallback Chain (optional) ────────────────────────────────────────── +# When the primary model fails, RustFox tries each fallback in order. +# Each entry is a full provider-prefixed model string. +# [fallback] +# chain = [ +# "openrouter/moonshotai/kimi-k2.6", +# "ollama/llama3.1", +# "lmstudio/qwen2.5-7b-instruct", +# ] +``` + +- [ ] **Step 2: Commit** + +```bash +git add config.example.toml +git commit -m "docs: add [[provider]] and [fallback] examples to config.example.toml" +``` + +--- + +### Self-Review Checklist + +- **Spec coverage:** Every section of the design spec has a corresponding task (including Β§7 config.example.toml β†’ Task 11): + - Β§3.1 Provider Trait β†’ Task 2 + - Β§3.2 Provider Types β†’ Tasks 3, 4, 5 + - Β§3.3 ProviderConfig β†’ Task 2 + - Β§3.4 ProviderRegistry β†’ Task 2 + - Β§3.5 LlmClient β†’ Task 6 + - Β§3.6 Fallback β†’ Task 7 (step 4) + - Β§4 Config β†’ Task 1 + - Β§5 Agent β†’ Task 7 + - Β§6 /models β†’ Task 9 + - Β§7 File changes β†’ covered + - Β§8 Error handling β†’ covered inline + - Β§9 Testing β†’ next task +- **Placeholder scan:** No TODOs or placeholders remain. All code blocks are complete. +- **Type consistency:** `ProviderConfig` defined in Task 2, used in Tasks 3/4/5/6/7. `ProviderRegistry::resolve_model()` returns `(&dyn Provider, &str)`, used consistently. diff --git a/docs/superpowers/specs/2026-06-22-multi-provider-design.md b/docs/superpowers/specs/2026-06-22-multi-provider-design.md new file mode 100644 index 0000000..74e7238 --- /dev/null +++ b/docs/superpowers/specs/2026-06-22-multi-provider-design.md @@ -0,0 +1,464 @@ +# Multi-Provider LLM Architecture β€” Design Spec + +**Date:** 2026-06-22 +**Status:** Draft +**Author:** RustFox Brainstorming Session + +## 1. Problem + +RustFox currently supports a single LLM provider (OpenRouter) via `[openrouter]` config. +This creates three limitations: + +1. **No fallback** β€” if OpenRouter is down, the bot is unusable +2. **No local models** β€” users cannot use Ollama, LM Studio, llama.cpp, or vLLM +3. **No provider mixing** β€” subagents always use the same endpoint as the main agent + +## 2. Goals + +- Support multiple LLM providers simultaneously (OpenRouter + local models) +- Each provider connects via an OpenAI-compatible `/chat/completions` endpoint +- Fallback chains: on provider failure, try the next in the chain +- Subagents can use any provider independently (e.g., main agent on Kimi, subagent on Ollama) +- Auto-discovery of local model lists where possible +- Backward compatible β€” existing `[openrouter]` config continues to work +- `/models` command gains multi-step providerβ†’model selection via inline keyboards + +## 3. Architecture + +### 3.1 Provider Trait + +New module `src/provider.rs` defines the core abstraction: + +```rust +#[async_trait] +pub trait Provider: Send + Sync { + fn name(&self) -> &str; + fn default_model(&self) -> &str; + fn supports_vision(&self) -> bool; + fn config(&self) -> &ProviderConfig; + + /// Send a chat completion request to this provider's endpoint. + /// `model` is already stripped of the provider prefix. + async fn chat_completion( + &self, + client: &reqwest::Client, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, + max_tokens: u32, + ) -> Result; + + /// Return available model IDs. Used by /models command. + async fn list_models(&self, client: &reqwest::Client) -> Result>; +} +``` + +### 3.2 Provider Types + +Three concrete implementations, all in `src/provider.rs`: + +| Provider | `list_models()` | Special behavior | +|---|---|---| +| `OpenRouterProvider` | `GET /v1/models` | Parameter sanitization, Kimi tool-call fallback | +| `OpenAICompatibleProvider` | `GET /v1/models` | Pure OpenAI protocol, `api_key` optional | +| `OllamaProvider` | `GET /api/tags` | Delegates chat to OpenAICompatibleProvider, custom discovery | + +`OpenAICompatibleProvider` handles LM Studio, llama.cpp, vLLM, Together AI, etc. β€” +any service exposing `/v1/chat/completions` with the OpenAI schema. + +`OllamaProvider` delegates its `chat_completion()` to an inner `OpenAICompatibleProvider` +(since Ollama's `/v1/chat/completions` is identical to the OpenAI protocol), but has its +own `list_models()` that calls `GET /api/tags` with a modified base URL path. +The `base_url` is stored as the full OpenAI-compatible path (e.g. `http://localhost:11434/v1`); +the Ollama provider strips a trailing `/v1`, `/v2`, or trailing slash then appends `/api/tags` +for discovery. If no versioned prefix is found, it appends `/api/tags` directly. +(e.g., `http://localhost:11434` β†’ `http://localhost:11434/api/tags`). + +### 3.3 ProviderConfig + +Deserialized from TOML, then used to construct a concrete Provider: + +```rust +#[derive(Debug, Clone)] +pub struct ProviderConfig { + pub name: String, // "ollama", "openrouter", etc. + pub provider_type: ProviderType, + pub base_url: String, + pub api_key: Option, + pub default_model: String, + pub supports_vision: bool, + pub max_tokens: u32, + pub discover_models: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum ProviderType { + OpenRouter, + OpenAICompatible, + Ollama, +} +``` + +### 3.4 ProviderRegistry + +Holds all configured providers and resolves model strings: + +```rust +pub struct ProviderRegistry { + providers: HashMap>, + default_provider: String, +} + +impl ProviderRegistry { + /// "ollama/llama3" β†’ (ollama_provider, "llama3") + /// "moonshotai/kimi-k2.6" β†’ (openrouter_provider, "moonshotai/kimi-k2.6") + /// (no provider named "moonshotai" β†’ falls through to default) + /// "llama3" (no slash) β†’ (default_provider, "llama3") + /// "nope/llama3" (unknown prefix) β†’ (default_provider, "nope/llama3") + pub fn resolve_model(&self, model: &str) -> (&dyn Provider, &str) { ... } + + pub fn get_provider(&self, name: &str) -> Option<&dyn Provider> { ... } + pub fn providers(&self) -> impl Iterator { ... } + pub fn default_provider_name(&self) -> &str { ... } +} +``` + +Model string resolution algorithm: +1. Split on first `/` +2. If first segment matches a known provider name β†’ use that provider, rest is model +3. If no match β†’ use default provider, full string is model + +This is backward compatible: `moonshotai/kimi-k2.6` with no provider named `moonshotai` +falls through to the default provider (openrouter) and sends `moonshotai/kimi-k2.6` as the +model name β€” exactly as today. + +### 3.5 LlmClient Changes + +LlmClient becomes a thin routing layer: + +```rust +pub struct LlmClient { + client: reqwest::Client, + registry: Arc, +} + +impl LlmClient { + pub async fn chat_completion_with_model( + &self, + messages: &[ChatMessage], + tools: &[ToolDefinition], + model: &str, // "ollama/llama3" or "moonshotai/kimi-k2.6" + ) -> Result { + let (provider, actual_model) = self.registry.resolve_model(model); + let max_tokens = provider.config().max_tokens; + provider.chat_completion( + &self.client, messages, tools, actual_model, max_tokens + ).await + .map(|mut completion| { + completion.model = model.to_string(); + completion + }) + } + + pub async fn list_all_models(&self) -> Vec { + // Iterate all providers, call list_models() for discoverable ones + } +} + +// --- Migration of existing API --- +// The old `chat_completion(&self, messages, tools)` method (no model param) is **removed**. +// It was only used internally; all callers now pass the model explicitly. +// `chat(&self, messages, tools)` stays as a convenience that reads from +// `self.registry.default_provider` for the model string. +// `chat_with_model()` is replaced by `chat_completion_with_model()`. +// `fetch_models()` becomes provider-specific: call `registry.get_provider("openrouter").list_models()`. + +/// A model as shown in the /models command, prefixed with its provider name. +pub struct ProviderModel { + pub provider: String, + pub model_id: String, // e.g. "llama3.1:8b" + pub qualified_id: String, // e.g. "ollama/llama3.1:8b" + pub description: String, // optional human-readable name from the provider + // OpenRouter: from ModelInfo.name + // Ollama: from /api/tags response name + // OpenAICompatible: from /v1/models name or empty +} +``` + +### 3.6 Fallback Chain + +Fallback lives in the Agent loop, not LlmClient. The agent's main loop and subagent loop +already have retry logic β€” we extend it to try different models on failure: + +```rust +// In process_message main loop: +let fallback_chain = &self.config.fallback.chain; // Vec + +for attempt in 0..=fallback_chain.len() { + let model = if attempt == 0 { + self.current_model.read().await.clone() + } else { + fallback_chain[attempt - 1].clone() + }; + + match self.llm.chat_completion_with_model(&prompt, &all_tools, &model).await { + Ok(completion) => { /* handle success */ break; } + Err(e) => { + warn!("Model '{}' failed: {}", model, e); + if attempt == fallback_chain.len() { + return Err(e); // all exhausted + } + continue; // try next fallback + } + } +} +``` + +Each model in the fallback chain is a full provider-prefixed string like +`"ollama/llama3.1"` β€” it resolves through `resolve_model` to whatever provider +owns that model. The same logic works for all providers. + +**After successful fallback**, the agent updates `current_model` to the working +fallback model in memory (for subsequent turns in the same conversation) but does +NOT persist to config.toml. The user can explicitly persist the working model +via `/model ` if desired. + +**Fallback applies only to the main agent loop**, not to subagent loops. Subagents +have an explicit model override (e.g., `model: "ollama/llama3"`) β€” if that provider +fails, the error propagates to the main agent, which can decide to retry the subagent +with a different model. This keeps subagent behavior predictable and avoids +surprising fallback within a delegated task. + +## 4. Config Changes + +### 4.1 New Config Struct Fields + +Add to `Config` in `src/config.rs`: + +```rust +#[serde(default)] +pub provider: Vec, + +#[serde(default)] +pub fallback: FallbackConfig, +``` + +```rust +pub struct ProviderSection { + pub name: String, + #[serde(rename = "type")] + pub provider_type: String, // "openrouter", "openai_compatible", "ollama" + // Unrecognized types cause a startup error + pub base_url: String, + pub api_key: Option, + #[serde(default = "default_model")] + pub model: String, + #[serde(default)] + pub supports_vision: bool, + #[serde(default = "default_max_tokens")] + pub max_tokens: u32, + #[serde(default)] + pub discover_models: bool, +} + +pub struct FallbackConfig { + #[serde(default)] + pub chain: Vec, +} +``` + +### 4.2 Backward Compatibility + +At startup, if the legacy `[openrouter]` section exists, it auto-creates a provider +named `"openrouter"` with those settings. The default provider is `"openrouter"` if +it exists, otherwise the first configured provider. + +If both `[openrouter]` AND `[[provider]] name = "openrouter"` are present, the explicit +`[[provider]]` entry wins and the legacy section is ignored (with a warning). +`set_model()` with an openrouter-prefixed model always modifies the same section that +was used to create the provider β€” there is no split-state path. + +### 4.3 Example config.toml + +```toml +# Legacy section β€” auto-creates "openrouter" provider (backward compat) +[openrouter] +api_key = "sk-..." +model = "moonshotai/kimi-k2.6" + +# Local Ollama (auto-discover models) +[[provider]] +name = "ollama" +type = "ollama" +base_url = "http://localhost:11434/v1" +model = "llama3.1" +discover_models = true + +# LM Studio (no api_key needed for local) +[[provider]] +name = "lmstudio" +type = "openai_compatible" +base_url = "http://localhost:1234/v1" +model = "qwen2.5-7b-instruct" + +# Fallback chain +[fallback] +chain = [ + "openrouter/moonshotai/kimi-k2.6", + "ollama/llama3.1", + "lmstudio/qwen2.5-7b-instruct", +] +``` + +## 5. Agent Integration + +### 5.1 Agent Constructor + +`Agent::new()` receives `Arc` and `LlmClient`: + +```rust +let llm = LlmClient::new(registry.clone()); +Self { + llm, + registry, + current_model: RwLock::new(initial_model), // full provider-prefixed string + ... +} +``` + +### 5.2 set_model() + +`set_model()` now: +1. Resolves the model string via `registry.resolve_model()` β€” always succeeds. + Additionally checks: if the model string contains a `/` but the prefix doesn't match + any known provider, log a warning (possible typo, but could be a valid model ID + like `moonshotai/kimi-k2.6` β€” never block the change). +2. Persists the full provider-prefixed model string (e.g., `"ollama/llama3.1"`) +3. Persists to config.toml as the `model` field of the corresponding provider section + (or the `[openrouter]` model if that's the provider) + +Config persistence follows the same approach as the current `set_model()`: +read TOML via `toml::from_str`, modify the `model` field in the right section, +write back via `toml::to_string_pretty`. This does NOT preserve comments/formatting +(consistent with existing behavior). If round-trip preservation is needed later, +a separate migration to `toml_edit` would cover all config writes at once. + +Model selection via `set_model()` persists the model to disk so it survives restarts. +At runtime, `current_model` is per-Agent (the bot has one agent instance, so it +appears "global" β€” affecting all conversations in that process). This is consistent +with the current behavior where model changes affect all users until the next restart. + +### 5.3 Subagent Model Resolution + +Subagent `model` overrides are full provider-prefixed strings like `"ollama/llama3"`. +The same `resolve_model()` path handles them β€” no special subagent logic needed. + +When a subagent is invoked via `invoke_agent` with `model: "ollama/llama3"`: +- The subagent loop calls `self.llm.chat_completion_with_model(..., "ollama/llama3")` +- LlmClient resolves to the `ollama` provider with model `llama3` +- The subagent runs entirely against that provider + +### 5.4 Per-Provider Vision + +The `supports_vision` flag moves from `config.openrouter.supports_vision` to +per-provider `ProviderConfig`. The agent checks this via the active provider: + +```rust +// In process_message, when deciding whether to send images or OCR: +let current = self.current_model.read().await; +let (provider, _) = self.registry.resolve_model(¤t); +if provider.config().supports_vision { + // Send base64 images as content parts +} else { + // OCR the images +} +``` + +## 6. `/models` Command Redesign + +### 6.1 Interactive Flow + +**Step 1 β€” `/models` with no args:** +If only one provider β†’ jump directly to Step 2 for that provider. +If multiple providers β†’ show inline keyboard: + +``` +Bot: Active model: openrouter/moonshotai/kimi-k2.6 + Select a provider: + [ollama] [openrouter] [lmstudio] + [❌ Cancel] +``` + +Callback format: `provider_select:{name}` + +**Step 2 β€” Provider selected:** + +*If provider has `discover_models=true` AND <= 20 models (e.g., Ollama):* +``` +Bot: Models on ollama (http://localhost:11434/v1): + [llama3.1:8b] [mistral:7b] [qwen2.5:7b] + [πŸ” Search all] [❌ Cancel] +``` + +*If provider has no discovery OR 300+ models (e.g., OpenRouter):* +``` +Bot: Models on openrouter (300+ available). + Send me a model name or ID to search for. + Examples: kimi, claude, gpt-4, gemini +``` + +**Step 3 β€” Model selection:** +Inline keyboard tap or text search β†’ `agent.set_model("ollama/llama3.1:8b")` + +``` +Bot: βœ… Model changed to "ollama/llama3.1:8b" +``` + +### 6.2 Implementation + +- New callback prefix `provider_select:{name}` handled in `handle_model_callback()` +- Existing `model_select:{id}` stores the full provider-prefixed string +- Search in `handle_model_search()` is scoped to the selected provider +- State tracked via `model_search_pending_{user_id}` includes the provider name +- `model_select:cancel` reverts to the provider picker + +## 7. File Changes Summary + +### New Files +| File | Purpose | +|---|---| +| `src/provider.rs` | `Provider` trait, `ProviderRegistry`, three impls | + +### Modified Files +| File | Changes | +|---|---| +| `src/config.rs` | Add `ProviderSection`, `FallbackConfig`; backward compat builder | +| `src/llm.rs` | `LlmClient` takes `Arc`, routing logic | +| `src/agent.rs` | Receive registry, fallback loop, subagent model resolution | +| `src/main.rs` | Build registry from config, pass to agent | +| `src/platform/telegram.rs` | `/models` multi-step providerβ†’model selection | +| `config.example.toml` | Document `[[provider]]` and `[fallback]` sections | + +## 8. Error Handling + +- **Unknown provider in model string**: `resolve_model()` always falls through to the default provider. `set_model()` may optionally warn about unrecognized prefixes as a typo check +- **Unknown `type` in config**: Startup error β€” unrecognized provider type is a hard config error +- **Provider connection failure**: Logged, fallback chain triggered +- **All fallbacks exhausted**: Error propagated to user as today +- **Auto-discovery failure**: Logged, provider still usable with its `default_model` +- **Vision on non-vision provider**: Falls back to OCR as today (per-provider `supports_vision` flag) + +## 9. Testing + +- `ProviderRegistry::resolve_model()` β€” various model string formats +- Fallback chain iteration +- Provider chat_completion serialization (each provider type) +- Callback handler parsing for new `provider_select:` prefix +- Backward compat config parsing + +## 10. Non-Goals + +- Rate limiting per provider (out of scope) +- Cost tracking per provider (out of scope) +- Dynamic provider discovery via mDNS (out of scope) +- Provider health checks / circuit breaker (future improvement) From c88b63686066c181b85fa658eacca95b8bcb1fbf Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Thu, 25 Jun 2026 15:54:28 +0800 Subject: [PATCH 09/10] feat: add fuzzy model matching via provider list_models in /models search --- src/platform/telegram.rs | 139 ++++++++++++++++++++++++++++++++++----- 1 file changed, 124 insertions(+), 15 deletions(-) diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index 56cc247..e1b065f 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -319,9 +319,12 @@ async fn handle_provider_model_select( Ok(()) } /// Accept a text model query and attempt to set the active model. -/// If the user previously selected a provider (via the inline keyboard), -/// the bare query is prefixed with that provider's name to form a -/// qualified model ID. +/// If the query is a bare name (e.g. "deepseek v4 flash"), fetches the +/// provider's model list via `list_models()` and does fuzzy matching to +/// resolve the actual model ID (e.g. "deepseek/deepseek-v4-flash"). +/// +/// If the user previously selected a provider via the inline keyboard, +/// the search is scoped to that provider. async fn handle_model_search( bot: Bot, chat_id: ChatId, @@ -329,32 +332,139 @@ async fn handle_model_search( query: &str, user_id: &str, ) -> ResponseResult<()> { - // Check if user has a stored provider from the interactive picker + // If query already has a known provider prefix, set directly + if let Some((prefix, _)) = query.split_once('/') { + if agent.registry.get_provider(prefix).is_some() { + return set_model_and_reply(bot, chat_id, agent, query).await; + } + } + + // Determine which provider to search let stored_provider = agent .memory .recall("settings", &format!("model_search_provider_{}", user_id)) .await .unwrap_or(None); - let qualified = if let Some(ref provider_name) = stored_provider { - // Clear the stored provider so it doesn't affect future searches + let provider_name = stored_provider + .clone() + .unwrap_or_else(|| agent.registry.default_provider_name().to_string()); + + // Clear stored provider so it doesn't affect future searches + if stored_provider.is_some() { agent .memory .forget("settings", &format!("model_search_provider_{}", user_id)) .await .ok(); - if query.contains('/') { - query.to_string() - } else { - format!("{}/{}", provider_name, query) + } + + let provider = match agent.registry.get_provider(&provider_name) { + Some(p) => p, + None => { + return set_model_and_reply( + bot, + chat_id, + agent, + &format!("{}/{}", provider_name, query), + ) + .await; } - } else { - query.to_string() }; - match agent.set_model(&qualified).await { + // Fetch model list and fuzzy match + match provider.list_models(&agent.llm.client).await { + Ok(models) if !models.is_empty() => { + let q = query + .to_lowercase() + .replace(['-', '_', '.', ' '], ""); + + // Exact match (full model ID) + if let Some(exact) = models + .iter() + .find(|m| m.to_lowercase() == query.to_lowercase()) + { + return set_model_and_reply( + bot, + chat_id, + agent, + &format!("{}/{}", provider_name, exact), + ) + .await; + } + + // Fuzzy match: normalize both sides and check containment + let mut matches: Vec<&String> = models + .iter() + .filter(|m| { + m.to_lowercase() + .replace(['-', '_', '.', ' '], "") + .contains(&q) + }) + .collect(); + matches.sort(); + matches.truncate(10); + + match matches.len() { + 0 => { + // No fuzzy match β€” try direct set anyway + return set_model_and_reply( + bot, + chat_id, + agent, + &format!("{}/{}", provider_name, query), + ) + .await; + } + 1 => { + return set_model_and_reply( + bot, + chat_id, + agent, + &format!("{}/{}", provider_name, matches[0]), + ) + .await; + } + _ => { + let mut reply = format!( + "Multiple models match '{}' on **{}**:\n\n", + query, provider_name + ); + for m in &matches { + reply.push_str(&format!("`{}/{m}`\n", provider_name)); + } + reply.push_str("\nUse `/models ` to set one."); + bot.send_message(chat_id, escape_text(&reply)) + .parse_mode(ParseMode::MarkdownV2) + .await?; + } + } + } + _ => { + // API unavailable or empty list β€” try direct set + return set_model_and_reply( + bot, + chat_id, + agent, + &format!("{}/{}", provider_name, query), + ) + .await; + } + } + + Ok(()) +} + +/// Set the model and send a success/failure reply. +async fn set_model_and_reply( + bot: Bot, + chat_id: ChatId, + agent: &Arc, + model_id: &str, +) -> ResponseResult<()> { + match agent.set_model(model_id).await { Ok(()) => { - let reply = format!("βœ… Model changed to `{}`", qualified); + let reply = format!("βœ… Model changed to `{}`", model_id); bot.send_message(chat_id, escape_text(&reply)) .parse_mode(ParseMode::MarkdownV2) .await?; @@ -368,7 +478,6 @@ async fn handle_model_search( .await?; } } - Ok(()) } From 241fbb86051b4ec4a99753b1733c69c4c3abcdfe Mon Sep 17 00:00:00 2001 From: "chinkan.ai" Date: Thu, 25 Jun 2026 16:12:39 +0800 Subject: [PATCH 10/10] fmt: fix chained method call formatting --- src/platform/telegram.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/platform/telegram.rs b/src/platform/telegram.rs index e1b065f..cb998ee 100644 --- a/src/platform/telegram.rs +++ b/src/platform/telegram.rs @@ -375,9 +375,7 @@ async fn handle_model_search( // Fetch model list and fuzzy match match provider.list_models(&agent.llm.client).await { Ok(models) if !models.is_empty() => { - let q = query - .to_lowercase() - .replace(['-', '_', '.', ' '], ""); + let q = query.to_lowercase().replace(['-', '_', '.', ' '], ""); // Exact match (full model ID) if let Some(exact) = models