From 178d529b10caeb52ae34ed8c93575da8b20e92b9 Mon Sep 17 00:00:00 2001 From: Felix Wunderlich Date: Thu, 5 Nov 2026 19:30:28 +0100 Subject: [PATCH] Add Opper as a model provider Add Opper, a European AI gateway with 700+ models behind one OpenAI-compatible API and one key, as a model provider. It follows the OpenRouter provider: a menu group with the same mix of models, a settings form where any other model can be typed in, an API key in Settings or OPPER_API_KEY, and a README entry. Model IDs are prefixed "opper/", since Opper's IDs are the models' own names (e.g. claude-sonnet-5-5). Each response is its own request to Opper's chat completions, as for OpenRouter. Disclosure: Opper is my startup. Co-Authored-By: Claude Opus 5.5 (1M context) --- README.md | 1 + chainforge/flask_app.py | 1 + .../react-server/src/GlobalSettingsModal.tsx | 9 + .../react-server/src/LLMListComponent.tsx | 2 + .../react-server/src/ModelSettingSchemas.tsx | 137 +++++++++ .../src/backend/__test__/opper.test.ts | 269 ++++++++++++++++++ chainforge/react-server/src/backend/models.ts | 19 ++ chainforge/react-server/src/backend/utils.ts | 103 ++++++- chainforge/react-server/src/store.tsx | 7 + 9 files changed, 546 insertions(+), 2 deletions(-) create mode 100644 chainforge/react-server/src/backend/__test__/opper.test.ts diff --git a/README.md b/README.md index 6234981cf..e653ad4ab 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ For detailed Docker documentation including architecture support, environment va - Anthropic - Google Gemini - DeepSeek +- [Opper](https://opper.ai) (a European AI gateway: 700+ models from 50+ providers, with one API key) - [HuggingFace](https://huggingface.co/docs/inference-providers) (Inference Providers and dedicated Endpoints) - Together.ai - [Ollama API](https://github.com/jmorganca/ollama) (locally-hosted models) diff --git a/chainforge/flask_app.py b/chainforge/flask_app.py index 370e5a23b..c5831b5cd 100644 --- a/chainforge/flask_app.py +++ b/chainforge/flask_app.py @@ -694,6 +694,7 @@ def fetchEnvironAPIKeys(): 'DEEPSEEK_API_KEY': 'DeepSeek', 'MINIMAX_API_KEY': 'MiniMax', 'OPENROUTER_API_KEY': 'OpenRouter', + 'OPPER_API_KEY': 'Opper', } d = { alias: os.environ.get(key) for key, alias in keymap.items() } ret = jsonify(d) diff --git a/chainforge/react-server/src/GlobalSettingsModal.tsx b/chainforge/react-server/src/GlobalSettingsModal.tsx index db07ab5cd..3ffe7c3c1 100644 --- a/chainforge/react-server/src/GlobalSettingsModal.tsx +++ b/chainforge/react-server/src/GlobalSettingsModal.tsx @@ -267,6 +267,7 @@ const GlobalSettingsModal = forwardRef( DeepSeek: "", MiniMax: "", OpenRouter: "", + Opper: "", Cohere: "", }, @@ -739,6 +740,14 @@ const GlobalSettingsModal = forwardRef( />
+ +
+ = { together: TOGETHER_PREFIX, openrouter: OPENROUTER_PREFIX, "openrouter-image": OPENROUTER_IMAGE_PREFIX, + opper: OPPER_PREFIX, hf: HUGGINGFACE_PREFIX, bedrock: BEDROCK_PREFIX, }; diff --git a/chainforge/react-server/src/ModelSettingSchemas.tsx b/chainforge/react-server/src/ModelSettingSchemas.tsx index 3b22029c0..dc1ec8639 100644 --- a/chainforge/react-server/src/ModelSettingSchemas.tsx +++ b/chainforge/react-server/src/ModelSettingSchemas.tsx @@ -926,6 +926,140 @@ export const OpenRouterImageSettings: ModelSettingsDict = { postprocessors: {}, }; +/** + * Models reached through Opper's OpenAI-compatible chat completions API. The + * listed models fill the model menu; any other model at https://opper.ai/models + * can be typed in. + */ +export const OpperSettings: ModelSettingsDict = { + fullName: "Opper", + schema: { + type: "object", + required: ["shortname"], + properties: { + shortname: { + type: "string", + title: "Nickname", + description: + "Unique identifier to appear in ChainForge. Keep it short.", + default: "Opper", + }, + model: { + type: "string", + title: "Model", + description: + "The Opper model to query. Pick a popular one, or type any model listed at https://opper.ai/models (e.g. claude-sonnet-5-5). A provider/model ID (e.g. anthropic/claude-sonnet-5-5) pins one route.", + // The same mix of frontier and cheap models as the OpenRouter menu. + enum: [ + "claude-sonnet-5-5", + "claude-opus-5-5", + "claude-sonnet-5", + "claude-haiku-4-5", + "gpt-6.1-sol", + "gpt-6-luna", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gemini-3.8-flash", + "gemini-3.1-flash-lite", + "grok-4.7", + "deepseek-v4-pro", + "deepseek-v4.1-flash", + "qwen3.8-max", + "qwen3.8-flash", + "kimi-k3", + ], + default: "claude-sonnet-5-5", + shortname_map: { + "claude-sonnet-5-5": "Claude Sonnet 5.5", + "claude-opus-5-5": "Claude Opus 5.5", + "claude-sonnet-5": "Claude Sonnet 5", + "claude-haiku-4-5": "Claude Haiku 4.5", + "gpt-6.1-sol": "GPT-6.1 Sol", + "gpt-6-luna": "GPT-6 Luna", + "gpt-5.4-mini": "GPT-5.4 Mini", + "gpt-5.4-nano": "GPT-5.4 Nano", + "gemini-3.8-flash": "Gemini 3.8 Flash", + "gemini-3.1-flash-lite": "Gemini 3.1 Flash-Lite", + "grok-4.7": "Grok 4.7", + "deepseek-v4-pro": "DeepSeek V4 Pro", + "deepseek-v4.1-flash": "DeepSeek V4.1 Flash", + "qwen3.8-max": "Qwen3.8 Max", + "qwen3.8-flash": "Qwen3.8 Flash", + "kimi-k3": "Kimi K3", + }, + }, + system_msg: { + type: "string", + title: "system_msg", + description: + "Many conversations begin with a system message to gently instruct the assistant. By default, ChainForge includes the suggested 'You are a helpful assistant.'", + default: "You are a helpful assistant.", + allow_empty_str: true, + }, + temperature: { + type: "number", + title: "temperature", + description: + "What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. Some models only take their default.", + default: 1, + minimum: 0, + maximum: 2, + multipleOf: 0.01, + }, + response_format: { + type: "string", + title: "response_format", + description: + "An object specifying the format that the model must output. Can be 'text' or 'json_object', or a JSON schema specifying structured outputs. In ChainForge, you should only specify text, json_object, or the verbatim JSON schema---do not add a JSON object with a 'type' parameter surrounding these values. Not every model supports structured outputs. IMPORTANT: when using JSON mode, you must also instruct the model to produce JSON yourself via a system or user message.", + default: "text", + }, + tools: { + type: "string", + title: "tools", + description: + "A list of JSON schema objects, each with 'name', 'description', and 'parameters' keys, which describe functions the model may generate JSON inputs for.", + default: "", + }, + tool_choice: { + type: "string", + title: "tool_choice", + description: + "Controls how the model responds to function calls. 'none' means the model does not call a function, and responds to the end-user. 'auto' means the model can pick between an end-user or calling a function. 'required' means the model must call one or more tools. Specifying a particular function name forces the model to call only that function. Leave blank for default behavior.", + default: "", + }, + stop: { + type: "string", + title: "stop sequences", + description: + 'Up to 4 sequences where the API will stop generating further tokens. Enclose stop sequences in double-quotes "" and use whitespace to separate them.', + default: "", + }, + max_tokens: { + type: "integer", + title: "max_tokens", + description: + "The maximum number of tokens to generate. Reasoning models may spend much of this on reasoning, so set it generously for them.", + }, + }, + }, + uiSchema: { + ...ChatGPTSettings.uiSchema, + model: { + "ui:help": + "Defaults to claude-sonnet-5-5. Type to enter any other Opper model ID.", + "ui:widget": "datalist", + }, + temperature: { + "ui:help": "Defaults to 1.0.", + "ui:widget": "range", + }, + max_tokens: { + "ui:help": "Defaults to blank (not sent).", + }, + }, + postprocessors: ChatGPTSettings.postprocessors, +}; + const DalleSettings: ModelSettingsDict = { // OpenAI shut down DALL·E on May 12, 2026. Kept so flows that use it still open. fullName: "Dall-E Image Models (OpenAI; shut down May 2026)", @@ -2451,6 +2585,7 @@ export const ModelSettings: Dict = { minimax: MiniMaxSettings, openrouter: OpenRouterSettings, "openrouter-image": OpenRouterImageSettings, + opper: OpperSettings, webllm: WebLLMSettings, }; @@ -2484,6 +2619,7 @@ export function baseModelToProvider(base_model: string): LLMProvider { minimax: LLMProvider.MiniMax, openrouter: LLMProvider.OpenRouter, "openrouter-image": LLMProvider.OpenRouter, + opper: LLMProvider.Opper, webllm: LLMProvider.WebLLM, }; return lookup[base_model] ?? LLMProvider.Custom; @@ -2516,6 +2652,7 @@ export function getSettingsSchemaForLLM( [LLMProvider.DeepSeek]: DeepSeekSettings, [LLMProvider.MiniMax]: MiniMaxSettings, [LLMProvider.OpenRouter]: OpenRouterSettings, + [LLMProvider.Opper]: OpperSettings, [LLMProvider.WebLLM]: WebLLMSettings, }; diff --git a/chainforge/react-server/src/backend/__test__/opper.test.ts b/chainforge/react-server/src/backend/__test__/opper.test.ts new file mode 100644 index 000000000..87284129f --- /dev/null +++ b/chainforge/react-server/src/backend/__test__/opper.test.ts @@ -0,0 +1,269 @@ +/* + * @jest-environment jsdom + */ + +// Same module stubs as openrouter.test.ts, so the provider calls can be +// tested without their dependencies. +jest.mock("../cache", () => ({ + __esModule: true, + default: class StorageCache { + static getInstance() { + return new StorageCache(); + } + }, + StringLookup: { get: (x: unknown) => x }, + MediaLookup: {}, +})); +jest.mock("@google/genai", () => ({ GoogleGenAI: jest.fn() })); +jest.mock("../pyodide/exec-py", () => ({ execPy: jest.fn() })); +jest.mock("../../store", () => ({ + __esModule: true, + default: { + getState: () => ({ AvailableLLMs: [], setAvailableLLMs: jest.fn() }), + }, +})); + +// eslint-disable-next-line import/first +import { beforeAll, describe, expect, test } from "@jest/globals"; +// eslint-disable-next-line import/first +import { + call_llm, + call_opper, + clear_api_keys, + extract_responses, + set_api_keys, +} from "../utils"; +// eslint-disable-next-line import/first +import { + LLMProvider, + RATE_LIMIT_BY_PROVIDER, + getProvider, + stripOpperPrefix, +} from "../models"; +// eslint-disable-next-line import/first +import { + ModelSettings, + baseModelToProvider, + getDefaultModelSettings, + getSettingsSchemaForLLM, +} from "../../ModelSettingSchemas"; +// eslint-disable-next-line import/first +import { Dict } from "../typing"; + +type Call = { url: string; init: RequestInit }; +let calls: Call[] = []; + +/** Replaces fetch with one returning the given responses in order (repeating the last). */ +const mockFetch = (...responses: { status?: number; body: unknown }[]) => { + calls = []; + let i = 0; + (globalThis as any).fetch = jest.fn( + async (url: string, init: RequestInit) => { + calls.push({ url, init }); + const { status = 200, body } = + responses[Math.min(i++, responses.length - 1)]; + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + }; + }, + ); +}; + +const jsonBody = (call: Call) => JSON.parse(call.init.body as string); +const headers = (call: Call) => call.init.headers as Record; + +const chatReply = (message: Dict, finish_reason = "stop") => ({ + choices: [{ message: { role: "assistant", ...message }, finish_reason }], +}); + +beforeAll(() => { + set_api_keys({ Opper: "op-test" }); +}); + +describe("recognizing Opper models", () => { + test("prefixed model IDs map to Opper, including typed-in ones", () => { + expect(getProvider("opper/claude-sonnet-5-5")).toBe(LLMProvider.Opper); + expect(getProvider("opper/anthropic/claude-sonnet-5-5")).toBe( + LLMProvider.Opper, + ); + expect(getProvider("opper/some-brand-new-model")).toBe(LLMProvider.Opper); + // The same names without the prefix belong to their own providers. + expect(getProvider("claude-sonnet-5-5")).toBe(LLMProvider.Anthropic); + }); + + test("the prefix is stripped for the API", () => { + expect(stripOpperPrefix("opper/gpt-5.4-mini")).toBe("gpt-5.4-mini"); + expect(stripOpperPrefix("opper/anthropic/claude-sonnet-5-5")).toBe( + "anthropic/claude-sonnet-5-5", + ); + }); + + test("base model, settings form, and rate limit", () => { + expect(baseModelToProvider("opper")).toBe(LLMProvider.Opper); + expect(getSettingsSchemaForLLM("opper/some-brand-new-model")).toBe( + ModelSettings.opper, + ); + expect(RATE_LIMIT_BY_PROVIDER[LLMProvider.Opper]).toBeDefined(); + }); + + test("the settings form names every listed model, and lists its default", () => { + const spec = ModelSettings.opper.schema.properties.model; + expect(spec.enum).toContain(spec.default); + const names = spec.shortname_map as Record; + for (const model of spec.enum as string[]) + expect(names[model]).toBeTruthy(); + }); +}); + +describe("Opper chat completions", () => { + test("sends the stripped model ID, the key, and only the settings that are set", async () => { + mockFetch({ body: chatReply({ content: "Hi!" }) }); + const [query, responses] = await call_opper( + "Hello", + "opper/claude-sonnet-5-5", + 1, + 0.5, + { + system_msg: "Be brief.", + max_tokens: NaN, // blank number field + stop: [], + tools: [], + tool_choice: "", + }, + ); + + expect(calls).toHaveLength(1); + expect(calls[0].url).toBe( + "https://api.opper.ai/v3/compat/chat/completions", + ); + expect(headers(calls[0]).Authorization).toBe("Bearer op-test"); + + const body = jsonBody(calls[0]); + expect(body.model).toBe("claude-sonnet-5-5"); + expect(body.temperature).toBe(0.5); + expect(body).not.toHaveProperty("n"); + for (const key of [ + "max_tokens", + "stop", + "tools", + "tool_choice", + "system_msg", + ]) + expect(body).not.toHaveProperty(key); + expect(body.messages[0]).toEqual({ role: "system", content: "Be brief." }); + expect(body.messages[body.messages.length - 1].role).toBe("user"); + + expect(query.model).toBe("claude-sonnet-5-5"); + expect( + extract_responses( + responses, + "opper/claude-sonnet-5-5", + LLMProvider.Opper, + ), + ).toEqual(["Hi!"]); + }); + + test("the form's defaults send only the model, temperature and messages", async () => { + const settings = getDefaultModelSettings("opper"); + expect(settings.max_tokens).toBeNull(); + mockFetch({ body: chatReply({ content: "Hi!" }) }); + await call_opper("Q", "opper/gpt-5.4-mini", 1, 1, settings); + expect(Object.keys(jsonBody(calls[0])).sort()).toEqual([ + "messages", + "model", + "temperature", + ]); + }); + + test("a plain-text response_format is left out, but a JSON one is sent", async () => { + mockFetch({ body: chatReply({ content: "Hi!" }) }); + await call_opper("Q", "opper/gpt-5.4-mini", 1, 1, { + response_format: { type: "text" }, + }); + expect(jsonBody(calls[0])).not.toHaveProperty("response_format"); + + mockFetch({ body: chatReply({ content: "{}" }) }); + await call_opper("Q", "opper/gpt-5.4-mini", 1, 1, { + response_format: { type: "json_object" }, + }); + expect(jsonBody(calls[0]).response_format).toEqual({ type: "json_object" }); + }); + + test("n responses are n requests", async () => { + mockFetch( + { body: chatReply({ content: "one" }) }, + { body: chatReply({ content: "two" }) }, + { body: chatReply({ content: "three" }) }, + ); + const [, responses] = await call_opper("Count", "opper/kimi-k3", 3); + expect(calls).toHaveLength(3); + expect( + extract_responses(responses, "opper/kimi-k3", LLMProvider.Opper), + ).toEqual(["one", "two", "three"]); + }); + + test("tool calls are extracted like OpenAI's", () => { + const reply = chatReply( + { + content: null, + tool_calls: [ + { function: { name: "get_weather", arguments: '{"city":"Oslo"}' } }, + ], + }, + "tool_calls", + ); + expect( + extract_responses([reply], "opper/gpt-6-luna", LLMProvider.Opper), + ).toEqual(['[[TOOLS]] get_weather {"city":"Oslo"}']); + }); + + test("a model that runs out of tokens before answering fails with advice", async () => { + mockFetch({ body: chatReply({ content: null }, "length") }); + await expect( + call_opper("Q", "opper/deepseek-v4-pro", 1, 1, { max_tokens: 50 }), + ).rejects.toThrow(/ran out of tokens before it answered/); + }); + + test("API errors surface their message", async () => { + mockFetch({ + status: 400, + body: { error: { code: "bad_request", message: "Unknown model" } }, + }); + await expect(call_opper("Q", "opper/not-a-model")).rejects.toThrow( + "Unknown model", + ); + }); + + test("a rejected key is named as the problem", async () => { + mockFetch({ status: 401, body: undefined }); + await expect(call_opper("Q", "opper/gpt-5.4-mini")).rejects.toThrow( + "Opper did not accept the API key", + ); + }); + + test("a missing key is named as the problem, before any request", async () => { + const env = process.env.OPPER_API_KEY; + delete process.env.OPPER_API_KEY; + clear_api_keys(); + mockFetch({ body: chatReply({ content: "Hi!" }) }); + try { + await expect(call_opper("Q", "opper/gpt-5.4-mini")).rejects.toThrow( + "Could not find an Opper API key", + ); + expect(calls).toHaveLength(0); + } finally { + if (env !== undefined) process.env.OPPER_API_KEY = env; + set_api_keys({ Opper: "op-test" }); + } + }); + + test("call_llm routes Opper models to its chat completions", async () => { + mockFetch({ body: chatReply({ content: "Hi" }) }); + await call_llm("opper/gemini-3.8-flash", LLMProvider.Opper, "Hello", 1, 1); + expect(calls[0].url).toBe( + "https://api.opper.ai/v3/compat/chat/completions", + ); + }); +}); diff --git a/chainforge/react-server/src/backend/models.ts b/chainforge/react-server/src/backend/models.ts index 30348032e..075b58b4e 100644 --- a/chainforge/react-server/src/backend/models.ts +++ b/chainforge/react-server/src/backend/models.ts @@ -310,6 +310,7 @@ export enum LLMProvider { DeepSeek = "deepseek", MiniMax = "minimax", OpenRouter = "openrouter", + Opper = "opper", Custom = "__custom", } @@ -541,6 +542,22 @@ export function stripOpenRouterPrefix(llm: LLM | string): string { return name; } +/** + * Opper serves hundreds of models, and users can type in any model listed at + * https://opper.ai/models. Its model IDs are the models' own names (e.g. + * "claude-sonnet-5-5"), so ChainForge prefixes them to map back to Opper + * rather than to the model's own lab. + */ +export const OPPER_PREFIX = "opper/"; + +/** The model ID that Opper expects, without ChainForge's prefix. */ +export function stripOpperPrefix(llm: LLM | string): string { + const name = llm.toString(); + return name.startsWith(OPPER_PREFIX) + ? name.substring(OPPER_PREFIX.length) + : name; +} + /** * Hugging Face serves hundreds of models through Inference Providers, and users * can type in any model ID listed at https://router.huggingface.co/v1/models @@ -592,6 +609,7 @@ export function getProvider(llm: LLM): LLMProvider | undefined { isOpenRouterImageModel(llm) ) return LLMProvider.OpenRouter; + else if (llm.toString().startsWith(OPPER_PREFIX)) return LLMProvider.Opper; else if (llm_name?.startsWith("OpenAI")) return LLMProvider.OpenAI; else if (llm_name?.startsWith("Azure")) return LLMProvider.Azure_OpenAI; else if (llm_name?.startsWith("GEMINI")) return LLMProvider.Google; @@ -654,6 +672,7 @@ export const RATE_LIMIT_BY_PROVIDER: { [key in LLMProvider]?: number } = { [LLMProvider.DeepSeek]: 1000, // DeepSeek does not constrain users atm but they might in the future. To be safe we are limiting it to 1000 queries per minute. [LLMProvider.MiniMax]: 1000, // MiniMax API rate limits are generous; 1000 RPM to be safe. [LLMProvider.OpenRouter]: 500, // OpenRouter sets no fixed limit for paid models, but the providers behind it do; 500 RPM to be safe. + [LLMProvider.Opper]: 500, // Same as OpenRouter: 500 RPM to be safe. [LLMProvider.Bedrock]: 100, // Bedrock quotas are per account, region and model; a conservative floor. }; diff --git a/chainforge/react-server/src/backend/utils.ts b/chainforge/react-server/src/backend/utils.ts index 906b418fa..d932e275e 100644 --- a/chainforge/react-server/src/backend/utils.ts +++ b/chainforge/react-server/src/backend/utils.ts @@ -19,6 +19,7 @@ import { stripHuggingFacePrefix, stripTogetherPrefix, stripOpenRouterPrefix, + stripOpperPrefix, } from "./models"; import { Dict, @@ -257,6 +258,7 @@ let TOGETHER_API_KEY = get_environ("TOGETHER_API_KEY"); let DEEPSEEK_API_KEY = get_environ("DEEPSEEK_API_KEY"); let MINIMAX_API_KEY = get_environ("MINIMAX_API_KEY"); let OPENROUTER_API_KEY = get_environ("OPENROUTER_API_KEY"); +let OPPER_API_KEY = get_environ("OPPER_API_KEY"); /** * The AWS SDK is a heavy dependency that reaches for web-stream globals as soon @@ -354,6 +356,7 @@ export function clear_api_keys(): void { DEEPSEEK_API_KEY = get_environ("DEEPSEEK_API_KEY"); MINIMAX_API_KEY = get_environ("MINIMAX_API_KEY"); OPENROUTER_API_KEY = get_environ("OPENROUTER_API_KEY"); + OPPER_API_KEY = get_environ("OPPER_API_KEY"); } /** @@ -397,6 +400,7 @@ export function set_api_keys(given_keys: Dict): void { if (key_is_present("DeepSeek")) DEEPSEEK_API_KEY = api_keys.DeepSeek; if (key_is_present("MiniMax")) MINIMAX_API_KEY = api_keys.MiniMax; if (key_is_present("OpenRouter")) OPENROUTER_API_KEY = api_keys.OpenRouter; + if (key_is_present("Opper")) OPPER_API_KEY = api_keys.Opper; } export function get_azure_openai_api_keys(): [ @@ -1273,6 +1277,97 @@ export async function call_openrouter_image_gen( ]; } +const OPPER_BASE_URL = "https://api.opper.ai/v3/compat"; + +/** + * Calls a model through Opper's OpenAI-compatible chat completions API. + * As with OpenRouter, each response is its own request, since `n` isn't taken. + */ +export async function call_opper( + prompt: string, + model: LLM, + n = 1, + temperature = 1.0, + params?: Dict, + should_cancel?: () => boolean, + images?: string[], +): Promise<[Dict, Dict]> { + if (!OPPER_API_KEY) + throw new Error( + "Could not find an Opper API key. Double-check that your API key is set in Settings or in your local environment.", + ); + + const modelname = stripOpperPrefix(model); + const settings: Dict = { ...params }; + strip_empty_chat_params(settings); + // Plain text is the default anyway, so it isn't sent. + if (settings.response_format?.type === "text") + delete settings.response_format; + + const chat_history: ChatHistory | undefined = settings.chat_history; + const system_msg: string | undefined = settings.system_msg; + delete settings.chat_history; + delete settings.system_msg; + + for (const [key, value] of Object.entries(settings)) + if (is_blank_setting(value)) delete settings[key]; + + const query: Dict = { model: modelname, temperature, ...settings }; + query.messages = await resolve_images_in_user_messages( + construct_chat_history(prompt, images, chat_history, system_msg), + "openai", + ); + + console.log(`Querying Opper model '${modelname}' (n=${n})...`); + + const responses: Dict[] = []; + while (responses.length < n) { + if (should_cancel && should_cancel()) throw new UserForcedPrematureExit(); + + const start = performance.now(); + const res = await fetch(`${OPPER_BASE_URL}/chat/completions`, { + method: "POST", + headers: { + Authorization: `Bearer ${OPPER_API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(query), + }); + + let payload: Dict | undefined; + try { + payload = await res.json(); + } catch { + payload = undefined; + } + if (res.status === 401) + throw new Error( + "Opper did not accept the API key. Check the key in Settings.", + ); + if (!res.ok || payload?.error) + throw new Error( + payload?.error?.message ?? `Opper request failed (HTTP ${res.status}).`, + ); + + const choice = payload?.choices?.[0]; + if (!payload || !choice) throw new Error("Opper returned no response."); + + // A reasoning model can spend its whole token budget thinking, leaving no + // answer. Say so, rather than recording a blank response. + const message = choice.message ?? {}; + const answered = message.content || message.tool_calls?.length > 0; + if (!answered && choice.finish_reason === "length") + throw new Error( + `${modelname} ran out of tokens before it answered. Raise max_tokens.`, + ); + + payload[LATENCY_KEY] = performance.now() - start; + responses.push(payload); + } + + return [query, responses]; +} + /** Whether an OpenAI model reasons: the o-series, and GPT-5 and later (but not their chat models). */ function is_openai_reasoning_model(model: string): boolean { return /^(o\d|gpt-[5-9])/.test(model) && !/-chat/.test(model); @@ -3412,6 +3507,7 @@ export async function call_llm( : isDecisionModel(llm) ? call_openrouter_decision : call_openrouter; + else if (llm_provider === LLMProvider.Opper) call_api = call_opper; if (call_api === undefined) throw new Error( `Adapter for Language model ${llm} and ${llm_provider} not found`, @@ -3503,8 +3599,9 @@ function _extract_openai_responses(response: Dict): Array { } /** - * Extracts the text of OpenRouter chat responses: one request per response, - * each in OpenAI's format. A blank answer is kept as "" rather than null. + * Extracts the text of OpenRouter (and Opper) chat responses: one request per + * response, each in OpenAI's format. A blank answer is kept as "" rather than + * null. */ function _extract_openrouter_chat_responses( responses: Array, @@ -3951,6 +4048,8 @@ export function extract_responses( if (isDecisionModel(llm)) return _extract_decision_responses(response as Dict[]); return _extract_openrouter_chat_responses(response as Dict[]); + case LLMProvider.Opper: + return _extract_openrouter_chat_responses(response as Dict[]); default: if ( Array.isArray(response) && diff --git a/chainforge/react-server/src/store.tsx b/chainforge/react-server/src/store.tsx index 82ffc3d2d..9b4163bc8 100644 --- a/chainforge/react-server/src/store.tsx +++ b/chainforge/react-server/src/store.tsx @@ -36,6 +36,7 @@ import { HuggingFaceSettings, OpenRouterImageSettings, OpenRouterSettings, + OpperSettings, TogetherChatSettings, } from "./ModelSettingSchemas"; import { @@ -46,6 +47,7 @@ import { OPENROUTER_IMAGE_PREFIX, OPENROUTER_PREFIX, OPENROUTER_EMOJI, + OPPER_PREFIX, openRouterEmoji, } from "./backend/models"; import { StringLookup } from "./backend/cache"; @@ -374,6 +376,11 @@ export const initLLMProviderMenu: (LLMSpec | LLMGroup)[] = [ }, ], }, + { + group: "Opper", + emoji: "🧭", + items: prefixedMenuItems(OpperSettings, OPPER_PREFIX, "opper", "🧭", 1.0), + }, { group: "HuggingFace", emoji: "🤗",