fix: keep requests within provider image limits

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@alphaxiv/agents",
-  "version": "0.7.11",
+  "version": "0.7.12",
   "license": "MIT",
   "fmt": {
     "lineWidth": 120

src/adapters/anthropic/adapter.ts

@@ -9,6 +9,7 @@ import {
   anthropicModelStructuredOutputSupport,
   anthropicModelThinkingSupport,
   type EffortLevel,
+  getAnthropicMaxImages,
   getAnthropicMessagesStreamConfig,
   type SupportedEffortLevel,
   type SupportedThinkingLevel,
@@ -99,6 +100,7 @@ export function anthropicModel<zO, zI, TModel extends AnthropicModels | (string
     ? (options.effort ?? modelConfig.effortSchema.parse(undefined))
     : options.effort;
   const thinkingDisplay = options.thinkingDisplay ?? "summarized";
+  const maxImages = getAnthropicMaxImages(options.model);
   const interleaved = options.interleaved;
   const streamConfig = modelConfig
     ? getAnthropicMessagesStreamConfig({
@@ -154,7 +156,7 @@ ${JSON.stringify(structuredOutput.originalJsonSchema, null, 2)}
       { history, instructions, tools, signal, output, cache: cacheDefault }: AdapterStreamOptions<zO, zI>,
     ): AdapterStreamIterator {
       const normalizedTools = normalizeAnthropicTools(tools);
-      const anthropicHistory = await getAnthropicHistory({ history, normalizedTools, signal });
+      const anthropicHistory = await getAnthropicHistory({ history, normalizedTools, signal, maxImages });
 
       // Tools mean an agent loop, which rereads its prefix every turn and profits from caching.
       // Without them a run is usually a single call, which would only pay the write premium.

src/adapters/anthropic/history.ts

@@ -2,7 +2,13 @@ import type Anthropic from "@anthropic-ai/sdk";
 import { isStructuredOutputRetryFeedback } from "../../constants.ts";
 import { normalizeToolName } from "../../tool.ts";
 import type { ChatItem } from "../../types.ts";
-import { fetchAttachmentText, IMAGE_MIME_TYPES, isTextLikeMimeType } from "../shared/media.ts";
+import {
+  fetchAttachmentText,
+  getOmittedImages,
+  isImageMimeType,
+  isTextLikeMimeType,
+  OMITTED_IMAGE_TEXT,
+} from "../shared/media.ts";
 import { ensureToolInputObject } from "../shared/tools.ts";
 import type { AnthropicToolMap } from "./utils.ts";
 
@@ -35,10 +41,13 @@ export async function getAnthropicHistory(options: {
   history: ChatItem[];
   normalizedTools: AnthropicToolMap[];
   signal: AbortSignal;
+  maxImages?: number;
 }): Promise<Anthropic.Messages.MessageParam[]> {
   const anthropicHistory: Anthropic.Messages.MessageParam[] = [];
   let anthropicToolFileBuffer: Anthropic.Messages.MessageParam[] = [];
 
+  const omittedImages = getOmittedImages(options.history, options.maxImages ?? Infinity);
+
   // Put all of the history in place
   for (const historyItem of options.history) {
     switch (historyItem.type) {
@@ -134,7 +143,9 @@ export async function getAnthropicHistory(options: {
       case "input_file":
       case "tool_result_file": {
         const pushBuffer = historyItem.type === "input_file" ? anthropicHistory : anthropicToolFileBuffer;
-        if ((IMAGE_MIME_TYPES as readonly string[]).includes(historyItem.kind)) {
+        if (omittedImages.has(historyItem)) {
+          pushBuffer.push({ role: "user", content: [{ type: "text", text: OMITTED_IMAGE_TEXT }] });
+        } else if (isImageMimeType(historyItem.kind)) {
           pushBuffer.push({
             role: "user",
             content: [{

src/adapters/anthropic/models.ts

@@ -211,6 +211,31 @@ export const anthropicModelStructuredOutputSupport = {
   "claude-sonnet-4-20250514": false,
 };
 
+/**
+ * Anthropic caps a request at 100 images on models with a 200k-token context window, and at 600 on the rest.
+ * @see https://platform.claude.com/docs/en/build-with-claude/vision
+ */
+const SMALL_CONTEXT_MODELS = new Set<string>(
+  [
+    "claude-opus-4-5",
+    "claude-opus-4-5-20251101",
+    "claude-sonnet-4-5",
+    "claude-sonnet-4-5-20250929",
+    "claude-haiku-4-5",
+    "claude-haiku-4-5-20251001",
+    "claude-opus-4-1",
+    "claude-opus-4-1-20250805",
+    "claude-opus-4-0",
+    "claude-opus-4-20250514",
+    "claude-sonnet-4-0",
+    "claude-sonnet-4-20250514",
+  ] satisfies AnthropicModels[],
+);
+
+export function getAnthropicMaxImages(model: string): number {
+  return SMALL_CONTEXT_MODELS.has(model) ? 100 : 600;
+}
+
 const thinkingBudgets: Record<Exclude<ThinkingLevel, "adaptive">, number> = {
   minimal: 1024,
   low: 2048,

src/adapters/azure_openai/adapter.ts

@@ -14,6 +14,13 @@ import {
 } from "../openai/models.ts";
 import type { ProviderFileStore } from "../../types.ts";
 
+/**
+ * Azure rejects a request with more images than this, counting the whole history.
+ * Pages of attached PDFs do not count.
+ * @see https://learn.microsoft.com/en-us/azure/foundry/openai/quotas-limits
+ */
+const AZURE_OPENAI_MAX_IMAGES = 50;
+
 /**
  * OpenAI models on Azure Foundry, addressed by model name rather than by deployment name.
  * A deployment named after a model id takes that name over, so never create one.
@@ -44,6 +51,7 @@ export function azureOpenAIModel<zO, zI, TModel extends OpenAIModels>(options: {
     reasoning: resolveOpenAIReasoning(options.model, options.effort),
     parallelToolCalls: options.parallelToolCalls,
     serviceTier: options.serviceTier,
+    maxImages: AZURE_OPENAI_MAX_IMAGES,
     // Responses on Azure only reads ids uploaded with this purpose, and Azure rejects `expires_after`.
     files: options.fileStore ? { purpose: "assistants", store: options.fileStore } : undefined,
   });

src/adapters/open_responses/adapter.ts

@@ -62,6 +62,8 @@ export function openResponsesModel<zO, zI>(options: {
   supportedMimeTypes?: string[];
   client?: OpenResponsesClient;
   files?: OpenResponsesFilesConfig;
+  /** Most images one request may carry. Past it, the oldest are sent as a text placeholder. */
+  maxImages?: number;
 }): Adapter<zO, zI> {
   const client = options.client ?? new OpenAI(options.openAIOptions);
   if (options.files && !client.files) {
@@ -109,6 +111,7 @@ export function openResponsesModel<zO, zI>(options: {
           supportedMimeTypes,
           toolCallReplays,
           files: files && { ...files, sent: sentFileIds, uploaded: uploadedFileIds },
+          maxImages: options.maxImages,
         });
         const pendingToolCallsByOutputIndex: PendingToolCall[] = [];
         const pendingToolCallsByItemId: Record<string, PendingToolCall> = {};

src/adapters/open_responses/history.ts

@@ -17,8 +17,10 @@ import {
   fetchRemoteFileAsDataUrl,
   fetchTextLikeFileAsTaggedText,
   getFileNameFromUrl,
-  IMAGE_MIME_TYPES,
+  getOmittedImages,
+  isImageMimeType,
   isTextLikeMimeType,
+  OMITTED_IMAGE_TEXT,
   PDF_MIME_TYPE,
   supportsMimeType,
   unsupportedMediaTypeError,
@@ -164,7 +166,7 @@ async function getOpenResponsesFileInput(
     throw unsupportedMediaTypeError(model, historyItem.kind);
   }
 
-  if (IMAGE_MIME_TYPES.some((mimeType) => mimeType === historyItem.kind)) {
+  if (isImageMimeType(historyItem.kind)) {
     if (files) {
       return {
         type: "input_image",
@@ -217,11 +219,29 @@ export async function getOpenResponsesHistory(options: {
   supportedMimeTypes?: string[];
   toolCallReplays?: Map<string, ToolCallReplay>;
   files?: OpenResponsesFiles;
+  maxImages?: number;
 }): Promise<ResponseInputItem[]> {
   const supportedMimeTypes = options.supportedMimeTypes ?? DEFAULT_SUPPORTED_MIME_TYPES;
   const responseHistory: ResponseInputItem[] = [];
   const calledToolUseIds = new Set<string>();
   const replayedReasoningItemIds = new Set<string>();
+  const omittedImages = getOmittedImages(options.history, options.maxImages ?? Infinity);
+
+  async function getFileInput(
+    historyItem: FileHistoryItem,
+  ): Promise<ResponseInputText | ResponseInputImage | ResponseInputFile> {
+    if (omittedImages.has(historyItem)) {
+      return { type: "input_text", text: OMITTED_IMAGE_TEXT };
+    }
+
+    return await getOpenResponsesFileInput(
+      options.model,
+      historyItem,
+      supportedMimeTypes,
+      options.signal,
+      options.files,
+    );
+  }
 
   for (const historyItem of options.history) {
     switch (historyItem.type) {
@@ -285,31 +305,15 @@ export async function getOpenResponsesHistory(options: {
         if (!calledToolUseIds.has(historyItem.tool_use_id)) break;
         const output = getOrCreateFunctionCallOutput(responseHistory, historyItem.tool_use_id);
         assert(typeof output.output !== "string");
-        output.output.push(
-          await getOpenResponsesFileInput(
-            options.model,
-            historyItem,
-            supportedMimeTypes,
-            options.signal,
-            options.files,
-          ),
-        );
+        output.output.push(await getFileInput(historyItem));
         break;
       }
       case "input_file":
         responseHistory.push({
           type: "message",
           role: "user",
           status: "completed",
-          content: [
-            await getOpenResponsesFileInput(
-              options.model,
-              historyItem,
-              supportedMimeTypes,
-              options.signal,
-              options.files,
-            ),
-          ],
+          content: [await getFileInput(historyItem)],
         });
         break;
       default:

src/adapters/openai/adapter.ts

@@ -5,7 +5,7 @@ import {
   type OpenResponsesServiceTier,
 } from "../open_responses/adapter.ts";
 import type { Adapter } from "../adapter.ts";
-import { getOpenAISupportedMimeTypes } from "./mimes.ts";
+import { getOpenAISupportedMimeTypes, OPENAI_MAX_IMAGES } from "./mimes.ts";
 import {
   type OpenAIModelModality,
   openAiModelReasoningSupport,
@@ -46,6 +46,7 @@ export function openAIModel<zO, zI, TModel extends OpenAIModels | (string & Reco
     reasoning: resolveOpenAIReasoning(options.model, options.effort),
     parallelToolCalls: options.parallelToolCalls,
     serviceTier: options.serviceTier,
+    maxImages: OPENAI_MAX_IMAGES,
     files: options.fileStore
       ? {
         purpose: "user_data",

src/adapters/openai/mimes.ts

@@ -8,6 +8,12 @@ export const OPENAI_IMAGE_INPUT_MIME_TYPES = [
   "image/gif",
 ] as const;
 
+/**
+ * OpenAI rejects a request with more images than this, counting the whole history.
+ * @see https://developers.openai.com/api/docs/guides/images-vision
+ */
+export const OPENAI_MAX_IMAGES = 1500;
+
 export const OPENAI_VISION_INPUT_MIME_TYPES = [
   "application/pdf",
 ] as const;

src/adapters/openai_completions/adapter.ts

@@ -51,6 +51,8 @@ export function openAICompletionsModel<zO, zI, TModel extends string>(options: {
   extraRequestBody?: OpenAICompletionsExtraRequestBody<TModel>;
   openAIOptions?: ClientOptions;
   client?: OpenAICompletionsClient;
+  /** Most images one request may carry. Past it, the oldest are sent as a text placeholder. */
+  maxImages?: number;
 }): Adapter<zO, zI> {
   const supportedMimeTypes = options.supportedMimeTypes ?? DEFAULT_SUPPORTED_MIME_TYPES;
   if (options.pdfSupport && !supportsMimeType("application/pdf", supportedMimeTypes)) {
@@ -83,6 +85,7 @@ export function openAICompletionsModel<zO, zI, TModel extends string>(options: {
         signal,
         supportedMimeTypes,
         pdfSupport: options.pdfSupport,
+        maxImages: options.maxImages,
       });
 
       const extraRequestBody = resolveOpenAICompletionsExtraRequestBody(options.extraRequestBody, {

src/adapters/openai_completions/history.ts

@@ -9,8 +9,10 @@ import {
   fetchTextLikeFileAsTaggedText,
   getContentLength,
   getFileNameFromUrl,
-  IMAGE_MIME_TYPES,
+  getOmittedImages,
+  isImageMimeType,
   isTextLikeMimeType,
+  OMITTED_IMAGE_TEXT,
   PDF_MIME_TYPE,
   supportsMimeType,
   unsupportedMediaTypeError,
@@ -45,7 +47,7 @@ async function getOpenAICompletionsFileMessage<TModel extends string>(
     throw unsupportedMediaTypeError(model, item.kind);
   }
 
-  if (IMAGE_MIME_TYPES.some((mimeType) => mimeType === item.kind)) {
+  if (isImageMimeType(item.kind)) {
     return {
       role: "user",
       content: [{
@@ -109,6 +111,7 @@ export async function getOpenAICompletionsHistory<TModel extends string>(options
   signal: AbortSignal;
   supportedMimeTypes?: string[];
   pdfSupport?: OpenAICompletionsPdfSupport<TModel>;
+  maxImages?: number;
 }): Promise<ChatCompletionMessageParam[]> {
   const supportedMimeTypes = options.supportedMimeTypes ?? DEFAULT_SUPPORTED_MIME_TYPES;
   const messages: ChatCompletionMessageParam[] = [{
@@ -155,6 +158,22 @@ export async function getOpenAICompletionsHistory<TModel extends string>(options
     turnReasoning = "";
   }
 
+  const omittedImages = getOmittedImages(options.history, options.maxImages ?? Infinity);
+
+  async function getFileMessage(item: FileHistoryItem): Promise<ChatCompletionMessageParam> {
+    if (omittedImages.has(item)) {
+      return { role: "user", content: OMITTED_IMAGE_TEXT };
+    }
+
+    return await getOpenAICompletionsFileMessage(
+      options.model,
+      item,
+      supportedMimeTypes,
+      options.pdfSupport,
+      options.signal,
+    );
+  }
+
   for (const historyItem of options.history) {
     switch (historyItem.type) {
       case "input_text":
@@ -216,29 +235,13 @@ export async function getOpenAICompletionsHistory<TModel extends string>(options
         if (!toolTurn.results.has(historyItem.tool_use_id)) {
           toolTurn.results.set(historyItem.tool_use_id, "");
         }
-        toolTurn.files.push(
-          await getOpenAICompletionsFileMessage(
-            options.model,
-            historyItem,
-            supportedMimeTypes,
-            options.pdfSupport,
-            options.signal,
-          ),
-        );
+        toolTurn.files.push(await getFileMessage(historyItem));
         break;
       }
       case "input_file":
         flushToolTurn();
         turnReasoning = "";
-        messages.push(
-          await getOpenAICompletionsFileMessage(
-            options.model,
-            historyItem,
-            supportedMimeTypes,
-            options.pdfSupport,
-            options.signal,
-          ),
-        );
+        messages.push(await getFileMessage(historyItem));
         break;
       default:
         historyItem satisfies never;

src/adapters/openrouter/adapter.ts

@@ -69,6 +69,8 @@ export function openrouterModel<zO, zI, TModel extends OpenRouterModels>(options
   appName?: string;
   extraRequestBody?: Record<string, unknown>;
   pdfSupport?: OpenAICompletionsPdfSupport<TModel>;
+  /** OpenRouter has no image limit of its own, so set the strictest limit among the providers it may route to. */
+  maxImages?: number;
 }): Adapter<zO, zI> {
   const adapter = openAICompletionsModel<zO, zI, TModel>({
     provider: "OpenRouter",
@@ -82,6 +84,7 @@ export function openrouterModel<zO, zI, TModel extends OpenRouterModels>(options
     pdfSupport: options.pdfSupport ??
       ((model) =>
         getOpenRouterNativePdfSupport(model) ? { mode: "native", maxSize: 4 * 1024 * 1024 } : { mode: "text" }),
+    maxImages: options.maxImages,
     extraRequestBody: () => ({
       ...(options.reasoning ? { reasoning: options.reasoning } : {}),
       ...(options.plugins?.length ? { plugins: options.plugins } : {}),