From 7ee83938fa245a45bf18de32193716eb604aea16 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:32:46 -0400 Subject: [PATCH 1/2] feat(ai): capture model calls with Rudimint When RUDIMINT_KEY is set, every Vercel AI Gateway call goes through Rudimint's gateway, which forwards it unchanged and keeps a copy for evaluation and training. Calls are tagged with their task, the thread as the session, the user, and the workspace, turn and model as metadata. Without the key, calls go straight to Vercel as before. --- .../workspaces/ai/chat/chat-endpoint.ts | 21 +++++++-- src/features/workspaces/ai/gateway.ts | 47 +++++++++++++++++-- .../providers/gemini-image-markdown.ts | 5 +- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src/features/workspaces/ai/chat/chat-endpoint.ts b/src/features/workspaces/ai/chat/chat-endpoint.ts index 762d213c..a80521f1 100644 --- a/src/features/workspaces/ai/chat/chat-endpoint.ts +++ b/src/features/workspaces/ai/chat/chat-endpoint.ts @@ -193,6 +193,13 @@ export async function handleAiChatTurn(input: { promptScope, userId, }; + // One Rudimint session per thread; each model call in a turn shares the turn's trace id. + const capture = (task: string) => ({ + task, + session: threadId, + user: userId, + metadata: { workspaceId: body.workspaceId, traceId: turn.streamId, modelId }, + }); const systemPrompt = buildAiChatSystemPrompt({ promptScope, timeZone: body.timeZone, @@ -227,7 +234,11 @@ export async function handleAiChatTurn(input: { try { result = await generateText({ - model: getWorkspaceAiLanguageModel(AI_CHAT_COMPACTION_MODEL_ID, env), + model: getWorkspaceAiLanguageModel( + AI_CHAT_COMPACTION_MODEL_ID, + env, + capture("chat-compaction"), + ), providerOptions: getWorkspaceAiGatewayProviderOptions({ modelId: AI_CHAT_COMPACTION_MODEL_ID, tags: ["task:chat-compaction"], @@ -349,7 +360,11 @@ export async function handleAiChatTurn(input: { let generated; try { - generated = await generateAIThreadTitle({ env, messages: [userMessage] }); + generated = await generateAIThreadTitle({ + env, + messages: [userMessage], + capture: capture("chat-title"), + }); } catch (error) { // The model call failing must still produce an event — a dead // title model would otherwise be invisible in the analytics. @@ -396,7 +411,7 @@ export async function handleAiChatTurn(input: { await hydrateAttachmentParts(context.messages, { userId, threadId }), ); const result = streamText({ - model: getWorkspaceAiLanguageModel(modelId, env), + model: getWorkspaceAiLanguageModel(modelId, env, capture("chat")), providerOptions: getWorkspaceAiGatewayProviderOptions({ modelId, thread: threadContext, diff --git a/src/features/workspaces/ai/gateway.ts b/src/features/workspaces/ai/gateway.ts index b13c55b8..a944f9a4 100644 --- a/src/features/workspaces/ai/gateway.ts +++ b/src/features/workspaces/ai/gateway.ts @@ -23,19 +23,30 @@ type WorkspaceAiProviderOptions = NonNullable< Parameters[0]["providerOptions"] >; +/** Which job, conversation and user a model call belongs to, as Rudimint records it. */ +export type RudimintCapture = { + task: string; + session?: string; + user?: string; + metadata?: Record; +}; + export function getWorkspaceAiLanguageModel( modelId: ReturnType, env: Cloudflare.Env, + capture?: RudimintCapture, ): LanguageModel { - return getWorkspaceAiLanguageModelForGatewayModel(getWorkspaceAiChatModel(modelId), env); + return getWorkspaceAiLanguageModelForGatewayModel(getWorkspaceAiChatModel(modelId), env, capture); } export function getWorkspaceAiLanguageModelForGatewayModel( gatewayModel: string, env: Cloudflare.Env, + capture?: RudimintCapture, ): LanguageModel { const gateway = createGateway({ apiKey: getVercelAiGatewayApiKey(env), + ...getRudimintGatewaySettings(env, capture), }); return wrapLanguageModel({ @@ -141,6 +152,28 @@ function getWorkspaceAiReasoningOptions( } } +// Rudimint forwards each call to Vercel AI Gateway unchanged and keeps a copy for +// evaluation and training. Without RUDIMINT_KEY, calls go to Vercel directly. +const RUDIMINT_GATEWAY_URL = "https://api.rudimint.com/vercel/v4/ai"; + +function getRudimintGatewaySettings(env: Cloudflare.Env, capture?: RudimintCapture) { + const key = (env as { RUDIMINT_KEY?: string }).RUDIMINT_KEY ?? process.env.RUDIMINT_KEY; + + if (!key) { + return {}; + } + + const headers: Record = { "x-rudimint-key": key }; + if (capture) { + headers["x-rudimint-task"] = capture.task; + if (capture.session) headers["x-rudimint-session"] = capture.session; + if (capture.user) headers["x-rudimint-user"] = capture.user; + if (capture.metadata) headers["x-rudimint-metadata"] = JSON.stringify(capture.metadata); + } + + return { baseURL: RUDIMINT_GATEWAY_URL, headers }; +} + function getVercelAiGatewayApiKey(env: Cloudflare.Env) { const apiKey = (env as { AI_GATEWAY_API_KEY?: string }).AI_GATEWAY_API_KEY ?? process.env.AI_GATEWAY_API_KEY; @@ -152,7 +185,11 @@ function getVercelAiGatewayApiKey(env: Cloudflare.Env) { return apiKey; } -export async function generateAIThreadTitle(input: { env: Cloudflare.Env; messages: UIMessage[] }) { +export async function generateAIThreadTitle(input: { + env: Cloudflare.Env; + messages: UIMessage[]; + capture?: RudimintCapture; +}) { const firstUserMessage = getFirstUserMessageText(input.messages); if (!firstUserMessage) { @@ -160,7 +197,11 @@ export async function generateAIThreadTitle(input: { env: Cloudflare.Env; messag } const result = await generateText({ - model: getWorkspaceAiLanguageModelForGatewayModel(AI_THREAD_TITLE_GATEWAY_MODEL, input.env), + model: getWorkspaceAiLanguageModelForGatewayModel( + AI_THREAD_TITLE_GATEWAY_MODEL, + input.env, + input.capture, + ), providerOptions: getAIThreadTitleGatewayProviderOptions(), instructions: "Produce a concise chat title for the user message. Two to six words. No quotes, no trailing punctuation.", diff --git a/src/features/workspaces/extraction/providers/gemini-image-markdown.ts b/src/features/workspaces/extraction/providers/gemini-image-markdown.ts index 1d70620e..c08734e7 100644 --- a/src/features/workspaces/extraction/providers/gemini-image-markdown.ts +++ b/src/features/workspaces/extraction/providers/gemini-image-markdown.ts @@ -38,7 +38,10 @@ export function createGeminiImageMarkdownProvider(env: Env): MarkdownExtractionP WORKSPACE_AI_CHAT_ATTACHMENT_POLICY.maxNormalizedFileSize, ); const result = await generateText({ - model: getWorkspaceAiLanguageModelForGatewayModel(IMAGE_EXTRACTION_GATEWAY_MODEL, env), + model: getWorkspaceAiLanguageModelForGatewayModel(IMAGE_EXTRACTION_GATEWAY_MODEL, env, { + task: "image-to-markdown", + metadata: { workspaceId: input.workspaceId, itemId: input.itemId }, + }), providerOptions: { gateway: { ...getWorkspaceAiGatewayTransportOptions(), From 70b8589420f0d85bb44ca77bab2a9f5b29b96b4a Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:56:31 -0400 Subject: [PATCH 2/2] fix(ai): load RUDIMINT_KEY and disclose Rudimint capture - Declare RUDIMINT_KEY in every secrets.required list so the Worker and local dev actually receive it. - Drop modelId from capture metadata; Rudimint records the model each call actually used, which differs for compaction and titles. - Name Rudimint in the privacy policy's providers and AI sections. --- src/features/workspaces/ai/chat/chat-endpoint.ts | 2 +- src/routes/privacy.tsx | 4 ++-- worker-configuration.d.ts | 7 +++++-- wrangler.jsonc | 3 +++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/features/workspaces/ai/chat/chat-endpoint.ts b/src/features/workspaces/ai/chat/chat-endpoint.ts index a80521f1..31c6e95e 100644 --- a/src/features/workspaces/ai/chat/chat-endpoint.ts +++ b/src/features/workspaces/ai/chat/chat-endpoint.ts @@ -198,7 +198,7 @@ export async function handleAiChatTurn(input: { task, session: threadId, user: userId, - metadata: { workspaceId: body.workspaceId, traceId: turn.streamId, modelId }, + metadata: { workspaceId: body.workspaceId, traceId: turn.streamId }, }); const systemPrompt = buildAiChatSystemPrompt({ promptScope, diff --git a/src/routes/privacy.tsx b/src/routes/privacy.tsx index 57ea89ec..86c1edd4 100644 --- a/src/routes/privacy.tsx +++ b/src/routes/privacy.tsx @@ -35,11 +35,11 @@ const privacyDocument = { }, { title: "Service providers", - body: "We use service providers to operate ThinkEx. Depending on the feature and environment, these may include Cloudflare for hosting, storage, databases, Durable Objects, Workers AI, Browser Rendering, email delivery, and sandboxed compute; Google for sign-in; Vercel AI Gateway and AI model providers such as OpenAI, Anthropic, and Google for AI responses; Firecrawl and Cloudflare Workers AI for file, web, and research extraction; PostHog for analytics, feedback, error reporting, session replay in deployed builds, and AI observability; The Context Company for AI observability; and Autumn for AI usage tracking. These providers process data only as needed to provide their services to ThinkEx Inc.", + body: "We use service providers to operate ThinkEx. Depending on the feature and environment, these may include Cloudflare for hosting, storage, databases, Durable Objects, Workers AI, Browser Rendering, email delivery, and sandboxed compute; Google for sign-in; Vercel AI Gateway and AI model providers such as OpenAI, Anthropic, and Google for AI responses; Firecrawl and Cloudflare Workers AI for file, web, and research extraction; PostHog for analytics, feedback, error reporting, session replay in deployed builds, and AI observability; The Context Company for AI observability; Rudimint, which records AI requests and responses so we can evaluate and improve the models behind ThinkEx; and Autumn for AI usage tracking. These providers process data only as needed to provide their services to ThinkEx Inc.", }, { title: "Workspace content and AI", - body: "Your workspace content remains yours. When you use AI features, ThinkEx may send your prompt, selected workspace context, extracted text, file-derived content, and tool outputs to the configured AI or extraction providers so they can return a result. Do not use ThinkEx for highly sensitive information unless you are comfortable with this processing.", + body: "Your workspace content remains yours. When you use AI features, ThinkEx may send your prompt, selected workspace context, extracted text, file-derived content, and tool outputs to the configured AI or extraction providers so they can return a result. AI requests and responses also pass through Rudimint, which keeps a copy so we can evaluate and improve the models ThinkEx uses. Do not use ThinkEx for highly sensitive information unless you are comfortable with this processing.", }, { title: "Cookies and local storage", diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 9119490c..aadb8feb 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: 904737024aa050fbb60baf63adacf826) +// Generated by Wrangler by running `wrangler types` (hash: 7c0c402166c020aab4d296d951ae26c2) // Runtime types generated with workerd@1.20260811.1 2026-03-24 nodejs_compat interface __BaseEnv_Env { WORKSPACE_FILES: R2Bucket; @@ -23,6 +23,7 @@ interface __BaseEnv_Env { R2_ACCESS_KEY_ID: string; R2_SECRET_ACCESS_KEY: string; WORKSPACE_UPLOAD_TOKEN_SECRET: string; + RUDIMINT_KEY: string; WorkspaceRoom: DurableObjectNamespace; DocumentSession: DurableObjectNamespace; OFFICE_PDF_CONVERTER: DurableObjectNamespace; @@ -58,6 +59,7 @@ declare namespace Cloudflare { R2_ACCESS_KEY_ID: string; R2_SECRET_ACCESS_KEY: string; WORKSPACE_UPLOAD_TOKEN_SECRET: string; + RUDIMINT_KEY: string; WorkspaceRoom: DurableObjectNamespace; DocumentSession: DurableObjectNamespace; OFFICE_PDF_CONVERTER: DurableObjectNamespace; @@ -87,6 +89,7 @@ declare namespace Cloudflare { R2_ACCESS_KEY_ID: string; R2_SECRET_ACCESS_KEY: string; WORKSPACE_UPLOAD_TOKEN_SECRET: string; + RUDIMINT_KEY: string; WorkspaceRoom: DurableObjectNamespace; DocumentSession: DurableObjectNamespace; OFFICE_PDF_CONVERTER: DurableObjectNamespace; @@ -101,7 +104,7 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } // Begin runtime types diff --git a/wrangler.jsonc b/wrangler.jsonc index 2eaffdb2..3e447fd5 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -102,6 +102,7 @@ "R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "WORKSPACE_UPLOAD_TOKEN_SECRET", + "RUDIMINT_KEY", ], }, "workflows": [ @@ -254,6 +255,7 @@ "R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "WORKSPACE_UPLOAD_TOKEN_SECRET", + "RUDIMINT_KEY", ], }, "ai": { @@ -369,6 +371,7 @@ "R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "WORKSPACE_UPLOAD_TOKEN_SECRET", + "RUDIMINT_KEY", ], }, "ai": {