feat(ai): capture model calls with Rudimint

src/features/workspaces/ai/chat/chat-endpoint.ts

@@ -193,6 +193,13 @@ export async function handleAiChatTurn(input: {
 			promptScope,
 			userId,
 		};
+		// One Rudimint session per thread; each model call in a turn shares the turn's trace id.
+		const capture = (task: string) => ({
+			task,
+			session: threadId,
+			user: userId,
+			metadata: { workspaceId: body.workspaceId, traceId: turn.streamId },
+		});
 		const systemPrompt = buildAiChatSystemPrompt({
 			promptScope,
 			timeZone: body.timeZone,
@@ -227,7 +234,11 @@ export async function handleAiChatTurn(input: {
 
 				try {
 					result = await generateText({
-						model: getWorkspaceAiLanguageModel(AI_CHAT_COMPACTION_MODEL_ID, env),
+						model: getWorkspaceAiLanguageModel(
+							AI_CHAT_COMPACTION_MODEL_ID,
+							env,
+							capture("chat-compaction"),
+						),
 						providerOptions: getWorkspaceAiGatewayProviderOptions({
 							modelId: AI_CHAT_COMPACTION_MODEL_ID,
 							tags: ["task:chat-compaction"],
@@ -349,7 +360,11 @@ export async function handleAiChatTurn(input: {
 							let generated;
 
 							try {
-								generated = await generateAIThreadTitle({ env, messages: [userMessage] });
+								generated = await generateAIThreadTitle({
+									env,
+									messages: [userMessage],
+									capture: capture("chat-title"),
+								});
 							} catch (error) {
 								// The model call failing must still produce an event — a dead
 								// title model would otherwise be invisible in the analytics.
@@ -396,7 +411,7 @@ export async function handleAiChatTurn(input: {
 					await hydrateAttachmentParts(context.messages, { userId, threadId }),
 				);
 				const result = streamText({
-					model: getWorkspaceAiLanguageModel(modelId, env),
+					model: getWorkspaceAiLanguageModel(modelId, env, capture("chat")),
 					providerOptions: getWorkspaceAiGatewayProviderOptions({
 						modelId,
 						thread: threadContext,

src/features/workspaces/ai/gateway.ts

@@ -23,19 +23,30 @@ type WorkspaceAiProviderOptions = NonNullable<
 	Parameters<typeof generateText>[0]["providerOptions"]
 >;
 
+/** Which job, conversation and user a model call belongs to, as Rudimint records it. */
+export type RudimintCapture = {
+	task: string;
+	session?: string;
+	user?: string;
+	metadata?: Record<string, string | undefined>;
+};
+
 export function getWorkspaceAiLanguageModel(
 	modelId: ReturnType<typeof resolveWorkspaceAiChatModelId>,
 	env: Cloudflare.Env,
+	capture?: RudimintCapture,
 ): LanguageModel {
-	return getWorkspaceAiLanguageModelForGatewayModel(getWorkspaceAiChatModel(modelId), env);
+	return getWorkspaceAiLanguageModelForGatewayModel(getWorkspaceAiChatModel(modelId), env, capture);
 }
 
 export function getWorkspaceAiLanguageModelForGatewayModel(
 	gatewayModel: string,
 	env: Cloudflare.Env,
+	capture?: RudimintCapture,
 ): LanguageModel {
 	const gateway = createGateway({
 		apiKey: getVercelAiGatewayApiKey(env),
+		...getRudimintGatewaySettings(env, capture),
 	});
 
 	return wrapLanguageModel({
@@ -141,6 +152,28 @@ function getWorkspaceAiReasoningOptions(
 	}
 }
 
+// Rudimint forwards each call to Vercel AI Gateway unchanged and keeps a copy for
+// evaluation and training. Without RUDIMINT_KEY, calls go to Vercel directly.
+const RUDIMINT_GATEWAY_URL = "https://api.rudimint.com/vercel/v4/ai";
+
+function getRudimintGatewaySettings(env: Cloudflare.Env, capture?: RudimintCapture) {
+	const key = (env as { RUDIMINT_KEY?: string }).RUDIMINT_KEY ?? process.env.RUDIMINT_KEY;
+
+	if (!key) {
+		return {};
+	}
+
+	const headers: Record<string, string> = { "x-rudimint-key": key };
+	if (capture) {
+		headers["x-rudimint-task"] = capture.task;
+		if (capture.session) headers["x-rudimint-session"] = capture.session;
+		if (capture.user) headers["x-rudimint-user"] = capture.user;
+		if (capture.metadata) headers["x-rudimint-metadata"] = JSON.stringify(capture.metadata);
+	}
+
+	return { baseURL: RUDIMINT_GATEWAY_URL, headers };
+}
+
 function getVercelAiGatewayApiKey(env: Cloudflare.Env) {
 	const apiKey =
 		(env as { AI_GATEWAY_API_KEY?: string }).AI_GATEWAY_API_KEY ?? process.env.AI_GATEWAY_API_KEY;
@@ -152,15 +185,23 @@ function getVercelAiGatewayApiKey(env: Cloudflare.Env) {
 	return apiKey;
 }
 
-export async function generateAIThreadTitle(input: { env: Cloudflare.Env; messages: UIMessage[] }) {
+export async function generateAIThreadTitle(input: {
+	env: Cloudflare.Env;
+	messages: UIMessage[];
+	capture?: RudimintCapture;
+}) {
 	const firstUserMessage = getFirstUserMessageText(input.messages);
 
 	if (!firstUserMessage) {
 		return undefined;
 	}
 
 	const result = await generateText({
-		model: getWorkspaceAiLanguageModelForGatewayModel(AI_THREAD_TITLE_GATEWAY_MODEL, input.env),
+		model: getWorkspaceAiLanguageModelForGatewayModel(
+			AI_THREAD_TITLE_GATEWAY_MODEL,
+			input.env,
+			input.capture,
+		),
 		providerOptions: getAIThreadTitleGatewayProviderOptions(),
 		instructions:
 			"Produce a concise chat title for the user message. Two to six words. No quotes, no trailing punctuation.",

src/features/workspaces/extraction/providers/gemini-image-markdown.ts

@@ -38,7 +38,10 @@ export function createGeminiImageMarkdownProvider(env: Env): MarkdownExtractionP
 				WORKSPACE_AI_CHAT_ATTACHMENT_POLICY.maxNormalizedFileSize,
 			);
 			const result = await generateText({
-				model: getWorkspaceAiLanguageModelForGatewayModel(IMAGE_EXTRACTION_GATEWAY_MODEL, env),
+				model: getWorkspaceAiLanguageModelForGatewayModel(IMAGE_EXTRACTION_GATEWAY_MODEL, env, {
+					task: "image-to-markdown",
+					metadata: { workspaceId: input.workspaceId, itemId: input.itemId },
+				}),
 				providerOptions: {
 					gateway: {
 						...getWorkspaceAiGatewayTransportOptions(),

src/routes/privacy.tsx

@@ -35,11 +35,11 @@ const privacyDocument = {
 		},
 		{
 			title: "Service providers",
-			body: "We use service providers to operate ThinkEx. Depending on the feature and environment, these may include Cloudflare for hosting, storage, databases, Durable Objects, Workers AI, Browser Rendering, email delivery, and sandboxed compute; Google for sign-in; Vercel AI Gateway and AI model providers such as OpenAI, Anthropic, and Google for AI responses; Firecrawl and Cloudflare Workers AI for file, web, and research extraction; PostHog for analytics, feedback, error reporting, session replay in deployed builds, and AI observability; The Context Company for AI observability; and Autumn for AI usage tracking. These providers process data only as needed to provide their services to ThinkEx Inc.",
+			body: "We use service providers to operate ThinkEx. Depending on the feature and environment, these may include Cloudflare for hosting, storage, databases, Durable Objects, Workers AI, Browser Rendering, email delivery, and sandboxed compute; Google for sign-in; Vercel AI Gateway and AI model providers such as OpenAI, Anthropic, and Google for AI responses; Firecrawl and Cloudflare Workers AI for file, web, and research extraction; PostHog for analytics, feedback, error reporting, session replay in deployed builds, and AI observability; The Context Company for AI observability; Rudimint, which records AI requests and responses so we can evaluate and improve the models behind ThinkEx; and Autumn for AI usage tracking. These providers process data only as needed to provide their services to ThinkEx Inc.",
 		},
 		{
 			title: "Workspace content and AI",
-			body: "Your workspace content remains yours. When you use AI features, ThinkEx may send your prompt, selected workspace context, extracted text, file-derived content, and tool outputs to the configured AI or extraction providers so they can return a result. Do not use ThinkEx for highly sensitive information unless you are comfortable with this processing.",
+			body: "Your workspace content remains yours. When you use AI features, ThinkEx may send your prompt, selected workspace context, extracted text, file-derived content, and tool outputs to the configured AI or extraction providers so they can return a result. AI requests and responses also pass through Rudimint, which keeps a copy so we can evaluate and improve the models ThinkEx uses. Do not use ThinkEx for highly sensitive information unless you are comfortable with this processing.",
 		},
 		{
 			title: "Cookies and local storage",

worker-configuration.d.ts

@@ -1,5 +1,5 @@
 /* eslint-disable */
-// Generated by Wrangler by running `wrangler types` (hash: 904737024aa050fbb60baf63adacf826)
+// Generated by Wrangler by running `wrangler types` (hash: 7c0c402166c020aab4d296d951ae26c2)
 // Runtime types generated with workerd@1.20260811.1 2026-03-24 nodejs_compat
 interface __BaseEnv_Env {
 	WORKSPACE_FILES: R2Bucket;
@@ -23,6 +23,7 @@ interface __BaseEnv_Env {
 	R2_ACCESS_KEY_ID: string;
 	R2_SECRET_ACCESS_KEY: string;
 	WORKSPACE_UPLOAD_TOKEN_SECRET: string;
+	RUDIMINT_KEY: string;
 	WorkspaceRoom: DurableObjectNamespace<import("./src/server").WorkspaceRoom>;
 	DocumentSession: DurableObjectNamespace<import("./src/server").DocumentSession>;
 	OFFICE_PDF_CONVERTER: DurableObjectNamespace<import("./src/server").OfficePdfConverter>;
@@ -58,6 +59,7 @@ declare namespace Cloudflare {
 		R2_ACCESS_KEY_ID: string;
 		R2_SECRET_ACCESS_KEY: string;
 		WORKSPACE_UPLOAD_TOKEN_SECRET: string;
+		RUDIMINT_KEY: string;
 		WorkspaceRoom: DurableObjectNamespace<import("./src/server").WorkspaceRoom>;
 		DocumentSession: DurableObjectNamespace<import("./src/server").DocumentSession>;
 		OFFICE_PDF_CONVERTER: DurableObjectNamespace<import("./src/server").OfficePdfConverter>;
@@ -87,6 +89,7 @@ declare namespace Cloudflare {
 		R2_ACCESS_KEY_ID: string;
 		R2_SECRET_ACCESS_KEY: string;
 		WORKSPACE_UPLOAD_TOKEN_SECRET: string;
+		RUDIMINT_KEY: string;
 		WorkspaceRoom: DurableObjectNamespace<import("./src/server").WorkspaceRoom>;
 		DocumentSession: DurableObjectNamespace<import("./src/server").DocumentSession>;
 		OFFICE_PDF_CONVERTER: DurableObjectNamespace<import("./src/server").OfficePdfConverter>;
@@ -101,7 +104,7 @@ type StringifyValues<EnvType extends Record<string, unknown>> = {
 	[Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string;
 };
 declare namespace NodeJS {
-	interface ProcessEnv extends StringifyValues<Pick<Cloudflare.Env, "FIRECRAWL_API_URL" | "R2_BUCKET_NAME" | "BETTER_AUTH_URL" | "BETTER_AUTH_SECRET" | "GOOGLE_CLIENT_ID" | "GOOGLE_CLIENT_SECRET" | "FIRECRAWL_API_KEY" | "AI_GATEWAY_API_KEY" | "AUTUMN_SECRET_KEY" | "R2_ACCOUNT_ID" | "R2_ACCESS_KEY_ID" | "R2_SECRET_ACCESS_KEY" | "WORKSPACE_UPLOAD_TOKEN_SECRET" | "AUTUMN_PROD_SECRET_KEY">> {}
+	interface ProcessEnv extends StringifyValues<Pick<Cloudflare.Env, "FIRECRAWL_API_URL" | "R2_BUCKET_NAME" | "BETTER_AUTH_URL" | "BETTER_AUTH_SECRET" | "GOOGLE_CLIENT_ID" | "GOOGLE_CLIENT_SECRET" | "FIRECRAWL_API_KEY" | "AI_GATEWAY_API_KEY" | "AUTUMN_SECRET_KEY" | "R2_ACCOUNT_ID" | "R2_ACCESS_KEY_ID" | "R2_SECRET_ACCESS_KEY" | "WORKSPACE_UPLOAD_TOKEN_SECRET" | "RUDIMINT_KEY" | "AUTUMN_PROD_SECRET_KEY">> {}
 }
 
 // Begin runtime types

wrangler.jsonc

@@ -102,6 +102,7 @@
 			"R2_ACCESS_KEY_ID",
 			"R2_SECRET_ACCESS_KEY",
 			"WORKSPACE_UPLOAD_TOKEN_SECRET",
+			"RUDIMINT_KEY",
 		],
 	},
 	"workflows": [
@@ -254,6 +255,7 @@
 					"R2_ACCESS_KEY_ID",
 					"R2_SECRET_ACCESS_KEY",
 					"WORKSPACE_UPLOAD_TOKEN_SECRET",
+					"RUDIMINT_KEY",
 				],
 			},
 			"ai": {
@@ -369,6 +371,7 @@
 					"R2_ACCESS_KEY_ID",
 					"R2_SECRET_ACCESS_KEY",
 					"WORKSPACE_UPLOAD_TOKEN_SECRET",
+					"RUDIMINT_KEY",
 				],
 			},
 			"ai": {