feat(chat): added streaming from chat panel & chat deploy (#321)

* added streaming for openai in chat panel

* added streaming for anthropic

* added streaming for cerebras + deepseek

* added streaming for xai and groq

* added streaming for chat deploy

* moved privacy to the bottom of settings modal, added layout for chat subdomain

* added logs for chat panel for openai with context passed along with stream

* added proper logging for all providers in the console and in the logs

* added unit tests

* added streaming for gemini models

* added logging formatting for chat deploy with streaming

* fix build issues

* remove extraneous logs

* fixed streaming not working with forced tool calls for all providers, fixed response formatting for streamed back responses without tool calls

Author: waleedlatif1

sim/app/api/chat/[subdomain]/route.ts

@@ -1,4 +1,4 @@
-import { NextRequest } from 'next/server'
+import { NextRequest, NextResponse } from 'next/server'
 import { eq } from 'drizzle-orm'
 import { createLogger } from '@/lib/logs/console-logger'
 import { db } from '@/db'
@@ -96,6 +96,28 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
       // Execute the workflow using our helper function
       const result = await executeWorkflowForChat(deployment.id, message)
 
+      // If the executor returned a ReadableStream, stream it directly to the client
+      if (result instanceof ReadableStream) {
+        const streamResponse = new NextResponse(result, {
+          status: 200,
+          headers: {
+            'Content-Type': 'text/plain; charset=utf-8',
+          },
+        })
+        return addCorsHeaders(streamResponse, request)
+      }
+      
+      // Handle StreamingExecution format
+      if (result && typeof result === 'object' && 'stream' in result && 'execution' in result) {
+        const streamResponse = new NextResponse(result.stream as ReadableStream, {
+          status: 200,
+          headers: {
+            'Content-Type': 'text/plain; charset=utf-8',
+          },
+        })
+        return addCorsHeaders(streamResponse, request)
+      }
+      
       // Format the result for the client
       // If result.content is an object, preserve it for structured handling
       // If it's text or another primitive, make sure it's accessible

sim/app/api/chat/utils.ts

@@ -11,6 +11,11 @@ import { Serializer } from '@/serializer'
 import { mergeSubblockState } from '@/stores/workflows/utils'
 import { persistExecutionLogs } from '@/lib/logs/execution-logger'
 import { buildTraceSpans } from '@/lib/logs/trace-spans'
+import { BlockLog } from '@/executor/types'
+
+declare global {
+  var __chatStreamProcessingTasks: Promise<{success: boolean, error?: any}>[] | undefined
+}
 
 const logger = createLogger('ChatAuthUtils')
 const isDevelopment = process.env.NODE_ENV === 'development'
@@ -393,16 +398,108 @@ export async function executeWorkflowForChat(chatId: string, message: string) {
   )
 
   // Create and execute the workflow - mimicking use-workflow-execution.ts
-  const executor = new Executor(
-    serializedWorkflow,
-    processedBlockStates,
-    decryptedEnvVars,
-    { input: message },
-    workflowVariables
-  )
-  
+  const executor = new Executor({
+    workflow: serializedWorkflow,
+    currentBlockStates: processedBlockStates,
+    envVarValues: decryptedEnvVars,
+    workflowInput: { input: message },
+    workflowVariables,
+    contextExtensions: {
+      // Always request streaming – the executor will downgrade gracefully if unsupported
+      stream: true,
+      selectedOutputIds: outputBlockIds,
+      edges: edges.map((e: any) => ({ source: e.source, target: e.target })),
+    },
+  })
+
   // Execute and capture the result
   const result = await executor.execute(workflowId)
+
+  // If the executor returned a ReadableStream, forward it directly for streaming
+  if (result instanceof ReadableStream) {
+    return result
+  }
+  
+  // Handle StreamingExecution format (combined stream + execution data)
+  if (result && typeof result === 'object' && 'stream' in result && 'execution' in result) {
+    // We need to stream the response to the client while *also* capturing the full
+    // content so that we can persist accurate logs once streaming completes.
+
+    // Duplicate the original stream – one copy goes to the client, the other we read
+    // server-side for log enrichment.
+    const [clientStream, loggingStream] = (result.stream as ReadableStream).tee()
+
+    // Kick off background processing to read the stream and persist enriched logs
+    const processingPromise = (async () => {
+      try {
+        // The stream is only used to properly drain it and prevent memory leaks
+        // All the execution data is already provided from the agent handler
+        // through the X-Execution-Data header
+        await drainStream(loggingStream)
+        
+        // No need to wait for a processing promise
+        // The execution-logger.ts will handle token estimation
+        
+        // We can use the execution data as-is since it's already properly structured
+        const executionData = result.execution as any
+
+        // Before persisting, clean up any response objects with zero tokens in agent blocks
+        // This prevents confusion in the console logs
+        if (executionData.logs && Array.isArray(executionData.logs)) {
+          executionData.logs.forEach((log: BlockLog) => {
+            if (log.blockType === 'agent' && log.output?.response) {
+              const response = log.output.response;
+              
+              // Check for zero tokens that will be estimated later
+              if (response.tokens && 
+                 (!response.tokens.completion || response.tokens.completion === 0) &&
+                 (!response.toolCalls || !response.toolCalls.list || response.toolCalls.list.length === 0)) {
+                
+                // Remove tokens from console display to avoid confusion
+                // They'll be properly estimated in the execution logger
+                delete response.tokens;
+              }
+            }
+          });
+        }
+
+        // Build trace spans and persist
+        const { traceSpans, totalDuration } = buildTraceSpans(executionData)
+        const enrichedResult = {
+          ...executionData,
+          traceSpans,
+          totalDuration,
+        }
+
+        const executionId = uuidv4()
+        await persistExecutionLogs(workflowId, executionId, enrichedResult, 'chat')
+        logger.debug(`[${requestId}] Persisted execution logs for streaming chat with ID: ${executionId}`)
+        
+        return { success: true }
+      } catch (error) {
+        logger.error(`[${requestId}] Failed to persist streaming chat execution logs:`, error)
+        return { success: false, error }
+      } finally {
+        // Ensure the stream is properly closed even if an error occurs
+        try {
+          const controller = new AbortController()
+          const signal = controller.signal
+          controller.abort()
+        } catch (cleanupError) {
+          logger.debug(`[${requestId}] Error during stream cleanup: ${cleanupError}`)
+        }
+      }
+    })()
+    
+    // Register this processing promise with a global handler or tracker if needed
+    // This allows the background task to be monitored or waited for in testing
+    if (typeof global.__chatStreamProcessingTasks !== 'undefined') {
+      global.__chatStreamProcessingTasks.push(processingPromise as Promise<{success: boolean, error?: any}>)
+    }
+
+    // Return the client-facing stream
+    return clientStream
+  }
 
   // Mark as chat execution in metadata
   if (result) {
@@ -412,7 +509,7 @@ export async function executeWorkflowForChat(chatId: string, message: string) {
     }
   }
 
-  // Persist execution logs using the 'chat' trigger type
+  // Persist execution logs using the 'chat' trigger type for non-streaming results
   try {
     // Build trace spans to enrich the logs (same as in use-workflow-execution.ts)
     const { traceSpans, totalDuration } = buildTraceSpans(result)
@@ -543,4 +640,20 @@ export async function executeWorkflowForChat(chatId: string, message: string) {
       type: 'workflow'
     }
   }
+}
+
+/**
+ * Utility function to properly drain a stream to prevent memory leaks
+ */
+async function drainStream(stream: ReadableStream): Promise<void> {
+  const reader = stream.getReader()
+  try {
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      // We don't need to do anything with the value, just drain the stream
+    }
+  } finally {
+    reader.releaseLock()
+  }
 } 

sim/app/api/providers/route.ts

@@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from 'next/server'
 import { createLogger } from '@/lib/logs/console-logger'
 import { executeProviderRequest } from '@/providers'
 import { getApiKey } from '@/providers/utils'
+import { StreamingExecution } from '@/executor/types'
 
 const logger = createLogger('ProvidersAPI')
 
@@ -24,6 +25,7 @@ export async function POST(request: NextRequest) {
       apiKey,
       responseFormat,
       workflowId,
+      stream,
     } = body
 
     let finalApiKey: string
@@ -48,8 +50,90 @@ export async function POST(request: NextRequest) {
       apiKey: finalApiKey,
       responseFormat,
       workflowId,
+      stream,
     })
 
+    // Check if the response is a StreamingExecution
+    if (response && typeof response === 'object' && 'stream' in response && 'execution' in response) {
+      const streamingExec = response as StreamingExecution
+      logger.info('Received StreamingExecution from provider')
+
+      // Extract the stream and execution data
+      const stream = streamingExec.stream
+      const executionData = streamingExec.execution
+
+      // Attach the execution data as a custom header
+      // We need to safely serialize the execution data to avoid circular references
+      let executionDataHeader
+      try {
+        // Create a safe version of execution data with the most important fields
+        const safeExecutionData = {
+          success: executionData.success,
+          output: {
+            response: {
+              // Sanitize content to remove non-ASCII characters that would cause ByteString errors
+              content: executionData.output?.response?.content 
+                ? String(executionData.output.response.content).replace(/[\u0080-\uFFFF]/g, '')
+                : '',
+              model: executionData.output?.response?.model,
+              tokens: executionData.output?.response?.tokens || {
+                prompt: 0,
+                completion: 0,
+                total: 0
+              },
+              // Sanitize any potential Unicode characters in tool calls
+              toolCalls: executionData.output?.response?.toolCalls
+                ? sanitizeToolCalls(executionData.output.response.toolCalls)
+                : undefined,
+              providerTiming: executionData.output?.response?.providerTiming,
+              cost: executionData.output?.response?.cost,
+            }
+          },
+          error: executionData.error,
+          logs: [], // Strip logs from header to avoid encoding issues
+          metadata: {
+            startTime: executionData.metadata?.startTime,
+            endTime: executionData.metadata?.endTime,
+            duration: executionData.metadata?.duration
+          },
+          isStreaming: true, // Always mark streaming execution data as streaming
+          blockId: executionData.logs?.[0]?.blockId,
+          blockName: executionData.logs?.[0]?.blockName,
+          blockType: executionData.logs?.[0]?.blockType,
+        }
+        executionDataHeader = JSON.stringify(safeExecutionData)
+      } catch (error) {
+        logger.error('Failed to serialize execution data:', error)
+        executionDataHeader = JSON.stringify({
+          success: executionData.success,
+          error: 'Failed to serialize full execution data'
+        })
+      }
+      
+      // Return the stream with execution data in a header
+      return new Response(stream, {
+        headers: {
+          'Content-Type': 'text/event-stream',
+          'Cache-Control': 'no-cache',
+          'Connection': 'keep-alive',
+          'X-Execution-Data': executionDataHeader
+        },
+      })
+    }
+    
+    // Check if the response is a ReadableStream for streaming
+    if (response instanceof ReadableStream) {
+      logger.info('Streaming response from provider')
+      return new Response(response, {
+        headers: {
+          'Content-Type': 'text/event-stream',
+          'Cache-Control': 'no-cache',
+          'Connection': 'keep-alive',
+        },
+      })
+    }
+
+    // Return regular JSON response for non-streaming
     return NextResponse.json(response)
   } catch (error) {
     logger.error('Provider request failed:', error)
@@ -59,3 +143,89 @@ export async function POST(request: NextRequest) {
     )
   }
 }
+
+/**
+ * Helper function to sanitize tool calls to remove Unicode characters
+ */
+function sanitizeToolCalls(toolCalls: any) {
+  // If it's an object with a list property, sanitize the list
+  if (toolCalls && typeof toolCalls === 'object' && Array.isArray(toolCalls.list)) {
+    return {
+      ...toolCalls,
+      list: toolCalls.list.map(sanitizeToolCall)
+    }
+  }
+  
+  // If it's an array, sanitize each item
+  if (Array.isArray(toolCalls)) {
+    return toolCalls.map(sanitizeToolCall)
+  }
+  
+  return toolCalls
+}
+
+/**
+ * Sanitize a single tool call to remove Unicode characters
+ */
+function sanitizeToolCall(toolCall: any) {
+  if (!toolCall || typeof toolCall !== 'object') return toolCall
+  
+  // Create a sanitized copy
+  const sanitized = { ...toolCall }
+  
+  // Sanitize any string fields that might contain Unicode
+  if (typeof sanitized.name === 'string') {
+    sanitized.name = sanitized.name.replace(/[\u0080-\uFFFF]/g, '')
+  }
+  
+  // Sanitize input/arguments
+  if (sanitized.input && typeof sanitized.input === 'object') {
+    sanitized.input = sanitizeObject(sanitized.input)
+  }
+  
+  if (sanitized.arguments && typeof sanitized.arguments === 'object') {
+    sanitized.arguments = sanitizeObject(sanitized.arguments)
+  }
+  
+  // Sanitize output/result
+  if (sanitized.output && typeof sanitized.output === 'object') {
+    sanitized.output = sanitizeObject(sanitized.output)
+  }
+  
+  if (sanitized.result && typeof sanitized.result === 'object') {
+    sanitized.result = sanitizeObject(sanitized.result)
+  }
+  
+  // Sanitize error message
+  if (typeof sanitized.error === 'string') {
+    sanitized.error = sanitized.error.replace(/[\u0080-\uFFFF]/g, '')
+  }
+  
+  return sanitized
+}
+
+/**
+ * Recursively sanitize an object to remove Unicode characters from strings
+ */
+function sanitizeObject(obj: any): any {
+  if (!obj || typeof obj !== 'object') return obj
+  
+  // Handle arrays
+  if (Array.isArray(obj)) {
+    return obj.map(item => sanitizeObject(item))
+  }
+  
+  // Handle objects
+  const result: any = {}
+  for (const [key, value] of Object.entries(obj)) {
+    if (typeof value === 'string') {
+      result[key] = value.replace(/[\u0080-\uFFFF]/g, '')
+    } else if (typeof value === 'object' && value !== null) {
+      result[key] = sanitizeObject(value)
+    } else {
+      result[key] = value
+    }
+  }
+  
+  return result
+}