feat(otel): added otel, persist settings to db, present user with the telemetry preferences & add privacy tab to settings (#318)

* feat(otel): added otel, persist settings to db, present user with the telemetry preferences & add privacy tab to settings

* updated telemetry endpoint

* add protected subdomains for chat deploy

* removed unused dependencies

* add execution telemetry logs for workflow-level and block-level logs, acknowledged PR comments

Author: waleedlatif1

.gitignore

@@ -65,4 +65,12 @@ docs/.contentlayer
 docs/.content-collections
 
 # database instantiation
-**/postgres_data/
+**/postgres_data/
+
+# file uploads
+uploads/
+
+# collector configuration
+collector-config.yaml
+docker-compose.collector.yml
+start-collector.sh

sim/app/api/chat/subdomains/validate/route.test.ts

@@ -162,6 +162,29 @@ describe('Subdomain Validation API Route', () => {
       subdomain: 'available-subdomain',
     })
   })
+  
+  it('should return available=false when subdomain is reserved', async () => {
+    vi.doMock('@/lib/auth', () => ({
+      getSession: vi.fn().mockResolvedValue({
+        user: { id: 'user-id' },
+      }),
+    }))
+
+    const req = new NextRequest('http://localhost:3000/api/chat/subdomains/validate?subdomain=telemetry')
+
+    const { GET } = await import('./route')
+
+    const response = await GET(req)
+    const data = await response.json()
+
+    expect(response.status).toBe(400)
+    expect(data).toHaveProperty('available', false)
+    expect(data).toHaveProperty('error', 'This subdomain is reserved')
+    expect(mockNextResponseJson).toHaveBeenCalledWith(
+      { available: false, error: 'This subdomain is reserved' },
+      { status: 400 }
+    )
+  })
 
   it('should return available=false when subdomain is already in use', async () => {
     vi.doMock('@/lib/auth', () => ({

sim/app/api/chat/subdomains/validate/route.ts

@@ -34,6 +34,18 @@ export async function GET(request: Request) {
         { status: 400 }
       )
     }
+    
+    // Protect reserved subdomains
+    const reservedSubdomains = ['telemetry', 'docs', 'api', 'admin', 'www', 'app', 'auth', 'blog', 'help', 'support'];
+    if (reservedSubdomains.includes(subdomain)) {
+      return NextResponse.json(
+        {
+          available: false,
+          error: 'This subdomain is reserved'
+        },
+        { status: 400 }
+      )
+    }
 
     // Query database to see if subdomain already exists
     const existingDeployment = await db

sim/app/api/telemetry/route.ts

@@ -0,0 +1,209 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { createLogger } from '@/lib/logs/console-logger'
+
+const logger = createLogger('TelemetryAPI')
+
+const ALLOWED_CATEGORIES = [
+  'page_view',
+  'feature_usage',
+  'performance',
+  'error',
+  'workflow',
+  'consent',
+]
+
+const DEFAULT_TIMEOUT = 5000 // 5 seconds timeout
+
+/**
+ * Validates telemetry data to ensure it doesn't contain sensitive information
+ */
+function validateTelemetryData(data: any): boolean {
+  if (!data || typeof data !== 'object') {
+    return false
+  }
+  
+  if (!data.category || !data.action) {
+    return false
+  }
+  
+  if (!ALLOWED_CATEGORIES.includes(data.category)) {
+    return false
+  }
+  
+  const jsonStr = JSON.stringify(data).toLowerCase()
+  const sensitivePatterns = [
+    /password/,
+    /token/,
+    /secret/,
+    /key/,
+    /auth/,
+    /credential/,
+    /private/,
+  ]
+  
+  return !sensitivePatterns.some(pattern => pattern.test(jsonStr))
+}
+
+/**
+ * Safely converts a value to string, handling undefined and null values
+ */
+function safeStringValue(value: any): string {
+  if (value === undefined || value === null) {
+    return ''
+  }
+  
+  try {
+    return String(value)
+  } catch (e) {
+    return ''
+  }
+}
+
+/**
+ * Creates a safe attribute object for OpenTelemetry
+ */
+function createSafeAttributes(data: Record<string, any>): Array<{key: string, value: {stringValue: string}}> {
+  if (!data || typeof data !== 'object') {
+    return []
+  }
+  
+  const attributes: Array<{key: string, value: {stringValue: string}}> = []
+  
+  Object.entries(data).forEach(([key, value]) => {
+    if (value !== undefined && value !== null && key) {
+      attributes.push({
+        key,
+        value: { stringValue: safeStringValue(value) }
+      })
+    }
+  })
+  
+  return attributes
+}
+
+/**
+ * Forwards telemetry data to OpenTelemetry collector
+ */
+async function forwardToCollector(data: any): Promise<boolean> {
+  if (!data || typeof data !== 'object') {
+    logger.error('Invalid telemetry data format')
+    return false
+  }
+  
+  const endpoint = process.env.TELEMETRY_ENDPOINT || 'https://telemetry.simstudio.ai/v1/traces'
+  const timeout = parseInt(process.env.TELEMETRY_TIMEOUT || '') || DEFAULT_TIMEOUT
+  
+  try {
+    const timestamp = Date.now() * 1000000
+    
+    const safeAttrs = createSafeAttributes(data)
+    
+    const serviceAttrs = [
+      { key: 'service.name', value: { stringValue: 'sim-studio' } },
+      { key: 'service.version', value: { stringValue: process.env.NEXT_PUBLIC_APP_VERSION || '0.1.0' } },
+      { key: 'deployment.environment', value: { stringValue: process.env.NODE_ENV || 'production' } }
+    ]
+    
+    const spanName = data.category && data.action ? `${data.category}.${data.action}` : 'telemetry.event'
+    
+    const payload = {
+      resourceSpans: [{
+        resource: {
+          attributes: serviceAttrs
+        },
+        instrumentationLibrarySpans: [{
+          spans: [{
+            name: spanName,
+            kind: 1,
+            startTimeUnixNano: timestamp,
+            endTimeUnixNano: timestamp + 1000000,
+            attributes: safeAttrs
+          }]
+        }]
+      }]
+    }
+    
+    // Safe debug log of the payload structure without sensitive data
+    logger.debug('Preparing to send telemetry payload', { 
+      endpoint,
+      hasAttributes: safeAttrs.length > 0,
+      attributeCount: safeAttrs.length
+    })
+    
+    // Create explicit AbortController for timeout
+    const controller = new AbortController()
+    const timeoutId = setTimeout(() => controller.abort(), timeout)
+    
+    try {
+      const options = {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify(payload),
+        signal: controller.signal
+      }
+      
+      const response = await fetch(endpoint, options)
+      clearTimeout(timeoutId)
+      
+      if (!response.ok) {
+        logger.error('Telemetry collector returned error', { 
+          status: response.status,
+          statusText: response.statusText
+        })
+        return false
+      }
+      
+      return true
+    } catch (fetchError) {
+      clearTimeout(timeoutId)
+      if (fetchError instanceof Error && fetchError.name === 'AbortError') {
+        logger.error('Telemetry request timed out', { endpoint })
+      } else {
+        logger.error('Failed to send telemetry to collector', fetchError)
+      }
+      return false
+    }
+  } catch (error) {
+    logger.error('Error preparing telemetry payload', error)
+    return false
+  }
+}
+
+/**
+ * Endpoint that receives telemetry events and forwards them to OpenTelemetry collector
+ */
+export async function POST(req: NextRequest) {
+  try {
+    let eventData
+    try {
+      eventData = await req.json()
+    } catch (parseError) {
+      return NextResponse.json(
+        { error: 'Invalid JSON in request body' },
+        { status: 400 }
+      )
+    }
+    
+    if (!validateTelemetryData(eventData)) {
+      return NextResponse.json(
+        { error: 'Invalid telemetry data format or contains sensitive information' },
+        { status: 400 }
+      )
+    }
+    
+    const forwarded = await forwardToCollector(eventData)
+    
+    return NextResponse.json({ 
+      success: true,
+      forwarded
+    })
+  } catch (error) {
+    logger.error('Error processing telemetry event', error)
+    return NextResponse.json(
+      { error: 'Failed to process telemetry event' },
+      { status: 500 }
+    )
+  }
+}