fix(kb): handle larger files in the kb (#2324)

* fix(kb): handle larger files in the kb

* fixed images on login page

Author: waleedlatif1

apps/sim/app/(landing)/components/nav/nav.tsx

@@ -135,6 +135,7 @@ export default function Nav({ hideAuthButtons = false, variant = 'landing' }: Na
               priority
               loading='eager'
               quality={100}
+              unoptimized
             />
           ) : (
             <Image

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx

@@ -466,7 +466,7 @@ export function KnowledgeBase({
    */
   const checkForDeadProcesses = async () => {
     const now = new Date()
-    const DEAD_PROCESS_THRESHOLD_MS = 150 * 1000
+    const DEAD_PROCESS_THRESHOLD_MS = 600 * 1000 // 10 minutes
 
     const staleDocuments = documents.filter((doc) => {
       if (doc.processingStatus !== 'processing' || !doc.processingStartedAt) {

apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts

@@ -423,6 +423,10 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
           return await uploadFileInChunks(file, presignedData, timeoutMs, fileIndex)
         }
 
+        if (presignedOverride?.directUploadSupported && presignedOverride.presignedUrl) {
+          return await uploadFileDirectly(file, presignedOverride, timeoutMs, controller, fileIndex)
+        }
+
         return await uploadFileThroughAPI(file, timeoutMs)
       } finally {
         clearTimeout(timeoutId)
@@ -510,7 +514,6 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
         if (event.lengthComputable && fileIndex !== undefined && !isCompleted) {
           const percentComplete = Math.round((event.loaded / event.total) * 100)
           setUploadProgress((prev) => {
-            // Only update if this file is still uploading
             if (prev.fileStatuses?.[fileIndex]?.status === 'uploading') {
               return {
                 ...prev,
@@ -638,7 +641,6 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
       })
 
       if (!partUrlsResponse.ok) {
-        // Abort the multipart upload if we can't get URLs
         await fetch('/api/files/multipart?action=abort', {
           method: 'POST',
           headers: { 'Content-Type': 'application/json' },
@@ -822,9 +824,6 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
     }
   }
 
-  /**
-   * Upload files using batch presigned URLs (works for both S3 and Azure Blob)
-   */
   /**
    * Uploads files in batches using presigned URLs
    */

apps/sim/lib/core/security/csp.ts

@@ -153,7 +153,7 @@ export function generateRuntimeCSP(): string {
     default-src 'self';
     script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.google.com https://apis.google.com https://assets.onedollarstats.com;
     style-src 'self' 'unsafe-inline' https://fonts.googleapis.com;
-    img-src 'self' data: blob: https://*.googleusercontent.com https://*.google.com https://*.atlassian.com https://cdn.discordapp.com https://*.githubusercontent.com ${brandLogoDomain} ${brandFaviconDomain};
+    img-src 'self' data: blob: https://*.googleusercontent.com https://*.google.com https://*.atlassian.com https://cdn.discordapp.com https://*.githubusercontent.com https://*.s3.amazonaws.com https://s3.amazonaws.com https://*.amazonaws.com https://*.blob.core.windows.net https://github.com/* ${brandLogoDomain} ${brandFaviconDomain};
     media-src 'self' blob:;
     font-src 'self' https://fonts.gstatic.com;
     connect-src 'self' ${appUrl} ${ollamaUrl} ${socketUrl} ${socketWsUrl} https://api.browser-use.com https://api.exa.ai https://api.firecrawl.dev https://*.googleapis.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.blob.core.windows.net https://api.github.com https://github.com/* https://*.atlassian.com https://*.supabase.co https://collector.onedollarstats.com ${dynamicDomainsStr};

apps/sim/lib/knowledge/documents/service.ts

@@ -1089,7 +1089,7 @@ export async function markDocumentAsFailedTimeout(
 ): Promise<{ success: boolean; processingDuration: number }> {
   const now = new Date()
   const processingDuration = now.getTime() - processingStartedAt.getTime()
-  const DEAD_PROCESS_THRESHOLD_MS = 150 * 1000
+  const DEAD_PROCESS_THRESHOLD_MS = 600 * 1000 // 10 minutes
 
   if (processingDuration <= DEAD_PROCESS_THRESHOLD_MS) {
     throw new Error('Document has not been processing long enough to be considered dead')

apps/sim/lib/uploads/utils/file-utils.ts

@@ -278,6 +278,7 @@ export function validateKnowledgeBaseFile(
 
 /**
  * Extract storage key from a file path
+ * Handles URLs like /api/files/serve/s3/key or /api/files/serve/blob/key
  */
 export function extractStorageKey(filePath: string): string {
   let pathWithoutQuery = filePath.split('?')[0]
@@ -292,7 +293,13 @@ export function extractStorageKey(filePath: string): string {
   }
 
   if (pathWithoutQuery.startsWith('/api/files/serve/')) {
-    return decodeURIComponent(pathWithoutQuery.substring('/api/files/serve/'.length))
+    let key = decodeURIComponent(pathWithoutQuery.substring('/api/files/serve/'.length))
+    if (key.startsWith('s3/')) {
+      key = key.substring(3)
+    } else if (key.startsWith('blob/')) {
+      key = key.substring(5)
+    }
+    return key
   }
   return pathWithoutQuery
 }

apps/sim/proxy.ts

@@ -144,7 +144,9 @@ export async function proxy(request: NextRequest) {
     if (hasActiveSession) {
       return NextResponse.redirect(new URL('/workspace', request.url))
     }
-    return NextResponse.next()
+    const response = NextResponse.next()
+    response.headers.set('Content-Security-Policy', generateRuntimeCSP())
+    return response
   }
 
   if (url.pathname.startsWith('/chat/')) {