imrpovement(kb): added client-side store, shared utils, background processing, retry with exponential backoff to knowledge base (#453)

* added background processing for file upload to knowledge base, added retry with exponential backoff, individual file retry, statuses

* added knowledge store

* added knowledge base selector and docs for knowledge block

* fixed build

* fix contributors page

* significantly improved error handling, typing, fault tolerance

* standardize file icon size

Author: waleedlatif1

apps/docs/content/docs/tools/knowledge.mdx

@@ -0,0 +1,108 @@
+---
+title: Knowledge
+description: Search knowledge
+---
+
+import { BlockInfoCard } from "@/components/ui/block-info-card"
+
+<BlockInfoCard 
+  type="knowledge"
+  color="#00B0B0"
+  icon={true}
+  iconSvg={`<svg className="block-icon"
+      
+      xmlns='http://www.w3.org/2000/svg'
+      
+      
+      viewBox='0 0 24 24'
+      fill='none'
+      stroke='currentColor'
+      strokeWidth='1.5'
+      strokeLinecap='round'
+      strokeLinejoin='round'
+    >
+      <path d='M21 10V8a2 2 0 0 0-1-1.73l-7-4a2 2 0 0 0-2 0l-7 4A2 2 0 0 0 3 8v8a2 2 0 0 0 1 1.73l7 4a2 2 0 0 0 2 0l2-1.14' />
+      <path d='m7.5 4.27 9 5.15' />
+      <polyline points='3.29 7 12 12 20.71 7' />
+      <line x1='12' x2='12' y1='22' y2='12' />
+      <circle cx='18.5' cy='15.5' r='2.5' />
+      <path d='M20.27 17.27 22 19' />
+    </svg>`}
+/>
+
+{/* MANUAL-CONTENT-START:intro */}
+Sim Studio's Knowledge Base is a powerful native feature that enables you to create, manage, and query custom knowledge bases directly within the platform. Using advanced AI embeddings and vector search technology, the Knowledge Base block allows you to build intelligent search capabilities into your workflows, making it easy to find and utilize relevant information across your organization.
+
+The Knowledge Base system provides a comprehensive solution for managing organizational knowledge through its flexible and scalable architecture. With its built-in vector search capabilities, teams can perform semantic searches that understand meaning and context, going beyond traditional keyword matching.
+
+Key features of the Knowledge Base include:
+
+- Semantic Search: Advanced AI-powered search that understands meaning and context, not just keywords
+- Vector Embeddings: Automatic conversion of text into high-dimensional vectors for intelligent similarity matching
+- Custom Knowledge Bases: Create and manage multiple knowledge bases for different purposes or departments
+- Flexible Content Types: Support for various document formats and content types
+- Real-time Updates: Immediate indexing of new content for instant searchability
+
+In Sim Studio, the Knowledge Base block enables your agents to perform intelligent semantic searches across your custom knowledge bases. This creates opportunities for automated information retrieval, content recommendations, and knowledge discovery as part of your AI workflows. The integration allows agents to search and retrieve relevant information programmatically, facilitating automated knowledge management tasks and ensuring that important information is easily accessible. By leveraging the Knowledge Base block, you can build intelligent agents that enhance information discovery while automating routine knowledge management tasks, improving team efficiency and ensuring consistent access to organizational knowledge.
+{/* MANUAL-CONTENT-END */}
+
+## Usage Instructions
+
+Perform semantic vector search across your knowledge base to find the most relevant content. Uses advanced AI embeddings to understand meaning and context, returning the most similar documents to your search query.
+
+
+
+## Tools
+
+### `knowledge_search`
+
+Search for similar content in a knowledge base using vector similarity
+
+#### Input
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `knowledgeBaseId` | string | Yes | ID of the knowledge base to search in |
+| `query` | string | Yes | Search query text |
+| `topK` | number | No | Number of most similar results to return \(1-100\) |
+
+#### Output
+
+| Parameter | Type |
+| --------- | ---- |
+| `results` | string |
+| `query` | string |
+| `knowledgeBaseId` | string |
+| `topK` | string |
+| `totalResults` | string |
+| `message` | string |
+
+
+
+## Block Configuration
+
+### Input
+
+| Parameter | Type | Required | Description | 
+| --------- | ---- | -------- | ----------- | 
+| `knowledgeBaseId` | string | Yes | Knowledge Base - Select knowledge base |
+
+
+
+### Outputs
+
+| Output | Type | Description |
+| ------ | ---- | ----------- |
+| `response` | object | Output from response |
+| ↳ `results` | json | results of the response |
+| ↳ `query` | string | query of the response |
+| ↳ `knowledgeBaseId` | string | knowledgeBaseId of the response |
+| ↳ `topK` | number | topK of the response |
+| ↳ `totalResults` | number | totalResults of the response |
+| ↳ `message` | string | message of the response |
+
+
+## Notes
+
+- Category: `blocks`
+- Type: `knowledge`

apps/docs/content/docs/tools/meta.json

@@ -21,6 +21,7 @@
     "image_generator",
     "jina",
     "jira",
+    "knowledge",
     "linear",
     "linkup",
     "mem0",

apps/docs/content/docs/tools/microsoft_teams.mdx

@@ -64,6 +64,9 @@ import { BlockInfoCard } from "@/components/ui/block-info-card"
         <stop offset='0' stopColor='#5a62c3' />
         <stop offset='.5' stopColor='#4d55bd' />
         <stop offset='1' stopColor='#3940ab' />
+        <stop offset='0' stopColor='#5a62c3' />
+        <stop offset='.5' stopColor='#4d55bd' />
+        <stop offset='1' stopColor='#3940ab' />
       </linearGradient>
       <path
         fill='url(#a)'

apps/docs/content/docs/tools/slack.mdx

@@ -1,6 +1,6 @@
 ---
 title: Slack
-description: Send a message to Slack
+description: Send messages to Slack
 ---
 
 import { BlockInfoCard } from "@/components/ui/block-info-card"
@@ -56,23 +56,24 @@ In Sim Studio, the Slack integration enables your agents to programmatically sen
 
 ## Usage Instructions
 
-Send messages to any Slack channel using OAuth authentication. Integrate automated notifications and alerts into your workflow to keep your team informed.
+Comprehensive Slack integration with OAuth authentication. Send formatted messages using Slack
 
 
 
 ## Tools
 
 ### `slack_message`
 
-Send messages to Slack channels or users through the Slack API. Enables direct communication and notifications with timestamp tracking and channel confirmation.
+Send messages to Slack channels or users through the Slack API. Supports Slack mrkdwn formatting.
 
 #### Input
 
 | Parameter | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
-| `apiKey` | string | Yes | Your Slack API token |
+| `botToken` | string | No | Bot token for Custom Bot |
+| `accessToken` | string | No | OAuth access token or bot token for Slack API |
 | `channel` | string | Yes | Target Slack channel \(e.g., #general\) |
-| `text` | string | Yes | Message text to send |
+| `text` | string | Yes | Message text to send \(supports Slack mrkdwn formatting\) |
 
 #### Output
 
@@ -89,7 +90,7 @@ Send messages to Slack channels or users through the Slack API. Enables direct c
 
 | Parameter | Type | Required | Description | 
 | --------- | ---- | -------- | ----------- | 
-| `apiKey` | string | Yes | OAuth Token - Enter your Slack OAuth token |
+| `operation` | string | Yes | Operation |
 
 
 

apps/sim/app/(landing)/contributors/page.tsx

@@ -433,7 +433,7 @@ export default function ContributorsPage() {
                     <ResponsiveContainer width='100%' height={300} className='sm:!h-[400px]'>
                       <BarChart
                         data={filteredContributors?.slice(0, showAllContributors ? undefined : 10)}
-                        margin={{ top: 10, right: 5, bottom: 45, left: 5 }}
+                        margin={{ top: 10, right: 10, bottom: 60, left: 10 }}
                         className='sm:!mx-2.5 sm:!mb-2.5'
                       >
                         <XAxis
@@ -448,11 +448,11 @@ export default function ContributorsPage() {
                             return (
                               <g transform={`translate(${x},${y})`}>
                                 <foreignObject
-                                  x='-12'
-                                  y='6'
-                                  width='24'
-                                  height='24'
-                                  className='sm:!x-[-16] sm:!y-[8] sm:!w-[32] sm:!h-[32]'
+                                  x='-16'
+                                  y='8'
+                                  width='32'
+                                  height='32'
+                                  style={{ overflow: 'visible' }}
                                 >
                                   <Avatar className='h-6 w-6 ring-1 ring-[#606060]/30 sm:h-8 sm:w-8'>
                                     <AvatarImage src={contributor?.avatar_url} />
@@ -464,8 +464,8 @@ export default function ContributorsPage() {
                               </g>
                             )
                           }}
-                          height={50}
-                          className='sm:!h-[60px] text-neutral-400'
+                          height={60}
+                          className='text-neutral-400'
                         />
                         <YAxis
                           stroke='currentColor'

apps/sim/app/api/files/presigned/route.ts

@@ -46,7 +46,7 @@ export async function POST(request: NextRequest) {
       Key: uniqueKey,
       ContentType: contentType,
       Metadata: {
-        originalName: fileName,
+        originalName: encodeURIComponent(fileName),
         uploadedAt: new Date().toISOString(),
       },
     })

apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/[chunkId]/route.ts

@@ -1,79 +1,21 @@
-import { and, eq, isNull } from 'drizzle-orm'
+import { eq } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { z } from 'zod'
 import { getSession } from '@/lib/auth'
 import { createLogger } from '@/lib/logs/console-logger'
 import { db } from '@/db'
-import { document, embedding, knowledgeBase } from '@/db/schema'
+import { embedding } from '@/db/schema'
+import { checkChunkAccess } from '../../../../../utils'
 
 const logger = createLogger('ChunkByIdAPI')
 
-// Schema for chunk updates
 const UpdateChunkSchema = z.object({
   content: z.string().min(1, 'Content is required').optional(),
   enabled: z.boolean().optional(),
   searchRank: z.number().min(0).optional(),
   qualityScore: z.number().min(0).max(1).optional(),
 })
 
-async function checkChunkAccess(
-  knowledgeBaseId: string,
-  documentId: string,
-  chunkId: string,
-  userId: string
-) {
-  // First check knowledge base access
-  const kb = await db
-    .select({
-      id: knowledgeBase.id,
-      userId: knowledgeBase.userId,
-    })
-    .from(knowledgeBase)
-    .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
-    .limit(1)
-
-  if (kb.length === 0) {
-    return { hasAccess: false, notFound: true, reason: 'Knowledge base not found' }
-  }
-
-  const kbData = kb[0]
-
-  // Check if user owns the knowledge base
-  if (kbData.userId !== userId) {
-    return { hasAccess: false, reason: 'Unauthorized knowledge base access' }
-  }
-
-  // Check if document exists and belongs to the knowledge base
-  const doc = await db
-    .select()
-    .from(document)
-    .where(
-      and(
-        eq(document.id, documentId),
-        eq(document.knowledgeBaseId, knowledgeBaseId),
-        isNull(document.deletedAt)
-      )
-    )
-    .limit(1)
-
-  if (doc.length === 0) {
-    return { hasAccess: false, notFound: true, reason: 'Document not found' }
-  }
-
-  // Check if chunk exists and belongs to the document
-  const chunk = await db
-    .select()
-    .from(embedding)
-    .where(and(eq(embedding.id, chunkId), eq(embedding.documentId, documentId)))
-    .limit(1)
-
-  if (chunk.length === 0) {
-    return { hasAccess: false, notFound: true, reason: 'Chunk not found' }
-  }
-
-  return { hasAccess: true, chunk: chunk[0], document: doc[0], knowledgeBase: kbData }
-}
-
 export async function GET(
   req: NextRequest,
   { params }: { params: Promise<{ id: string; documentId: string; chunkId: string }> }
@@ -95,14 +37,13 @@ export async function GET(
       session.user.id
     )
 
-    if (accessCheck.notFound) {
-      logger.warn(
-        `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}, Chunk=${chunkId}`
-      )
-      return NextResponse.json({ error: accessCheck.reason }, { status: 404 })
-    }
-
     if (!accessCheck.hasAccess) {
+      if (accessCheck.notFound) {
+        logger.warn(
+          `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}, Chunk=${chunkId}`
+        )
+        return NextResponse.json({ error: accessCheck.reason }, { status: 404 })
+      }
       logger.warn(
         `[${requestId}] User ${session.user.id} attempted unauthorized chunk access: ${accessCheck.reason}`
       )
@@ -144,14 +85,13 @@ export async function PUT(
       session.user.id
     )
 
-    if (accessCheck.notFound) {
-      logger.warn(
-        `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}, Chunk=${chunkId}`
-      )
-      return NextResponse.json({ error: accessCheck.reason }, { status: 404 })
-    }
-
     if (!accessCheck.hasAccess) {
+      if (accessCheck.notFound) {
+        logger.warn(
+          `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}, Chunk=${chunkId}`
+        )
+        return NextResponse.json({ error: accessCheck.reason }, { status: 404 })
+      }
       logger.warn(
         `[${requestId}] User ${session.user.id} attempted unauthorized chunk update: ${accessCheck.reason}`
       )
@@ -235,14 +175,13 @@ export async function DELETE(
       session.user.id
     )
 
-    if (accessCheck.notFound) {
-      logger.warn(
-        `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}, Chunk=${chunkId}`
-      )
-      return NextResponse.json({ error: accessCheck.reason }, { status: 404 })
-    }
-
     if (!accessCheck.hasAccess) {
+      if (accessCheck.notFound) {
+        logger.warn(
+          `[${requestId}] ${accessCheck.reason}: KB=${knowledgeBaseId}, Doc=${documentId}, Chunk=${chunkId}`
+        )
+        return NextResponse.json({ error: accessCheck.reason }, { status: 404 })
+      }
       logger.warn(
         `[${requestId}] User ${session.user.id} attempted unauthorized chunk deletion: ${accessCheck.reason}`
       )