Feat/rag foundation clean

.gitignore

@@ -54,4 +54,5 @@ node_modules
 !.env.example
 *.pem
 
-.gitignore
+.gitignore
+.pnpm-store/*

README.md

@@ -151,4 +151,4 @@ pnpm lint       # Biome によるコード検査
 pnpm format     # Biome によるコードフォーマット
 ```
 
-aaaaa
+c

infra/.env.example

@@ -10,4 +10,12 @@ CONFLUENCE_SPACES=*
 CONFLUENCE_APP_KEY=*
 CONFLUENCE_APP_SECRET=*
 CONFLUENCE_ACCESS_TOKEN=*
-CONFLUENCE_REFRESH_TOKEN=*
+CONFLUENCE_REFRESH_TOKEN=*
+
+# Image RAG batch indexing
+IMAGE_INDEX_BUCKET=
+IMAGE_INDEX_INPUT_PREFIX=documents/images/
+IMAGE_INDEX_OUTPUT_KEY=documents/image-index/image-index.jsonl
+IMAGE_INDEX_VLM_MODEL_ID=jp.anthropic.claude-sonnet-4-5-20250929-v1:0
+IMAGE_INDEX_PUBLIC_BASE_URL=
+IMAGE_INDEX_MAX_IMAGES=200

infra/README_BEDROCK_KB.md

@@ -101,6 +101,39 @@ bedrockKb: {
    aws bedrock start-ingestion-job --knowledge-base-id [kb-id] --data-source-id [ds-id]
    ```
 
+## 画像RAG（事前バッチ）手順
+
+`documents/images/` に置いた画像をVLMで説明文・タグ化し、`documents/image-index/` にJSONLを生成します。
+
+1. 画像をS3に配置:
+
+   ```bash
+   aws s3 cp --recursive ./images s3://<bucket-name>/documents/images/
+   ```
+
+2. 画像索引バッチを実行:
+
+   ```bash
+   cd infra
+   pnpm image:index:batch
+   ```
+
+3. Knowledge Base ingestionを実行:
+
+   ```bash
+   aws bedrock-agent start-ingestion-job \
+     --knowledge-base-id <kb-id> \
+     --data-source-id <data-source-id>
+   ```
+
+4. スモークテスト:
+
+   ```text
+   みかん
+   ```
+
+   検索結果に `imageUrl` と `caption` が含まれていれば成功です。
+
 ## 主要な機能
 
 ### VPC設定

infra/bin/infra.ts

@@ -3,57 +3,32 @@ import * as cdk from "aws-cdk-lib";
 import * as dotenv from "dotenv";
 import { getConfig } from "../lib/config/environmental_config";
 import { AmazonBedrockKbStack } from "../lib/stack/bedrock-kb-stack";
-import { SecretsStack } from "../lib/stack/secrets-stack";
+import { SageMakerStack } from "../lib/stack/sagemaker-stack"; // 1. 追加したスタックをインポート
 
 dotenv.config();
 
 const app = new cdk.App();
 
-// 環境名を取得
 const stage = app.node.tryGetContext("stage") || "test";
 const stagePrefix = stage.charAt(0).toUpperCase() + stage.slice(1);
-
-// 環境設定を取得（エントリーポイントでのみ呼び出す）
 const config = getConfig(stage);
 
 const env = {
   account: process.env.CDK_DEFAULT_ACCOUNT ?? process.env.AWS_ACCOUNT,
   region: process.env.CDK_DEFAULT_REGION ?? process.env.AWS_REGION,
 };
 
-// Secrets スタック（認証情報を管理）
-const secretsStack = new SecretsStack(app, `SecretsStack${stagePrefix}`, {
+// 2. Bedrock Stack を先に定義（S3バケット等の情報を後続に渡すため）
+const bedrockKbStack = new AmazonBedrockKbStack(app, `BedrockKbStack${stagePrefix}`, {
   stage,
-  confluence: config.bedrockKb?.confluence
-    ? {
-        confluenceAppKey: config.bedrockKb.confluence.confluenceAppKey,
-        confluenceAppSecret: config.bedrockKb.confluence.confluenceAppSecret,
-        confluenceAccessToken:
-          config.bedrockKb.confluence.confluenceAccessToken,
-        confluenceRefreshToken:
-          config.bedrockKb.confluence.confluenceRefreshToken,
-      }
-    : undefined,
   env,
 });
 
-// Bedrock Knowledge Base スタック
-const bedrockKbConfig = config.bedrockKb;
-if (!bedrockKbConfig) {
-  throw new Error(
-    `Bedrock KB configuration is not defined for environment: ${stage}`,
-  );
-}
-
-new AmazonBedrockKbStack(app, `BedrockKbStack${stagePrefix}`, {
+// 3. SageMaker Stack を定義
+// 必要に応じて、bedrockKbStack で作成したバケットの参照などを props で渡します
+new SageMakerStack(app, `SageMakerStack${stagePrefix}`, {
   stage,
-  confluence:
-    bedrockKbConfig.confluence && secretsStack.confluenceSecretArn
-      ? {
-          secretArn: secretsStack.confluenceSecretArn,
-          hostUrl: bedrockKbConfig.confluence.hostUrl,
-          spaces: bedrockKbConfig.confluence.spaces,
-        }
-      : undefined,
   env,
-});
+  // 作業に必要であれば、bedrockKbStackからバケット情報を渡す設計にします
+  // dataSourceBucket: bedrockKbStack.dataSourceBucket 
+});

infra/lambda/sync-kb/index.ts

@@ -0,0 +1,22 @@
+import { BedrockAgentClient, StartIngestionJobCommand } from "@aws-sdk/client-bedrock-agent";
+
+const client = new BedrockAgentClient({});
+
+export const handler = async (event: any) => {
+  console.log("S3 Event received:", JSON.stringify(event, null, 2));
+
+  try {
+    const command = new StartIngestionJobCommand({
+      knowledgeBaseId: process.env.KNOWLEDGE_BASE_ID,
+      dataSourceId: process.env.DATA_SOURCE_ID,
+    });
+
+    const response = await client.send(command);
+    console.log("Bedrock KB Sync started:", response.ingestionJob?.ingestionJobId);
+
+    return { statusCode: 200, body: "Sync initiated" };
+  } catch (error) {
+    console.error("Error starting Bedrock KB sync:", error);
+    throw error;
+  }
+};

infra/lib/config/environmental_config.ts

@@ -84,7 +84,7 @@ const environmentConfigs: { [key: string]: EnvironmentConfig } = {
     },
     bedrockKb: {
       embeddingModelArn:
-        "arn:aws:bedrock:ap-northeast-1::foundation-model/amazon.titan-embed-text-v1",
+        "arn:aws:bedrock:ap-northeast-1::foundation-model/amazon.titan-embed-text-v2",
       aurora: {
         instanceType: "t3.medium",
         version: "16.4",

infra/lib/lambda/code-chunking/index.ts

@@ -132,6 +132,7 @@ function detectLanguage(filePath: string): SupportedLanguage | "text" | null {
     case "md":
     case "txt":
     case "json":
+    case "jsonl":
     case "yaml":
     case "yml":
     case "xml":
@@ -219,6 +220,56 @@ function chunkTextByParagraph(
   return chunks;
 }
 
+function chunkImageIndexJsonl(
+  content: string,
+  filePath: string,
+): Array<{ text: string; metadata: Record<string, string> }> {
+  const chunks: Array<{ text: string; metadata: Record<string, string> }> = [];
+  const lines = content
+    .split("\n")
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0);
+
+  for (const [index, line] of lines.entries()) {
+    try {
+      const parsed = JSON.parse(line) as {
+        imageUrl?: string;
+        sourceKey?: string;
+        mimeType?: string;
+        caption?: string;
+        tags?: string[];
+      };
+
+      if (!parsed.imageUrl || !parsed.caption) {
+        continue;
+      }
+
+      const tags = Array.isArray(parsed.tags)
+        ? parsed.tags.filter((tag) => typeof tag === "string")
+        : [];
+      const text = `画像説明: ${parsed.caption}\nタグ: ${tags.join(", ")}\nimageUrl: ${parsed.imageUrl}`;
+
+      chunks.push({
+        text,
+        metadata: {
+          type: "image",
+          filePath,
+          imageUrl: parsed.imageUrl,
+          caption: parsed.caption,
+          tags: tags.join(", "),
+          sourceKey: parsed.sourceKey ?? "",
+          mimeType: parsed.mimeType ?? "",
+          lineNumber: (index + 1).toString(),
+        },
+      });
+    } catch (error) {
+      console.log(`Failed to parse JSONL line in ${filePath}:`, error);
+    }
+  }
+
+  return chunks;
+}
+
 export const handler = async (
   event: BedrockCustomTransformationEvent,
 ): Promise<BedrockCustomTransformationOutput> => {
@@ -258,7 +309,11 @@ export const handler = async (
 
     let chunks: Array<{ text: string; metadata: Record<string, string> }> = [];
 
-    if (!language) {
+    if (filePath.startsWith("documents/image-index/") && filePath.endsWith(".jsonl")) {
+      console.log(`Chunking ${filePath} as image-index jsonl`);
+      chunks = chunkImageIndexJsonl(sourceCode, filePath);
+      console.log(`Generated ${chunks.length} image metadata chunks`);
+    } else if (!language) {
       console.log(`Unsupported file type for ${filePath}, skipping`);
       // 空のチャンクで処理を続行
       chunks = [];