Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 150 additions & 0 deletions cmd/kb/cmd_enrich.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
package main

import (
"context"
"fmt"
"os"
"os/signal"
"strings"
"syscall"

"github.com/spf13/cobra"

"github.com/knowledge-broker/knowledge-broker/internal/config"
"github.com/knowledge-broker/knowledge-broker/internal/enrich"
"github.com/knowledge-broker/knowledge-broker/pkg/model"
)

func enrichCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "enrich",
Short: "Run LLM enrichment on already-ingested fragments, then re-embed",
Long: `Enrich runs LLM-based chunk enrichment on fragments that have already been
ingested. After enrichment, fragments are re-embedded with the enriched content.

Requires at least one --source flag in the format type/name (e.g. git/owner/repo,
filesystem/path/to/dir). Multiple --source flags are supported.

By default, fragments already enriched with the same model and prompt version
are skipped. Use --force to re-enrich all fragments.`,
RunE: func(cmd *cobra.Command, args []string) error {
cfg := loadConfig(cmd).Config
debugMode := isDebug(cmd)
logger := newLogger(debugMode)
client := httpClient(logger, debugMode)

sources, _ := cmd.Flags().GetStringArray("source")
force, _ := cmd.Flags().GetBool("force")
enrichModel, _ := cmd.Flags().GetString("enrich-model")
promptVersion, _ := cmd.Flags().GetString("prompt-version")

if len(sources) == 0 {
return fmt.Errorf("at least one --source flag is required (format: type/name, e.g. git/owner/repo)")
}

// Parse source specs into type/name pairs.
type sourceSpec struct {
sourceType string
sourceName string
}
var specs []sourceSpec
for _, s := range sources {
idx := strings.Index(s, "/")
if idx < 1 {
return fmt.Errorf("invalid --source format %q: expected type/name (e.g. git/owner/repo)", s)
}
specs = append(specs, sourceSpec{
sourceType: s[:idx],
sourceName: s[idx+1:],
})
}

s, err := openStore(cfg)
if err != nil {
return fmt.Errorf("open store: %w", err)
}
defer s.Close()

ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()

if err := ensureOllama(ctx, cmd, cfg, true); err != nil {
return err
}

emb := newEmbedder(cfg, client)

if enrichModel == "" {
enrichModel = cfg.EnrichModel
}
enricher := enrich.NewOllamaEnricher(cfg.OllamaURL, enrichModel, promptVersion, client, logger)
modelName := enricher.Model()
// Set the global prompt version for cache keying.
if promptVersion == "" {
promptVersion = enrich.DefaultPromptVersion
}

for _, spec := range specs {
// Verify the source exists.
src, err := s.GetSource(ctx, spec.sourceType, spec.sourceName)
if err != nil {
return fmt.Errorf("get source %s/%s: %w", spec.sourceType, spec.sourceName, err)
}
if src == nil {
return fmt.Errorf("source %s/%s not found", spec.sourceType, spec.sourceName)
}

frags, err := s.GetFragmentsBySource(ctx, spec.sourceName)
if err != nil {
return fmt.Errorf("get fragments for %s/%s: %w", spec.sourceType, spec.sourceName, err)
}

if len(frags) == 0 {
fmt.Fprintf(os.Stderr, "No fragments found for %s/%s, skipping\n", spec.sourceType, spec.sourceName)
continue
}

// Filter out already-enriched fragments unless --force.
if !force {
var toEnrich []model.SourceFragment
skipped := 0
for _, f := range frags {
if f.EnrichedContent != "" &&
f.EnrichmentModel == modelName &&
f.EnrichmentVersion == promptVersion {
skipped++
continue
}
toEnrich = append(toEnrich, f)
}
if skipped > 0 {
fmt.Fprintf(os.Stderr, " Skipping %d already-enriched fragments (model=%s, version=%s)\n",
skipped, modelName, promptVersion)
}
frags = toEnrich
}

if len(frags) == 0 {
fmt.Fprintf(os.Stderr, "All fragments for %s/%s already enriched, skipping (use --force to re-enrich)\n",
spec.sourceType, spec.sourceName)
continue
}

fmt.Fprintf(os.Stderr, "Enriching %d fragments from %s/%s...\n",
len(frags), spec.sourceType, spec.sourceName)

if err := reEnrichFragments(ctx, s, emb, enricher, frags, logger); err != nil {
return fmt.Errorf("enrich %s/%s: %w", spec.sourceType, spec.sourceName, err)
}
}

return nil
},
}
cmd.Flags().StringArray("source", nil, "Source to enrich in type/name format (e.g. git/owner/repo, repeatable)")
cmd.Flags().Bool("force", false, "Re-enrich all fragments, even if already enriched with same model and prompt version")
cmd.Flags().String("enrich-model", "", "Ollama model for chunk enrichment (default: from config)")
cmd.Flags().String("prompt-version", "", "Enrichment prompt version: v1 (full rewrite), v2 (append keywords)")
cmd.Flags().String("db", "", config.DBFlagUsage)
return cmd
}
8 changes: 6 additions & 2 deletions cmd/kb/cmd_ingest.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@ func ingestCmd() *cobra.Command {

watchMode, _ := cmd.Flags().GetBool("watch")
description, _ := cmd.Flags().GetString("description")
skipEnrichment, _ := cmd.Flags().GetBool("skip-enrichment")
skipEnrichmentFlag, _ := cmd.Flags().GetBool("skip-enrichment")
enrichFlag, _ := cmd.Flags().GetBool("enrich")
// --skip-enrichment defaults to true; --enrich overrides it to false.
skipEnrichment := skipEnrichmentFlag && !enrichFlag
enrichModel, _ := cmd.Flags().GetString("enrich-model")
reEnrich, _ := cmd.Flags().GetBool("re-enrich")
promptVersion, _ := cmd.Flags().GetString("prompt-version")
Expand Down Expand Up @@ -494,7 +497,8 @@ func ingestCmd() *cobra.Command {
cmd.Flags().Bool("watch", false, "Watch for file changes and re-ingest automatically (local sources only)")
cmd.Flags().Bool("parallel", false, "Ingest multiple sources in parallel (default: sequential)")
cmd.Flags().String("description", "", "Human-readable description of this source (shown to agents via MCP)")
cmd.Flags().Bool("skip-enrichment", false, "Skip LLM chunk enrichment (faster ingestion)")
cmd.Flags().Bool("skip-enrichment", true, "Skip LLM chunk enrichment (default: true)")
cmd.Flags().Bool("enrich", false, "Enable LLM chunk enrichment during ingestion")
cmd.Flags().String("enrich-model", "", "Ollama model for chunk enrichment (default: qwen2.5:0.5b)")
cmd.Flags().Bool("re-enrich", false, "Re-run enrichment on already-ingested chunks, then re-embed")
cmd.Flags().String("prompt-version", "", "Enrichment prompt version: v1 (full rewrite), v2 (append keywords)")
Expand Down
1 change: 1 addition & 0 deletions cmd/kb/cmd_serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ func serveCmd() *cobra.Command {
OllamaURL: cfg.OllamaURL,
EnrichModel: cfg.EnrichModel,
WorkerCount: cfg.WorkerCount,
SkipEnrichment: true,
MaxChunkSize: cfg.MaxChunkSize,
ChunkOverlap: cfg.ChunkOverlap,
}
Expand Down
1 change: 1 addition & 0 deletions cmd/kb/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ func main() {

root.AddCommand(versionCmd())
root.AddCommand(ingestCmd())
root.AddCommand(enrichCmd())
root.AddCommand(queryCmd())
root.AddCommand(serveCmd())
root.AddCommand(exportCmd())
Expand Down
Loading