Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion internal/store/sqlite.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,27 @@ func NewSQLiteStore(dbPath string, embeddingDim int) (*SQLiteStore, error) {
return &SQLiteStore{db: db, embeddingDim: embeddingDim}, nil
}

// schemaReady checks whether the database schema is already initialized and
// the stored embedding dimension matches. When true, we can skip all write
// operations in initSchema, avoiding write-lock contention with concurrent
// writers (e.g. an in-progress ingestion).
func schemaReady(db *sql.DB, embeddingDim int) bool {
var val string
err := db.QueryRow("SELECT value FROM metadata WHERE key = 'embedding_dim'").Scan(&val)
return err == nil && val == fmt.Sprintf("%d", embeddingDim)
}

// initSchema runs migrations, creates virtual tables, and validates metadata.
// All operations are idempotent so safe to retry.
// All operations are idempotent so safe to retry. If the schema is already
// fully initialized, only a read is performed (no write lock needed).
func initSchema(db *sql.DB, embeddingDim int) error {
// Fast path: if schema already exists with the correct embedding dim,
// skip all writes. This lets read-only callers (e.g. "kb sources list")
// open the store without acquiring a write lock.
if schemaReady(db, embeddingDim) {
return nil
}

// Run schema migrations.
if _, err := db.Exec(migrationSQL); err != nil {
return fmt.Errorf("run migrations: %w", err)
Expand Down
34 changes: 34 additions & 0 deletions internal/store/sqlite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,40 @@ func TestMetadataValidation(t *testing.T) {
}
}

// TestConcurrentOpenDuringWrite verifies that opening a second store handle
// (e.g. for "kb sources list") succeeds even when another connection holds a
// long-running write transaction (e.g. ingestion). This is the scenario
// reported in GitHub issue #12.
func TestConcurrentOpenDuringWrite(t *testing.T) {
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")

// First open initializes the schema.
s1, err := NewSQLiteStore(dbPath, 4)
if err != nil {
t.Fatalf("first open: %v", err)
}
defer s1.Close()

// Start a long-running write transaction to simulate ingestion.
tx, err := s1.db.Begin()
if err != nil {
t.Fatalf("begin tx: %v", err)
}
defer tx.Rollback()
if _, err := tx.Exec("INSERT INTO fragments (id, raw_content, source_type, source_path, source_uri, content_date, file_type, checksum) VALUES ('hold', 'data', 'test', '/x', 'file:///x', '2024-01-01T00:00:00Z', 'txt', 'aaa')"); err != nil {
t.Fatalf("insert in tx: %v", err)
}

// Second open should succeed without blocking on the write lock,
// because schemaReady sees the metadata via a read and skips writes.
s2, err := NewSQLiteStore(dbPath, 4)
if err != nil {
t.Fatalf("second open while write tx held: %v", err)
}
s2.Close()
}

func TestUpsertAndGetFragments(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
Expand Down
Loading