diff --git a/README.md b/README.md index 0d02b27..5167e17 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,12 @@ The module's options (`database`, `timeout`, `busy_timeout`, `foreign_keys`, environment variables (`SQLITE_*`) or per-option `get_` / `set_` accessor builtins, and serve as defaults for `connect` / `connect_remote`. Two opt-in host levers (`max_rows` and `NewModuleWithFileAccess`) bound what an -untrusted script can reach; both default to off. See the +untrusted script can reach; both default to off. `timeout` bounds each database +operation (a per-query deadline, and a cancellation point tied to the script +thread) — this is what stops an unreachable remote from hanging the host. The +connection PRAGMAs (`foreign_keys`, `journal_mode`, …) are applied to every +pooled connection, and an in-memory database is served from a single connection +so its schema and data persist across queries. See the [Configuration section of docs/API.md](docs/API.md#configuration) for the full option table, defaults, accessors, and host-hardening details. diff --git a/database.go b/database.go index d399ed8..70bdd1e 100644 --- a/database.go +++ b/database.go @@ -1,10 +1,13 @@ package sqlite import ( + "context" "database/sql" "fmt" + "time" "github.com/1set/starlet/dataconv" + "github.com/starpkg/base/util" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" ) @@ -19,14 +22,23 @@ type database struct { db *sql.DB maxRows int restrictFileAccess bool + opTimeout time.Duration // per-operation deadline (0 = none) +} + +// opContext derives the context bounding a single operation from the calling +// script thread (so a cancelled thread aborts the query) plus the configured +// per-operation timeout. The caller must invoke the returned cancel func. +func (db *database) opContext(thread *starlark.Thread) (context.Context, context.CancelFunc) { + return util.OpContext(thread, db.opTimeout) } // newDatabaseInstance creates a new Starlark database instance. -func newDatabaseInstance(db *sql.DB, maxRows int, restrictFileAccess bool) *starlarkstruct.Module { +func newDatabaseInstance(db *sql.DB, maxRows int, restrictFileAccess bool, opTimeout time.Duration) *starlarkstruct.Module { dbi := &database{ db: db, maxRows: maxRows, restrictFileAccess: restrictFileAccess, + opTimeout: opTimeout, } // Create dictionary of methods @@ -108,7 +120,9 @@ func (db *database) execute(thread *starlark.Thread, fn *starlark.Builtin, args } // Execute the query - result, err := db.db.Exec(sqlQuery.query, sqlQuery.params...) + ctx, cancel := db.opContext(thread) + defer cancel() + result, err := db.db.ExecContext(ctx, sqlQuery.query, sqlQuery.params...) if err != nil { return nil, fmt.Errorf("failed to execute query: %w", err) } @@ -132,7 +146,9 @@ func (db *database) batch(thread *starlark.Thread, fn *starlark.Builtin, args st } // Begin transaction - tx, err := db.db.Begin() + ctx, cancel := db.opContext(thread) + defer cancel() + tx, err := db.db.BeginTx(ctx, nil) if err != nil { return nil, fmt.Errorf("failed to begin transaction: %w", err) } @@ -205,7 +221,7 @@ func (db *database) batch(thread *starlark.Thread, fn *starlark.Builtin, args st } // Execute the query within transaction - result, err := tx.Exec(sqlQuery.query, sqlQuery.params...) + result, err := tx.ExecContext(ctx, sqlQuery.query, sqlQuery.params...) if err != nil { tx.Rollback() return nil, fmt.Errorf("query %d: failed to execute: %w", i, err) @@ -255,7 +271,9 @@ func (db *database) query(thread *starlark.Thread, fn *starlark.Builtin, args st } // Execute the query - rows, err := db.db.Query(sqlQuery.query, sqlQuery.params...) + ctx, cancel := db.opContext(thread) + defer cancel() + rows, err := db.db.QueryContext(ctx, sqlQuery.query, sqlQuery.params...) if err != nil { return nil, fmt.Errorf("failed to execute query: %w", err) } @@ -282,7 +300,9 @@ func (db *database) queryOne(thread *starlark.Thread, fn *starlark.Builtin, args } // Execute the query - rows, err := db.db.Query(sqlQuery.query, sqlQuery.params...) + ctx, cancel := db.opContext(thread) + defer cancel() + rows, err := db.db.QueryContext(ctx, sqlQuery.query, sqlQuery.params...) if err != nil { return nil, fmt.Errorf("failed to execute query: %w", err) } @@ -297,14 +317,31 @@ func (db *database) begin(thread *starlark.Thread, fn *starlark.Builtin, args st return nil, err } - // Begin transaction - tx, err := db.db.Begin() + // Acquire the connection under a DEADLINE (the per-operation timeout), so that + // acquiring it can't block the host indefinitely when the pool is busy — e.g. + // a single-connection in-memory database whose sole connection is held by + // another open transaction. + acquireCtx, acquireCancel := db.opContext(thread) + defer acquireCancel() + conn, err := db.db.Conn(acquireCtx) + if err != nil { + return nil, fmt.Errorf("failed to begin transaction: %w", err) + } + // The transaction spans multiple later calls (execute → commit), so its + // context must live until commit/rollback and must NOT carry the per-operation + // deadline (which would abort a legitimately long transaction). Cancellation- + // only still propagates a cancelled script thread; the cancel func and the + // dedicated connection are handed to the transaction and released on finish. + lifeCtx, lifeCancel := util.OpContext(thread, 0) + tx, err := conn.BeginTx(lifeCtx, nil) if err != nil { + lifeCancel() + conn.Close() return nil, fmt.Errorf("failed to begin transaction: %w", err) } // Create transaction object - return newTransactionInstance(tx, db.maxRows), nil + return newTransactionInstance(tx, db.maxRows, db.opTimeout, conn, lifeCancel), nil } // attach attaches another database with an alias. @@ -324,7 +361,9 @@ func (db *database) attach(thread *starlark.Thread, fn *starlark.Builtin, args s // Execute ATTACH DATABASE statement query := fmt.Sprintf("ATTACH DATABASE ? AS %s", quoteName(alias)) - _, err := db.db.Exec(query, database) + ctx, cancel := db.opContext(thread) + defer cancel() + _, err := db.db.ExecContext(ctx, query, database) if err != nil { return nil, fmt.Errorf("failed to attach database: %w", err) } @@ -343,7 +382,9 @@ func (db *database) detach(thread *starlark.Thread, fn *starlark.Builtin, args s // Execute DETACH DATABASE statement query := fmt.Sprintf("DETACH DATABASE %s", quoteName(alias)) - _, err := db.db.Exec(query) + ctx, cancel := db.opContext(thread) + defer cancel() + _, err := db.db.ExecContext(ctx, query) if err != nil { return nil, fmt.Errorf("failed to detach database: %w", err) } @@ -358,7 +399,9 @@ func (db *database) tables(thread *starlark.Thread, fn *starlark.Builtin, args s } // Query tables - rows, err := db.db.Query("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") + ctx, cancel := db.opContext(thread) + defer cancel() + rows, err := db.db.QueryContext(ctx, "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") if err != nil { return nil, fmt.Errorf("failed to query tables: %w", err) } @@ -394,7 +437,9 @@ func (db *database) tableInfo(thread *starlark.Thread, fn *starlark.Builtin, arg } // Query table info - rows, err := db.db.Query(fmt.Sprintf("PRAGMA table_info(%s)", quoteName(table))) + ctx, cancel := db.opContext(thread) + defer cancel() + rows, err := db.db.QueryContext(ctx, fmt.Sprintf("PRAGMA table_info(%s)", quoteName(table))) if err != nil { return nil, fmt.Errorf("failed to query table info: %w", err) } @@ -459,7 +504,9 @@ func (db *database) indices(thread *starlark.Thread, fn *starlark.Builtin, args } // Query indices - rows, err := db.db.Query("SELECT name, sql FROM sqlite_master WHERE type='index' AND tbl_name=?", table) + ctx, cancel := db.opContext(thread) + defer cancel() + rows, err := db.db.QueryContext(ctx, "SELECT name, sql FROM sqlite_master WHERE type='index' AND tbl_name=?", table) if err != nil { return nil, fmt.Errorf("failed to query indices: %w", err) } @@ -504,7 +551,9 @@ func (db *database) tableExists(thread *starlark.Thread, fn *starlark.Builtin, a // Query to check if table exists var count int - err := db.db.QueryRow("SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&count) + ctx, cancel := db.opContext(thread) + defer cancel() + err := db.db.QueryRowContext(ctx, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&count) if err != nil { return nil, fmt.Errorf("failed to check if table exists: %w", err) } @@ -522,13 +571,15 @@ func (db *database) prepare(thread *starlark.Thread, fn *starlark.Builtin, args } // Prepare statement - stmt, err := db.db.Prepare(query) + ctx, cancel := db.opContext(thread) + defer cancel() + stmt, err := db.db.PrepareContext(ctx, query) if err != nil { return nil, fmt.Errorf("failed to prepare statement: %w", err) } // Create prepared statement object - return newPreparedStatementInstance(stmt, db.maxRows), nil + return newPreparedStatementInstance(stmt, db.maxRows, db.opTimeout), nil } // prepareQuery prepares a SQL query statement. @@ -541,11 +592,13 @@ func (db *database) prepareQuery(thread *starlark.Thread, fn *starlark.Builtin, } // Prepare statement - stmt, err := db.db.Prepare(query) + ctx, cancel := db.opContext(thread) + defer cancel() + stmt, err := db.db.PrepareContext(ctx, query) if err != nil { return nil, fmt.Errorf("failed to prepare query: %w", err) } // Create prepared query object - return newPreparedQueryInstance(stmt, db.maxRows), nil + return newPreparedQueryInstance(stmt, db.maxRows, db.opTimeout), nil } diff --git a/docs/API.md b/docs/API.md index 3a01873..34c40aa 100644 --- a/docs/API.md +++ b/docs/API.md @@ -35,7 +35,7 @@ parameter omitted falls back to the corresponding module config option. **Parameters:** - `database` (string): Database path or `:memory:` for in-memory (default: module config) -- `timeout` (float): Connection timeout in seconds (default: module config) +- `timeout` (float): Per-operation deadline in seconds — each query/statement is bounded by it and cancels with the script thread. Omit or pass 0 to use the module default; set the module `timeout` config (or `SQLITE_TIMEOUT`) to 0 to disable the deadline. - `busy_timeout` (float): Busy timeout in seconds (default: module config) - `foreign_keys` (bool): Enable foreign key constraints (default: module config) - `journal_mode` (string): Journal mode (default: module config) @@ -998,7 +998,7 @@ accessor — never a getter — but this module has none.) | Option | Getter | Setter | Type | Env var | Default | Description | |--------|--------|--------|------|---------|---------|-------------| | `database` | `get_database` | `set_database` | string | `SQLITE_DATABASE` | `:memory:` | Path to the SQLite database (use `:memory:` for in-memory) | -| `timeout` | `get_timeout` | `set_timeout` | float | `SQLITE_TIMEOUT` | `30.0` | Connection timeout in seconds | +| `timeout` | `get_timeout` | `set_timeout` | float | `SQLITE_TIMEOUT` | `30.0` | Per-operation deadline in seconds (per-query; cancels with the script thread; 0 = none) | | `busy_timeout` | `get_busy_timeout` | `set_busy_timeout` | float | `SQLITE_BUSY_TIMEOUT` | `5.0` | Busy timeout in seconds | | `foreign_keys` | `get_foreign_keys` | `set_foreign_keys` | bool | `SQLITE_FOREIGN_KEYS` | `true` | Enable foreign key constraints | | `journal_mode` | `get_journal_mode` | `set_journal_mode` | string | `SQLITE_JOURNAL_MODE` | `DELETE` | Journal mode (WAL, DELETE, TRUNCATE, PERSIST, MEMORY, OFF) | diff --git a/example_test.go b/example_test.go index 70640ca..7cc98fc 100644 --- a/example_test.go +++ b/example_test.go @@ -1,12 +1,15 @@ package sqlite import ( + "context" + "database/sql" "fmt" "os" "path/filepath" "strings" "sync/atomic" "testing" + "time" "github.com/1set/starlet" "github.com/starpkg/base" @@ -2477,3 +2480,156 @@ main() return NewModule().LoadModule() }) } + +// --- STAR-75 / STAR-76: connection-pool semantics + cancellable operations --- + +// TestMemoryDBSingleConnection verifies an in-memory database keeps its schema +// and data: because the pool is pinned to one connection, a later query never +// lands on a second (empty) physical connection ("no such table" / lost rows). +func TestMemoryDBSingleConnection(t *testing.T) { + db, err := openDatabase(":memory:", 5, true, "MEMORY", "NORMAL", -2000) + if err != nil { + t.Fatalf("openDatabase: %v", err) + } + defer db.Close() + if got := db.Stats().MaxOpenConnections; got != 1 { + t.Errorf("in-memory pool MaxOpenConnections = %d, want 1 (pinned)", got) + } + if _, err := db.Exec("CREATE TABLE t (id INTEGER)"); err != nil { + t.Fatalf("create: %v", err) + } + if _, err := db.Exec("INSERT INTO t VALUES (1)"); err != nil { + t.Fatalf("insert: %v", err) + } + for i := 0; i < 25; i++ { + var n int + if err := db.QueryRow("SELECT count(*) FROM t").Scan(&n); err != nil { + t.Fatalf("query %d: %v (memory data lost across connections)", i, err) + } + if n != 1 { + t.Fatalf("query %d: count=%d, want 1", i, n) + } + } +} + +// TestForeignKeysAcrossConnections verifies foreign_keys is enforced on EVERY +// pooled connection (set via the DSN, not once with db.Exec) — otherwise a later +// connection falls back to SQLite's default OFF and constraint violations +// silently succeed. +func TestForeignKeysAcrossConnections(t *testing.T) { + path := filepath.Join(t.TempDir(), "fk.db") + db, err := openDatabase(path, 5, true, "WAL", "NORMAL", -2000) + if err != nil { + t.Fatalf("openDatabase: %v", err) + } + defer db.Close() + db.SetMaxOpenConns(4) + // Deterministically open (and hold) several DISTINCT physical connections, so + // this actually exercises a fresh connection rather than reusing one idle + // connection — then confirm foreign_keys is ON on every one of them. + const n = 4 + conns := make([]*sql.Conn, 0, n) + for i := 0; i < n; i++ { + c, err := db.Conn(context.Background()) + if err != nil { + t.Fatalf("acquire conn %d: %v", i, err) + } + defer c.Close() + conns = append(conns, c) + } + for i, c := range conns { + var v int + if err := c.QueryRowContext(context.Background(), "PRAGMA foreign_keys").Scan(&v); err != nil { + t.Fatalf("conn %d pragma read: %v", i, err) + } + if v != 1 { + t.Errorf("foreign_keys=%d on pooled connection %d, want 1", v, i) + } + } +} + +// TestQueryRespectsThreadCancellation verifies a query is bound to the script +// thread's context, so a cancelled thread aborts it (the fix for uncancellable +// remote/slow queries that could hang the host). +func TestQueryRespectsThreadCancellation(t *testing.T) { + db, err := openDatabase(":memory:", 5, true, "MEMORY", "NORMAL", -2000) + if err != nil { + t.Fatalf("openDatabase: %v", err) + } + defer db.Close() + if _, err := db.Exec("CREATE TABLE t (id INTEGER)"); err != nil { + t.Fatalf("create: %v", err) + } + mod := newDatabaseInstance(db, 0, false, 0) + qVal, err := mod.Attr("query") + if err != nil || qVal == nil { + t.Fatalf("query attr: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() // already cancelled + thread := &starlark.Thread{} + thread.SetLocal("context", ctx) + _, callErr := starlark.Call(thread, qVal.(*starlark.Builtin), starlark.Tuple{starlark.String("SELECT * FROM t")}, nil) + if callErr == nil || !strings.Contains(callErr.Error(), "context canceled") { + t.Errorf("query on a cancelled thread: err = %v, want it to mention context canceled", callErr) + } +} + +// TestInMemoryDSNClassification locks the file-access-gate classifier: only true +// in-memory / private-temp forms are in-memory; a bare path (even one containing +// "?mode=memory" or ":memory:") is a real disk file and must NOT be misclassified +// (that would let a restricted script reach disk). +func TestInMemoryDSNClassification(t *testing.T) { + inMemory := []string{"", ":memory:", "file:", "file:?cache=shared", "file::memory:", "file::memory:?cache=shared"} + // Only the path decides disk-vs-memory; a mode=memory query never reclassifies + // a disk path (Go/SQLite query parsing disagree — a gate bypass), and a + // non-URI ":memory:?…" is a disk filename to SQLite. + onDisk := []string{ + "test.db", "/etc/passwd", "file:/etc/passwd", "file:/etc/passwd?x=:memory:", + "file::memory:evil", "foo.db?mode=memory", ":memory:?cache=shared", + "file:named?mode=memory", "file:/etc/passwd?mode=memory&mode=rw%00%ZZ", + "file:/etc/passwd?mode=memory&mode%00junk=rw", + } + for _, s := range inMemory { + if !isInMemoryDSN(s) { + t.Errorf("isInMemoryDSN(%q) = false, want true", s) + } + } + for _, s := range onDisk { + if isInMemoryDSN(s) { + t.Errorf("isInMemoryDSN(%q) = true, want false (a disk file must not pass the in-memory gate)", s) + } + } +} + +// TestNestedBeginHonorsTimeout verifies that acquiring a connection for a second +// transaction on a single-connection in-memory database is bounded by the +// per-operation timeout — it errors instead of blocking the host indefinitely +// when the sole connection is held by an open transaction. +func TestNestedBeginHonorsTimeout(t *testing.T) { + db, err := openDatabase(":memory:", 5, true, "MEMORY", "NORMAL", -2000) + if err != nil { + t.Fatalf("openDatabase: %v", err) + } + defer db.Close() + mod := newDatabaseInstance(db, 0, false, 200*time.Millisecond) // short op deadline + beginV, err := mod.Attr("begin") + if err != nil { + t.Fatalf("begin attr: %v", err) + } + begin := beginV.(*starlark.Builtin) + th := &starlark.Thread{} // Background context, like a default starlet Run + if _, err := starlark.Call(th, begin, nil, nil); err != nil { + t.Fatalf("first begin: %v", err) // holds the sole connection + } + done := make(chan error, 1) + go func() { _, e := starlark.Call(th, begin, nil, nil); done <- e }() + select { + case e := <-done: + if e == nil { + t.Error("second begin unexpectedly succeeded on a single-connection DB") + } + case <-time.After(5 * time.Second): + t.Error("second begin blocked >5s — the per-operation timeout was not honored") + } +} diff --git a/go.mod b/go.mod index 4c38632..acc80a6 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,15 @@ module github.com/starpkg/sqlite go 1.20 require ( - github.com/1set/starlet v0.2.2 - github.com/starpkg/base v0.1.1 + github.com/1set/starlet v0.2.3 + github.com/starpkg/base v0.1.2 github.com/tursodatabase/libsql-client-go v0.0.0-20260528064733-9d5d30a29a60 go.starlark.net v0.0.0-20260324133313-ffb3f39dd27a modernc.org/sqlite v1.31.1 ) require ( - github.com/1set/starlight v0.2.0 // indirect + github.com/1set/starlight v0.2.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/coder/websocket v1.8.12 // indirect diff --git a/go.sum b/go.sum index ad808db..7c4906c 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ -github.com/1set/starlet v0.2.2 h1:TWEbuY5O3291GvRwxHLMiJCSGWhI1ZMdXe+nCtBejW0= -github.com/1set/starlet v0.2.2/go.mod h1:3Sz9ToVkumS1OMojGmb37iLqd69UVTr1PWFOXaiC+rA= -github.com/1set/starlight v0.2.0 h1:W9yulJYANolyMLMOH0M4xcW8RQVpi68opSdNmoipcic= -github.com/1set/starlight v0.2.0/go.mod h1:o9KiJBpy92daHyNHBUwS0nFIjjLxLM/XmRxAZf4FIaE= +github.com/1set/starlet v0.2.3 h1:9YuS1Pa7ND6wetV2GRgY8RG7yLn30OqUY+4g4bk2/2Q= +github.com/1set/starlet v0.2.3/go.mod h1:txkHyqtkqb1j9zBM0lOTwf6oW9vwtnsoLyUfqe9UeQo= +github.com/1set/starlight v0.2.1 h1:07XiRr4XJ9p8rSL/A4XDC7sDInGszpXIP5jASCIZo10= +github.com/1set/starlight v0.2.1/go.mod h1:o9KiJBpy92daHyNHBUwS0nFIjjLxLM/XmRxAZf4FIaE= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -37,8 +37,8 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6Ng github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/spyzhov/ajson v0.9.6 h1:iJRDaLa+GjhCDAt1yFtU/LKMtLtsNVKkxqlpvrHHlpQ= github.com/spyzhov/ajson v0.9.6/go.mod h1:a6oSw0MMb7Z5aD2tPoPO+jq11ETKgXUr2XktHdT8Wt8= -github.com/starpkg/base v0.1.1 h1:AeNOEIyuyQJoQjgU+AxAE3+1ebgLlsF1HmA49tdTOtU= -github.com/starpkg/base v0.1.1/go.mod h1:0QLSawzBUCFLNk1jMRpzNs6AZH9SVpumd2J7WLOPrR0= +github.com/starpkg/base v0.1.2 h1:LUPfBAj5JMGsau6p3uXqsEDRm7Eh5gBtsWno36QyMfw= +github.com/starpkg/base v0.1.2/go.mod h1:A8xGHH1ijQ6raz8w7MhC+4zaJwVz7XzlJjyjky2Q2dA= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tursodatabase/libsql-client-go v0.0.0-20260528064733-9d5d30a29a60 h1:TfQEwhr0Q9t+Bgs0TNk2eHZ9EGD107Mimic0kcoGS1M= diff --git a/operations.go b/operations.go index 27af24a..50d5b69 100644 --- a/operations.go +++ b/operations.go @@ -1,6 +1,7 @@ package sqlite import ( + "context" "database/sql" "errors" "fmt" @@ -60,8 +61,25 @@ func isTableAlreadyExistsError(err error) bool { return false } +// execAffecting runs an INSERT/UPDATE/DELETE-style statement under the thread's +// operation context and returns the number of affected rows as a Starlark int. +// failMsg labels an execution failure. +func (db *database) execAffecting(thread *starlark.Thread, failMsg, query string, params ...interface{}) (starlark.Value, error) { + ctx, cancel := db.opContext(thread) + defer cancel() + result, err := db.db.ExecContext(ctx, query, params...) + if err != nil { + return nil, fmt.Errorf("%s: %w", failMsg, err) + } + rowsAffected, err := result.RowsAffected() + if err != nil { + return nil, fmt.Errorf("failed to get rows affected: %w", err) + } + return starlark.MakeInt64(rowsAffected), nil +} + // createTable creates a new table with the specified columns, optional constraints, and indexes. -func (db *database) createTable(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func (db *database) createTable(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var table string var columns *starlark.Dict var constraintsVal starlark.Value @@ -77,24 +95,37 @@ func (db *database) createTable(_ *starlark.Thread, fn *starlark.Builtin, args s return nil, err } + // Build CREATE TABLE statement + query, err := buildCreateTableStatement(table, columns, constraintsVal) + if err != nil { + return nil, err + } + // Begin transaction for atomicity - tx, err := db.db.Begin() + ctx, cancel := db.opContext(thread) + defer cancel() + tx, err := db.db.BeginTx(ctx, nil) if err != nil { return nil, fmt.Errorf("failed to begin transaction: %w", err) } - // Build CREATE TABLE statement + // Execute the CREATE TABLE statement and its indexes within the transaction + return execCreateTable(ctx, tx, query, table, indexesVal, existOk) +} + +// buildCreateTableStatement assembles the CREATE TABLE SQL from the column +// definitions and optional table-level constraints. +func buildCreateTableStatement(table string, columns *starlark.Dict, constraintsVal starlark.Value) (string, error) { + // Build column definitions columnDefs, err := buildColumnDefinitions(columns) if err != nil { - tx.Rollback() - return nil, err + return "", err } // Process table-level constraints tableConstraints, err := processTableConstraints(constraintsVal) if err != nil { - tx.Rollback() - return nil, err + return "", err } // Combine column definitions and table constraints @@ -102,12 +133,15 @@ func (db *database) createTable(_ *starlark.Thread, fn *starlark.Builtin, args s allDefinitions = append(allDefinitions, columnDefs...) allDefinitions = append(allDefinitions, tableConstraints...) - // Create SQL statement - query := fmt.Sprintf("CREATE TABLE %s (%s)", quoteName(table), strings.Join(allDefinitions, ", ")) + return fmt.Sprintf("CREATE TABLE %s (%s)", quoteName(table), strings.Join(allDefinitions, ", ")), nil +} +// execCreateTable runs the CREATE TABLE statement and its indexes inside tx, +// committing on success and rolling back on any failure. When existOk is set, +// a "table already exists" error is treated as a no-op success. +func execCreateTable(ctx context.Context, tx *sql.Tx, query, table string, indexesVal starlark.Value, existOk bool) (starlark.Value, error) { // Execute the CREATE TABLE statement - _, err = tx.Exec(query) - if err != nil { + if _, err := tx.ExecContext(ctx, query); err != nil { tx.Rollback() // Handle exist_ok case: ignore "table already exists" errors if existOk && isTableAlreadyExistsError(err) { @@ -117,8 +151,7 @@ func (db *database) createTable(_ *starlark.Thread, fn *starlark.Builtin, args s } // Process and create indexes - err = createTableIndexes(tx, table, indexesVal) - if err != nil { + if err := createTableIndexes(ctx, tx, table, indexesVal); err != nil { tx.Rollback() return nil, err } @@ -132,7 +165,7 @@ func (db *database) createTable(_ *starlark.Thread, fn *starlark.Builtin, args s } // dropTable drops a table. -func (db *database) dropTable(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func (db *database) dropTable(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var table string if err := starlark.UnpackArgs(fn.Name(), args, kwargs, @@ -144,7 +177,9 @@ func (db *database) dropTable(_ *starlark.Thread, fn *starlark.Builtin, args sta query := fmt.Sprintf("DROP TABLE %s", quoteName(table)) // Execute the statement - _, err := db.db.Exec(query) + ctx, cancel := db.opContext(thread) + defer cancel() + _, err := db.db.ExecContext(ctx, query) if err != nil { return nil, fmt.Errorf("failed to drop table: %w", err) } @@ -153,7 +188,7 @@ func (db *database) dropTable(_ *starlark.Thread, fn *starlark.Builtin, args sta } // truncateTable removes all rows from a table. -func (db *database) truncateTable(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func (db *database) truncateTable(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var table string if err := starlark.UnpackArgs(fn.Name(), args, kwargs, @@ -165,22 +200,11 @@ func (db *database) truncateTable(_ *starlark.Thread, fn *starlark.Builtin, args query := fmt.Sprintf("DELETE FROM %s", quoteName(table)) // Execute the statement - result, err := db.db.Exec(query) - if err != nil { - return nil, fmt.Errorf("failed to truncate table: %w", err) - } - - // Get affected rows - rowsAffected, err := result.RowsAffected() - if err != nil { - return nil, fmt.Errorf("failed to get rows affected: %w", err) - } - - return starlark.MakeInt64(rowsAffected), nil + return db.execAffecting(thread, "failed to truncate table", query) } // insert inserts a record into a table. -func (db *database) insert(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func (db *database) insert(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var table string var values *starlark.Dict @@ -220,7 +244,9 @@ func (db *database) insert(_ *starlark.Thread, fn *starlark.Builtin, args starla strings.Join(placeholders, ", ")) // Execute the statement - result, err := db.db.Exec(query, params...) + ctx, cancel := db.opContext(thread) + defer cancel() + result, err := db.db.ExecContext(ctx, query, params...) if err != nil { return nil, fmt.Errorf("failed to insert record: %w", err) } @@ -240,7 +266,7 @@ func (db *database) insert(_ *starlark.Thread, fn *starlark.Builtin, args starla } // insertMany inserts multiple records into a table. -func (db *database) insertMany(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func (db *database) insertMany(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var table string var valuesList *starlark.List @@ -255,86 +281,102 @@ func (db *database) insertMany(_ *starlark.Thread, fn *starlark.Builtin, args st return starlark.MakeInt(0), nil } - // Get first row to determine columns - firstRowVal := valuesList.Index(0) + // Extract column names from the first row + originalColumnNames, quotedColumnNames, err := extractInsertColumns(valuesList.Index(0)) + if err != nil { + return nil, err + } + + // Begin transaction + ctx, cancel := db.opContext(thread) + defer cancel() + tx, err := db.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("failed to begin transaction: %w", err) + } + + // Build and prepare the INSERT statement + query := buildInsertManyQuery(table, quotedColumnNames) + stmt, err := tx.PrepareContext(ctx, query) + if err != nil { + tx.Rollback() + return nil, fmt.Errorf("failed to prepare statement: %w", err) + } + defer stmt.Close() + + // Insert each row + totalRows, err := execInsertManyRows(ctx, tx, stmt, valuesList, originalColumnNames) + if err != nil { + return nil, err + } + + // Commit transaction + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("failed to commit transaction: %w", err) + } + + return starlark.MakeInt64(totalRows), nil +} +// extractInsertColumns reads the column names from the first row dictionary, +// returning both the original (unquoted) names for value lookup and the quoted +// names for SQL. +func extractInsertColumns(firstRowVal starlark.Value) ([]string, []string, error) { firstRow, ok := firstRowVal.(*starlark.Dict) if !ok { - return nil, fmt.Errorf("values must be a list of dictionaries") + return nil, nil, fmt.Errorf("values must be a list of dictionaries") } - // Extract column names from first row var originalColumnNames []string // Store original (unquoted) column names var quotedColumnNames []string // Store quoted column names for SQL for _, tuple := range firstRow.Items() { colName, ok := tuple.Index(0).(starlark.String) if !ok { - return nil, fmt.Errorf("column name must be a string") + return nil, nil, fmt.Errorf("column name must be a string") } sColName := string(colName) originalColumnNames = append(originalColumnNames, sColName) quotedColumnNames = append(quotedColumnNames, quoteName(sColName)) } - // Begin transaction - tx, err := db.db.Begin() - if err != nil { - return nil, fmt.Errorf("failed to begin transaction: %w", err) - } + return originalColumnNames, quotedColumnNames, nil +} - // Build SQL statement +// buildInsertManyQuery builds the parameterized INSERT statement for insertMany. +func buildInsertManyQuery(table string, quotedColumnNames []string) string { placeholders := strings.Repeat("?, ", len(quotedColumnNames)) // Use count of columns placeholders = placeholders[:len(placeholders)-2] // Remove trailing ", " - query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", + return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", quoteName(table), strings.Join(quotedColumnNames, ", "), // Use quoted names for SQL placeholders) +} - // Prepare statement - stmt, err := tx.Prepare(query) - if err != nil { - tx.Rollback() - return nil, fmt.Errorf("failed to prepare statement: %w", err) - } - defer stmt.Close() - - // Insert each row +// execInsertManyRows executes the prepared statement for every row, rolling back +// tx and returning an error on the first failure, and returns the total number +// of affected rows on success. +func execInsertManyRows(ctx context.Context, tx *sql.Tx, stmt *sql.Stmt, valuesList *starlark.List, originalColumnNames []string) (int64, error) { var totalRows int64 for i := 0; i < valuesList.Len(); i++ { - rowVal := valuesList.Index(i) - - row, ok := rowVal.(*starlark.Dict) + row, ok := valuesList.Index(i).(*starlark.Dict) if !ok { tx.Rollback() - return nil, fmt.Errorf("values must be a list of dictionaries") + return 0, fmt.Errorf("values must be a list of dictionaries") } // Extract values in the same order as originalColumnNames - var params []interface{} - for _, originalCol := range originalColumnNames { // Iterate using original names for lookup - val, found, err := row.Get(starlark.String(originalCol)) // Use original name for Get - if err != nil { - tx.Rollback() - return nil, err - } - if !found { - tx.Rollback() - return nil, fmt.Errorf("column %s missing in row %d", originalCol, i) - } - sqlVal, err := starlarkToSQLiteValue(val) - if err != nil { - tx.Rollback() - return nil, err - } - params = append(params, sqlVal) + params, err := buildInsertRowParams(row, originalColumnNames, i) + if err != nil { + tx.Rollback() + return 0, err } // Execute statement - result, err := stmt.Exec(params...) + result, err := stmt.ExecContext(ctx, params...) if err != nil { tx.Rollback() - return nil, fmt.Errorf("failed to insert row %d: %w", i, err) + return 0, fmt.Errorf("failed to insert row %d: %w", i, err) } // Count affected rows @@ -344,16 +386,33 @@ func (db *database) insertMany(_ *starlark.Thread, fn *starlark.Builtin, args st } } - // Commit transaction - if err := tx.Commit(); err != nil { - return nil, fmt.Errorf("failed to commit transaction: %w", err) + return totalRows, nil +} + +// buildInsertRowParams looks up each column's value in row (in column order) and +// converts it to a SQLite value. rowIndex labels a missing-column error. +func buildInsertRowParams(row *starlark.Dict, originalColumnNames []string, rowIndex int) ([]interface{}, error) { + var params []interface{} + for _, originalCol := range originalColumnNames { // Iterate using original names for lookup + val, found, err := row.Get(starlark.String(originalCol)) // Use original name for Get + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("column %s missing in row %d", originalCol, rowIndex) + } + sqlVal, err := starlarkToSQLiteValue(val) + if err != nil { + return nil, err + } + params = append(params, sqlVal) } - return starlark.MakeInt64(totalRows), nil + return params, nil } // update updates records in a table. -func (db *database) update(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func (db *database) update(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var table string var values *starlark.Dict var whereVal starlark.Value @@ -405,43 +464,17 @@ func (db *database) update(_ *starlark.Thread, fn *starlark.Builtin, args starla } // Execute the statement - result, err := db.db.Exec(query, params...) - if err != nil { - return nil, fmt.Errorf("failed to update records: %w", err) - } - - // Get affected rows - rowsAffected, err := result.RowsAffected() - if err != nil { - return nil, fmt.Errorf("failed to get rows affected: %w", err) - } - - return starlark.MakeInt64(rowsAffected), nil + return db.execAffecting(thread, "failed to update records", query, params...) } -// upsert inserts a record if it doesn't exist, or updates it if it does. -func (db *database) upsert(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { - var table string - var values *starlark.Dict - var keyColumnsVal starlark.Value - - if err := starlark.UnpackArgs(fn.Name(), args, kwargs, - "table", &table, - "values", &values, - "conflict_columns", &keyColumnsVal); err != nil { - return nil, err - } - - // Extract column names and values - var columns []string - var placeholders []string - var params []interface{} - var updateClauses []string - +// buildUpsertClauses walks the values dictionary and returns the quoted column +// names, the placeholder list, the bound parameters, and the ON CONFLICT DO +// UPDATE SET clauses (each mapping a column to its excluded value). +func buildUpsertClauses(values *starlark.Dict) (columns, placeholders []string, params []interface{}, updateClauses []string, err error) { for _, tuple := range values.Items() { colName, ok := tuple.Index(0).(starlark.String) if !ok { - return nil, fmt.Errorf("column name must be a string") + return nil, nil, nil, nil, fmt.Errorf("column name must be a string") } // Add column name @@ -451,7 +484,7 @@ func (db *database) upsert(_ *starlark.Thread, fn *starlark.Builtin, args starla placeholders = append(placeholders, "?") val, err := starlarkToSQLiteValue(tuple.Index(1)) if err != nil { - return nil, err + return nil, nil, nil, nil, err } params = append(params, val) @@ -460,6 +493,28 @@ func (db *database) upsert(_ *starlark.Thread, fn *starlark.Builtin, args starla quoteName(string(colName)), quoteName(string(colName)))) } + return columns, placeholders, params, updateClauses, nil +} + +// upsert inserts a record if it doesn't exist, or updates it if it does. +func (db *database) upsert(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var table string + var values *starlark.Dict + var keyColumnsVal starlark.Value + + if err := starlark.UnpackArgs(fn.Name(), args, kwargs, + "table", &table, + "values", &values, + "conflict_columns", &keyColumnsVal); err != nil { + return nil, err + } + + // Extract column names and values + columns, placeholders, params, updateClauses, err := buildUpsertClauses(values) + if err != nil { + return nil, err + } + // Extract conflict columns using extractColumns conflictTarget, err := extractColumns(keyColumnsVal) if err != nil { @@ -479,22 +534,11 @@ func (db *database) upsert(_ *starlark.Thread, fn *starlark.Builtin, args starla strings.Join(updateClauses, ", ")) // Execute the statement - result, err := db.db.Exec(query, params...) - if err != nil { - return nil, fmt.Errorf("failed to upsert record: %w", err) - } - - // Get affected rows - rowsAffected, err := result.RowsAffected() - if err != nil { - return nil, fmt.Errorf("failed to get rows affected: %w", err) - } - - return starlark.MakeInt64(rowsAffected), nil + return db.execAffecting(thread, "failed to upsert record", query, params...) } // delete deletes records from a table. -func (db *database) delete(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func (db *database) delete(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var table string var whereVal starlark.Value @@ -519,18 +563,7 @@ func (db *database) delete(_ *starlark.Thread, fn *starlark.Builtin, args starla } // Execute the statement - result, err := db.db.Exec(query, params...) - if err != nil { - return nil, fmt.Errorf("failed to delete records: %w", err) - } - - // Get affected rows - rowsAffected, err := result.RowsAffected() - if err != nil { - return nil, fmt.Errorf("failed to get rows affected: %w", err) - } - - return starlark.MakeInt64(rowsAffected), nil + return db.execAffecting(thread, "failed to delete records", query, params...) } // extractColumns extracts column names from a Starlark value, @@ -634,7 +667,7 @@ func parseWhereClause(whereVal starlark.Value) (string, []interface{}, error) { } // selectRecords selects records from a table. -func (db *database) selectRecords(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func (db *database) selectRecords(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var table string var columnsVal starlark.Value var whereVal starlark.Value @@ -706,7 +739,9 @@ func (db *database) selectRecords(_ *starlark.Thread, fn *starlark.Builtin, args } // Execute the query - rows, err := db.db.Query(query, params...) + ctx, cancel := db.opContext(thread) + defer cancel() + rows, err := db.db.QueryContext(ctx, query, params...) if err != nil { return nil, fmt.Errorf("failed to select records: %w", err) } @@ -716,7 +751,7 @@ func (db *database) selectRecords(_ *starlark.Thread, fn *starlark.Builtin, args } // count counts records in a table. -func (db *database) count(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func (db *database) count(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var table string var whereVal starlark.Value @@ -742,7 +777,9 @@ func (db *database) count(_ *starlark.Thread, fn *starlark.Builtin, args starlar // Execute the query var count int64 - if err := db.db.QueryRow(query, params...).Scan(&count); err != nil { + ctx, cancel := db.opContext(thread) + defer cancel() + if err := db.db.QueryRowContext(ctx, query, params...).Scan(&count); err != nil { return nil, fmt.Errorf("failed to count records: %w", err) } @@ -873,7 +910,7 @@ func processTableConstraints(constraintsVal starlark.Value) ([]string, error) { } // createTableIndexes creates indexes for a table from a Starlark value. -func createTableIndexes(tx *sql.Tx, tableName string, indexesVal starlark.Value) error { +func createTableIndexes(ctx context.Context, tx *sql.Tx, tableName string, indexesVal starlark.Value) error { if indexesVal == nil || indexesVal == starlark.None { return nil } @@ -925,7 +962,7 @@ func createTableIndexes(tx *sql.Tx, tableName string, indexesVal starlark.Value) } // Execute the index creation - _, err = tx.Exec(indexSQL) + _, err = tx.ExecContext(ctx, indexSQL) if err != nil { return fmt.Errorf("failed to create index: %w", err) } diff --git a/prepared.go b/prepared.go index 4cdeb63..9976fee 100644 --- a/prepared.go +++ b/prepared.go @@ -1,10 +1,13 @@ package sqlite import ( + "context" "database/sql" "fmt" + "time" "github.com/1set/starlet/dataconv" + "github.com/starpkg/base/util" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" ) @@ -16,13 +19,21 @@ import ( // preparedStmt represents a prepared statement struct with methods // directly attached, providing cleaner and more idiomatic Go code. type preparedStmt struct { - stmt *sql.Stmt - maxRows int + stmt *sql.Stmt + maxRows int + opTimeout time.Duration +} + +// opContext derives the context bounding a single operation from the calling +// script thread plus the configured per-operation timeout. The caller must +// invoke the returned cancel func. +func (ps *preparedStmt) opContext(thread *starlark.Thread) (context.Context, context.CancelFunc) { + return util.OpContext(thread, ps.opTimeout) } // newPreparedStatementInstance creates a new Starlark prepared statement instance. -func newPreparedStatementInstance(stmt *sql.Stmt, maxRows int) *starlarkstruct.Module { - ps := &preparedStmt{stmt: stmt, maxRows: maxRows} +func newPreparedStatementInstance(stmt *sql.Stmt, maxRows int, opTimeout time.Duration) *starlarkstruct.Module { + ps := &preparedStmt{stmt: stmt, maxRows: maxRows, opTimeout: opTimeout} // Create dictionary of methods dict := starlark.StringDict{ @@ -34,8 +45,8 @@ func newPreparedStatementInstance(stmt *sql.Stmt, maxRows int) *starlarkstruct.M } // newPreparedQueryInstance creates a new Starlark prepared query instance. -func newPreparedQueryInstance(stmt *sql.Stmt, maxRows int) *starlarkstruct.Module { - ps := &preparedStmt{stmt: stmt, maxRows: maxRows} +func newPreparedQueryInstance(stmt *sql.Stmt, maxRows int, opTimeout time.Duration) *starlarkstruct.Module { + ps := &preparedStmt{stmt: stmt, maxRows: maxRows, opTimeout: opTimeout} // Create dictionary of methods dict := starlark.StringDict{ @@ -67,7 +78,9 @@ func (ps *preparedStmt) execute(thread *starlark.Thread, fn *starlark.Builtin, a } // Execute the statement - result, err := ps.stmt.Exec(goParams...) + ctx, cancel := ps.opContext(thread) + defer cancel() + result, err := ps.stmt.ExecContext(ctx, goParams...) if err != nil { return nil, fmt.Errorf("failed to execute statement: %w", err) } @@ -110,7 +123,9 @@ func (ps *preparedStmt) query(thread *starlark.Thread, fn *starlark.Builtin, arg } // Execute the query - rows, err := ps.stmt.Query(goParams...) + ctx, cancel := ps.opContext(thread) + defer cancel() + rows, err := ps.stmt.QueryContext(ctx, goParams...) if err != nil { return nil, fmt.Errorf("failed to execute query: %w", err) } @@ -135,7 +150,9 @@ func (ps *preparedStmt) queryOne(thread *starlark.Thread, fn *starlark.Builtin, } // Execute the query - rows, err := ps.stmt.Query(goParams...) + ctx, cancel := ps.opContext(thread) + defer cancel() + rows, err := ps.stmt.QueryContext(ctx, goParams...) if err != nil { return nil, fmt.Errorf("failed to execute query: %w", err) } diff --git a/sqlite.go b/sqlite.go index 446d23e..6d7444b 100644 --- a/sqlite.go +++ b/sqlite.go @@ -5,12 +5,14 @@ import ( "database/sql" "fmt" "net/url" + "strconv" "strings" "time" "github.com/1set/starlet" "github.com/1set/starlet/dataconv/types" "github.com/starpkg/base" + "github.com/starpkg/base/util" "github.com/tursodatabase/libsql-client-go/libsql" "go.starlark.net/starlark" _ "modernc.org/sqlite" @@ -28,7 +30,7 @@ const ( // Default configuration values const ( - defaultTimeout = 30.0 // Default connection timeout in seconds. + defaultTimeout = 30.0 // Default per-operation deadline in seconds (0 = no deadline). defaultBusyTimeout = 5.0 // Default busy timeout in seconds. SQLite will wait for this duration if the database is locked. defaultDatabase = ":memory:" // Default database path. Use ":memory:" for an in-memory database, or provide a file path. defaultForeignKeys = true // Default for foreign key constraints. Set to true to enable, false to disable. Enabling enforces referential integrity. @@ -70,7 +72,7 @@ type Module struct { func NewModule() *Module { return newModuleWithOptions( genConfigOption(configKeyDatabase, "Path to SQLite database (use :memory: for in-memory)", defaultDatabase), - genConfigOption(configKeyTimeout, "Connection timeout in seconds", defaultTimeout), + genConfigOption(configKeyTimeout, "Per-operation deadline in seconds (0 = no deadline)", defaultTimeout), genConfigOption(configKeyBusyTimeout, "Busy timeout in seconds", defaultBusyTimeout), genConfigOption(configKeyForeignKeys, "Enable foreign key constraints", defaultForeignKeys), genConfigOption(configKeyJournalMode, "Journal mode (WAL, DELETE, TRUNCATE, PERSIST, MEMORY, OFF)", defaultJournalMode), @@ -90,10 +92,7 @@ func NewModuleWithFileAccess(allowed bool) *Module { // genConfigOption creates a configuration option with common settings. // It sets up the name, description, default value, and environment variable. func genConfigOption[T any](name, description string, defaultValue T) *base.ConfigOption[T] { - return base.NewConfigOption(defaultValue). - WithName(name). - WithDescription(description). - WithEnvVar(strings.ToUpper(ModuleName + "_" + name)) + return base.NewNamedConfigOption(ModuleName, name, description, defaultValue) } // newModuleWithOptions creates a Module with the given configuration options. @@ -201,19 +200,38 @@ func (m *Module) connect(thread *starlark.Thread, fn *starlark.Builtin, args sta maxRows := m.ext.GetInt(configKeyMaxRows, defaultMaxRows) // Create a new database connection - db, err := openDatabase(database, timeout.GoFloat(), busyTimeout.GoFloat(), foreignKeysValue, journalMode, synchronous, cacheSize) + db, err := openDatabase(database, busyTimeout.GoFloat(), foreignKeysValue, journalMode, synchronous, cacheSize) if err != nil { return nil, fmt.Errorf("failed to open database: %w", err) } - // Create and return the database object - return newDatabaseInstance(db, maxRows, m.restrictFileAccess), nil + // Create and return the database object. `timeout` now bounds each operation + // (via a per-query context), replacing its former misuse as the connection's + // max lifetime. + return newDatabaseInstance(db, maxRows, m.restrictFileAccess, secondsToDuration(timeout.GoFloat())), nil +} + +// secondsToDuration converts a fractional-seconds timeout into a time.Duration. +// A non-positive value means "no per-operation deadline". +func secondsToDuration(sec float64) time.Duration { + if sec <= 0 { + return 0 + } + return time.Duration(sec * float64(time.Second)) } // connectRemote opens a connection to a remote libSQL server — a self-hosted // sqld or Turso Cloud — over the pure-Go libSQL client (no cgo). The returned // object exposes the same query/exec/table API as a local connection, because // the libSQL remote client is a standard database/sql driver. +// +// Cancellation caveat: queries (Ping/Query/Exec) over the HTTP(S)/Turso client +// honour the operation context, but the pinned libSQL driver on Go 1.20 does not +// thread a context into COMMIT/ROLLBACK, nor into a ws://wss:// connection dial +// (a fixed background dial). So a commit/rollback, or a reconnect over ws/wss, +// can still outlast the per-operation deadline / thread cancellation until the +// driver's own timeout. The documented HTTP(S)/Turso paths are the unaffected +// common case. func (m *Module) connectRemote(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var dbURL, authToken string if err := starlark.UnpackArgs(fn.Name(), args, kwargs, @@ -236,13 +254,19 @@ func (m *Module) connectRemote(thread *starlark.Thread, fn *starlark.Builtin, ar } db := sql.OpenDB(connector) - if err := db.Ping(); err != nil { + opTimeout := secondsToDuration(m.ext.GetFloat(configKeyTimeout, defaultTimeout)) + // A remote connect/query is a network round-trip: bound the Ping (and, below, + // every query) with a context so an unreachable or slow host cannot block the + // host goroutine indefinitely and so a cancelled script thread aborts it. + pingCtx, cancel := util.OpContext(thread, opTimeout) + defer cancel() + if err := db.PingContext(pingCtx); err != nil { db.Close() return nil, fmt.Errorf("failed to connect to remote database: %w", err) } maxRows := m.ext.GetInt(configKeyMaxRows, defaultMaxRows) - return newDatabaseInstance(db, maxRows, m.restrictFileAccess), nil + return newDatabaseInstance(db, maxRows, m.restrictFileAccess, opTimeout), nil } // isInMemoryDSN reports whether a SQLite DSN refers to an in-memory (or private @@ -250,58 +274,144 @@ func (m *Module) connectRemote(thread *starlark.Thread, fn *starlark.Builtin, ar // strict: a loose substring match on ":memory:" / "mode=memory" would let a file // DSN such as "file:/etc/passwd?x=:memory:" slip past the file-access restriction. func isInMemoryDSN(s string) bool { - switch { - case s == "", s == ":memory:": - // Empty = private temporary database; ":memory:" = the in-memory database. - return true - case strings.HasPrefix(s, "file::memory:"): - // Named in-memory database, e.g. "file::memory:?cache=shared". + // Empty = private temporary database; ":memory:" = the in-memory database. + if s == "" || s == ":memory:" { return true } - // Honour "mode=memory" only as a real query parameter, not as a substring. - if i := strings.IndexByte(s, '?'); i >= 0 { - if q, err := url.ParseQuery(s[i+1:]); err == nil && q.Get("mode") == "memory" { - return true - } + // A DSN without the "file:" scheme is a literal filename to SQLite — even one + // containing "?mode=memory" or ":memory:" is a real on-disk file, so it must + // NOT be treated as in-memory (that was a file-access bypass). + if !strings.HasPrefix(s, "file:") { + return false + } + u, err := url.Parse(s) + if err != nil { + return false + } + // Only the PATH decides disk-vs-memory. We deliberately do NOT trust a + // mode=memory query parameter to reclassify a disk path: Go's url parsing and + // SQLite's DSN parsing disagree on malformed/edge queries (%00 truncation, + // duplicate keys), and any such disagreement on a disk path would be a + // file-access-gate bypass. An empty path is a private temporary database; + // a ":memory:" path (file::memory:[?…]) is the in-memory database. + path := u.Opaque + if path == "" { + path = u.Path } - return false + return path == "" || path == ":memory:" } -// openDatabase creates a new SQLite database connection with the given options. -func openDatabase(connStr string, timeout, busyTimeout float64, foreignKeys bool, journalMode, synchronous string, cacheSize int) (*sql.DB, error) { - // Open database connection - db, err := sql.Open("sqlite", connStr) +// openDatabase creates a new local SQLite database connection. The connection +// PRAGMAs are set through the DSN (modernc's _pragma= parameter) so they apply +// to EVERY physical connection the pool opens; applying them once with db.Exec +// only configures a single borrowed connection, leaving every later connection +// on SQLite's defaults (notably foreign_keys OFF, so constraint-violating writes +// silently succeed). An in-memory or private-temporary database is additionally +// pinned to one never-recycled connection, because its schema and data live only +// inside that connection — a second (empty) connection or a recycled one would +// otherwise make earlier data vanish ("no such table"). +func openDatabase(connStr string, busyTimeout float64, foreignKeys bool, journalMode, synchronous string, cacheSize int) (*sql.DB, error) { + if !pragmaIdentifier(journalMode) { + return nil, fmt.Errorf("invalid journal_mode %q", journalMode) + } + if !pragmaIdentifier(synchronous) { + return nil, fmt.Errorf("invalid synchronous %q", synchronous) + } + settings := pragmaSettings(busyTimeout, foreignKeys, journalMode, synchronous, cacheSize) + + memory := isInMemoryDSN(connStr) + dsn := connStr + if !memory { + // A file database is served by a POOL of connections; carry the PRAGMAs in + // the DSN so every connection the pool opens is configured identically. + dsn = appendPragmas(connStr, settings) + } + db, err := sql.Open("sqlite", dsn) if err != nil { return nil, err } - - // Verify database connection with Ping - // This will catch database file accessibility issues early - if err := db.Ping(); err != nil { + if memory { + err = configureMemoryConnection(db, settings) + } else { + err = db.Ping() + } + if err != nil { db.Close() return nil, err } + return db, nil +} - // Configure connection options - pragmas := []string{ - fmt.Sprintf("PRAGMA busy_timeout = %d", int(busyTimeout*1000)), - fmt.Sprintf("PRAGMA journal_mode = %s", journalMode), - fmt.Sprintf("PRAGMA synchronous = %s", synchronous), - fmt.Sprintf("PRAGMA cache_size = %d", cacheSize), - fmt.Sprintf("PRAGMA foreign_keys = %d", boolToInt(foreignKeys)), +// configureMemoryConnection pins an in-memory / private-temporary database to a +// single never-recycled connection (so its schema and data don't vanish when a +// second connection opens or the first is retired) and applies the PRAGMAs +// directly to that one connection — which also verifies it opens, sidestepping +// the DSN quirk that an empty connStr would be misparsed. +func configureMemoryConnection(db *sql.DB, settings [][2]string) error { + db.SetMaxOpenConns(1) + db.SetConnMaxLifetime(0) + db.SetConnMaxIdleTime(0) + for _, stmt := range pragmaStatements(settings) { + if _, err := db.Exec(stmt); err != nil { + return fmt.Errorf("failed to apply %s: %w", stmt, err) + } } + return nil +} - // Execute all pragmas - for _, pragma := range pragmas { - _, err = db.Exec(pragma) - if err != nil { - db.Close() - return nil, fmt.Errorf("failed to execute %s: %w", pragma, err) +// pragmaIdentifier reports whether v is a bare alphanumeric PRAGMA keyword/value, +// so it is safe to place into a PRAGMA statement or the DSN _pragma= expression +// without risking injection (a value like "WAL); DROP …" is rejected). +func pragmaIdentifier(v string) bool { + if v == "" { + return false + } + for _, r := range v { + if !isASCIIAlphanumeric(r) { + return false } } + return true +} - // Set timeout - db.SetConnMaxLifetime(time.Duration(timeout * float64(time.Second))) +// isASCIIAlphanumeric reports whether r is an ASCII letter or digit. +func isASCIIAlphanumeric(r rune) bool { + return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' +} - return db, nil +// pragmaSettings returns the ordered (name, value) connection PRAGMAs. journal_mode +// and synchronous must already be validated by pragmaIdentifier; the rest are +// integer-formatted. +func pragmaSettings(busyTimeout float64, foreignKeys bool, journalMode, synchronous string, cacheSize int) [][2]string { + return [][2]string{ + {"busy_timeout", strconv.Itoa(int(busyTimeout * 1000))}, + {"foreign_keys", strconv.Itoa(boolToInt(foreignKeys))}, + {"journal_mode", journalMode}, + {"synchronous", synchronous}, + {"cache_size", strconv.Itoa(cacheSize)}, + } +} + +// appendPragmas builds the DSN carrying the PRAGMAs as modernc _pragma= +// parameters so every pooled (file) connection is configured identically. +func appendPragmas(connStr string, settings [][2]string) string { + parts := make([]string, len(settings)) + for i, s := range settings { + parts[i] = "_pragma=" + s[0] + "(" + s[1] + ")" + } + sep := "?" + if strings.Contains(connStr, "?") { + sep = "&" + } + return connStr + sep + strings.Join(parts, "&") +} + +// pragmaStatements returns the PRAGMA statements for the single-connection +// (in-memory) path. +func pragmaStatements(settings [][2]string) []string { + stmts := make([]string, len(settings)) + for i, s := range settings { + stmts[i] = "PRAGMA " + s[0] + " = " + s[1] + } + return stmts } diff --git a/transaction.go b/transaction.go index 3f94dd4..17e1b3b 100644 --- a/transaction.go +++ b/transaction.go @@ -1,10 +1,14 @@ package sqlite import ( + "context" "database/sql" "fmt" + "runtime" + "time" "github.com/1set/starlet/dataconv" + "github.com/starpkg/base/util" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" ) @@ -130,13 +134,52 @@ func (r *OperationResult) Index(i int) starlark.Value { // transaction represents a database transaction. type transaction struct { - tx *sql.Tx - maxRows int + tx *sql.Tx + maxRows int + opTimeout time.Duration + // conn is the dedicated connection the transaction runs on. begin() acquires + // it under a deadline (so acquisition can't block the host forever when the + // single-connection in-memory pool is busy) and hands it here to be released + // on finish; the transaction itself then runs under a cancellation-only + // context that a per-statement deadline could not safely bound. + conn *sql.Conn + // cancel releases the transaction-lifetime context created in begin(). It is + // separate from the per-statement contexts: cancelling it aborts the whole + // transaction, so it is called only once the transaction ends (commit or + // rollback), never when an individual statement returns. + cancel context.CancelFunc +} + +// finish aborts the transaction context and returns its connection to the pool +// (idempotent). Cancelling first makes database/sql roll back any still-open +// transaction before the connection is released. +func (tx *transaction) finish() { + if tx.cancel != nil { + tx.cancel() + } + if tx.conn != nil { + tx.conn.Close() + tx.conn = nil + } +} + +// opContext derives the context bounding a single operation from the calling +// script thread plus the configured per-operation timeout. The caller must +// invoke the returned cancel func. +func (tx *transaction) opContext(thread *starlark.Thread) (context.Context, context.CancelFunc) { + return util.OpContext(thread, tx.opTimeout) } // newTransactionInstance creates a new Starlark transaction instance. -func newTransactionInstance(tx *sql.Tx, maxRows int) *starlarkstruct.Module { - txObj := &transaction{tx: tx, maxRows: maxRows} +func newTransactionInstance(tx *sql.Tx, maxRows int, opTimeout time.Duration, conn *sql.Conn, cancel context.CancelFunc) *starlarkstruct.Module { + txObj := &transaction{tx: tx, maxRows: maxRows, opTimeout: opTimeout, conn: conn, cancel: cancel} + // Safety net for a transaction the script begins but never commits/rolls back: + // the host Machine may run under an uncancelled context.Background(), so + // without this the lifetime context (and the connection it pins on a + // single-connection in-memory database) would leak until process exit. When + // the abandoned object is collected, cancelling its context makes database/sql + // roll the transaction back and return the connection to the pool. + runtime.SetFinalizer(txObj, func(t *transaction) { t.finish() }) // Create dictionary of methods dict := starlark.StringDict{ @@ -168,7 +211,9 @@ func (tx *transaction) execute(thread *starlark.Thread, fn *starlark.Builtin, ar } // Execute the query within transaction - result, err := tx.tx.Exec(sqlQuery.query, sqlQuery.params...) + ctx, cancel := tx.opContext(thread) + defer cancel() + result, err := tx.tx.ExecContext(ctx, sqlQuery.query, sqlQuery.params...) if err != nil { return newErrorResult(fmt.Errorf("failed to execute transaction query: %w", err)), nil } @@ -200,7 +245,9 @@ func (tx *transaction) query(thread *starlark.Thread, fn *starlark.Builtin, args } // Execute the query within transaction - rows, err := tx.tx.Query(sqlQuery.query, sqlQuery.params...) + ctx, cancel := tx.opContext(thread) + defer cancel() + rows, err := tx.tx.QueryContext(ctx, sqlQuery.query, sqlQuery.params...) if err != nil { return newErrorResult(fmt.Errorf("failed to execute transaction query: %w", err)), nil } @@ -232,7 +279,9 @@ func (tx *transaction) queryOne(thread *starlark.Thread, fn *starlark.Builtin, a } // Execute the query within transaction - rows, err := tx.tx.Query(sqlQuery.query, sqlQuery.params...) + ctx, cancel := tx.opContext(thread) + defer cancel() + rows, err := tx.tx.QueryContext(ctx, sqlQuery.query, sqlQuery.params...) if err != nil { return newErrorResult(fmt.Errorf("failed to execute transaction query: %w", err)), nil } @@ -252,6 +301,7 @@ func (tx *transaction) commit(thread *starlark.Thread, fn *starlark.Builtin, arg return nil, err } + defer tx.finish() if err := tx.tx.Commit(); err != nil { return newErrorResult(fmt.Errorf("failed to commit transaction: %w", err)), nil } @@ -267,6 +317,7 @@ func (tx *transaction) rollback(thread *starlark.Thread, fn *starlark.Builtin, a return nil, err } + defer tx.finish() if err := tx.tx.Rollback(); err != nil { return nil, fmt.Errorf("failed to rollback transaction: %w", err) } diff --git a/utils.go b/utils.go index 1fc9a0e..b329b55 100644 --- a/utils.go +++ b/utils.go @@ -202,8 +202,13 @@ func processQueryOneRow(rows *sql.Rows) (starlark.Value, error) { return nil, fmt.Errorf("failed to get column names: %w", err) } - // Check if we have a row + // Check if we have a row. A false Next() can also mean the iteration failed + // (cancellation, deadline, driver error), so surface rows.Err() rather than + // silently reporting a successful no-row result. if !rows.Next() { + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error during query iteration: %w", err) + } return starlark.None, nil }