From 9e5c386ae9e6672f13b385fd96984d41d9665a85 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 23 Oct 2026 00:04:46 -0700 Subject: [PATCH 1/2] build(deps): fix SQLite lock-wait cancellation --- CHANGELOG.md | 2 ++ README.md | 4 ++- go.mod | 6 ++-- go.sum | 20 ++++++------- store/cancellation_test.go | 59 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 store/cancellation_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index bc0eb90..4ead927 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Update SQLite to v1.61.0 and its required libc v1.79.0 runtime so writes waiting for database locks honor context cancellation and deadlines; retain the Go 1.27.0 minimum. + ## 0.17.2 - 2026-10-20 **Highlights:** Rebuild with Go 1.27.3 to fix the reachable crypto/x509 certificate-validation vulnerability. diff --git a/README.md b/README.md index 495f6ab..c775452 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,9 @@ make check This runs module tidiness, formatting, vet, dead-code and vulnerability checks, unit tests, and race tests with `GOWORK=off`. See [CONTRIBUTING.md](CONTRIBUTING.md) for the compatibility rules. -Keep `modernc.org/libc` at the exact version required by the selected `modernc.org/sqlite` module. SQLite v1.60.1 requires libc v1.77.1; its runtime dependency must be updated together with SQLite, not independently. +Keep `modernc.org/libc` at the exact version required by the selected `modernc.org/sqlite` module. SQLite v1.61.0 requires libc v1.79.0; its runtime dependency must be updated together with SQLite, not independently. + +SQLite operations waiting for a write lock honor their context cancellation or deadline, including during the default five-second busy timeout. ## License diff --git a/go.mod b/go.mod index 065c754..7bffc11 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/mattn/go-isatty v0.0.24 github.com/pelletier/go-toml/v2 v2.4.3 golang.org/x/sys v0.49.0 - modernc.org/sqlite v1.60.1 + modernc.org/sqlite v1.61.0 ) require ( @@ -26,7 +26,6 @@ require ( github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/dustin/go-humanize v1.1.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect - github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 // indirect github.com/google/uuid v1.6.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.1 // indirect github.com/mattn/go-localereader v0.0.1 // indirect @@ -40,8 +39,7 @@ require ( github.com/xo/terminfo v1.2.0 // indirect golang.org/x/crypto v0.58.0 // indirect golang.org/x/text v0.43.0 // indirect - golang.org/x/tools v0.52.0 // indirect - modernc.org/libc v1.77.1 // indirect + modernc.org/libc v1.79.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index 82a55cc..9d10d66 100644 --- a/go.sum +++ b/go.sum @@ -73,20 +73,20 @@ golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= -modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= -modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= -modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA= +modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys= +modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4= +modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= -modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o= +modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs= -modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0= +modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8= +modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= @@ -95,8 +95,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw= -modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0= +modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY= +modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/store/cancellation_test.go b/store/cancellation_test.go new file mode 100644 index 0000000..b3cbd53 --- /dev/null +++ b/store/cancellation_test.go @@ -0,0 +1,59 @@ +package store + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" +) + +func TestWriteLockWaitHonorsContextDeadline(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "archive.db") + holder, err := Open(ctx, Options{Path: path, Schema: `create table things(value text)`}) + if err != nil { + t.Fatal(err) + } + defer holder.Close() + waiter, err := Open(ctx, Options{Path: path}) + if err != nil { + t.Fatal(err) + } + defer waiter.Close() + + tx, err := holder.DB().BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, `insert into things values('held')`); err != nil { + t.Fatal(err) + } + + writeCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + start := time.Now() + _, err = waiter.DB().ExecContext(writeCtx, `insert into things values('waiting')`) + elapsed := time.Since(start) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("blocked write error = %v, want deadline exceeded", err) + } + // Leave ample scheduling headroom while rejecting the five-second busy wait. + if elapsed > 2*time.Second { + t.Fatalf("blocked write took %v after a 100ms deadline", elapsed) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + if _, err := waiter.DB().ExecContext(ctx, `insert into things values('retry')`); err != nil { + t.Fatalf("write after releasing lock: %v", err) + } + var count int + if err := waiter.DB().QueryRowContext(ctx, `select count(*) from things`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatalf("row count = %d, want only the retry", count) + } +} From 5ccd29c7611c89c4cee32f37184abfe8c9fd4db0 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 23 Oct 2026 14:09:07 -0700 Subject: [PATCH 2/2] test: qualify SQLite upgrade against migrated Gitcrawl --- .github/scripts/README.md | 18 +++++++++--------- .github/workflows/ci.yml | 2 +- README.md | 6 ++++++ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/scripts/README.md b/.github/scripts/README.md index db5e3b6..7133168 100644 --- a/.github/scripts/README.md +++ b/.github/scripts/README.md @@ -20,15 +20,15 @@ The automatic job deliberately advances its Crawlkit API baseline from `IngestManifest.Warnings` and `ArchiveSnapshot.Warnings`, which the older baseline does not provide. The job uses the published source directly, without API shims. -Its Gitcrawl pin advances from `dab4d849f9562306da41267cda2b2fd6868a41a8` to -`0a4cefe1858b5a8b8c1138260252b09589c4294f`. This is a -[34-commit refresh](https://github.com/openclaw/gitcrawl/compare/dab4d849f9562306da41267cda2b2fd6868a41a8...0a4cefe1858b5a8b8c1138260252b09589c4294f), -including CLI and storage refactors, archive admission, export and sync fixes, -GraphQL history, and dependency and release updates. It also includes the -fixture-only Git maintenance isolation in -[Gitcrawl PR215](https://github.com/openclaw/gitcrawl/pull/215). -This qualifies current downstream compatibility; it does not establish the -cause or repair of the historical Git 2.55 cleanup failure. +Its Gitcrawl pin advances from `0a4cefe1858b5a8b8c1138260252b09589c4294f` to +`440c46c04877b4d85a21061e123c7798957d8380`, including the explicit JSON-subquery +conversion in [Gitcrawl PR271](https://github.com/openclaw/gitcrawl/pull/271). +SQLite 3.54 no longer preserves the JSON subtype through a scalar subquery; +the snapshot privacy assertion now uses `json(...)` to keep the same nested +array on both SQLite 3.53 and 3.54. Production sanitization and the expected +privacy result are unchanged. The pin also includes the intervening Gitcrawl +CLI, storage, dependency, and release updates, retaining its fixture-only Git +maintenance isolation. Both complete CLI suites remain required, against the published baseline and the exact candidate head. Selected-app manual qualification retains baseline diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4b1fbd..a5a98f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: env: CRAWLKIT_BASELINE_SHA: 493f7470c5e366b52b228e49ed08807f6c81dbf4 CRAWLKIT_CANDIDATE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - GITCRAWL_SHA: 0a4cefe1858b5a8b8c1138260252b09589c4294f + GITCRAWL_SHA: 440c46c04877b4d85a21061e123c7798957d8380 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: diff --git a/README.md b/README.md index c775452..cfaf9f4 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,12 @@ Keep `modernc.org/libc` at the exact version required by the selected `modernc.o SQLite operations waiting for a write lock honor their context cancellation or deadline, including during the default five-second busy timeout. +When nesting JSON produced by a scalar subquery inside another JSON constructor, +mark the subquery result explicitly with `json(...)`, for example +`json_array(json((select json_group_array(value) from items)))`. SQLite 3.54 +does not preserve the JSON subtype through that subquery boundary. Explicit +conversion also works on SQLite 3.53 and keeps arrays from becoming quoted strings. + ## License [MIT](LICENSE)