From 0b65b049c6dcc0e305260d6685132d38f240d1d3 Mon Sep 17 00:00:00 2001 From: Pablo Ontiveros Date: Thu, 10 Sep 2026 21:54:31 -0600 Subject: [PATCH 1/4] fix(search): requeue search_echo=0 Codex shell wrapper calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend `unmarkedDirectSearchCallSQL` to also requeue pre-#80 Codex `shell` tool calls whose serialized text is a direct Backscroll search, mirroring how `isCodexDirectSearchCall` recognizes the shell argv shape at ingest time (argv is exactly [, -c|-lc, "backscroll search ..."]). The new GLOB clause matches `shell command=[*sh","-c|-lc","backscroll search"]` (bare) or `shell command=[*sh","-c|-lc","backscroll search..."]` (with extra args), requiring the shell binary basename to end in `sh` via the [[]*sh char class escape. A paired NOT GLOB excludes 4+ argv-element shell calls (whose argv[2] starts with `backscroll search`) so the SQL does not over-match cases the reader would reject via len(Command)==3 — otherwise requeueing would loop forever (SQL matches every sync, reader never marks, no convergence). Regression test `TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellCall` covers the 3 requeue shapes (bare -c, with-args -c, with-args -lc) and the 5 non-requeue shapes (different command, wrong flag, 4-element -c/-lc, and the trailing-whitespace 4-element over-match guard). --- internal/storage/search.go | 24 ++++++++++- internal/storage/search_echo_test.go | 60 ++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/internal/storage/search.go b/internal/storage/search.go index a7efe93..9523022 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -439,11 +439,33 @@ func unmarkedDirectSearchCallSQL(alias string) string { sep := "'[' || " + asciiWhitespaceSQL + " || ']'" bash := "'[Bb][Aa][Ss][Hh]' || " + sep + " || 'command=backscroll' || " + sep + " || 'search'" execCmd := "'exec_command' || " + sep + " || 'cmd=backscroll' || " + sep + " || 'search'" + // Codex shell wrapper form: argv is exactly [, -c|-lc, "backscroll search ..."] + // serialized as shellcommand=[*sh","-c|-lc","backscroll search"] (bare) or + // shellcommand=[*sh","-c|-lc","backscroll search..."] (with extra args). + shellC := "'shell' || " + sep + " || 'command=[[]*sh' || '\",\"' || '-c' || '\",\"' || 'backscroll search'" + shellLC := "'shell' || " + sep + " || 'command=[[]*sh' || '\",\"' || '-lc' || '\",\"' || 'backscroll search'" glob := func(prefix string) string { return trimmed + " GLOB (" + prefix + ") OR " + trimmed + " GLOB (" + prefix + " || " + sep + " || '*')" } + // shellGlob matches the bare form (prefix + '"]') or the with-args form + // (prefix + sep + '*' + '"]'). The trailing '"]' anchors the closing of the + // JSON string for argv[2] and the array. + shellGlob := func(prefix string) string { + return trimmed + " GLOB (" + prefix + ` || '"]') OR ` + + trimmed + " GLOB (" + prefix + " || " + sep + ` || '*"]')` + } + // shellExtra excludes 4+ argv-element shell calls (e.g. argv[2]="backscroll search ", + // argv[3]="bar") that the with-args '*"]' would otherwise over-match. The reader's + // isCodexDirectSearchCall rejects them via len(Command)==3, so requeueing them + // would loop forever: the SQL matches every sync and the reader never marks. + shellExtra := func(flag string) string { + notPattern := "'shell' || " + sep + " || 'command=[[]*sh' || '\",\"' || '" + flag + "' || '\",\"' || 'backscroll search*' || '\",\"*'" + return alias + ".text NOT GLOB (" + notPattern + ")" + } return alias + ".content_type = 'tool' AND COALESCE(" + alias + ".search_echo, 0) = 0 AND (" + - glob(bash) + " OR " + glob(execCmd) + ")" + glob(bash) + " OR " + glob(execCmd) + " OR " + + shellGlob(shellC) + " OR " + shellGlob(shellLC) + ")" + + " AND " + shellExtra("-c") + " AND " + shellExtra("-lc") } // mergeRRF uses Reciprocal Rank Fusion to merge two ranked lists by position, diff --git a/internal/storage/search_echo_test.go b/internal/storage/search_echo_test.go index b98ce7a..9f4ee99 100644 --- a/internal/storage/search_echo_test.go +++ b/internal/storage/search_echo_test.go @@ -133,6 +133,66 @@ func TestPendingSearchEchoPathsRequeuesZeroValuedDirectCalls(t *testing.T) { } } +func TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellCall(t *testing.T) { + db, err := Open(filepath.Join(t.TempDir(), "index.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + // Each fixture stores one tool row at search_echo=0 to simulate a pre-#80 + // Codex writer. The requeue expectation mirrors isCodexDirectSearchCall's + // argv shape: argv is exactly [, -c|-lc, "backscroll search ..."]. + files := []IndexedFile{ + // Requeued: bare 3-element -c shell direct search. + {Source: "session", SourcePath: "shell_bare_c.jsonl", Hash: "h1", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: `shell command=["sh","-c","backscroll search"]`, ContentType: "tool"}, + }}, + // Requeued: 3-element -c with extra args. + {Source: "session", SourcePath: "shell_args_c.jsonl", Hash: "h2", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: `shell command=["sh","-c","backscroll search --text orchard"]`, ContentType: "tool"}, + }}, + // Requeued: 3-element -lc with /bin/bash path. + {Source: "session", SourcePath: "shell_args_lc.jsonl", Hash: "h3", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: `shell command=["/bin/bash","-lc","backscroll search --text orchard"]`, ContentType: "tool"}, + }}, + // NOT requeued: different command (argv[2]="ls"). + {Source: "session", SourcePath: "shell_ls.jsonl", Hash: "h4", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: `shell command=["sh","-c","ls"]`, ContentType: "tool"}, + }}, + // NOT requeued: wrong flag (argv[1]="-x"). + {Source: "session", SourcePath: "shell_xflag.jsonl", Hash: "h5", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: `shell command=["sh","-x","backscroll search"]`, ContentType: "tool"}, + }}, + // NOT requeued: 4-element argv with trailing 4th element. + {Source: "session", SourcePath: "shell_four.jsonl", Hash: "h6", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: `shell command=["sh","-c","backscroll search","bar"]`, ContentType: "tool"}, + }}, + // NOT requeued: 4-element argv with trailing whitespace + 4th element (over-match guard). + {Source: "session", SourcePath: "shell_four_ws.jsonl", Hash: "h7", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: `shell command=["sh","-c","backscroll search ","bar"]`, ContentType: "tool"}, + }}, + // NOT requeued: 4-element argv with -lc flag. + {Source: "session", SourcePath: "shell_four_lc.jsonl", Hash: "h8", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: `shell command=["bash","-lc","backscroll search","bar"]`, ContentType: "tool"}, + }}, + } + if err := db.SyncFiles(files); err != nil { + t.Fatal(err) + } + // Force every row to search_echo=0 so only the new SQL clause can requeue them. + if _, err := db.db.Exec(`UPDATE search_items SET search_echo=0`); err != nil { + t.Fatal(err) + } + got, err := db.PendingSearchEchoPaths() + if err != nil { + t.Fatal(err) + } + want := []string{"shell_args_c.jsonl", "shell_args_lc.jsonl", "shell_bare_c.jsonl"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("pending=%v want %v", got, want) + } +} + func TestEchoProvenanceDoesNotAffectProse(t *testing.T) { db, err := Open(filepath.Join(t.TempDir(), "index.db")) if err != nil { From c359571e1e7101b45ab291715b03c8202f1f3ee1 Mon Sep 17 00:00:00 2001 From: Pablo Ontiveros Date: Thu, 10 Sep 2026 22:17:29 -0600 Subject: [PATCH 2/4] fix(search): mirror every JSON-escape separator between 'backscroll' and 'search' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer finding on PR #87: the reader's `isCodexDirectSearchCommand` accepts ANY whitespace separator between the 'backscroll' and 'search' tokens (it uses `strings.Fields`, which splits on every unicode.IsSpace rune), but the serialized text carries whatever the JSON encoder produced for the original argv[2]. For ASCII whitespace that's a literal char (space), a JSON single-letter escape (\t, \n, \f, \r), or — for every other unicode.IsSpace rune — a JSON \uXXXX escape (\u000b for vertical tab, \u0085 for NEL, \u00a0 for NBSP, etc.). The previous SQL only matched the literal-space case, so any pre-#80 Codex `shell` row whose stored text was `shell command=[..., "backscroll\\tsearch orchard"]` (or any other escape form) stayed at search_echo=0 forever and leaked into unfiltered recall. Fix: emit a separate bare + with-args GLOB pair per separator form (whitespace class, then four single-letter escapes, then a \uXXXX char class), each with its own NOT GLOB guard for the 4-element over-match trap. SQLite (modernc.org/sqlite) does not process backslash escapes in string literals by default, so the JSON backslash is built via `char(92) || ''` rather than `'\\'`. Regression test `TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellSerializedWhitespace` builds each fixture's stored text via `readers.SerializeToolInput` from a real accepted argv triple (no hand-written `\\t`/\\n strings), then verifies that every accepted separator form requeues and that 4-element calls with tab or NBSP separators still do NOT requeue (the over-match guard holds per form). --- internal/storage/search.go | 72 ++++++++++++++------ internal/storage/search_echo_test.go | 99 ++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 21 deletions(-) diff --git a/internal/storage/search.go b/internal/storage/search.go index 9523022..2daf295 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -439,33 +439,63 @@ func unmarkedDirectSearchCallSQL(alias string) string { sep := "'[' || " + asciiWhitespaceSQL + " || ']'" bash := "'[Bb][Aa][Ss][Hh]' || " + sep + " || 'command=backscroll' || " + sep + " || 'search'" execCmd := "'exec_command' || " + sep + " || 'cmd=backscroll' || " + sep + " || 'search'" - // Codex shell wrapper form: argv is exactly [, -c|-lc, "backscroll search ..."] - // serialized as shellcommand=[*sh","-c|-lc","backscroll search"] (bare) or - // shellcommand=[*sh","-c|-lc","backscroll search..."] (with extra args). - shellC := "'shell' || " + sep + " || 'command=[[]*sh' || '\",\"' || '-c' || '\",\"' || 'backscroll search'" - shellLC := "'shell' || " + sep + " || 'command=[[]*sh' || '\",\"' || '-lc' || '\",\"' || 'backscroll search'" glob := func(prefix string) string { return trimmed + " GLOB (" + prefix + ") OR " + trimmed + " GLOB (" + prefix + " || " + sep + " || '*')" } - // shellGlob matches the bare form (prefix + '"]') or the with-args form - // (prefix + sep + '*' + '"]'). The trailing '"]' anchors the closing of the - // JSON string for argv[2] and the array. - shellGlob := func(prefix string) string { - return trimmed + " GLOB (" + prefix + ` || '"]') OR ` + - trimmed + " GLOB (" + prefix + " || " + sep + ` || '*"]')` - } - // shellExtra excludes 4+ argv-element shell calls (e.g. argv[2]="backscroll search ", - // argv[3]="bar") that the with-args '*"]' would otherwise over-match. The reader's - // isCodexDirectSearchCall rejects them via len(Command)==3, so requeueing them - // would loop forever: the SQL matches every sync and the reader never marks. - shellExtra := func(flag string) string { - notPattern := "'shell' || " + sep + " || 'command=[[]*sh' || '\",\"' || '" + flag + "' || '\",\"' || 'backscroll search*' || '\",\"*'" + // Codex shell wrapper form: argv is exactly [, -c|-lc, "backscroll search ..."]. + // SerializeToolInput renders the JSON-encoded argv as compact JSON, so the + // separator between the 'backscroll' and 'search' tokens depends on what the + // original argv[2] looked like: a literal space (the common case), a JSON + // single-letter escape (\t, \n, \f, \r) for the matching ASCII whitespace, + // or a JSON \uXXXX escape for any other unicode.IsSpace rune (vertical tab, + // NEL, NBSP, em/en spaces, line/paragraph separators). The reader's + // isCodexDirectSearchCall uses strings.Fields on the unescaped command, so + // it accepts every separator form — the SQL must mirror that exactly, or + // the row stays at search_echo=0 forever. + shellPrefixC := "'shell' || " + sep + " || 'command=[[]*sh' || '\",\"' || '-c' || '\",\"' || 'backscroll'" + shellPrefixLC := "'shell' || " + sep + " || 'command=[[]*sh' || '\",\"' || '-lc' || '\",\"' || 'backscroll'" + // shellSeparators lists the SQL fragments that evaluate to the separator text + // between 'backscroll' and 'search' in the serialized third element. The + // single-letter escapes and the \uXXXX char class cover every JSON encoding + // of a rune that strings.Fields would split on. SQLite (via modernc.org/sqlite) + // does not process backslash escapes in string literals by default, so we build + // the JSON backslash via char(92) (92 = ASCII '\') and concatenate. + backslash := "char(92)" + shellSeparators := []string{ + sep, // literal whitespace + backslash + " || 't'", + backslash + " || 'n'", + backslash + " || 'f'", + backslash + " || 'r'", + backslash + " || 'u[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]'", + } + // shellForms returns the bare + with-args GLOB clauses for one (prefix, + // separator) pair. The trailing '"]' anchors the closing of the JSON + // string for argv[2] and the array. + shellForms := func(prefix, separator string) string { + p := prefix + " || " + separator + return trimmed + " GLOB (" + p + ` || '"]') OR ` + + trimmed + " GLOB (" + p + ` || '*"]')` + } + // shellFormNotExtra returns the NOT GLOB guard for one (prefix, separator) + // pair. Excludes 4+ argv-element shell calls (e.g. argv[2]="backscroll search ", + // argv[3]="bar") that the reader rejects via len(Command)==3, so requeuing + // them would loop forever: SQL matches every sync, reader never marks. + shellFormNotExtra := func(prefix, separator string) string { + notPattern := prefix + " || " + separator + " || '*' || '\",\"*'" return alias + ".text NOT GLOB (" + notPattern + ")" } + orParts := []string{glob(bash), glob(execCmd)} + notParts := make([]string, 0, len(shellSeparators)*2) + for _, separator := range shellSeparators { + orParts = append(orParts, shellForms(shellPrefixC, separator)) + orParts = append(orParts, shellForms(shellPrefixLC, separator)) + notParts = append(notParts, shellFormNotExtra(shellPrefixC, separator)) + notParts = append(notParts, shellFormNotExtra(shellPrefixLC, separator)) + } return alias + ".content_type = 'tool' AND COALESCE(" + alias + ".search_echo, 0) = 0 AND (" + - glob(bash) + " OR " + glob(execCmd) + " OR " + - shellGlob(shellC) + " OR " + shellGlob(shellLC) + ")" + - " AND " + shellExtra("-c") + " AND " + shellExtra("-lc") + strings.Join(orParts, " OR ") + ")" + + " AND " + strings.Join(notParts, " AND ") } // mergeRRF uses Reciprocal Rank Fusion to merge two ranked lists by position, diff --git a/internal/storage/search_echo_test.go b/internal/storage/search_echo_test.go index 9f4ee99..9ef5558 100644 --- a/internal/storage/search_echo_test.go +++ b/internal/storage/search_echo_test.go @@ -3,14 +3,43 @@ package storage import ( "context" "database/sql" + "encoding/json" + "fmt" "path/filepath" "reflect" + "strings" "testing" "github.com/pablontiv/backscroll/internal/compat" "github.com/pablontiv/backscroll/internal/models" + "github.com/pablontiv/backscroll/internal/readers" ) +// shellText serializes a Codex shell argv triple via readers.SerializeToolInput +// so the stored text mirrors what the reader would actually persist for an +// accepted argv shape — in particular, any whitespace separator between the +// 'backscroll' and 'search' tokens gets JSON-escaped (\t, \n, \u00a0, …) by the +// encoder, which is what the SQL selection must match. +func shellText(t *testing.T, shellBin, flag, argv2 string) string { + t.Helper() + argsJSON := fmt.Sprintf(`{"command":[%q,%q,%q]}`, shellBin, flag, argv2) + return readers.SerializeToolInput("shell", json.RawMessage(argsJSON)) +} + +// shellTextN serializes a Codex shell call with an arbitrary argv length. Used +// for 4+ element fixtures the reader rejects via len(Command)==3, whose +// over-match guard must still hold for every JSON-escape separator form. +func shellTextN(t *testing.T, shellBin, flag string, argvRest ...string) string { + t.Helper() + quoted := make([]string, 0, 2+len(argvRest)) + quoted = append(quoted, fmt.Sprintf("%q", shellBin), fmt.Sprintf("%q", flag)) + for _, a := range argvRest { + quoted = append(quoted, fmt.Sprintf("%q", a)) + } + argsJSON := fmt.Sprintf(`{"command":[%s]}`, strings.Join(quoted, ",")) + return readers.SerializeToolInput("shell", json.RawMessage(argsJSON)) +} + func TestV15EchoBackfillPreservesPerennialIdentity(t *testing.T) { path := createFixtureDatabase(t, "v14.sql") db, err := openWithoutSetup(path) @@ -193,6 +222,76 @@ func TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellCall(t *testing.T) { } } +// TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellSerializedWhitespace +// mirrors the reader's strings.Fields acceptance for any whitespace separator +// between 'backscroll' and 'search' inside the shell argv's third element. +// Each fixture is built via readers.SerializeToolInput from a real accepted +// argv triple, so the stored text carries the JSON-escape shape the encoder +// produces for each whitespace form (literal \t, \n, \r, \f; the GLOB \uXXXX +// class for NBSP and other unicode.IsSpace runes). The SQL must requeue +// search_echo=0 rows in every accepted shape, and must NOT requeue +// 4-element calls whose argv[2] starts with 'backscroll' followed by any of +// those separators (the over-match guard per separator form). +func TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellSerializedWhitespace(t *testing.T) { + db, err := Open(filepath.Join(t.TempDir(), "index.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + files := []IndexedFile{ + // Requeued: literal space between backscroll and search (control case). + {Source: "session", SourcePath: "sep_space.jsonl", Hash: "h1", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll search orchard"), ContentType: "tool"}, + }}, + // Requeued: JSON \t escape (tab between backscroll and search). + {Source: "session", SourcePath: "sep_tab.jsonl", Hash: "h2", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\tsearch orchard"), ContentType: "tool"}, + }}, + // Requeued: JSON \n escape. + {Source: "session", SourcePath: "sep_newline.jsonl", Hash: "h3", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\nsearch orchard"), ContentType: "tool"}, + }}, + // Requeued: JSON \r escape. + {Source: "session", SourcePath: "sep_cr.jsonl", Hash: "h4", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\rsearch orchard"), ContentType: "tool"}, + }}, + // Requeued: JSON \f escape. + {Source: "session", SourcePath: "sep_ff.jsonl", Hash: "h5", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\fsearch orchard"), ContentType: "tool"}, + }}, + // Requeued: JSON \u00a0 escape (NBSP — covered by the \uXXXX GLOB class). + {Source: "session", SourcePath: "sep_nbsp.jsonl", Hash: "h6", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\u00a0search orchard"), ContentType: "tool"}, + }}, + // Requeued: -lc flag with JSON \t escape and a shell binary path ending in 'sh'. + {Source: "session", SourcePath: "sep_tab_lc.jsonl", Hash: "h7", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellText(t, "/bin/bash", "-lc", "backscroll\tsearch orchard"), ContentType: "tool"}, + }}, + // NOT requeued: 4-element argv with tab separator + 4th element (over-match guard). + {Source: "session", SourcePath: "sep_tab_four.jsonl", Hash: "h8", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellTextN(t, "sh", "-c", "backscroll\tsearch ", "bar"), ContentType: "tool"}, + }}, + // NOT requeued: 4-element argv with NBSP separator + 4th element. + {Source: "session", SourcePath: "sep_nbsp_four.jsonl", Hash: "h9", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellTextN(t, "sh", "-c", "backscroll\u00a0search ", "bar"), ContentType: "tool"}, + }}, + } + if err := db.SyncFiles(files); err != nil { + t.Fatal(err) + } + if _, err := db.db.Exec(`UPDATE search_items SET search_echo=0`); err != nil { + t.Fatal(err) + } + got, err := db.PendingSearchEchoPaths() + if err != nil { + t.Fatal(err) + } + want := []string{"sep_cr.jsonl", "sep_ff.jsonl", "sep_nbsp.jsonl", "sep_newline.jsonl", "sep_space.jsonl", "sep_tab.jsonl", "sep_tab_lc.jsonl"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("pending=%v want %v", got, want) + } +} + func TestEchoProvenanceDoesNotAffectProse(t *testing.T) { db, err := Open(filepath.Join(t.TempDir(), "index.db")) if err != nil { From c0829757ba898109004262b076ab936cc96a64ea Mon Sep 17 00:00:00 2001 From: Pablo Ontiveros Date: Thu, 10 Sep 2026 22:36:51 -0600 Subject: [PATCH 3/4] refactor(search): move Codex shell echo check from SQL GLOB to Go-side decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 2 reviewer's finding: the round-2 SQL GLOB (enumerating \t/\n/\f/\r + a \\uXXXX char class) was correct for control chars but fundamentally unsound for every other unicode.IsSpace rune (NBSP U+00A0, en/em spaces U+2002-2003, NEL U+0085, etc) — real JSON encoders (Go's encoding/json and Rust serde_json) only escape control chars (<0x20) and emit everything else as raw UTF-8 bytes, so a SQL GLOB has provably no way to keep up with the open-ended whitespace encoding space. This round stops trying to enumerate separator byte sequences in SQL and instead decodes the JSON argv in Go, applying the EXACT predicate the reader already uses at ingest time. The two call sites now share a single source of truth so they cannot drift apart again: internal/directsearch/directsearch.go (new) IsDirectSearchCommand(command string) bool IsCodexDirectSearchCall(tool, arguments string) bool internal/readers/claude_reader.go isDirectSearchInput now calls directsearch.IsDirectSearchCommand internal/readers/codex_reader.go isCodexDirectSearchCall now wraps directsearch.IsCodexDirectSearchCall (so the in-tree tests + readers package still own the predicate call site) internal/storage/search.go unmarkedDirectSearchCallSQL drops the Codex-shell clauses (kept only bash / exec_cmd, whose text is simple key=value tokens with no JSON encoding). The shell broad-filter clause moved into PendingSearchEchoPaths as an additional OR branch. internal/storage/queries.go PendingSearchEchoPaths (via filterShellEchoZeroPaths / pendingSearchEchoShellMatches) now selects rows whose text starts with 'shell command=[' in SQL, then decodes the JSON array and calls directsearch.IsCodexDirectSearchCall('shell', ). paths are kept when ANY of their echo-zero shell rows passes the strict predicate. Regression test fixes the round-1 fixture-bug the reviewer flagged: shellText / shellTextN now build the stored text via a real json.Marshal round-trip into readers.SerializeToolInput, not fmt.Sprintf+%q — so the NBSP / en-space / em-space cases serialize as the raw UTF-8 bytes real JSON produces, not the Go strconv.Quote '\\u00a0' escape that never appears on disk. Added 3 new Unicode-whitespace fixtures (en space, em space, NBSP+'/bin/bash' -lc) and rebalanced the existing set. --- internal/directsearch/directsearch.go | 61 +++++++++++++ internal/readers/claude_reader.go | 11 +-- internal/readers/codex_reader.go | 36 ++------ internal/storage/queries.go | 125 +++++++++++++++++++++++++- internal/storage/search.go | 65 +++----------- internal/storage/search_echo_test.go | 99 ++++++++++++-------- 6 files changed, 264 insertions(+), 133 deletions(-) create mode 100644 internal/directsearch/directsearch.go diff --git a/internal/directsearch/directsearch.go b/internal/directsearch/directsearch.go new file mode 100644 index 0000000..66f5f35 --- /dev/null +++ b/internal/directsearch/directsearch.go @@ -0,0 +1,61 @@ +// Package directsearch holds the shared predicates the Codex ingest path +// and the storage replay path both rely on to decide whether a stored tool +// call is a direct `backscroll search` invocation. Keeping a single source of +// truth here is the only way the SQL "what's pending requeue?" predicate and +// the reader's "did this row get marked?" predicate can stay in lockstep — +// every prior SQL GLOB attempt to enumerate separator byte-sequences missed at +// least one real encoding (Unicode whitespace that strings.Fields accepts but +// JSON escapes in non-uniform ways), so the replay check now decodes the JSON +// argv and re-runs the exact strings.Fields-based acceptance that ingest +// already uses. +package directsearch + +import ( + "encoding/json" + "path" + "strings" +) + +// IsDirectSearchCommand is the one command boundary every reader shares: the +// raw command text must start with the bare `backscroll search` tokens. +// Absolute paths, env/shell wrappers, and other subcommands are not echoes. +// It mirrors strings.Fields splitting, so any unicode.IsSpace separator +// between the two tokens is accepted. +func IsDirectSearchCommand(command string) bool { + fields := strings.Fields(command) + return len(fields) >= 2 && fields[0] == "backscroll" && fields[1] == "search" +} + +// IsCodexDirectSearchCall recognizes Codex's own direct shell invocations of +// `backscroll search`: an `exec_command` whose raw `cmd` starts with the bare +// tokens, or a `shell` call whose argv is exactly a shell, `-c`/`-lc`, and +// that same command string. `arguments` is the JSON object the codex rollout +// stored (e.g. `{"command":["sh","-c","backscroll search"]}` or +// `{"cmd":"backscroll search"}`). +func IsCodexDirectSearchCall(tool, arguments string) bool { + switch tool { + case "exec_command": + var obj struct { + Cmd string `json:"cmd"` + } + if json.Unmarshal([]byte(arguments), &obj) != nil { + return false + } + return IsDirectSearchCommand(obj.Cmd) + case "shell": + var obj struct { + Command []string `json:"command"` + } + if json.Unmarshal([]byte(arguments), &obj) != nil || len(obj.Command) != 3 { + return false + } + if !strings.HasSuffix(path.Base(obj.Command[0]), "sh") { + return false + } + if obj.Command[1] != "-c" && obj.Command[1] != "-lc" { + return false + } + return IsDirectSearchCommand(obj.Command[2]) + } + return false +} diff --git a/internal/readers/claude_reader.go b/internal/readers/claude_reader.go index ff35b9e..7602a13 100644 --- a/internal/readers/claude_reader.go +++ b/internal/readers/claude_reader.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/pablontiv/backscroll/internal/directsearch" "github.com/pablontiv/backscroll/internal/input_config" "github.com/pablontiv/backscroll/internal/models" "github.com/pablontiv/backscroll/internal/sync" @@ -250,15 +251,7 @@ func isDirectSearchInput(tool string, input json.RawMessage) bool { if err := json.Unmarshal(input, &obj); err != nil { return false } - return isDirectSearchCommand(obj.Command) -} - -// isDirectSearchCommand is the one command boundary every reader shares: the -// raw command text must start with the bare `backscroll search` tokens. -// Absolute paths, env/shell wrappers, and other subcommands are not echoes. -func isDirectSearchCommand(command string) bool { - fields := strings.Fields(command) - return len(fields) >= 2 && fields[0] == "backscroll" && fields[1] == "search" + return directsearch.IsDirectSearchCommand(obj.Command) } func classifyText(text string) string { diff --git a/internal/readers/codex_reader.go b/internal/readers/codex_reader.go index 5848e7e..087d1e4 100644 --- a/internal/readers/codex_reader.go +++ b/internal/readers/codex_reader.go @@ -2,10 +2,10 @@ package readers import ( "encoding/json" - "path" "strings" "time" + "github.com/pablontiv/backscroll/internal/directsearch" "github.com/pablontiv/backscroll/internal/input_config" "github.com/pablontiv/backscroll/internal/models" "github.com/pablontiv/backscroll/internal/sync" @@ -125,37 +125,11 @@ func codexCallOf(item codexItem) codexCall { return codexCall{} } -// isCodexDirectSearchCall recognizes Codex's own direct shell invocations of -// `backscroll search`: an `exec_command` whose raw `cmd` starts with the bare -// tokens, or a `shell` call whose argv is exactly a shell, `-c`/`-lc`, and -// that same command string. The wrapper form is Codex-reader-local; the -// shared command boundary itself never widens. +// isCodexDirectSearchCall is now a thin wrapper around the shared predicate +// in internal/directsearch. The storage replay path uses the same function, +// so the two call sites cannot drift apart. func isCodexDirectSearchCall(tool, arguments string) bool { - switch tool { - case "exec_command": - var obj struct { - Cmd string `json:"cmd"` - } - if json.Unmarshal([]byte(arguments), &obj) != nil { - return false - } - return isDirectSearchCommand(obj.Cmd) - case "shell": - var obj struct { - Command []string `json:"command"` - } - if json.Unmarshal([]byte(arguments), &obj) != nil || len(obj.Command) != 3 { - return false - } - if !strings.HasSuffix(path.Base(obj.Command[0]), "sh") { - return false - } - if obj.Command[1] != "-c" && obj.Command[1] != "-lc" { - return false - } - return isDirectSearchCommand(obj.Command[2]) - } - return false + return directsearch.IsCodexDirectSearchCall(tool, arguments) } func codexMessage(item codexItem, ts time.Time, reasoning bool) (models.Message, bool) { diff --git a/internal/storage/queries.go b/internal/storage/queries.go index 17a1279..e8916c2 100644 --- a/internal/storage/queries.go +++ b/internal/storage/queries.go @@ -9,6 +9,7 @@ import ( "time" "github.com/pablontiv/backscroll/internal/categories" + "github.com/pablontiv/backscroll/internal/directsearch" "github.com/pablontiv/backscroll/internal/projects" "github.com/pablontiv/backscroll/internal/sequences" ) @@ -989,6 +990,16 @@ func (d *Database) StalePaths(currentVersion int) ([]string, error) { // It also requeues surviving sources whose tool rows were stored as search_echo=0 // with a serialized direct search call (pre-#80 Codex/OpenCode writes), so // identity pairing can mark the paired result. Output-only rows stay unmatched. +// +// The bash and exec_cmd cases are filtered in SQL — their stored text is the +// simple `name key=value` token form with no JSON encoding, so a GLOB prefix +// is faithful. The Codex shell case is filtered in Go (see +// filterShellEchoZeroPaths / pendingSearchEchoShellMatches): its arguments +// are JSON-encoded and the on-disk separator between 'backscroll' and +// 'search' can be any form strings.Fields accepts but the JSON encoder +// leaves in the wild (literal space, \t/\n/\f/\r, \uXXXX, or raw UTF-8 bytes +// for non-control whitespace). Trying to enumerate every byte sequence in +// SQL GLOB is provably unbounded; decoding the JSON argv in Go is not. func (d *Database) PendingSearchEchoPaths() ([]string, error) { return d.stalePaths(0, true) } @@ -1002,8 +1013,12 @@ func (d *Database) stalePaths(currentVersion int, echoOnly bool) ([]string, erro AND ((? AND (search_items.extraction_version IS NULL OR search_items.extraction_version < ?)) OR search_items.search_echo IS NULL` if echoOnly { + // Broad filter for shell candidates (tool rows whose text starts with + // `shell command=[`); the strict isCodexDirectSearchCall check happens + // in Go after we know which rows are candidates. query += ` - OR (` + unmarkedDirectSearchCallSQL("search_items") + `)` + OR (` + unmarkedDirectSearchCallSQL("search_items") + `) + OR (search_items.content_type = 'tool' AND COALESCE(search_items.search_echo, 0) = 0 AND search_items.text LIKE 'shell command=[%')` } query += `) ORDER BY indexed_files.last_indexed ASC, search_items.source_path ASC @@ -1028,9 +1043,117 @@ func (d *Database) stalePaths(currentVersion int, echoOnly bool) ([]string, erro return nil, fmt.Errorf("iterate stale paths: %w", err) } + if echoOnly { + filtered, err := d.filterShellEchoZeroPaths(paths) + if err != nil { + return nil, err + } + paths = filtered + } + return paths, nil } +// filterShellEchoZeroPaths drops paths whose only echo-zero shell candidate +// row is rejected by directsearch.IsCodexDirectSearchCall. A path survives if +// it has a v15 NULL search_echo row (those are always kept), a non-shell +// echo-zero candidate (bash / exec_cmd) that already passed the SQL filter, +// or at least one echo-zero shell row that the strict reader predicate +// accepts. +func (d *Database) filterShellEchoZeroPaths(paths []string) ([]string, error) { + if len(paths) == 0 { + return paths, nil + } + keep := make(map[string]bool, len(paths)) + for _, path := range paths { + // Reason 1: v15 NULL search_echo backlog — always keep. + var nullHit int + err := d.db.QueryRow(` + SELECT 1 FROM search_items + WHERE source_path = ? AND search_echo IS NULL + LIMIT 1 + `, path).Scan(&nullHit) + if err != nil && err != sql.ErrNoRows { + return nil, fmt.Errorf("check NULL backlog for %s: %w", path, err) + } + if nullHit == 1 { + keep[path] = true + continue + } + // Reason 2: non-shell echo-zero survivor (bash / exec_cmd GLOB). + var globHit int + err = d.db.QueryRow(` + SELECT 1 FROM search_items + WHERE source_path = ? + AND content_type = 'tool' + AND COALESCE(search_echo, 0) = 0 + AND text NOT LIKE 'shell command=[%' + AND (`+unmarkedDirectSearchCallSQL("search_items")+`) + LIMIT 1 + `, path).Scan(&globHit) + if err != nil && err != sql.ErrNoRows { + return nil, fmt.Errorf("check non-shell echo-zero for %s: %w", path, err) + } + if globHit == 1 { + keep[path] = true + continue + } + // Reason 3: echo-zero shell candidate. Apply the strict reader + // predicate — keep the path if ANY row matches. + rows, err := d.db.Query(` + SELECT text FROM search_items + WHERE source_path = ? + AND content_type = 'tool' + AND COALESCE(search_echo, 0) = 0 + AND text LIKE 'shell command=[%' + `, path) + if err != nil { + return nil, fmt.Errorf("query shell rows for %s: %w", path, err) + } + matched := false + for rows.Next() { + var text string + if err := rows.Scan(&text); err != nil { + _ = rows.Close() + return nil, fmt.Errorf("scan shell row for %s: %w", path, err) + } + if pendingSearchEchoShellMatches(text) { + matched = true + break + } + } + _ = rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate shell rows for %s: %w", path, err) + } + if matched { + keep[path] = true + } + } + out := make([]string, 0, len(keep)) + for _, p := range paths { + if keep[p] { + out = append(out, p) + } + } + return out, nil +} + +// pendingSearchEchoShellMatches is the Go-side check for whether a stored +// Codex shell tool row (text starting with `shell command=`) is a direct +// `backscroll search` call. The text after `shell command=` is the JSON array +// SerializeToolInput produced from the rollout's `arguments`; we wrap it in +// the object shape directsearch.IsCodexDirectSearchCall expects and call the +// exact same predicate the reader uses at ingest time. +func pendingSearchEchoShellMatches(text string) bool { + const prefix = "shell command=" + if !strings.HasPrefix(text, prefix) { + return false + } + args := `{"command":` + text[len(prefix):] + `}` + return directsearch.IsCodexDirectSearchCall("shell", args) +} + // ReresolveProjects iterates all distinct source_paths where project='unknown' or project IS NULL, // calls the resolver function for each path, and updates ALL rows for that path with the returned project ID. // If resolver returns empty string or "unknown", the source_path is skipped and rows remain unchanged. diff --git a/internal/storage/search.go b/internal/storage/search.go index 2daf295..d91c724 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -434,6 +434,17 @@ func directBackscrollSearchEchoSQL(alias string) string { // readers wrote false as 0, so those rows are not in the v15 NULL backlog. // Requeueing the call's source file lets identity pairing mark the result; // output shape is never matched here. +// +// Only the bash and exec_cmd cases are matched in SQL: their stored text uses +// simple `name key=value` tokens with no JSON encoding, so a GLOB prefix is +// faithful. The Codex shell case has its own predicate in Go (see +// PendingSearchEchoPaths and directsearch.IsCodexDirectSearchCall) because +// its arguments are JSON-encoded and the on-disk separator between +// 'backscroll' and 'search' can be any form strings.Fields accepts but the +// JSON encoder leaves in the wild (literal space, \t/\n/\f/\r, \uXXXX, or +// raw UTF-8 bytes for non-control whitespace). Trying to enumerate every +// byte sequence in SQL GLOB is provably unbounded; decoding the JSON argv +// in Go is not. func unmarkedDirectSearchCallSQL(alias string) string { trimmed := "ltrim(" + alias + ".text, " + asciiWhitespaceSQL + ")" sep := "'[' || " + asciiWhitespaceSQL + " || ']'" @@ -442,60 +453,8 @@ func unmarkedDirectSearchCallSQL(alias string) string { glob := func(prefix string) string { return trimmed + " GLOB (" + prefix + ") OR " + trimmed + " GLOB (" + prefix + " || " + sep + " || '*')" } - // Codex shell wrapper form: argv is exactly [, -c|-lc, "backscroll search ..."]. - // SerializeToolInput renders the JSON-encoded argv as compact JSON, so the - // separator between the 'backscroll' and 'search' tokens depends on what the - // original argv[2] looked like: a literal space (the common case), a JSON - // single-letter escape (\t, \n, \f, \r) for the matching ASCII whitespace, - // or a JSON \uXXXX escape for any other unicode.IsSpace rune (vertical tab, - // NEL, NBSP, em/en spaces, line/paragraph separators). The reader's - // isCodexDirectSearchCall uses strings.Fields on the unescaped command, so - // it accepts every separator form — the SQL must mirror that exactly, or - // the row stays at search_echo=0 forever. - shellPrefixC := "'shell' || " + sep + " || 'command=[[]*sh' || '\",\"' || '-c' || '\",\"' || 'backscroll'" - shellPrefixLC := "'shell' || " + sep + " || 'command=[[]*sh' || '\",\"' || '-lc' || '\",\"' || 'backscroll'" - // shellSeparators lists the SQL fragments that evaluate to the separator text - // between 'backscroll' and 'search' in the serialized third element. The - // single-letter escapes and the \uXXXX char class cover every JSON encoding - // of a rune that strings.Fields would split on. SQLite (via modernc.org/sqlite) - // does not process backslash escapes in string literals by default, so we build - // the JSON backslash via char(92) (92 = ASCII '\') and concatenate. - backslash := "char(92)" - shellSeparators := []string{ - sep, // literal whitespace - backslash + " || 't'", - backslash + " || 'n'", - backslash + " || 'f'", - backslash + " || 'r'", - backslash + " || 'u[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]'", - } - // shellForms returns the bare + with-args GLOB clauses for one (prefix, - // separator) pair. The trailing '"]' anchors the closing of the JSON - // string for argv[2] and the array. - shellForms := func(prefix, separator string) string { - p := prefix + " || " + separator - return trimmed + " GLOB (" + p + ` || '"]') OR ` + - trimmed + " GLOB (" + p + ` || '*"]')` - } - // shellFormNotExtra returns the NOT GLOB guard for one (prefix, separator) - // pair. Excludes 4+ argv-element shell calls (e.g. argv[2]="backscroll search ", - // argv[3]="bar") that the reader rejects via len(Command)==3, so requeuing - // them would loop forever: SQL matches every sync, reader never marks. - shellFormNotExtra := func(prefix, separator string) string { - notPattern := prefix + " || " + separator + " || '*' || '\",\"*'" - return alias + ".text NOT GLOB (" + notPattern + ")" - } - orParts := []string{glob(bash), glob(execCmd)} - notParts := make([]string, 0, len(shellSeparators)*2) - for _, separator := range shellSeparators { - orParts = append(orParts, shellForms(shellPrefixC, separator)) - orParts = append(orParts, shellForms(shellPrefixLC, separator)) - notParts = append(notParts, shellFormNotExtra(shellPrefixC, separator)) - notParts = append(notParts, shellFormNotExtra(shellPrefixLC, separator)) - } return alias + ".content_type = 'tool' AND COALESCE(" + alias + ".search_echo, 0) = 0 AND (" + - strings.Join(orParts, " OR ") + ")" + - " AND " + strings.Join(notParts, " AND ") + glob(bash) + " OR " + glob(execCmd) + ")" } // mergeRRF uses Reciprocal Rank Fusion to merge two ranked lists by position, diff --git a/internal/storage/search_echo_test.go b/internal/storage/search_echo_test.go index 9ef5558..76ac0ce 100644 --- a/internal/storage/search_echo_test.go +++ b/internal/storage/search_echo_test.go @@ -4,10 +4,8 @@ import ( "context" "database/sql" "encoding/json" - "fmt" "path/filepath" "reflect" - "strings" "testing" "github.com/pablontiv/backscroll/internal/compat" @@ -15,15 +13,25 @@ import ( "github.com/pablontiv/backscroll/internal/readers" ) -// shellText serializes a Codex shell argv triple via readers.SerializeToolInput -// so the stored text mirrors what the reader would actually persist for an -// accepted argv shape — in particular, any whitespace separator between the -// 'backscroll' and 'search' tokens gets JSON-escaped (\t, \n, \u00a0, …) by the -// encoder, which is what the SQL selection must match. +// shellText serializes a Codex shell argv triple via a real json.Marshal +// round-trip into readers.SerializeToolInput, so the stored text mirrors what +// the CodexReader actually persists for an accepted argv shape. The earlier +// fmt.Sprintf+%q fixture used Go's strconv.Quote escaping, which escapes +// NBSP (U+00A0) as `\u00a0` text — but real JSON encoders (Go's encoding/json +// and Rust serde_json) only escape control chars (<0x20) and emit other +// unicode.IsSpace runes as their raw UTF-8 bytes. Building fixtures through +// a real json.Marshal round-trip catches encoding-divergence bugs the string +// quoting cannot. func shellText(t *testing.T, shellBin, flag, argv2 string) string { t.Helper() - argsJSON := fmt.Sprintf(`{"command":[%q,%q,%q]}`, shellBin, flag, argv2) - return readers.SerializeToolInput("shell", json.RawMessage(argsJSON)) + args := struct { + Command []string `json:"command"` + }{Command: []string{shellBin, flag, argv2}} + raw, err := json.Marshal(args) + if err != nil { + t.Fatal(err) + } + return readers.SerializeToolInput("shell", raw) } // shellTextN serializes a Codex shell call with an arbitrary argv length. Used @@ -31,13 +39,15 @@ func shellText(t *testing.T, shellBin, flag, argv2 string) string { // over-match guard must still hold for every JSON-escape separator form. func shellTextN(t *testing.T, shellBin, flag string, argvRest ...string) string { t.Helper() - quoted := make([]string, 0, 2+len(argvRest)) - quoted = append(quoted, fmt.Sprintf("%q", shellBin), fmt.Sprintf("%q", flag)) - for _, a := range argvRest { - quoted = append(quoted, fmt.Sprintf("%q", a)) + argv := append([]string{shellBin, flag}, argvRest...) + args := struct { + Command []string `json:"command"` + }{Command: argv} + raw, err := json.Marshal(args) + if err != nil { + t.Fatal(err) } - argsJSON := fmt.Sprintf(`{"command":[%s]}`, strings.Join(quoted, ",")) - return readers.SerializeToolInput("shell", json.RawMessage(argsJSON)) + return readers.SerializeToolInput("shell", raw) } func TestV15EchoBackfillPreservesPerennialIdentity(t *testing.T) { @@ -223,15 +233,16 @@ func TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellCall(t *testing.T) { } // TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellSerializedWhitespace -// mirrors the reader's strings.Fields acceptance for any whitespace separator -// between 'backscroll' and 'search' inside the shell argv's third element. -// Each fixture is built via readers.SerializeToolInput from a real accepted -// argv triple, so the stored text carries the JSON-escape shape the encoder -// produces for each whitespace form (literal \t, \n, \r, \f; the GLOB \uXXXX -// class for NBSP and other unicode.IsSpace runes). The SQL must requeue -// search_echo=0 rows in every accepted shape, and must NOT requeue -// 4-element calls whose argv[2] starts with 'backscroll' followed by any of -// those separators (the over-match guard per separator form). +// walks the reader's strings.Fields acceptance for every unicode.IsSpace +// separator between 'backscroll' and 'search' inside the shell argv's third +// element. Each fixture is built via a real json.Marshal round-trip into +// readers.SerializeToolInput, so the stored text carries whatever the JSON +// encoder actually produces — control chars (U+0009/000A/000D/000C) escape +// to \t/\n/\r/\f, but every other unicode.IsSpace rune (NBSP U+00A0, NEL +// U+0085, en/em spaces U+2002-2003, …) survives as raw UTF-8 bytes that no +// SQL GLOB can keep up with. The replay check must therefore do what the +// reader does — decode the JSON argv and re-run strings.Fields — and this +// test pins that behavior down. func TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellSerializedWhitespace(t *testing.T) { db, err := Open(filepath.Join(t.TempDir(), "index.db")) if err != nil { @@ -239,42 +250,52 @@ func TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellSerializedWhitespace( } defer db.Close() files := []IndexedFile{ - // Requeued: literal space between backscroll and search (control case). + // Requeued: literal space (control case). {Source: "session", SourcePath: "sep_space.jsonl", Hash: "h1", Messages: []IndexedMessage{ {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll search orchard"), ContentType: "tool"}, }}, - // Requeued: JSON \t escape (tab between backscroll and search). + // Requeued: JSON \t escape (U+0009 < 0x20 → escapes). {Source: "session", SourcePath: "sep_tab.jsonl", Hash: "h2", Messages: []IndexedMessage{ {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\tsearch orchard"), ContentType: "tool"}, }}, - // Requeued: JSON \n escape. + // Requeued: JSON \n escape (U+000A < 0x20). {Source: "session", SourcePath: "sep_newline.jsonl", Hash: "h3", Messages: []IndexedMessage{ {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\nsearch orchard"), ContentType: "tool"}, }}, - // Requeued: JSON \r escape. + // Requeued: JSON \r escape (U+000D < 0x20). {Source: "session", SourcePath: "sep_cr.jsonl", Hash: "h4", Messages: []IndexedMessage{ {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\rsearch orchard"), ContentType: "tool"}, }}, - // Requeued: JSON \f escape. + // Requeued: JSON \f escape (U+000C < 0x20). {Source: "session", SourcePath: "sep_ff.jsonl", Hash: "h5", Messages: []IndexedMessage{ {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\fsearch orchard"), ContentType: "tool"}, }}, - // Requeued: JSON \u00a0 escape (NBSP — covered by the \uXXXX GLOB class). + // Requeued: NBSP (U+00A0) — raw UTF-8 bytes (0xc2 0xa0), no \uXXXX escape. + // This is the case the round-2 SQL GLOB missed. {Source: "session", SourcePath: "sep_nbsp.jsonl", Hash: "h6", Messages: []IndexedMessage{ {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\u00a0search orchard"), ContentType: "tool"}, }}, - // Requeued: -lc flag with JSON \t escape and a shell binary path ending in 'sh'. - {Source: "session", SourcePath: "sep_tab_lc.jsonl", Hash: "h7", Messages: []IndexedMessage{ - {Ordinal: 0, Role: "assistant", Text: shellText(t, "/bin/bash", "-lc", "backscroll\tsearch orchard"), ContentType: "tool"}, + // Requeued: en space (U+2002) — raw UTF-8 bytes (0xe2 0x80 0x82). + {Source: "session", SourcePath: "sep_en_space.jsonl", Hash: "h7", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\u2002search orchard"), ContentType: "tool"}, }}, - // NOT requeued: 4-element argv with tab separator + 4th element (over-match guard). - {Source: "session", SourcePath: "sep_tab_four.jsonl", Hash: "h8", Messages: []IndexedMessage{ - {Ordinal: 0, Role: "assistant", Text: shellTextN(t, "sh", "-c", "backscroll\tsearch ", "bar"), ContentType: "tool"}, + // Requeued: em space (U+2003) — raw UTF-8 bytes (0xe2 0x80 0x83). + {Source: "session", SourcePath: "sep_em_space.jsonl", Hash: "h8", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellText(t, "sh", "-c", "backscroll\u2003search orchard"), ContentType: "tool"}, + }}, + // Requeued: NBSP with -lc flag and /bin/bash path (cross-flag sanity). + {Source: "session", SourcePath: "sep_nbsp_lc.jsonl", Hash: "h9", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellText(t, "/bin/bash", "-lc", "backscroll\u00a0search orchard"), ContentType: "tool"}, }}, - // NOT requeued: 4-element argv with NBSP separator + 4th element. - {Source: "session", SourcePath: "sep_nbsp_four.jsonl", Hash: "h9", Messages: []IndexedMessage{ + // NOT requeued: 4-element argv with NBSP separator + 4th element + // (over-match guard holds for raw UTF-8 separators too). + {Source: "session", SourcePath: "sep_nbsp_four.jsonl", Hash: "h10", Messages: []IndexedMessage{ {Ordinal: 0, Role: "assistant", Text: shellTextN(t, "sh", "-c", "backscroll\u00a0search ", "bar"), ContentType: "tool"}, }}, + // NOT requeued: 4-element argv with tab separator + 4th element. + {Source: "session", SourcePath: "sep_tab_four.jsonl", Hash: "h11", Messages: []IndexedMessage{ + {Ordinal: 0, Role: "assistant", Text: shellTextN(t, "sh", "-c", "backscroll\tsearch ", "bar"), ContentType: "tool"}, + }}, } if err := db.SyncFiles(files); err != nil { t.Fatal(err) @@ -286,7 +307,7 @@ func TestPendingSearchEchoPathsRequeuesZeroValuedCodexShellSerializedWhitespace( if err != nil { t.Fatal(err) } - want := []string{"sep_cr.jsonl", "sep_ff.jsonl", "sep_nbsp.jsonl", "sep_newline.jsonl", "sep_space.jsonl", "sep_tab.jsonl", "sep_tab_lc.jsonl"} + want := []string{"sep_cr.jsonl", "sep_em_space.jsonl", "sep_en_space.jsonl", "sep_ff.jsonl", "sep_nbsp.jsonl", "sep_nbsp_lc.jsonl", "sep_newline.jsonl", "sep_space.jsonl", "sep_tab.jsonl"} if !reflect.DeepEqual(got, want) { t.Fatalf("pending=%v want %v", got, want) } From b2ddbf1555dad84f6cdf0e6d840d6a5c3a5997ae Mon Sep 17 00:00:00 2001 From: Pablo Ontiveros Date: Thu, 10 Sep 2026 22:58:36 -0600 Subject: [PATCH 4/4] fix(search): decode Codex shell argv JSON from any token position Round 3 reviewer identified two bugs in how the requeue path reconstructs JSON from the stored toolfmt text, both in internal/storage/queries.go: Bug A: pendingSearchEchoShellMatches built "{\"command\":" + text[len(\"shell command=\"):] + \"}\" \u2014 it assumed everything after 'command=' was exactly the JSON array. But SerializeToolInput emits ALL of a shell call's fields as sorted 'key=value' tokens. Real Codex shell calls carry workdir and/or timeout_ms and/or additional_permissions; the reader ignores them when deciding SearchEcho, but the requeue path's string-concatenation turned those trailing tokens into invalid JSON like '{\"command\":[...,"backscroll search\"] timeout_ms=10000 workdir=/tmp}' which json.Unmarshal rejects, the path is silently dropped, and a legitimate search_echo=0 echo call never requeues. Fix: decode exactly ONE JSON value starting right after 'command=' via json.NewDecoder(strings.NewReader(remainder)).Decode(&commandArray) \u2014 the decoder stops after reading one complete value and ignores whatever non-JSON key=value text follows it. We only ever need the command array, not the full original arguments object. Bug B: the broad SQL prefilter in stalePaths / filterShellEchoZeroPaths used LIKE 'shell command=[%', assuming 'command' is the first serialized key. SerializeToolInput sorts keys alphabetically, so a shell call carrying 'additional_permissions' serializes as 'shell additional_permissions={...} command=[...]' and the LIKE prefilter misses that row entirely. Fix: broaden the admission check to LIKE 'shell %' (any content_type='tool' search_echo=0 row starting with the bare shell tool-name token reaches the Go-side check, which locates the ' command=' token boundary anywhere in the sorted key=value list). Regression test TestPendingSearchEchoPathsRequeuesCodexShellRoundTrip builds each fixture via a full CodexReader.Parse round-trip of an inline rollout JSONL with extra fields (workdir, timeout_ms, additional_permissions in either position), not via SerializeToolInput directly. Covers 6 cases: control (command-only), workdir+timeout_ms after command, additional_permissions BEFORE command (the worst case), workdir BEFORE command (also sorts before), wrong command (must NOT requeue), and four-element argv (must NOT requeue, the over-match guard still holds). Also added internal/directsearch/directsearch_test.go with direct unit coverage for IsDirectSearchCommand and IsCodexDirectSearchCall (the shared predicates both readers ingest and storage replay rely on) so the predicate behavior is pinned independently of the storage and reader packages. AGENTS.md Module Layout and Package Layout updated for the new internal/directsearch package. --- AGENTS.md | 2 + internal/directsearch/directsearch_test.go | 176 +++++++++++++++++++++ internal/storage/queries.go | 52 ++++-- internal/storage/search_echo_test.go | 93 +++++++++++ 4 files changed, 308 insertions(+), 15 deletions(-) create mode 100644 internal/directsearch/directsearch_test.go diff --git a/AGENTS.md b/AGENTS.md index d834899..174a053 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,6 +55,7 @@ cmd/backscroll/ internal/ ├── config/ — config resolution: backscroll.toml → ~/.config → env → defaults ├── compat/ — stateless schema-shape inspection, release lineage catalog, migration plans, and canonical recovery planning +├── directsearch/ — shared direct-search predicates (IsDirectSearchCommand, IsCodexDirectSearchCall) so readers ingest path and storage replay path use the exact same strings.Fields / argv-shape acceptance ├── input_config/ — input manifest loading, discovery, and legacy session-dirs compatibility ├── models/ — domain types: SessionRecord, MessageContent, ParsedFile, SearchResult, Stats ├── sync/ — WalkDir, SHA-256 dedup, JSONL parsing, noise filtering, content-type classification @@ -235,6 +236,7 @@ Workflows delegate to [pablontiv/crossbeam](https://github.com/pablontiv/crossbe github.com/pablontiv/backscroll/cmd/backscroll — CLI entrypoint github.com/pablontiv/backscroll/internal/config — Config structs and resolution github.com/pablontiv/backscroll/internal/compat — Stateless schema inspection, release lineage catalog, migration planning, and canonical recovery planning +github.com/pablontiv/backscroll/internal/directsearch — Shared direct-search predicates (IsDirectSearchCommand, IsCodexDirectSearchCall) used by readers and storage replay github.com/pablontiv/backscroll/internal/input_config — Input manifest loading, discovery, and legacy session-dirs compatibility github.com/pablontiv/backscroll/internal/models — Domain types and SearchEngine interface github.com/pablontiv/backscroll/internal/sync — Session parsing and noise filtering diff --git a/internal/directsearch/directsearch_test.go b/internal/directsearch/directsearch_test.go new file mode 100644 index 0000000..6a36243 --- /dev/null +++ b/internal/directsearch/directsearch_test.go @@ -0,0 +1,176 @@ +package directsearch + +import "testing" + +func TestIsDirectSearchCommand(t *testing.T) { + tests := []struct { + name, command string + want bool + }{ + {"bare", "backscroll search", true}, + {"with_flags", "backscroll search --text orchard", true}, + {"only_two_tokens", "backscroll search", true}, + {"missing_search", "backscroll", false}, + {"missing_backscroll", "search", false}, + {"empty", "", false}, + {"whitespace_separated", "backscroll\tsearch", true}, + {"newline_separated", "backscroll\nsearch", true}, + {"nbsp_separated", "backscroll\u00a0search", true}, + {"em_space_separated", "backscroll\u2003search", true}, + {"wrapped_in_another", "env backscroll search", false}, // Fields[0]="env" not "backscroll" + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsDirectSearchCommand(tt.command); got != tt.want { + t.Errorf("IsDirectSearchCommand(%q) = %v, want %v", tt.command, got, tt.want) + } + }) + } +} + +func TestIsCodexDirectSearchCall_Shell(t *testing.T) { + tests := []struct { + name, tool, arguments string + want bool + }{ + { + name: "bare_three_element_c", + tool: "shell", + arguments: `{"command":["sh","-c","backscroll search"]}`, + want: true, + }, + { + name: "three_element_with_extra_flags", + tool: "shell", + arguments: `{"command":["sh","-c","backscroll search --text orchard"]}`, + want: true, + }, + { + name: "three_element_lc", + tool: "shell", + arguments: `{"command":["bash","-lc","backscroll search --text orchard"]}`, + want: true, + }, + { + name: "shell_binary_path", + tool: "shell", + arguments: `{"command":["/bin/bash","-c","backscroll search"]}`, + want: true, + }, + { + name: "four_elements_rejected", + tool: "shell", + arguments: `{"command":["sh","-c","backscroll search","extra"]}`, + want: false, + }, + { + name: "wrong_flag", + tool: "shell", + arguments: `{"command":["sh","-x","backscroll search"]}`, + want: false, + }, + { + name: "different_command", + tool: "shell", + arguments: `{"command":["sh","-c","ls"]}`, + want: false, + }, + { + name: "not_a_shell_binary", + tool: "shell", + arguments: `{"command":["python","-c","backscroll search"]}`, + want: false, + }, + { + name: "arguments_extra_fields_decoded", + tool: "shell", + arguments: `{"additional_permissions":{"network":false},"command":["bash","-lc","backscroll search"]}`, + want: true, + }, + { + name: "arguments_extra_fields_in_storage_position", + tool: "shell", + arguments: `{"command":["sh","-c","backscroll search"],"workdir":"/tmp","timeout_ms":10000}`, + want: true, + }, + { + name: "malformed_arguments", + tool: "shell", + arguments: `not json`, + want: false, + }, + { + name: "command_separator_tab_accepted", + tool: "shell", + arguments: `{"command":["sh","-c","backscroll\tsearch"]}`, + want: true, + }, + { + name: "command_separator_nbsp_accepted", + tool: "shell", + arguments: `{"command":["sh","-c","backscroll\u00a0search"]}`, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsCodexDirectSearchCall(tt.tool, tt.arguments); got != tt.want { + t.Errorf("IsCodexDirectSearchCall(%q, %q) = %v, want %v", tt.tool, tt.arguments, got, tt.want) + } + }) + } +} + +func TestIsCodexDirectSearchCall_ExecCommand(t *testing.T) { + tests := []struct { + name, tool, arguments string + want bool + }{ + { + name: "bare", + tool: "exec_command", + arguments: `{"cmd":"backscroll search --text orchard"}`, + want: true, + }, + { + name: "no_args", + tool: "exec_command", + arguments: `{"cmd":"backscroll search"}`, + want: true, + }, + { + name: "different_command", + tool: "exec_command", + arguments: `{"cmd":"rg orchard"}`, + want: false, + }, + { + name: "malformed_arguments", + tool: "exec_command", + arguments: `not json`, + want: false, + }, + { + name: "missing_cmd_key", + tool: "exec_command", + arguments: `{}`, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsCodexDirectSearchCall(tt.tool, tt.arguments); got != tt.want { + t.Errorf("IsCodexDirectSearchCall(%q, %q) = %v, want %v", tt.tool, tt.arguments, got, tt.want) + } + }) + } +} + +func TestIsCodexDirectSearchCall_UnknownTool(t *testing.T) { + if got := IsCodexDirectSearchCall("unknown", `{"command":["sh","-c","backscroll search"]}`); got { + t.Errorf("unknown tool should not match, got true") + } + if got := IsCodexDirectSearchCall("", `{"command":["sh","-c","backscroll search"]}`); got { + t.Errorf("empty tool should not match, got true") + } +} diff --git a/internal/storage/queries.go b/internal/storage/queries.go index e8916c2..5b23653 100644 --- a/internal/storage/queries.go +++ b/internal/storage/queries.go @@ -3,6 +3,7 @@ package storage import ( "context" "database/sql" + "encoding/json" "fmt" "sort" "strings" @@ -1013,12 +1014,16 @@ func (d *Database) stalePaths(currentVersion int, echoOnly bool) ([]string, erro AND ((? AND (search_items.extraction_version IS NULL OR search_items.extraction_version < ?)) OR search_items.search_echo IS NULL` if echoOnly { - // Broad filter for shell candidates (tool rows whose text starts with - // `shell command=[`); the strict isCodexDirectSearchCall check happens - // in Go after we know which rows are candidates. + // Broad filter for shell candidates (any tool row whose text starts + // with the bare `shell` tool-name token). The strict + // isCodexDirectSearchCall check happens in Go via + // pendingSearchEchoShellMatches, which locates the `command=` token + // boundary inside the sorted key=value list (it may not be the first + // key — e.g. `additional_permissions` sorts before `command`) and + // decodes just the JSON array that follows. query += ` OR (` + unmarkedDirectSearchCallSQL("search_items") + `) - OR (search_items.content_type = 'tool' AND COALESCE(search_items.search_echo, 0) = 0 AND search_items.text LIKE 'shell command=[%')` + OR (search_items.content_type = 'tool' AND COALESCE(search_items.search_echo, 0) = 0 AND search_items.text LIKE 'shell %')` } query += `) ORDER BY indexed_files.last_indexed ASC, search_items.source_path ASC @@ -1087,7 +1092,7 @@ func (d *Database) filterShellEchoZeroPaths(paths []string) ([]string, error) { WHERE source_path = ? AND content_type = 'tool' AND COALESCE(search_echo, 0) = 0 - AND text NOT LIKE 'shell command=[%' + AND text NOT LIKE 'shell %' AND (`+unmarkedDirectSearchCallSQL("search_items")+`) LIMIT 1 `, path).Scan(&globHit) @@ -1105,7 +1110,7 @@ func (d *Database) filterShellEchoZeroPaths(paths []string) ([]string, error) { WHERE source_path = ? AND content_type = 'tool' AND COALESCE(search_echo, 0) = 0 - AND text LIKE 'shell command=[%' + AND text LIKE 'shell %' `, path) if err != nil { return nil, fmt.Errorf("query shell rows for %s: %w", path, err) @@ -1140,18 +1145,35 @@ func (d *Database) filterShellEchoZeroPaths(paths []string) ([]string, error) { } // pendingSearchEchoShellMatches is the Go-side check for whether a stored -// Codex shell tool row (text starting with `shell command=`) is a direct -// `backscroll search` call. The text after `shell command=` is the JSON array -// SerializeToolInput produced from the rollout's `arguments`; we wrap it in -// the object shape directsearch.IsCodexDirectSearchCall expects and call the -// exact same predicate the reader uses at ingest time. +// Codex shell tool row is a direct `backscroll search` call. +// +// SerializeToolInput emits the rollout's `arguments` as a space-joined +// `key=value` token list with keys sorted alphabetically. Real Codex shell +// calls carry not just `command` but also `workdir`, `timeout_ms`, and +// potentially `additional_permissions` or `sandbox` — so `command=` is not +// guaranteed to be the first key. We locate the ` command=` token boundary +// anywhere in the text and feed only what follows to a json.Decoder, which +// stops after reading one complete JSON value (the array). Whatever +// (already-serialized, non-JSON) `key=value` text follows the array is +// ignored. The decoded array is then wrapped into the object shape +// directsearch.IsCodexDirectSearchCall expects and fed to the exact same +// predicate the reader uses at ingest time. func pendingSearchEchoShellMatches(text string) bool { - const prefix = "shell command=" - if !strings.HasPrefix(text, prefix) { + const token = " command=" + idx := strings.Index(text, token) + if idx < 0 { + return false + } + remainder := text[idx+len(token):] + var commandArray []string + if err := json.NewDecoder(strings.NewReader(remainder)).Decode(&commandArray); err != nil { + return false + } + args, err := json.Marshal(map[string][]string{"command": commandArray}) + if err != nil { return false } - args := `{"command":` + text[len(prefix):] + `}` - return directsearch.IsCodexDirectSearchCall("shell", args) + return directsearch.IsCodexDirectSearchCall("shell", string(args)) } // ReresolveProjects iterates all distinct source_paths where project='unknown' or project IS NULL, diff --git a/internal/storage/search_echo_test.go b/internal/storage/search_echo_test.go index 76ac0ce..953c16e 100644 --- a/internal/storage/search_echo_test.go +++ b/internal/storage/search_echo_test.go @@ -4,11 +4,14 @@ import ( "context" "database/sql" "encoding/json" + "os" "path/filepath" "reflect" + "strings" "testing" "github.com/pablontiv/backscroll/internal/compat" + "github.com/pablontiv/backscroll/internal/input_config" "github.com/pablontiv/backscroll/internal/models" "github.com/pablontiv/backscroll/internal/readers" ) @@ -327,3 +330,93 @@ func TestEchoProvenanceDoesNotAffectProse(t *testing.T) { t.Fatalf("prose filtered: %v %v", got, err) } } + +// TestPendingSearchEchoPathsRequeuesCodexShellRoundTrip is the regression +// fixture for the round-3 reviewer finding. Real Codex shell calls carry +// not just `command` but also `workdir`, `timeout_ms`, and potentially +// `additional_permissions` — Codex's own ShellToolCallParams schema. +// SerializeToolInput sorts the keys alphabetically, so a call with +// `additional_permissions` serializes as +// `shell additional_permissions={...} command=[...] workdir=/tmp timeout_ms=10000`, +// not `shell command=[...] workdir=/tmp timeout_ms=10000`. The requeue path +// must accept both orderings (Bug B: broaden the admission past the literal +// `shell command=` prefix and locate the `command=` token boundary inside +// the sorted list) and must read only the JSON array that follows, +// ignoring the trailing key=value tokens (Bug A: don't try to wrap the +// entire remainder as a JSON object). +// +// The fixture is generated via a full CodexReader.Parse round-trip of an +// inline rollout JSONL — not via SerializeToolInput directly, not via a +// hand-written string — so it exercises the same storage path the production +// ingest uses. +func TestPendingSearchEchoPathsRequeuesCodexShellRoundTrip(t *testing.T) { + db, err := Open(filepath.Join(t.TempDir(), "index.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + // Three shapes: command-only (control), command + workdir + timeout_ms, + // and the worst case — additional_permissions sorts before command, so + // the legacy `text LIKE 'shell command=[%'` prefilter would miss it. + cases := []struct { + name, argsJSON string + requeue bool + }{ + {"control", `{"command":["sh","-c","backscroll search --text orchard"]}`, true}, + {"workdir_timeout_ms", `{"command":["sh","-c","backscroll search"],"workdir":"/tmp","timeout_ms":10000}`, true}, + {"additional_permissions_first", `{"additional_permissions":{"network":false},"command":["bash","-lc","backscroll search --text orchard"]}`, true}, + {"workdir_first", `{"workdir":"/tmp","command":["sh","-c","backscroll search"]}`, true}, + {"wrong_command", `{"command":["sh","-c","ls"],"workdir":"/tmp","timeout_ms":10000}`, false}, + {"four_elements", `{"command":["sh","-c","backscroll search","bar"],"workdir":"/tmp"}`, false}, + } + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + jsonl := "{\"ordinal\":0,\"timestamp\":\"2026-09-01T12:00:00Z\",\"type\":\"session_meta\",\"payload\":{\"cwd\":\"/synthetic/test\"}}\n" + + "{\"ordinal\":1,\"timestamp\":\"2026-09-01T12:00:01Z\",\"type\":\"response_item\",\"payload\":{\"type\":\"function_call\",\"name\":\"shell\",\"call_id\":\"test-call\",\"arguments\":\"" + strings.ReplaceAll(tc.argsJSON, "\"", "\\\"") + "\"}}\n" + dir := t.TempDir() + path := filepath.Join(dir, "codex.jsonl") + if err := os.WriteFile(path, []byte(jsonl), 0o644); err != nil { + t.Fatal(err) + } + parsed, err := (&readers.CodexReader{}).Parse(path, input_config.InputDefinition{}) + if err != nil { + t.Fatal(err) + } + var storedText string + for _, rec := range parsed.Records { + if rec.ContentType == "tool" && strings.HasPrefix(rec.Content, "shell ") { + storedText = rec.Content + break + } + } + if storedText == "" { + t.Fatal("shell record not parsed") + } + db2, err := Open(filepath.Join(t.TempDir(), "db-"+tc.name+".db")) + if err != nil { + t.Fatal(err) + } + defer db2.Close() + sourcePath := "shell_" + tc.name + ".jsonl" + if err := db2.SyncFiles([]IndexedFile{{Source: "session", SourcePath: sourcePath, Hash: "h", Messages: []IndexedMessage{{Ordinal: 0, Role: "assistant", Text: storedText, ContentType: "tool"}}}}); err != nil { + t.Fatal(err) + } + if _, err := db2.db.Exec(`UPDATE search_items SET search_echo=0`); err != nil { + t.Fatal(err) + } + got, err := db2.PendingSearchEchoPaths() + if err != nil { + t.Fatal(err) + } + if tc.requeue { + if !reflect.DeepEqual(got, []string{sourcePath}) { + t.Errorf("case %d (%s): pending=%v want [%s]", i, tc.name, got, sourcePath) + } + } else { + if len(got) != 0 { + t.Errorf("case %d (%s): pending=%v want []", i, tc.name, got) + } + } + }) + } +}