From 2c026f5a8d4eebe5e46be2233e3867c1eb2e8d5b Mon Sep 17 00:00:00 2001 From: Florian DAVID Date: Tue, 6 Oct 2026 15:05:51 +0200 Subject: [PATCH 1/2] fix: route layerdir through ENVIRON in bytesof(), not -v (#475) The ambiguous-vocabulary-keyword collision report in rebuild-tsv.sh built layerdir in bash and passed it to awk via `-v layerdir=...`. awk's -v decodes backslash escapes in the value it receives -- the same mistake #424 fixed for JIT_BASE/tools_base/vocab_base and #437 fixed for GENERIC_WORDS_FILE/GENERIC_WORDS_FILES, left untouched at this one other site. On a backslash-bearing CLAUDE_PROJECT_DIR, bytesof()'s getline against the mangled path silently failed to open, so every entry in that layer reported "unmeasured" instead of its real byte count. layerdir now travels through ENVIRON instead, matching the route GENERIC_WORDS_FILES already uses in this same file. Co-Authored-By: Claude Sonnet 5 --- .../475.fixed.bytesof-layerdir-environ.md | 1 + scripts/rebuild-tsv.sh | 13 ++++++-- tests/test-entry-bytes.sh | 31 +++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 changelog.d/475.fixed.bytesof-layerdir-environ.md diff --git a/changelog.d/475.fixed.bytesof-layerdir-environ.md b/changelog.d/475.fixed.bytesof-layerdir-environ.md new file mode 100644 index 00000000..a63633dc --- /dev/null +++ b/changelog.d/475.fixed.bytesof-layerdir-environ.md @@ -0,0 +1 @@ +- `rebuild-tsv.sh`'s ambiguous-vocabulary-keyword collision report fed `layerdir` to its `bytesof()` helper through `awk -v`, which decodes backslash escapes in the value it receives -- the same mistake #424 fixed for `JIT_BASE`/`tools_base`/`vocab_base` and #437 fixed for `GENERIC_WORDS_FILE`/`GENERIC_WORDS_FILES`, left untouched in this one other spot. On a backslash-bearing `CLAUDE_PROJECT_DIR`, every file in that report's layer opened against the mangled path and failed, so every entry was reported "unmeasured" instead of its real byte count. `layerdir` now travels through `ENVIRON` instead (#475). diff --git a/scripts/rebuild-tsv.sh b/scripts/rebuild-tsv.sh index 2be0caf4..1a89d742 100755 --- a/scripts/rebuild-tsv.sh +++ b/scripts/rebuild-tsv.sh @@ -1409,7 +1409,16 @@ out=$( [ -L "$layerdir" ] && continue # Dimension included, like every other layer label this script prints (#150). layer="vocabulary/$(jit_report_name "$(basename "$layerdir")")" - LC_ALL=C awk -F'\t' -v layerdir="$layerdir" -v layer="$layer" "$JIT_AWK_REPORT_NAME"' + # #475: layerdir used to travel in via `-v layerdir=...`, and -v PROCESSES backslash + # escapes in the value it receives -- the exact mistake #424 fixed for JIT_BASE/ + # tools_base/vocab_base and #437 fixed for GENERIC_WORDS_FILE/GENERIC_WORDS_FILES, in + # this same file. On a backslash-bearing CLAUDE_PROJECT_DIR, bytesof()'s getline + # against the mangled path silently failed to open and every entry in that layer + # reported "unmeasured" instead of its real byte count. layerdir now travels through + # ENVIRON instead, the same route GENERIC_WORDS_FILES already uses a few hundred + # lines below -- never -v, since ENVIRON hands the awk program the bytes bash built, + # undecoded. + layerdir="$layerdir" LC_ALL=C awk -F'\t' -v layer="$layer" "$JIT_AWK_REPORT_NAME"' function bytesof(path, b, line, rc, first) { if (path in bcache) return bcache[path] b = 0; first = 1 @@ -1426,7 +1435,7 @@ out=$( return b } $1 != "" && $2 != "" { - printf "%s\t%d\t%s[%s]\n", $1, bytesof(layerdir "/" $2), jit_report_name($2), layer + printf "%s\t%d\t%s[%s]\n", $1, bytesof(ENVIRON["layerdir"] "/" $2), jit_report_name($2), layer } ' "$tsv" done | LC_ALL=C awk -F'\t' -v floor="$COLLISION_BYTES_FLOOR" "$JIT_AWK_REPORT_KEYWORD"' diff --git a/tests/test-entry-bytes.sh b/tests/test-entry-bytes.sh index 9af1155a..a1e484db 100644 --- a/tests/test-entry-bytes.sh +++ b/tests/test-entry-bytes.sh @@ -418,6 +418,37 @@ for ENG in $ENGINES; do rm -f "$ERR" rm -rf "$PROJ" + # ============================================================================ + # 475 - layerdir fed to bytesof() via -v, mangled by a backslash-bearing path + # ============================================================================ + # Same defect class as #424/#437 elsewhere in this file: a CLAUDE_PROJECT_DIR-derived + # path handed to awk via -v has its escapes DECODED, so a literal backslash in the + # path mangles it before bytesof()'s getline ever opens the file -- every entry in + # that layer is then reported unmeasured instead of its real byte count. Positive + # control first, inside the SAME fixture: the collision report must still fire on the + # shared keyword (so the absence of the bad note below is not just a report that never + # ran), and only the backslash case may show the "could not be measured" note. + echo "" + echo "=== 475 [$ENG]: a backslash-bearing vocabulary layer path still gets real byte counts ===" + ROOT475=$(mktemp -d) + BACKSLASH475="$(printf '\\')" + PROJ="$ROOT475/x${BACKSLASH475}zz475" + mkdir -p "$PROJ/.claude/jit-context/vocabulary/00-manual" 2> /dev/null + if [ -d "$PROJ/.claude/jit-context/vocabulary/00-manual" ]; then + V="$PROJ/.claude/jit-context/vocabulary/00-manual" + printf -- '---\ntitle: T475a\nkeywords: dpt475\n---\nfirst entry body for 475\n' > "$V/a475.md" + printf -- '---\ntitle: T475b\nkeywords: dpt475\n---\nsecond entry body for 475\n' > "$V/b475.md" + ERR475=$(mktemp) + PATH="$ENGINE_BIN/$ENG:$PATH" CLAUDE_PROJECT_DIR="$PROJ" JIT_CONTEXT_COLLISION_BYTES=1 LC_ALL=C \ + bash "$SCRIPTS/rebuild-tsv.sh" > /dev/null 2> "$ERR475" + assert_has "the collision report still fires on the shared keyword" "$ERR475" "dpt475" + assert_lacks "every entry in the backslash-bearing layer is measured, not reported unmeasured (#475)" "$ERR475" "could not be measured" + rm -f "$ERR475" + else + echo " SKIPPED: could not create a directory containing a literal backslash on this filesystem." + fi + rm -rf "$ROOT475" + # ============================================================================ # 156 - jit_clip() rewrote values nobody asked it to rewrite # ============================================================================ From 90f207346d2be43dcc3f3f2c423efffc2e152eb1 Mon Sep 17 00:00:00 2001 From: Florian DAVID Date: Tue, 6 Oct 2026 15:36:07 +0200 Subject: [PATCH 2/2] chore: retire trap.d/424.rebuild-tsv-v-escape.md, both items now fixed (#475) Item 1 (layerdir/bytesof()) is the bug this branch's previous commit fixed. Item 2 (GENERIC_WORDS_FILE/wf in the generic-word classifier's BEGIN block) was already fixed before this branch started -- it reads ENVIRON["GENERIC_WORDS_FILES"] at HEAD, not -v wf=.... Caught by self-review (Explore spawn) of this branch's own diff: the fragment still described item 1 as open after the fix landed. Co-Authored-By: Claude Sonnet 5 --- trap.d/424.rebuild-tsv-v-escape.md | 31 ------------------------------ 1 file changed, 31 deletions(-) delete mode 100644 trap.d/424.rebuild-tsv-v-escape.md diff --git a/trap.d/424.rebuild-tsv-v-escape.md b/trap.d/424.rebuild-tsv-v-escape.md deleted file mode 100644 index 67ea104b..00000000 --- a/trap.d/424.rebuild-tsv-v-escape.md +++ /dev/null @@ -1,31 +0,0 @@ -Same defect class as #424 (a CLAUDE_PROJECT_DIR-derived path passed to awk via -v, -silently escape-decoded), found by Explore's self-review of that PR in -scripts/rebuild-tsv.sh, in two more places, neither touched by #424: - -1. The ambiguous-vocabulary-keyword collision report builds `layerdir` (a glob under - `VOCAB_BASE="$JIT_BASE/vocabulary"`, itself CLAUDE_PROJECT_DIR-relative) and hands - it to awk via `-v layerdir=...`. The awk function `bytesof()` then does - `getline line < (layerdir "/" $2)` using the -v-decoded (potentially mangled) path. - On a backslash-bearing CLAUDE_PROJECT_DIR, every open fails and every entry in that - layer is reported as "unmeasured" instead of its real byte count -- a reporting-only - break in a tool a maintainer runs by hand, not a hook that runs in a stranger's - session. - -2. The generic-word classifier pass builds `-v wf="$GENERIC_WORDS_FILE"` (also - CLAUDE_PROJECT_DIR-relative or override-derived) and reads it with - `getline w < wf` inside the awk BEGIN block. A mangled `wf` leaves the `seen[]` - lookup table empty, so every keyword silently classifies as non-generic -- - this still returns one verdict per keyword (so it passes rebuild-tsv.sh's own - count-mismatch guard undetected), just the wrong verdict for all of them. - -Fix shape: same as jit-misses.sh's own fragment (#424.jit-misses-generic-words-v-escape.md) --- route layerdir/wf through ENVIRON, matching the technique #424 already used for -JIT_BASE/tools_base/vocab_base. - -Not fixed as part of #424 itself: a third file, a maintainer-only tool (not a -per-session hook), and each needs its own fixture to pin (a real byte-count -divergence for #1, a keyword-classification divergence for #2). - -Source: Explore's self-review of PR fixing #423/#424, scripts/rebuild-tsv.sh (grepped -around VOCAB_BASE/layerdir/bytesof and GENERIC_WORDS_FILE/wf -- not cited by line -number, see this repo's own line-citation rule).