diff --git a/changelog.d/475.fixed.bytesof-layerdir-environ.md b/changelog.d/475.fixed.bytesof-layerdir-environ.md new file mode 100644 index 00000000..a63633dc --- /dev/null +++ b/changelog.d/475.fixed.bytesof-layerdir-environ.md @@ -0,0 +1 @@ +- `rebuild-tsv.sh`'s ambiguous-vocabulary-keyword collision report fed `layerdir` to its `bytesof()` helper through `awk -v`, which decodes backslash escapes in the value it receives -- the same mistake #424 fixed for `JIT_BASE`/`tools_base`/`vocab_base` and #437 fixed for `GENERIC_WORDS_FILE`/`GENERIC_WORDS_FILES`, left untouched in this one other spot. On a backslash-bearing `CLAUDE_PROJECT_DIR`, every file in that report's layer opened against the mangled path and failed, so every entry was reported "unmeasured" instead of its real byte count. `layerdir` now travels through `ENVIRON` instead (#475). diff --git a/scripts/rebuild-tsv.sh b/scripts/rebuild-tsv.sh index 2be0caf4..1a89d742 100755 --- a/scripts/rebuild-tsv.sh +++ b/scripts/rebuild-tsv.sh @@ -1409,7 +1409,16 @@ out=$( [ -L "$layerdir" ] && continue # Dimension included, like every other layer label this script prints (#150). layer="vocabulary/$(jit_report_name "$(basename "$layerdir")")" - LC_ALL=C awk -F'\t' -v layerdir="$layerdir" -v layer="$layer" "$JIT_AWK_REPORT_NAME"' + # #475: layerdir used to travel in via `-v layerdir=...`, and -v PROCESSES backslash + # escapes in the value it receives -- the exact mistake #424 fixed for JIT_BASE/ + # tools_base/vocab_base and #437 fixed for GENERIC_WORDS_FILE/GENERIC_WORDS_FILES, in + # this same file. On a backslash-bearing CLAUDE_PROJECT_DIR, bytesof()'s getline + # against the mangled path silently failed to open and every entry in that layer + # reported "unmeasured" instead of its real byte count. layerdir now travels through + # ENVIRON instead, the same route GENERIC_WORDS_FILES already uses a few hundred + # lines below -- never -v, since ENVIRON hands the awk program the bytes bash built, + # undecoded. + layerdir="$layerdir" LC_ALL=C awk -F'\t' -v layer="$layer" "$JIT_AWK_REPORT_NAME"' function bytesof(path, b, line, rc, first) { if (path in bcache) return bcache[path] b = 0; first = 1 @@ -1426,7 +1435,7 @@ out=$( return b } $1 != "" && $2 != "" { - printf "%s\t%d\t%s[%s]\n", $1, bytesof(layerdir "/" $2), jit_report_name($2), layer + printf "%s\t%d\t%s[%s]\n", $1, bytesof(ENVIRON["layerdir"] "/" $2), jit_report_name($2), layer } ' "$tsv" done | LC_ALL=C awk -F'\t' -v floor="$COLLISION_BYTES_FLOOR" "$JIT_AWK_REPORT_KEYWORD"' diff --git a/tests/test-entry-bytes.sh b/tests/test-entry-bytes.sh index 9af1155a..a1e484db 100644 --- a/tests/test-entry-bytes.sh +++ b/tests/test-entry-bytes.sh @@ -418,6 +418,37 @@ for ENG in $ENGINES; do rm -f "$ERR" rm -rf "$PROJ" + # ============================================================================ + # 475 - layerdir fed to bytesof() via -v, mangled by a backslash-bearing path + # ============================================================================ + # Same defect class as #424/#437 elsewhere in this file: a CLAUDE_PROJECT_DIR-derived + # path handed to awk via -v has its escapes DECODED, so a literal backslash in the + # path mangles it before bytesof()'s getline ever opens the file -- every entry in + # that layer is then reported unmeasured instead of its real byte count. Positive + # control first, inside the SAME fixture: the collision report must still fire on the + # shared keyword (so the absence of the bad note below is not just a report that never + # ran), and only the backslash case may show the "could not be measured" note. + echo "" + echo "=== 475 [$ENG]: a backslash-bearing vocabulary layer path still gets real byte counts ===" + ROOT475=$(mktemp -d) + BACKSLASH475="$(printf '\\')" + PROJ="$ROOT475/x${BACKSLASH475}zz475" + mkdir -p "$PROJ/.claude/jit-context/vocabulary/00-manual" 2> /dev/null + if [ -d "$PROJ/.claude/jit-context/vocabulary/00-manual" ]; then + V="$PROJ/.claude/jit-context/vocabulary/00-manual" + printf -- '---\ntitle: T475a\nkeywords: dpt475\n---\nfirst entry body for 475\n' > "$V/a475.md" + printf -- '---\ntitle: T475b\nkeywords: dpt475\n---\nsecond entry body for 475\n' > "$V/b475.md" + ERR475=$(mktemp) + PATH="$ENGINE_BIN/$ENG:$PATH" CLAUDE_PROJECT_DIR="$PROJ" JIT_CONTEXT_COLLISION_BYTES=1 LC_ALL=C \ + bash "$SCRIPTS/rebuild-tsv.sh" > /dev/null 2> "$ERR475" + assert_has "the collision report still fires on the shared keyword" "$ERR475" "dpt475" + assert_lacks "every entry in the backslash-bearing layer is measured, not reported unmeasured (#475)" "$ERR475" "could not be measured" + rm -f "$ERR475" + else + echo " SKIPPED: could not create a directory containing a literal backslash on this filesystem." + fi + rm -rf "$ROOT475" + # ============================================================================ # 156 - jit_clip() rewrote values nobody asked it to rewrite # ============================================================================ diff --git a/trap.d/424.rebuild-tsv-v-escape.md b/trap.d/424.rebuild-tsv-v-escape.md deleted file mode 100644 index 67ea104b..00000000 --- a/trap.d/424.rebuild-tsv-v-escape.md +++ /dev/null @@ -1,31 +0,0 @@ -Same defect class as #424 (a CLAUDE_PROJECT_DIR-derived path passed to awk via -v, -silently escape-decoded), found by Explore's self-review of that PR in -scripts/rebuild-tsv.sh, in two more places, neither touched by #424: - -1. The ambiguous-vocabulary-keyword collision report builds `layerdir` (a glob under - `VOCAB_BASE="$JIT_BASE/vocabulary"`, itself CLAUDE_PROJECT_DIR-relative) and hands - it to awk via `-v layerdir=...`. The awk function `bytesof()` then does - `getline line < (layerdir "/" $2)` using the -v-decoded (potentially mangled) path. - On a backslash-bearing CLAUDE_PROJECT_DIR, every open fails and every entry in that - layer is reported as "unmeasured" instead of its real byte count -- a reporting-only - break in a tool a maintainer runs by hand, not a hook that runs in a stranger's - session. - -2. The generic-word classifier pass builds `-v wf="$GENERIC_WORDS_FILE"` (also - CLAUDE_PROJECT_DIR-relative or override-derived) and reads it with - `getline w < wf` inside the awk BEGIN block. A mangled `wf` leaves the `seen[]` - lookup table empty, so every keyword silently classifies as non-generic -- - this still returns one verdict per keyword (so it passes rebuild-tsv.sh's own - count-mismatch guard undetected), just the wrong verdict for all of them. - -Fix shape: same as jit-misses.sh's own fragment (#424.jit-misses-generic-words-v-escape.md) --- route layerdir/wf through ENVIRON, matching the technique #424 already used for -JIT_BASE/tools_base/vocab_base. - -Not fixed as part of #424 itself: a third file, a maintainer-only tool (not a -per-session hook), and each needs its own fixture to pin (a real byte-count -divergence for #1, a keyword-classification divergence for #2). - -Source: Explore's self-review of PR fixing #423/#424, scripts/rebuild-tsv.sh (grepped -around VOCAB_BASE/layerdir/bytesof and GENERIC_WORDS_FILE/wf -- not cited by line -number, see this repo's own line-citation rule).