From 36e275c5030f8d4596963448e09eeaf4069b4dd3 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:08:37 +0100 Subject: [PATCH] feat(githooks): docstring scanner leg B, date-gated touched ratio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds leg B to `.githooks/docstring-scan.sh --check`: documented / (documented + undocumented) over ALL touched (added + modified, never skipped) functions below DOCSTRING_THRESHOLD (default 80) is a violation — the question CodeRabbit's Docstring Coverage check asks. Leg A (an added undocumented function blocks) is unchanged. Phase-in uses the self-flipping gate shape of scripts/check-docs-presence.sh: ENFORCE_DOCSTRINGS_FROM (default 2026-11-01, owner ruling 2026-10-01) and a DOCS_TODAY seam. Before the cutoff a leg-B violation is a WARN line on stderr and leaves the exit code alone; on/after it, --check exits 1. Malformed dates or threshold exit 2, so a bad value cannot silently disarm the gate. Zero touched functions give no leg-B verdict, said explicitly. The verdict is decided in integers (d*100 < T*n), so exactly the threshold passes. SUMMARY gains `threshold=T legb=pass|fail|n/a`, appended at the end only; the one known consumer (the ~/.claude docstring Stop hook) drops SUMMARY and reads only added/undocumented rows, and runs without --check. Tests: leg-A assertions now run with leg B disarmed (threshold 0) so they do not flip on the cutoff date; new leg-B fixtures cover before/on/after the cutoff, exactly 80%, 79.17%, zero touched, skipped-in-either-direction, and malformed configuration. Two known-answer controls (fetched by full SHA, unfetchable = FAILURE) match CodeRabbit's verdicts: panll#136 and nextgen-databases#107 both measure 2 of 22 touched functions documented. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Hw7qg3u9PAP6b2oKyVTSVC --- .githooks/docstring-scan.sh | 86 ++++++++++++++++-- scripts/tests/docstring-scan-test.sh | 126 ++++++++++++++++++++++++++- 2 files changed, 200 insertions(+), 12 deletions(-) diff --git a/.githooks/docstring-scan.sh b/.githooks/docstring-scan.sh index c978548e0..4061334d2 100755 --- a/.githooks/docstring-scan.sh +++ b/.githooks/docstring-scan.sh @@ -14,17 +14,32 @@ # docstring-scan.sh --worktree uncommitted work vs HEAD (untracked files count as added) # docstring-scan.sh --staged the index vs HEAD (pre-commit) # docstring-scan.sh --range BASE..HEAD a commit range (CI, calibration) -# add --check to exit 1 when a NEWLY-ADDED function is undocumented. +# add --check to enforce two legs: +# leg A exit 1 when a NEWLY-ADDED function is undocumented (always enforced). +# leg B exit 1 when documented / (documented + undocumented) over ALL touched (added and +# modified, never skipped) functions is below DOCSTRING_THRESHOLD percent — the ratio +# CodeRabbit's Docstring Coverage check asks about. Phased in by a self-flipping date: +# before ENFORCE_DOCSTRINGS_FROM a leg-B violation is a WARN line on stderr and leaves +# the exit code alone; ON and after that date it exits 1. Zero touched functions give +# NO leg-B verdict (said explicitly on stderr): a vacuous ratio is neither pass nor fail. +# +# Environment (the shipped policy is the default in each case): +# DOCSTRING_THRESHOLD integer 0-100, default 80 (leg B passes at exactly the threshold). +# ENFORCE_DOCSTRINGS_FROM YYYY-MM-DD, default 2026-11-01; leg B blocks ON this date. +# DOCS_TODAY YYYY-MM-DD; overrides "now" so both sides of the cutoff are testable. +# A malformed value in any of them is exit 2: it must never silently disarm the gate. # # Output (stdout): one TSV row per touched function, then one SUMMARY line. # pathlinesymboladded|modifieddocumented|undocumented # path---skipped (a changed source file in an unsupported language) # SUMMARY files=N functions=N documented=N undocumented=N added_undocumented=N skipped=N coverage=P% +# threshold=T legb=pass|fail|n/a (one line; fields are only ever appended) # The denominator is always printed: "0 undocumented" out of 0 functions is a vacuous pass and must # not read the same as a real one. # -# Exit: 0 report produced (and, with --check, no added undocumented function); 1 --check found an -# added undocumented function; 2 usage or git error. +# Exit: 0 report produced (and, with --check, no enforced leg failed); 1 --check found an added +# undocumented function (leg A) or, on/after the cutoff, a touched ratio below threshold (leg B); +# 2 usage, configuration or git error. # # Tier 1 (this version): shell. Every other source extension reports as SKIPPED — never as # documented. Documentation files (.adoc/.md/.rst/.txt-less docs) are ignored outright, as @@ -39,12 +54,41 @@ while [ $# -gt 0 ]; do --staged) MODE=staged ;; --range) MODE=range; RANGE="${2:-}"; shift ;; --check) CHECK=1 ;; - -h|--help) sed -n '2,31p' "$0"; exit 0 ;; + -h|--help) sed -n '2,46p' "$0"; exit 0 ;; *) printf 'docstring-scan: unknown argument: %s\n' "$1" >&2; exit 2 ;; esac shift done [ -n "$MODE" ] || { printf 'docstring-scan: one of --worktree, --staged, --range BASE..HEAD is required\n' >&2; exit 2; } + +# Leg B policy (owner ruling 2026-10-01). The cutoff is a real date that flips itself: on it, leg B +# starts failing --check with no further edit to this file. Gate shape copied from +# scripts/check-docs-presence.sh. +DOCSTRING_THRESHOLD="${DOCSTRING_THRESHOLD:-80}" +ENFORCE_DOCSTRINGS_FROM="${ENFORCE_DOCSTRINGS_FROM:-2026-11-01}" +TODAY="${DOCS_TODAY:-$(date -u +%Y-%m-%d)}" + +# A malformed date would make the lexicographic comparison below silently choose the grace branch +# forever, turning leg B into a fake gate. Refuse to run rather than run un-armed. +# Succeed when the argument has the YYYY-MM-DD shape. +valid_date() { + case "$1" in + [0-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9]) return 0 ;; + *) return 1 ;; + esac +} +# Exit 2 unless the named setting holds a YYYY-MM-DD date. +require_date() { + if ! valid_date "$2"; then + printf "docstring-scan: %s='%s' is not YYYY-MM-DD; refusing to run: an unparseable cutoff would silently disarm leg B\n" "$1" "$2" >&2 + exit 2 + fi +} +require_date ENFORCE_DOCSTRINGS_FROM "$ENFORCE_DOCSTRINGS_FROM" +require_date DOCS_TODAY "$TODAY" +# A strict pattern, not arithmetic: "080" is an octal error in bash and "1e2" is not a number. +[[ "$DOCSTRING_THRESHOLD" =~ ^(0|[1-9][0-9]?|100)$ ]] || { + printf "docstring-scan: DOCSTRING_THRESHOLD='%s' is not an integer 0-100\n" "$DOCSTRING_THRESHOLD" >&2; exit 2; } git rev-parse --git-dir >/dev/null 2>&1 || { printf 'docstring-scan: not inside a git repository\n' >&2; exit 2; } G=(git -c core.quotePath=false) @@ -214,8 +258,34 @@ if [ "$functions" -gt 0 ]; then else cov="n/a" fi -printf 'SUMMARY files=%d functions=%d documented=%d undocumented=%d added_undocumented=%d skipped=%d coverage=%s%%\n' \ - "$files" "$functions" "$documented" "$undocumented" "$added_undoc" "$skipped" "$cov" -[ "$CHECK" = 1 ] && [ "$added_undoc" -gt 0 ] && exit 1 -exit 0 +# Leg B verdict, in integers so exactly-the-threshold is unambiguous: fail iff d/n < T/100. +# The denominator is touched documented + undocumented functions; skipped files contribute none. +if [ "$functions" -eq 0 ]; then + legb="n/a" +elif [ $((documented * 100)) -lt $((DOCSTRING_THRESHOLD * functions)) ]; then + legb=fail +else + legb=pass +fi +printf 'SUMMARY files=%d functions=%d documented=%d undocumented=%d added_undocumented=%d skipped=%d coverage=%s%% threshold=%d legb=%s\n' \ + "$files" "$functions" "$documented" "$undocumented" "$added_undoc" "$skipped" "$cov" "$DOCSTRING_THRESHOLD" "$legb" + +[ "$CHECK" = 1 ] || exit 0 +rc=0 +[ "$added_undoc" -gt 0 ] && rc=1 +case "$legb" in + n/a) + printf 'docstring-scan: leg B: no verdict: 0 touched functions (a vacuous ratio is neither a pass nor a fail)\n' >&2 ;; + fail) + # String comparison is sound: both operands are YYYY-MM-DD, format-validated above. + if [[ "$TODAY" < "$ENFORCE_DOCSTRINGS_FROM" ]]; then + printf 'docstring-scan: WARN leg B: %d of %d touched functions documented (%s%%) is below %d%%; NOT YET ENFORCED: this becomes a blocking failure on %s (today is %s)\n' \ + "$documented" "$functions" "$cov" "$DOCSTRING_THRESHOLD" "$ENFORCE_DOCSTRINGS_FROM" "$TODAY" >&2 + else + printf 'docstring-scan: FAIL leg B: %d of %d touched functions documented (%s%%) is below %d%% (enforced since %s)\n' \ + "$documented" "$functions" "$cov" "$DOCSTRING_THRESHOLD" "$ENFORCE_DOCSTRINGS_FROM" >&2 + rc=1 + fi ;; +esac +exit "$rc" diff --git a/scripts/tests/docstring-scan-test.sh b/scripts/tests/docstring-scan-test.sh index 5f5232521..97965b048 100755 --- a/scripts/tests/docstring-scan-test.sh +++ b/scripts/tests/docstring-scan-test.sh @@ -48,6 +48,39 @@ EOF # Run the scanner in the current repository. scan() { bash "$SCANNER" "$@" 2>&1; } +# Run the scanner with leg B disarmed (threshold 0), so a leg-A assertion holds on either side of +# the leg-B cutoff instead of flipping red on ENFORCE_DOCSTRINGS_FROM. +scan_a() { DOCSTRING_THRESHOLD=0 scan "$@"; } + +# Run the scanner with --check under a pinned "today" and print its exit code. +rc_on() { local today="$1" r=0; shift; DOCS_TODAY="$today" bash "$SCANNER" "$@" --check >/dev/null 2>&1 || r=$?; echo "$r"; } + +# Write n documented and m undocumented shell functions to the named file. +mkfns() { + local file="$1" n="$2" m="$3" i + : > "$file" + for ((i = 0; i < n; i++)); do printf '# Documented %d.\nd%d() { :; }\n' "$i" "$i" >> "$file"; done + for ((i = 0; i < m; i++)); do printf 'u%d() { :; }\n' "$i" >> "$file"; done +} + +# Known-answer control: scan a range of another public estate repo and compare with the +# CodeRabbit Docstring Coverage verdict measured at the head CodeRabbit actually reviewed. +# Both commits are fetched by full SHA into a throwaway repo (so --depth=1 harms nothing); +# an unfetchable commit is a FAILURE, never a skip. +remote_control() { + local label="$1" repo="$2" base="$3" head="$4" want_fn="$5" want_doc="$6" d="$WORK/ctl-$1" o + rm -rf "$d"; git init -q "$d" && git -C "$d" config core.hooksPath /dev/null + git -C "$d" fetch --quiet --no-tags --depth=1 "https://github.com/$repo" "$base" "$head" 2>/dev/null || true + if git -C "$d" cat-file -e "$base^{commit}" 2>/dev/null && git -C "$d" cat-file -e "$head^{commit}" 2>/dev/null; then + o="$(cd "$d" && scan --range "$base..$head")" + check "$label functions" "$want_fn" "$(field "$o" functions)" + check "$label documented" "$want_doc" "$(field "$o" documented)" + check "$label leg B fails" fail "$(field "$o" legb)" + else + # A skip is not a pass: an unfetchable calibration commit must not report the known-answer control as green. + bad "$label commits present (fetch by SHA from $repo failed)" "${base:0:12}..${head:0:12} reachable" "absent" + fi +} echo "== calibration: standards PR #1034 at 1cc72cdc80c9 (CodeRabbit: 13 functions / 2 files / 0.00% / 3 skipped)" # The calibration commit is PR #1034's PRE-SQUASH head: it lives only under refs/pull/1034/head, @@ -68,6 +101,12 @@ else bad "calibration commit present (fetch by SHA from refs/pull/1034/head failed)" "1cc72cdc80c9 reachable" "absent" fi +echo "== calibration: leg-B ratio against CodeRabbit (2 of 22 touched functions documented, both)" +remote_control "panll#136" hyperpolymath/panll \ + 964f9563070129b2860b9e9e627a28b6dca05b02 61141fb353225ceacb42fd502f21bfd0b0074893 22 2 +remote_control "nextgen-databases#107" hyperpolymath/nextgen-databases \ + fc12a2eb856ff2bed2b91f6df38dbac5d6663257 e12e4c71855c1314f528b7fdf833786c6c06630a 22 2 + echo "== planted positive: a new undocumented function blocks under --check" newrepo cat > new.sh <<'EOF' @@ -76,7 +115,7 @@ cat > new.sh <<'EOF' good() { :; } undoc() { :; } EOF -out="$(scan --worktree)"; rc=0; scan --worktree --check >/dev/null || rc=$? +out="$(scan --worktree)"; rc=0; scan_a --worktree --check >/dev/null || rc=$? check "untracked new file is scanned" 2 "$(field "$out" functions)" check "good is documented" added/documented "$(row "$out" good)" check "undoc is undocumented" added/undocumented "$(row "$out" undoc)" @@ -85,7 +124,7 @@ check "--check exits 1" 1 "$rc" echo "== negative control: touching only a documented function is silent" newrepo sed -i 's/echo hello/echo hello world/' base.sh -out="$(scan --worktree)"; rc=0; scan --worktree --check >/dev/null || rc=$? +out="$(scan --worktree)"; rc=0; scan_a --worktree --check >/dev/null || rc=$? check "one touched function" 1 "$(field "$out" functions)" check "undocumented is zero" 0 "$(field "$out" undocumented)" check "--check exits 0" 0 "$rc" @@ -93,12 +132,12 @@ check "--check exits 0" 0 "$rc" echo "== added vs modified: editing an undocumented legacy body reports but does not block" newrepo sed -i 's/echo old/echo older/' base.sh -out="$(scan --worktree)"; rc=0; scan --worktree --check >/dev/null || rc=$? +out="$(scan --worktree)"; rc=0; scan_a --worktree --check >/dev/null || rc=$? check "legacy is modified/undocumented" modified/undocumented "$(row "$out" legacy)" check "untouched greet is not reported" "" "$(row "$out" greet)" check "--check exits 0 on modified-only" 0 "$rc" printf 'fresh() {\n :\n}\n' >> base.sh -rc=0; scan --worktree --check >/dev/null || rc=$? +rc=0; scan_a --worktree --check >/dev/null || rc=$? check "adding an undocumented function beside it blocks" 1 "$rc" echo "== predicate edges" @@ -162,6 +201,85 @@ out="$(scan --worktree)" check "path with spaces is scanned" "added/undocumented" "$(row "$out" x)" check "non-ASCII path is scanned" "added/documented" "$(row "$out" y)" +# Create a repo whose change MODIFIES n documented and m undocumented functions (none added), +# so leg A stays silent and only leg B can decide the exit code. +legb_repo() { + newrepo + mkfns t.sh "$1" "$2" + git add t.sh && git commit -qm fns + sed -i 's/{ :; }/{ : touched; }/' t.sh +} + +echo "== leg B: touched-function ratio, phased in by a self-flipping date" +legb_repo 1 1 +out="$(scan --worktree)" +check "1 of 2 touched documented is a leg-B fail" fail "$(field "$out" legb)" +check "no added function, so leg A is silent" 0 "$(field "$out" added_undocumented)" +check "SUMMARY carries the threshold" 80 "$(field "$out" threshold)" +check "before the cutoff a violation does not block" 0 "$(rc_on 2026-10-31 --worktree)" +warn="$(DOCS_TODAY=2026-10-31 bash "$SCANNER" --worktree --check 2>&1 >/dev/null)" +case "$warn" in *"WARN leg B"*"2026-11-01"*) ok "before the cutoff it WARNs on stderr naming the cutoff" ;; + *) bad "before the cutoff it WARNs on stderr naming the cutoff" "WARN leg B ... 2026-11-01" "$warn" ;; esac +check "ON the shipped cutoff date it blocks" 1 "$(rc_on 2026-11-01 --worktree)" +check "after the cutoff it blocks" 1 "$(rc_on 2027-03-01 --worktree)" +check "ENFORCE_DOCSTRINGS_FROM moves the cutoff" 1 "$(ENFORCE_DOCSTRINGS_FROM=2026-01-01 rc_on 2026-06-01 --worktree)" +rc=0; DOCS_TODAY=2027-03-01 bash "$SCANNER" --worktree >/dev/null 2>&1 || rc=$? +check "without --check a leg-B fail never changes rc" 0 "$rc" + +legb_repo 4 1 +out="$(scan --worktree)" +check "exactly 80% (4 of 5) passes" pass "$(field "$out" legb)" +check "exactly 80% does not block after the cutoff" 0 "$(rc_on 2027-03-01 --worktree)" + +legb_repo 19 5 +out="$(scan --worktree)" +check "79.17% (19 of 24) coverage" 79.17% "$(field "$out" coverage)" +check "79.17% fails leg B" fail "$(field "$out" legb)" +check "79.17% blocks after the cutoff" 1 "$(rc_on 2027-03-01 --worktree)" +check "DOCSTRING_THRESHOLD=75 lets 79.17% pass" 0 "$(DOCSTRING_THRESHOLD=75 rc_on 2027-03-01 --worktree)" + +echo "== leg B: zero touched functions is no verdict, not a pass or a fail" +newrepo +printf 'fn main() {}\n' > main.rs +out="$(scan --worktree)" +check "zero touched gives legb=n/a" n/a "$(field "$out" legb)" +check "zero touched does not block after the cutoff" 0 "$(rc_on 2027-03-01 --worktree)" +nv="$(DOCS_TODAY=2027-03-01 bash "$SCANNER" --worktree --check 2>&1 >/dev/null)" +case "$nv" in *"leg B: no verdict: 0 touched functions"*) ok "zero touched says 'no verdict' explicitly" ;; + *) bad "zero touched says 'no verdict' explicitly" "leg B: no verdict: 0 touched functions" "$nv" ;; esac + +echo "== leg B: a skipped file never counts as documented" +# 3 documented + 1 undocumented = 75% (fail). Counting the skipped file as documented would give +# 4 of 5 = 80% (pass), so this fixture flips if skipped ever leaks into the ratio. +legb_repo 3 1 +printf 'fn main() {}\n' > main.rs +out="$(scan --worktree)" +check "the skipped file is reported" 1 "$(field "$out" skipped)" +check "it adds nothing to the denominator" 4 "$(field "$out" functions)" +check "3 of 4 fails leg B despite the skipped file" fail "$(field "$out" legb)" +check "and blocks after the cutoff" 1 "$(rc_on 2027-03-01 --worktree)" +# The other direction: 4 of 5 = 80% (pass). Counting the skipped file as UNdocumented would give +# 4 of 6 = 66.67% (fail). +legb_repo 4 1 +printf 'fn main() {}\n' > main.rs +out="$(scan --worktree)" +check "4 of 5 passes leg B beside a skipped file" pass "$(field "$out" legb)" +check "and does not block after the cutoff" 0 "$(rc_on 2027-03-01 --worktree)" + +echo "== leg B: malformed configuration is an error, never a silent disarm" +for bad_date in 2026-11 11/01/2026 tomorrow ' 2026-11-01'; do + rc=0; ENFORCE_DOCSTRINGS_FROM="$bad_date" bash "$SCANNER" --worktree --check >/dev/null 2>&1 || rc=$? + check "ENFORCE_DOCSTRINGS_FROM='$bad_date' exits 2" 2 "$rc" +done +rc=0; DOCS_TODAY=20261101 bash "$SCANNER" --worktree --check >/dev/null 2>&1 || rc=$? +check "DOCS_TODAY='20261101' exits 2" 2 "$rc" +for bad_t in abc 101 080 -1 80.5 ' 80'; do + rc=0; DOCSTRING_THRESHOLD="$bad_t" bash "$SCANNER" --worktree --check >/dev/null 2>&1 || rc=$? + check "DOCSTRING_THRESHOLD='$bad_t' exits 2" 2 "$rc" +done +rc=0; DOCSTRING_THRESHOLD=abc bash "$SCANNER" --worktree >/dev/null 2>&1 || rc=$? +check "a malformed threshold is an error without --check too" 2 "$rc" + echo "== errors are loud" rc=0; (cd "$WORK" && bash "$SCANNER" --worktree >/dev/null 2>&1) || rc=$? check "outside a git repository exits 2" 2 "$rc"