diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index f22608b6d..a31232556 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -78,6 +78,8 @@ workflows: - 'erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124' - 'github/codeql-action@b96794f015dfd88f77b49b1c93e0fa7110f94c63' '.github/workflows/hypatia-scan.yml': [] + '.github/workflows/ijson-jcs.yml': + - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' '.github/workflows/instant-sync.yml': - 'peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697' '.github/workflows/k9-contractile.yml': diff --git a/.github/workflows/ijson-jcs.yml b/.github/workflows/ijson-jcs.yml new file mode 100644 index 000000000..d3cbfe14e --- /dev/null +++ b/.github/workflows/ijson-jcs.yml @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: MPL-2.0 +# Local dogfood gate for 3-practice/JSON-POLICY.adoc (owner ruling D306): +# every .json / .jsonl is I-JSON (RFC 7493) + JCS (RFC 8785) canonical. +# REPORT-ONLY: the scan lists non-canonical files and exits 0. The test step +# is what can fail, and it proves the scan can tell the three verdicts apart. +name: ijson-jcs conformance + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + ijson-jcs: + name: Report non-canonical JSON (report-only) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # Pinned to a commit: ijson-jcs has no releases yet. + - run: cargo install --git https://github.com/hyperpolymath/ijson-jcs --rev 0fdeea716fd558e9807362acf6f8202d56d8d88f --locked ijson-jcs + - run: bash scripts/tests/check-ijson-jcs-test.sh + - run: scripts/check-ijson-jcs.sh . diff --git a/.github/workflows/self-test.yml b/.github/workflows/self-test.yml index a4238f0f6..b05307e60 100644 --- a/.github/workflows/self-test.yml +++ b/.github/workflows/self-test.yml @@ -51,6 +51,11 @@ jobs: sudo apt-get update -qq sudo apt-get install -y --no-install-recommends ripgrep libxml2-utils ruby ruby-minitest + # scripts/tests/check-ijson-jcs-test.sh needs the real tool; it exits 2 + # without it. Same pin as ijson-jcs.yml (JSON-POLICY §3). + - name: Install ijson-jcs + run: cargo install --git https://github.com/hyperpolymath/ijson-jcs --rev 0fdeea716fd558e9807362acf6f8202d56d8d88f --locked ijson-jcs + - name: Run tests/*.sh and scripts/tests/*.sh run: bash scripts/run-shell-test-suite.sh diff --git a/3-practice/JSON-POLICY.adoc b/3-practice/JSON-POLICY.adoc new file mode 100644 index 000000000..588100e2a --- /dev/null +++ b/3-practice/JSON-POLICY.adoc @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += Hyperpolymath JSON Policy: I-JSON + JCS +Jonathan D.A. Jewell +:toc: +:toc-placement: preamble + +Canonical policy for how JSON and JSON Lines are *written* across all +hyperpolymath and metadatastician repositories. + +Ruled by the owner on 2026-10-06 as D306 (recorded on `standards#787`). As in +`YAML-POLICY.adoc`, every rule below says whether it is *in force*, and what +enforces it. A rule with no tool behind it is labelled as such. + +== 0. The measurement this rests on + +Taken 2026-10-07 over this repository at `9f68971b`, with +`scripts/check-ijson-jcs.sh` and `ijson-jcs` built from `0fdeea71`: + +[cols="4,2,2,6",options="header"] +|=== +| Population | Files | Canonical | Notes + +| tracked `.json` +| 61 +| 0 +| None is invalid; every one fails only on layout or key order. + +| tracked `.jsonl` +| 1 +| 0 +| `.hypatia/activity.jsonl`, bot-appended. +|=== + +Nothing in this repository conforms yet. That is why the gate in §3 starts +report-only. + +== 1. Rule J-1: JSON is I-JSON, serialised by JCS + +*Status: IN FORCE for every new or rewritten file. Enforced by: report-only +gate (§3). Not yet blocking.* + +Every `.json` file the estate authors is + +. an I-JSON message (RFC 7493): UTF-8, no duplicate member names, numbers + representable as IEEE 754 doubles, no lone surrogates; and +. serialised in the JSON Canonicalization Scheme (RFC 8785): members sorted by + UTF-16 code unit, no insignificant whitespace, ECMAScript number and string + serialisation; +. followed by exactly one line feed. + +The tool is `hyperpolymath/ijson-jcs`: +`ijson-jcs check FILE` (exit 0 canonical, 1 not canonical, 2 invalid), +`ijson-jcs fix FILE` (rewrites in place), `ijson-jcs canon` (stdin to stdout). + +*File extensions do not change.* There is no `.ijson` or `.ijsonl`; the rule is +about content (D306). + +== 2. Rule J-2: JSON Lines is one canonical record per line + +*Status: IN FORCE for every new or rewritten file. Enforced by: report-only +gate (§3).* + +A `.jsonl` file is a sequence of J-1 records, one per line, each followed by a +line feed. No blank lines. `ijson-jcs` reads whole documents only, so the gate +checks each line with `ijson-jcs canon`. + +== 3. The gate + +`scripts/check-ijson-jcs.sh [--enforce] [PATH...]`, run by +`.github/workflows/ijson-jcs.yml` on every push and pull request. + +* *Default, report-only.* Lists every non-canonical or invalid file and a + summary, emits one `::notice`, exits 0. +* `--enforce` exits 1 on any finding. It exists for repositories that have + finished converting, and for the test. +* Exit 2 means the check could not run (tool or path missing). A gate that + cannot look does not report a pass. +* `scripts/tests/check-ijson-jcs-test.sh` proves the gate can fail: planted + canonical, non-canonical and invalid files of both kinds, plus the carve-out. + The workflow runs it before the scan. + +Making the gate blocking here, or adding it to the estate's reusable gates, is +a separate owner decision. It should follow conversion, not precede it. + +== 4. Carve-out: tool-demanded JSONC + +Some tools demand a file named `*.json` whose content is JSON with comments: +`devcontainer.json`, `.vscode/*.json`. Those comments carry SPDX headers, and +canonicalising would delete them. The gate skips these paths and counts the +skips in its summary. + +Adding a path to the carve-out needs the same evidence: a tool that demands the +name, and content that is not JSON. + +== 5. Conformance of the tool itself + +Measured 2026-10-07 on `ijson-jcs@0fdeea71`: + +* its own suite: 38 unit, 11 CLI and 3 doc tests pass; +* the RFC 8785 §3.2.3 sample canonicalises correctly (CLI test + `rfc8785_sample_canonicalises`); +* RFC 8785 Appendix B number vectors: *23 of 24 pass*. The failure is + `43143ff3c1cb0959`: expected `1424953923781206.2`, emitted + `1424953923781206.3`. Both strings round-trip to the same double; ECMAScript + picks the one closer to the exact value, and the tool does not; +* `NaN`, `Infinity` and `1e400` are rejected (exit 2), as I-JSON requires. + +The failure affects only doubles whose two shortest round-trip forms are a +tie. Estate JSON rarely carries such numbers, but a byte-exact hash over one +would differ from another JCS implementation's. + +== 6. What this document does not rule + +* *Bot-written files.* Some JSON is written by tools that rewrite it in their + own layout: `package.json` (`bun add`), `.hypatia-baseline.json`, scanner + output. Converting those is undone on the next tool run. Whether they are + carved out, or the tools are changed, is open. +* *Mass reformatting.* Convert a repository's JSON in a PR scoped to that + repository, after checking each consumer reads it by parsing, not by line. + Do not reformat estate-wide outside a PR that adds the gate. +* *JSON Schema files.* They are JSON and fall under J-1. This document does not + rule on schema dialect. + +== 7. Sources + +* RFC 7493, The I-JSON Message Format. +* RFC 8785, JSON Canonicalization Scheme (JCS). +* `hyperpolymath/ijson-jcs`. +* Owner ruling D306, `standards#787`. +* `developer/AGENTS.md` §2a, the transition table. + +== 8. Change log + +[cols="2,3,8",options="header"] +|=== +| Date | Change | Detail + +| 2026-10-07 +| Created +| J-1 and J-2 from D306. Report-only gate and its test. §0 measurement and §5 + tool conformance taken the same day. +|=== diff --git a/scripts/check-ijson-jcs.sh b/scripts/check-ijson-jcs.sh new file mode 100755 index 000000000..e32f3bfec --- /dev/null +++ b/scripts/check-ijson-jcs.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Report JSON and JSON Lines files that are not I-JSON (RFC 7493) + JCS +# (RFC 8785) canonical, per 3-practice/JSON-POLICY.adoc (owner ruling D306). +# +# Usage: check-ijson-jcs.sh [--enforce] [PATH...] +# +# Default is REPORT-ONLY: it lists every non-canonical or invalid file and a +# summary, then exits 0. --enforce exits 1 when any file is non-canonical or +# invalid; it exists for repositories that have finished converting, and for +# this script's own test. Exit 2 always means the check could not run (missing +# tool, missing path): a gate that cannot look must not report a pass. +# +# .json files go through `ijson-jcs check`. .jsonl files are checked one line +# at a time with `ijson-jcs canon`, because the tool reads whole documents only. +# Tool-demanded JSONC (devcontainer.json, .vscode/*.json) is skipped and counted: +# canonicalising would delete its comments, including SPDX headers. +set -euo pipefail + +IJSON_JCS="${IJSON_JCS:-ijson-jcs}" +enforce=0 +if [ "${1:-}" = "--enforce" ]; then + enforce=1 + shift +fi +if [ "$#" -eq 0 ]; then + set -- . +fi +if ! command -v "$IJSON_JCS" >/dev/null 2>&1; then + printf '%s: not found; install hyperpolymath/ijson-jcs or set IJSON_JCS\n' "$IJSON_JCS" >&2 + exit 2 +fi +# A mistyped path must not turn into an empty, passing scan. +for path in "$@"; do + if [ ! -e "$path" ]; then + printf '%s: scan path does not exist\n' "$path" >&2 + exit 2 + fi +done + +# List the JSON and JSON Lines files under the given paths, NUL-separated, +# skipping VCS metadata and regenerable build trees. +list_files() { + find "$@" \( -name .git -o -name node_modules -o -name target -o -name _build \) -prune \ + -o -type f \( -name '*.json' -o -name '*.jsonl' \) -print0 +} + +# Succeed when a file is tool-demanded JSONC that the policy carves out. +is_jsonc_carve_out() { + case "$1" in + */devcontainer.json | devcontainer.json | */.vscode/*.json | .vscode/*.json) return 0 ;; + esac + return 1 +} + +# Classify one .json file; print CANONICAL, NOT_CANONICAL or INVALID. +# `ijson-jcs check` accepts a file with no final newline, so a missing one is +# checked here: J-1 requires exactly one. +classify_json() { + local rc=0 + "$IJSON_JCS" check "$1" >/dev/null 2>&1 || rc=$? + if [ "$rc" = 0 ] && [ "$(tail -c 1 "$1" | od -An -c | tr -d ' ')" != '\n' ]; then + rc=1 + fi + case "$rc" in + 0) echo CANONICAL ;; + 1) echo NOT_CANONICAL ;; + *) echo INVALID ;; + esac +} + +# Classify one .jsonl file line by line; print CANONICAL, NOT_CANONICAL or +# INVALID. A blank line or an unparseable line is INVALID; a line whose +# canonical form differs, or a missing final newline, is NOT_CANONICAL. +classify_jsonl() { + local file="$1" line canon verdict=CANONICAL + if [ -s "$file" ] && [ "$(tail -c 1 "$file" | od -An -c | tr -d ' ')" != '\n' ]; then + verdict=NOT_CANONICAL + fi + while IFS= read -r line || [ -n "$line" ]; do + if [ -z "$line" ]; then + echo INVALID + return + fi + if ! canon="$(printf '%s' "$line" | "$IJSON_JCS" canon 2>/dev/null)"; then + echo INVALID + return + fi + if [ "$canon" != "$line" ]; then + verdict=NOT_CANONICAL + fi + done <"$file" + echo "$verdict" +} + +total=0 canonical=0 noncanonical=0 invalid=0 skipped=0 +while IFS= read -r -d '' file; do + file="${file#./}" + if is_jsonc_carve_out "$file"; then + skipped=$((skipped + 1)) + continue + fi + total=$((total + 1)) + case "$file" in + *.jsonl) verdict="$(classify_jsonl "$file")" ;; + *) verdict="$(classify_json "$file")" ;; + esac + case "$verdict" in + CANONICAL) canonical=$((canonical + 1)) ;; + NOT_CANONICAL) noncanonical=$((noncanonical + 1)); printf 'NOT CANONICAL %s\n' "$file" ;; + *) invalid=$((invalid + 1)); printf 'INVALID %s\n' "$file" ;; + esac +done < <(list_files "$@" | sort -z) + +summary="ijson-jcs: ${total} file(s) checked, ${canonical} canonical, ${noncanonical} not canonical, ${invalid} invalid; ${skipped} JSONC carve-out(s) skipped" +echo "$summary" +if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::notice title=ijson-jcs (report-only)::${summary}. Fix with: ijson-jcs fix " +fi +if [ "$enforce" -eq 1 ] && [ $((noncanonical + invalid)) -gt 0 ]; then + exit 1 +fi +exit 0 diff --git a/scripts/tests/check-ijson-jcs-test.sh b/scripts/tests/check-ijson-jcs-test.sh new file mode 100755 index 000000000..984fbc526 --- /dev/null +++ b/scripts/tests/check-ijson-jcs-test.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# +# Proves check-ijson-jcs.sh CAN FAIL, and classifies each file for the right +# reason: canonical, not canonical and invalid, for both .json and .jsonl, with +# the JSONC carve-out skipped. Report-only mode must exit 0 whatever it finds; +# --enforce must exit 1 on any finding; a missing tool or path must exit 2. +# +# Needs the ijson-jcs binary on PATH or in $IJSON_JCS. +set -euo pipefail +SCRIPT="$(cd "$(dirname "$0")/.." && pwd)/check-ijson-jcs.sh" +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT + +pass=0; fail=0 +# Run the checker (with any extra flags) over $WORK/t and compare its exit code. +expect() { # expect