feat(canonical): JCS (RFC 8785) canonical JSON, verified against ijson-jcs; the KYAML ratchet

scripts/kyaml-format.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MPL-2.0
+# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) <j.d.a.jewell@open.ac.uk>
+#
+# kyaml-format.sh — format arbitrary YAML as KYAML, or check that it already is
+# (YAML-POLICY Y-3; KYAML adoption step 3, standards#1022).
+#
+# The rewriter is `yq -p yaml -o kyaml '.'`, verbatim — the one rewriter the
+# step-2 comment-preservation proof (tools/yaml-comment-proof, standards#1021)
+# covers. That proof does not generalise to any other rewriter, so this script
+# deliberately adds none: no post-processing, no re-indenting, no sed.
+#
+# Usage:
+#   kyaml-format.sh [--check] FILE...
+#
+#   (default)  rewrite each FILE in place as KYAML when it is not already.
+#   --check    rewrite nothing; exit 1 if any FILE is not already KYAML.
+#
+# Exit contract:
+#   0 = every FILE is KYAML (after the rewrite, in format mode)
+#   1 = --check: at least one FILE is not KYAML (each one is named)
+#   2 = refused: no FILE given, a FILE is missing or unparseable, a FILE has no
+#       final newline, or the rewrite would change the parsed data. A refusal
+#       never rewrites that FILE and never counts as clean.
+#
+# Why the final-newline refusal: a `|` block scalar that ends at EOF with no
+# line break is read differently by go-yaml (yq) and eemeli/yaml — measured on
+# k9-contractile.yml, the trailing "\n" of its last run: body exists for one
+# parser and not the other. Formatting such a file would silently pick one
+# reading, so the script refuses it and names the fix instead.
+set -uo pipefail
+
+MODE=format
+if [ "${1:-}" = "--check" ]; then
+  MODE=check
+  shift
+fi
+
+if [ "$#" -eq 0 ]; then
+  echo "kyaml-format: no files given (usage: kyaml-format.sh [--check] FILE...)" >&2
+  exit 2
+fi
+
+command -v yq >/dev/null 2>&1 || { echo "kyaml-format: yq is not on PATH" >&2; exit 2; }
+
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+
+# canonical_data <file> — prints the file's parsed value as key-sorted JSON;
+# fails when the file does not parse.
+canonical_data() {
+  yq -p yaml -o json -I 0 'sort_keys(..)' "$1"
+}
+
+# kyaml_of <file> <out> — writes the KYAML rendering of <file> to <out>;
+# fails when <file> does not parse.
+kyaml_of() {
+  yq -p yaml -o kyaml '.' "$1" > "$2"
+}
+
+# refuse <file> <reason> — reports a file this script will not judge.
+refuse() {
+  echo "::error file=$1::kyaml-format: REFUSED $1 — $2" >&2
+  refused=$((refused + 1))
+}
+
+refused=0 drift=0 clean=0 rewritten=0
+i=0
+for f in "$@"; do
+  i=$((i + 1))
+  out="$TMP/$i.kyaml"
+  if [ ! -f "$f" ]; then
+    refuse "$f" "no such file"; continue
+  fi
+  if [ -s "$f" ] && [ -n "$(tail -c1 "$f")" ]; then
+    refuse "$f" "no final newline; add one first (a trailing block scalar is ambiguous without it)"; continue
+  fi
+  if ! kyaml_of "$f" "$out" 2>"$TMP/err"; then
+    refuse "$f" "does not parse: $(head -1 "$TMP/err")"; continue
+  fi
+  if cmp -s "$f" "$out"; then
+    clean=$((clean + 1)); continue
+  fi
+  # The rewrite is only acceptable when it preserves the data exactly.
+  if ! before=$(canonical_data "$f") || ! after=$(canonical_data "$out") || [ "$before" != "$after" ]; then
+    refuse "$f" "the KYAML rendering does not parse to the same data"; continue
+  fi
+  if [ "$MODE" = check ]; then
+    echo "::error file=$f::kyaml-format: $f is not KYAML — run scripts/kyaml-format.sh $f" >&2
+    drift=$((drift + 1))
+  else
+    cat "$out" > "$f"
+    echo "kyaml-format: rewrote $f"
+    rewritten=$((rewritten + 1))
+  fi
+done
+
+total=$#
+if [ "$refused" -gt 0 ]; then
+  echo "kyaml-format: $refused of $total file(s) REFUSED — not judged, not clean" >&2
+  exit 2
+fi
+if [ "$drift" -gt 0 ]; then
+  echo "kyaml-format: $drift of $total file(s) are not KYAML" >&2
+  exit 1
+fi
+if [ "$MODE" = check ]; then
+  echo "kyaml-format: all $total file(s) are KYAML"
+else
+  echo "kyaml-format: $total file(s) KYAML ($rewritten rewritten, $clean already clean)"
+fi
+exit 0

src/MetaManifold.jl

@@ -33,6 +33,9 @@ include("analysis/diversity.jl")
 include("analysis/analysis.jl")
 include("analysis/differential.jl")
 
+# Canonical JSON: deterministic bytes for hashing and content-addressing.
+include("canonical.jl")
+
 # Web server
 include("server/server.jl")
 

src/canonical.jl

@@ -0,0 +1,203 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) <j.d.a.jewell@open.ac.uk>
+#
+# JCS (RFC 8785) canonical JSON: deterministic bytes for hashing, signing and
+# content-addressing. The same value always yields the same byte string, so a
+# chart spec or an analysis result can be hashed where it is produced and the
+# hash compared across machines, Julia versions and refactorings that build
+# the same object in a different order.
+#
+# What this implements, and what it deliberately does not:
+#
+#   * Object keys are sorted by UTF-16 code unit, the rule that makes an
+#     astral-plane key sort BEFORE a key such as U+FF21, because its high
+#     surrogate (U+D800..U+DBFF) is a smaller code unit than U+FF21.
+#   * Float64 values serialize per ECMAScript's Number::toString, the rules
+#     RFC 8785 builds on: shortest round-trip digits, positional notation for
+#     1e-6 <= |x| < 1e21, exponent notation outside it, a signed exponent
+#     ("1e+30"), and -0.0 collapsing to 0.
+#   * Integers outside I-JSON's exact range (-(2^53)+1 .. 2^53-1) are
+#     refused, as the estate's ijson-jcs crate refuses them, because a
+#     consumer cannot read them back as the same number.
+#   * NaN and Inf are refused for the same reason.
+#
+# The implementation is verified against the estate's ijson-jcs crate
+# (Rust, RFC 7493 + RFC 8785): the fixtures in
+# test/fixtures/jcs_oracle.jsonl were produced BY that crate, and
+# test/unit/test_canonical.jl asserts this module reproduces them
+# byte-for-byte. The crate is the oracle; this is the second
+# implementation, and any disagreement between them is a bug in one of
+# them.
+
+module Canonical
+
+
+using SHA
+
+export jcs_bytes, jcs_sha256
+
+# The I-JSON exact-integer range.
+const SAFE_INT_MAX = 2^53 - 1
+
+"""
+    jcs_bytes(x) -> Vector{UInt8}
+
+The RFC 8785 canonical form of `x`, as bytes. Accepts the JSON-shaped
+values the server produces: `Dict` and `JSON3.Object` (key order is
+irrelevant; it is sorted away), `AbstractVector` and `JSON3.Array` (order
+is preserved, as JCS preserves array order), `AbstractString`, `Integer`,
+`AbstractFloat`, `Bool` and `nothing`. Floats serialize per ECMAScript's
+Number::toString; integers outside the exact I-JSON range, and NaN or Inf
+floats, throw `ArgumentError`.
+"""
+function jcs_bytes(x)
+    io = IOBuffer()
+    _write!(io, x)
+    take!(io)
+end
+
+"""
+    jcs_sha256(x) -> String
+
+The SHA-256 of `jcs_bytes(x)`, as lowercase hex -- the content address of
+a JSON value: two producers that build the same value hash identically
+whatever order they built it in.
+"""
+jcs_sha256(x) = bytes2hex(sha256(jcs_bytes(x)))
+
+# Object keys sort by UTF-16 code unit, which for astral-plane characters
+# means their surrogate pair, a smaller first unit than any BMP code point
+# at or above U+E000. transcoding the key to UInt16 gives exactly those
+# units to compare.
+_utf16_units(s::AbstractString) = transcode(UInt16, String(s))
+
+_write!(io, ::Nothing) = print(io, "null")
+_write!(io, b::Bool) = print(io, b ? "true" : "false")
+
+function _write!(io, n::Integer)
+    # I-JSON: only |n| <= 2^53-1 survives a JSON round trip exactly.
+    -SAFE_INT_MAX <= n <= SAFE_INT_MAX || throw(ArgumentError(
+        "canonical JSON refuses the integer $n: it is outside the exact " *
+        "IEEE-754 binary64 range I-JSON allows"))
+    print(io, n)
+end
+
+function _write!(io, x::AbstractFloat)
+    _write_float!(io, Float64(x))
+end
+
+function _write!(io, s::AbstractString)
+    print(io, '"')
+    for c in s
+        _write_char!(io, c)
+    end
+    print(io, '"')
+end
+
+# ECMAScript's JSON.stringify escapes exactly these: the two mandatory
+# characters, the five with short forms, and every other control
+# character as \u00xx with lowercase hex.
+function _write_char!(io, c::Char)
+    if c == '"'
+        print(io, "\\\"")
+    elseif c == '\\'
+        print(io, "\\\\")
+    elseif c == '\b'
+        print(io, "\\b")
+    elseif c == '\t'
+        print(io, "\\t")
+    elseif c == '\n'
+        print(io, "\\n")
+    elseif c == '\f'
+        print(io, "\\f")
+    elseif c == '\r'
+        print(io, "\\r")
+    elseif c < ' '
+        print(io, "\\u", string(UInt32(c); base=16, pad=4))
+    else
+        print(io, c)
+    end
+end
+
+# AbstractDict (Dict and JSON3.Object alike): sorted keys, then each pair.
+function _write!(io, d::AbstractDict)
+    print(io, '{')
+    first = true
+    for k in sort!(collect(String.(keys(d))); by = _utf16_units)
+        first || print(io, ',')
+        first = false
+        _write!(io, string(k))
+        print(io, ':')
+        _write!(io, d[k])
+    end
+    print(io, '}')
+end
+
+# AbstractVector (Vector and JSON3.Array alike): order preserved.
+function _write!(io, a::AbstractVector)
+    print(io, '[')
+    for (i, v) in enumerate(a)
+        i > 1 && print(io, ',')
+        _write!(io, v)
+    end
+    print(io, ']')
+end
+
+# RFC 8785 §3.2.2.3: ECMAScript's Number::toString. Julia's own repr gives
+# the shortest round-trip digits (Ryu, the same family ES specifies), so
+# the work is reformatting those digits under ES's notation rules.
+function _write_float!(io, x::Float64)
+    isnan(x) && throw(ArgumentError(
+        "canonical JSON refuses NaN: no consumer reads it back as the same value"))
+    isinf(x) && throw(ArgumentError(
+        "canonical JSON refuses Inf: no consumer reads it back as the same value"))
+    if x == 0
+        print(io, "0")   # +0.0 and -0.0 both collapse to 0, as ECMAScript does.
+        return
+    end
+    if signbit(x)
+        print(io, '-')
+        x = -x
+    end
+    digits, n = _shortest_digits(x)
+    k = length(digits)
+    if n > 21 || n <= -6
+        # Exponential: one digit, then the fraction, then e with an
+        # explicit sign (ES prints "1e+30", not "1e30").
+        print(io, digits[1])
+        k > 1 && (print(io, '.'); print(io, digits[2:end]))
+        e = n - 1
+        print(io, 'e', e < 0 ? '-' : '+', string(abs(e)))
+    elseif n >= k
+        # An integer too big for the fraction: digits followed by zeros.
+        print(io, digits, "0"^(n - k))
+    elseif n > 0
+        # Digits with the point inside them.
+        print(io, digits[1:n], '.', digits[n+1:end])
+    else
+        # Below one, positional as far as ES goes: 0.000001, not 1e-6.
+        print(io, "0.", "0"^(-n), digits)
+    end
+end
+
+# The shortest round-trip decimal form of x as (digits, n) with
+# x = 0.digits * 10^n: repr yields either "123", "4.5" or "1.5e-10".
+function _shortest_digits(x::Float64)
+    s = repr(x)
+    mantissa, exponent = occursin('e', s) ? (split(s, 'e') .|> String) : (s, "")
+    e = isempty(exponent) ? 0 : parse(Int, exponent)
+    if occursin('.', mantissa)
+        intpart, fracpart = split(mantissa, '.') .|> String
+    else
+        intpart, fracpart = mantissa, ""
+    end
+    digits = string(intpart, fracpart)
+    n = length(intpart) + e
+    # Trailing zeros are notation, not value: 1.0e30 is digits "1", n 31.
+    while length(digits) > 1 && endswith(digits, '0')
+        digits = digits[1:end-1]
+    end
+    digits, n
+end
+
+end # module

test/fixtures/jcs_oracle.jsonl

@@ -0,0 +1,11 @@
+{"input":{"b":1,"a":2,"c":[]},"expected":"{\"a\":2,\"b\":1,\"c\":[]}"}
+{"input":{"€":"euro","Ａ":"fullwidth A","a":"first"},"expected":"{\"a\":\"first\",\"€\":\"euro\",\"Ａ\":\"fullwidth A\"}"}
+{"input":{"😀":"astral first?","Ａ":"high bmp"},"expected":"{\"😀\":\"astral first?\",\"Ａ\":\"high bmp\"}"}
+{"input":{"d":[3,1,2],"e":{"z":1,"y":{"x":[true,false,null]}}},"expected":"{\"d\":[3,1,2],\"e\":{\"y\":{\"x\":[true,false,null]},\"z\":1}}"}
+{"input":[4.50,24499999999999998,1E+30,0.000001,1E-7,1E+21,1E+20,1E-33,-0.0,0.0,9.9999999999999997E+22],"expected":"[4.5,24500000000000000,1e+30,0.000001,1e-7,1e+21,100000000000000000000,1e-33,0,0,1e+23]"}
+{"input":[0.000001,1E-7,1.5E-10,9007199254740991,-42],"expected":"[0.000001,1e-7,1.5e-10,9007199254740991,-42]"}
+{"input":{"s\":with\"quotes":"tab\there\nnewline","esc\\":"\\backslash","ctl":"\u0001\u001f"},"expected":"{\"ctl\":\"\\u0001\\u001f\",\"esc\\\\\":\"\\\\backslash\",\"s\\\":with\\\"quotes\":\"tab\\there\\nnewline\"}"}
+{"input":{"nested":{"arr":[{"n":1.0},{"m":2}],"deep":{"deeper":{"deepest":"value"}}}},"expected":"{\"nested\":{\"arr\":[{\"n\":1},{\"m\":2}],\"deep\":{\"deeper\":{\"deepest\":\"value\"}}}}"}
+{"input":{"literals":true,"n":null,"f":false,"empty_obj":{},"empty_arr":[]},"expected":"{\"empty_arr\":[],\"empty_obj\":{},\"f\":false,\"literals\":true,\"n\":null}"}
+{"input":[5E-324,1.7976931348623157E+308,2.2250738585072014E-308,0.3,0.1,0.30000000000000004],"expected":"[5e-324,1.7976931348623157e+308,2.2250738585072014e-308,0.3,0.1,0.30000000000000004]"}
+{"input":{"Plotly":{"layout":{"title":{"text":"CO₂"}},"data":[{"x":[1,2,3],"y":[4.5,5.5,6.5],"type":"scatter"}]}},"expected":"{\"Plotly\":{\"data\":[{\"type\":\"scatter\",\"x\":[1,2,3],\"y\":[4.5,5.5,6.5]}],\"layout\":{\"title\":{\"text\":\"CO₂\"}}}}"}

test/runtests.jl

@@ -54,6 +54,7 @@ const UNIT_FILES = [
     "test_permanova.jl",
     "test_write_gates.jl",
     "test_server_security.jl",
+    "test_canonical.jl",
 ]
 
 # The files named on the command line ("test_routes.jl", "routes" or