diff --git a/scripts/kyaml-format.sh b/scripts/kyaml-format.sh new file mode 100755 index 00000000..145b157d --- /dev/null +++ b/scripts/kyaml-format.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# kyaml-format.sh — format arbitrary YAML as KYAML, or check that it already is +# (YAML-POLICY Y-3; KYAML adoption step 3, standards#1022). +# +# The rewriter is `yq -p yaml -o kyaml '.'`, verbatim — the one rewriter the +# step-2 comment-preservation proof (tools/yaml-comment-proof, standards#1021) +# covers. That proof does not generalise to any other rewriter, so this script +# deliberately adds none: no post-processing, no re-indenting, no sed. +# +# Usage: +# kyaml-format.sh [--check] FILE... +# +# (default) rewrite each FILE in place as KYAML when it is not already. +# --check rewrite nothing; exit 1 if any FILE is not already KYAML. +# +# Exit contract: +# 0 = every FILE is KYAML (after the rewrite, in format mode) +# 1 = --check: at least one FILE is not KYAML (each one is named) +# 2 = refused: no FILE given, a FILE is missing or unparseable, a FILE has no +# final newline, or the rewrite would change the parsed data. A refusal +# never rewrites that FILE and never counts as clean. +# +# Why the final-newline refusal: a `|` block scalar that ends at EOF with no +# line break is read differently by go-yaml (yq) and eemeli/yaml — measured on +# k9-contractile.yml, the trailing "\n" of its last run: body exists for one +# parser and not the other. Formatting such a file would silently pick one +# reading, so the script refuses it and names the fix instead. +set -uo pipefail + +MODE=format +if [ "${1:-}" = "--check" ]; then + MODE=check + shift +fi + +if [ "$#" -eq 0 ]; then + echo "kyaml-format: no files given (usage: kyaml-format.sh [--check] FILE...)" >&2 + exit 2 +fi + +command -v yq >/dev/null 2>&1 || { echo "kyaml-format: yq is not on PATH" >&2; exit 2; } + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +# canonical_data — prints the file's parsed value as key-sorted JSON; +# fails when the file does not parse. +canonical_data() { + yq -p yaml -o json -I 0 'sort_keys(..)' "$1" +} + +# kyaml_of — writes the KYAML rendering of to ; +# fails when does not parse. +kyaml_of() { + yq -p yaml -o kyaml '.' "$1" > "$2" +} + +# refuse — reports a file this script will not judge. +refuse() { + echo "::error file=$1::kyaml-format: REFUSED $1 — $2" >&2 + refused=$((refused + 1)) +} + +refused=0 drift=0 clean=0 rewritten=0 +i=0 +for f in "$@"; do + i=$((i + 1)) + out="$TMP/$i.kyaml" + if [ ! -f "$f" ]; then + refuse "$f" "no such file"; continue + fi + if [ -s "$f" ] && [ -n "$(tail -c1 "$f")" ]; then + refuse "$f" "no final newline; add one first (a trailing block scalar is ambiguous without it)"; continue + fi + if ! kyaml_of "$f" "$out" 2>"$TMP/err"; then + refuse "$f" "does not parse: $(head -1 "$TMP/err")"; continue + fi + if cmp -s "$f" "$out"; then + clean=$((clean + 1)); continue + fi + # The rewrite is only acceptable when it preserves the data exactly. + if ! before=$(canonical_data "$f") || ! after=$(canonical_data "$out") || [ "$before" != "$after" ]; then + refuse "$f" "the KYAML rendering does not parse to the same data"; continue + fi + if [ "$MODE" = check ]; then + echo "::error file=$f::kyaml-format: $f is not KYAML — run scripts/kyaml-format.sh $f" >&2 + drift=$((drift + 1)) + else + cat "$out" > "$f" + echo "kyaml-format: rewrote $f" + rewritten=$((rewritten + 1)) + fi +done + +total=$# +if [ "$refused" -gt 0 ]; then + echo "kyaml-format: $refused of $total file(s) REFUSED — not judged, not clean" >&2 + exit 2 +fi +if [ "$drift" -gt 0 ]; then + echo "kyaml-format: $drift of $total file(s) are not KYAML" >&2 + exit 1 +fi +if [ "$MODE" = check ]; then + echo "kyaml-format: all $total file(s) are KYAML" +else + echo "kyaml-format: $total file(s) KYAML ($rewritten rewritten, $clean already clean)" +fi +exit 0 diff --git a/src/MetaManifold.jl b/src/MetaManifold.jl index 31db2a5c..b5f13a59 100644 --- a/src/MetaManifold.jl +++ b/src/MetaManifold.jl @@ -33,6 +33,9 @@ include("analysis/diversity.jl") include("analysis/analysis.jl") include("analysis/differential.jl") +# Canonical JSON: deterministic bytes for hashing and content-addressing. +include("canonical.jl") + # Web server include("server/server.jl") diff --git a/src/canonical.jl b/src/canonical.jl new file mode 100644 index 00000000..04363037 --- /dev/null +++ b/src/canonical.jl @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# JCS (RFC 8785) canonical JSON: deterministic bytes for hashing, signing and +# content-addressing. The same value always yields the same byte string, so a +# chart spec or an analysis result can be hashed where it is produced and the +# hash compared across machines, Julia versions and refactorings that build +# the same object in a different order. +# +# What this implements, and what it deliberately does not: +# +# * Object keys are sorted by UTF-16 code unit, the rule that makes an +# astral-plane key sort BEFORE a key such as U+FF21, because its high +# surrogate (U+D800..U+DBFF) is a smaller code unit than U+FF21. +# * Float64 values serialize per ECMAScript's Number::toString, the rules +# RFC 8785 builds on: shortest round-trip digits, positional notation for +# 1e-6 <= |x| < 1e21, exponent notation outside it, a signed exponent +# ("1e+30"), and -0.0 collapsing to 0. +# * Integers outside I-JSON's exact range (-(2^53)+1 .. 2^53-1) are +# refused, as the estate's ijson-jcs crate refuses them, because a +# consumer cannot read them back as the same number. +# * NaN and Inf are refused for the same reason. +# +# The implementation is verified against the estate's ijson-jcs crate +# (Rust, RFC 7493 + RFC 8785): the fixtures in +# test/fixtures/jcs_oracle.jsonl were produced BY that crate, and +# test/unit/test_canonical.jl asserts this module reproduces them +# byte-for-byte. The crate is the oracle; this is the second +# implementation, and any disagreement between them is a bug in one of +# them. + +module Canonical + + +using SHA + +export jcs_bytes, jcs_sha256 + +# The I-JSON exact-integer range. +const SAFE_INT_MAX = 2^53 - 1 + +""" + jcs_bytes(x) -> Vector{UInt8} + +The RFC 8785 canonical form of `x`, as bytes. Accepts the JSON-shaped +values the server produces: `Dict` and `JSON3.Object` (key order is +irrelevant; it is sorted away), `AbstractVector` and `JSON3.Array` (order +is preserved, as JCS preserves array order), `AbstractString`, `Integer`, +`AbstractFloat`, `Bool` and `nothing`. Floats serialize per ECMAScript's +Number::toString; integers outside the exact I-JSON range, and NaN or Inf +floats, throw `ArgumentError`. +""" +function jcs_bytes(x) + io = IOBuffer() + _write!(io, x) + take!(io) +end + +""" + jcs_sha256(x) -> String + +The SHA-256 of `jcs_bytes(x)`, as lowercase hex -- the content address of +a JSON value: two producers that build the same value hash identically +whatever order they built it in. +""" +jcs_sha256(x) = bytes2hex(sha256(jcs_bytes(x))) + +# Object keys sort by UTF-16 code unit, which for astral-plane characters +# means their surrogate pair, a smaller first unit than any BMP code point +# at or above U+E000. transcoding the key to UInt16 gives exactly those +# units to compare. +_utf16_units(s::AbstractString) = transcode(UInt16, String(s)) + +_write!(io, ::Nothing) = print(io, "null") +_write!(io, b::Bool) = print(io, b ? "true" : "false") + +function _write!(io, n::Integer) + # I-JSON: only |n| <= 2^53-1 survives a JSON round trip exactly. + -SAFE_INT_MAX <= n <= SAFE_INT_MAX || throw(ArgumentError( + "canonical JSON refuses the integer $n: it is outside the exact " * + "IEEE-754 binary64 range I-JSON allows")) + print(io, n) +end + +function _write!(io, x::AbstractFloat) + _write_float!(io, Float64(x)) +end + +function _write!(io, s::AbstractString) + print(io, '"') + for c in s + _write_char!(io, c) + end + print(io, '"') +end + +# ECMAScript's JSON.stringify escapes exactly these: the two mandatory +# characters, the five with short forms, and every other control +# character as \u00xx with lowercase hex. +function _write_char!(io, c::Char) + if c == '"' + print(io, "\\\"") + elseif c == '\\' + print(io, "\\\\") + elseif c == '\b' + print(io, "\\b") + elseif c == '\t' + print(io, "\\t") + elseif c == '\n' + print(io, "\\n") + elseif c == '\f' + print(io, "\\f") + elseif c == '\r' + print(io, "\\r") + elseif c < ' ' + print(io, "\\u", string(UInt32(c); base=16, pad=4)) + else + print(io, c) + end +end + +# AbstractDict (Dict and JSON3.Object alike): sorted keys, then each pair. +function _write!(io, d::AbstractDict) + print(io, '{') + first = true + for k in sort!(collect(String.(keys(d))); by = _utf16_units) + first || print(io, ',') + first = false + _write!(io, string(k)) + print(io, ':') + _write!(io, d[k]) + end + print(io, '}') +end + +# AbstractVector (Vector and JSON3.Array alike): order preserved. +function _write!(io, a::AbstractVector) + print(io, '[') + for (i, v) in enumerate(a) + i > 1 && print(io, ',') + _write!(io, v) + end + print(io, ']') +end + +# RFC 8785 §3.2.2.3: ECMAScript's Number::toString. Julia's own repr gives +# the shortest round-trip digits (Ryu, the same family ES specifies), so +# the work is reformatting those digits under ES's notation rules. +function _write_float!(io, x::Float64) + isnan(x) && throw(ArgumentError( + "canonical JSON refuses NaN: no consumer reads it back as the same value")) + isinf(x) && throw(ArgumentError( + "canonical JSON refuses Inf: no consumer reads it back as the same value")) + if x == 0 + print(io, "0") # +0.0 and -0.0 both collapse to 0, as ECMAScript does. + return + end + if signbit(x) + print(io, '-') + x = -x + end + digits, n = _shortest_digits(x) + k = length(digits) + if n > 21 || n <= -6 + # Exponential: one digit, then the fraction, then e with an + # explicit sign (ES prints "1e+30", not "1e30"). + print(io, digits[1]) + k > 1 && (print(io, '.'); print(io, digits[2:end])) + e = n - 1 + print(io, 'e', e < 0 ? '-' : '+', string(abs(e))) + elseif n >= k + # An integer too big for the fraction: digits followed by zeros. + print(io, digits, "0"^(n - k)) + elseif n > 0 + # Digits with the point inside them. + print(io, digits[1:n], '.', digits[n+1:end]) + else + # Below one, positional as far as ES goes: 0.000001, not 1e-6. + print(io, "0.", "0"^(-n), digits) + end +end + +# The shortest round-trip decimal form of x as (digits, n) with +# x = 0.digits * 10^n: repr yields either "123", "4.5" or "1.5e-10". +function _shortest_digits(x::Float64) + s = repr(x) + mantissa, exponent = occursin('e', s) ? (split(s, 'e') .|> String) : (s, "") + e = isempty(exponent) ? 0 : parse(Int, exponent) + if occursin('.', mantissa) + intpart, fracpart = split(mantissa, '.') .|> String + else + intpart, fracpart = mantissa, "" + end + digits = string(intpart, fracpart) + n = length(intpart) + e + # Trailing zeros are notation, not value: 1.0e30 is digits "1", n 31. + while length(digits) > 1 && endswith(digits, '0') + digits = digits[1:end-1] + end + digits, n +end + +end # module diff --git a/test/fixtures/jcs_oracle.jsonl b/test/fixtures/jcs_oracle.jsonl new file mode 100644 index 00000000..a61e6e37 --- /dev/null +++ b/test/fixtures/jcs_oracle.jsonl @@ -0,0 +1,11 @@ +{"input":{"b":1,"a":2,"c":[]},"expected":"{\"a\":2,\"b\":1,\"c\":[]}"} +{"input":{"€":"euro","Ａ":"fullwidth A","a":"first"},"expected":"{\"a\":\"first\",\"€\":\"euro\",\"Ａ\":\"fullwidth A\"}"} +{"input":{"😀":"astral first?","Ａ":"high bmp"},"expected":"{\"😀\":\"astral first?\",\"Ａ\":\"high bmp\"}"} +{"input":{"d":[3,1,2],"e":{"z":1,"y":{"x":[true,false,null]}}},"expected":"{\"d\":[3,1,2],\"e\":{\"y\":{\"x\":[true,false,null]},\"z\":1}}"} +{"input":[4.50,24499999999999998,1E+30,0.000001,1E-7,1E+21,1E+20,1E-33,-0.0,0.0,9.9999999999999997E+22],"expected":"[4.5,24500000000000000,1e+30,0.000001,1e-7,1e+21,100000000000000000000,1e-33,0,0,1e+23]"} +{"input":[0.000001,1E-7,1.5E-10,9007199254740991,-42],"expected":"[0.000001,1e-7,1.5e-10,9007199254740991,-42]"} +{"input":{"s\":with\"quotes":"tab\there\nnewline","esc\\":"\\backslash","ctl":"\u0001\u001f"},"expected":"{\"ctl\":\"\\u0001\\u001f\",\"esc\\\\\":\"\\\\backslash\",\"s\\\":with\\\"quotes\":\"tab\\there\\nnewline\"}"} +{"input":{"nested":{"arr":[{"n":1.0},{"m":2}],"deep":{"deeper":{"deepest":"value"}}}},"expected":"{\"nested\":{\"arr\":[{\"n\":1},{\"m\":2}],\"deep\":{\"deeper\":{\"deepest\":\"value\"}}}}"} +{"input":{"literals":true,"n":null,"f":false,"empty_obj":{},"empty_arr":[]},"expected":"{\"empty_arr\":[],\"empty_obj\":{},\"f\":false,\"literals\":true,\"n\":null}"} +{"input":[5E-324,1.7976931348623157E+308,2.2250738585072014E-308,0.3,0.1,0.30000000000000004],"expected":"[5e-324,1.7976931348623157e+308,2.2250738585072014e-308,0.3,0.1,0.30000000000000004]"} +{"input":{"Plotly":{"layout":{"title":{"text":"CO₂"}},"data":[{"x":[1,2,3],"y":[4.5,5.5,6.5],"type":"scatter"}]}},"expected":"{\"Plotly\":{\"data\":[{\"type\":\"scatter\",\"x\":[1,2,3],\"y\":[4.5,5.5,6.5]}],\"layout\":{\"title\":{\"text\":\"CO₂\"}}}}"} diff --git a/test/runtests.jl b/test/runtests.jl index d107e559..77e15bcf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -54,6 +54,7 @@ const UNIT_FILES = [ "test_permanova.jl", "test_write_gates.jl", "test_server_security.jl", + "test_canonical.jl", ] # The files named on the command line ("test_routes.jl", "routes" or diff --git a/test/unit/test_canonical.jl b/test/unit/test_canonical.jl new file mode 100644 index 00000000..1a9bb01b --- /dev/null +++ b/test/unit/test_canonical.jl @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# Unit tests for the JCS (RFC 8785) canonical JSON writer. The oracle is the +# estate's ijson-jcs crate (Rust): every fixture in +# test/fixtures/jcs_oracle.jsonl was PRODUCED by that crate, and this file +# asserts MetaManifold.Canonical reproduces it byte-for-byte. Any +# disagreement is a bug in one of the two implementations, and the fixture +# names the case. +using MetaManifold +using MetaManifold.Canonical: jcs_bytes, jcs_sha256 +using JSON3, SHA, Test + +@testset "canonical JSON (JCS)" begin + # Byte-for-byte against the Rust crate's output, over the fixture + # corpus: key sorting (including the astral-before-high-BMP UTF-16 + # case), ES number formatting, string escaping, nesting, literals. + fixtures = joinpath(@__DIR__, "..", "fixtures", "jcs_oracle.jsonl") + for (i, line) in enumerate(eachline(fixtures)) + case = JSON3.read(line) + @testset "oracle case $i" begin + value = case.input + # The input arrives already parsed; the expected is the oracle's text. + @test String(jcs_bytes(value)) == case.expected + end + end + + @testset "key order is irrelevant, bytes are not" begin + a = Dict("b" => 1, "a" => Dict("y" => [1.0, 2], "x" => "s")) + b = Dict("a" => Dict("x" => "s", "y" => [1.0, 2]), "b" => 1) + @test jcs_bytes(a) == jcs_bytes(b) + # ...but array order is preserved, as JCS requires. + c = Dict("a" => [2, 1]) + @test jcs_bytes(a) != jcs_bytes(c) + end + + @testset "the astral key sorts before the high BMP key" begin + # U+1F600 arrives as the surrogate pair D83D DE00; UTF-16 code-unit + # order puts it before U+FF21. Naive code-point order would agree + # here; naive byte order would too -- this case pins the rule. + d = Dict("\uff21" => "high bmp", "\U0001f600" => "astral") + @test String(jcs_bytes(d)) == "{\"😀\":\"astral\",\"Ａ\":\"high bmp\"}" + end + + @testset "float notation follows ECMAScript" begin + @test String(jcs_bytes([4.50, 2.4499999999999998e16, 1e30, 1e-6, 1e-7, + 1e21, 1e20, -0.0, 5e-324])) == + "[4.5,24500000000000000,1e+30,0.000001,1e-7,1e+21,100000000000000000000,0,5e-324]" + end + + @testset "refusals" begin + @test_throws ArgumentError jcs_bytes(2^53) + @test_throws ArgumentError jcs_bytes(-(2^53)) + @test_throws ArgumentError jcs_bytes(Inf) + @test_throws ArgumentError jcs_bytes(NaN) + @test jcs_bytes(2^53 - 1) == b"9007199254740991" + end + + @testset "a content address is stable" begin + # The use this exists for: a chart-like spec built in any key + # order hashes identically, and the hash can be recorded where the + # bytes cannot. + spec = Dict( + "layout" => Dict("title" => Dict("text" => "CO₂")), + "data" => [Dict("x" => [1, 2, 3], "y" => [4.5, 5.5, 6.5], + "type" => "scatter")]) + same = Dict( + "data" => [Dict("type" => "scatter", "y" => [4.5, 5.5, 6.5], + "x" => [1, 2, 3])], + "layout" => Dict("title" => Dict("text" => "CO₂"))) + @test jcs_sha256(spec) == jcs_sha256(same) + @test jcs_sha256(spec) == bytes2hex(sha256(jcs_bytes(spec))) + end +end diff --git a/test/unit/test_install_pins.jl b/test/unit/test_install_pins.jl index 43ba8412..9c340b57 100644 --- a/test/unit/test_install_pins.jl +++ b/test/unit/test_install_pins.jl @@ -383,3 +383,21 @@ end end end end + +@testset "KYAML is the authoring dialect, ratcheted" begin + # YAML-POLICY Y-3 (standards, owner rulings D280/D307): KYAML is the + # estate's authoring dialect; a *.kyaml file that is not in KYAML form + # is a violation, an unconverted *.yml is migration backlog. The + # formatter is the standards repo's kyaml-format.sh, verbatim: the one + # rewriter the step-2 comment-preservation proof covers. Tool-named + # files (workflows, dependabot, codecov) keep their extensions; the + # shared config defaults convert as an upstream unit, not unilaterally. + formatter = joinpath(REPO_ROOT, "scripts", "kyaml-format.sh") + tracked = split(readchomp(`git -C $(REPO_ROOT) ls-files`), "\n") + kyamls = [joinpath(REPO_ROOT, f) for f in tracked if endswith(f, ".kyaml")] + for path in kyamls + proc = run(pipeline(ignorestatus(`bash $formatter --check $path`), + stdout = devnull, stderr = devnull)) + @test success(proc) + end +end