diff --git a/.github/workflows/format-readiness.yml b/.github/workflows/format-readiness.yml new file mode 100644 index 00000000..fa40fe50 --- /dev/null +++ b/.github/workflows/format-readiness.yml @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: MPL-2.0 +# Preparatory evidence only: this does not approve kyq or migrate any runtime IO. +name: Format readiness evidence + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + reference: + name: Format harness and pinned JSON reference + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Test harness failure boundaries + env: + PYTHONDONTWRITEBYTECODE: '1' + run: python3 -m unittest discover -s scripts/tests -p 'test_format*.py' -v + - name: Inventory candidates without declaring compliance + run: python3 scripts/formats/inventory.py > "$RUNNER_TEMP/format-boundaries.tsv" + - name: Read immutable JSON reference pins + id: pins + run: | + python3 - <<'PY' + import os, re, tomllib + with open('config/format-tools.toml', 'rb') as stream: + pin = tomllib.load(stream)['ijson_jcs'] + if pin['repository'] != 'https://github.com/hyperpolymath/ijson-jcs': + raise SystemExit('unexpected reference repository') + if not re.fullmatch(r'[0-9a-f]{40}', pin['revision']): + raise SystemExit('reference must be an immutable commit') + if not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+', pin['rust_toolchain']): + raise SystemExit('Rust must be an explicit release') + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(f"revision={pin['revision']}\nrust={pin['rust_toolchain']}\n") + PY + - name: Checkout pinned reference source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: hyperpolymath/ijson-jcs + ref: ${{ steps.pins.outputs.revision }} + path: .format-reference + persist-credentials: false + - name: Build the real reference CLI with locked dependencies + env: + REFERENCE_REVISION: ${{ steps.pins.outputs.revision }} + RUST_VERSION: ${{ steps.pins.outputs.rust }} + run: | + set -euo pipefail + test "$(git -C .format-reference rev-parse HEAD)" = "$REFERENCE_REVISION" + rustup toolchain install "$RUST_VERSION" --profile minimal + cargo "+${RUST_VERSION}" install --path .format-reference --locked --root "$RUNNER_TEMP/format-reference" + - name: Exercise original bytes against the installed reference + env: + PYTHONDONTWRITEBYTECODE: '1' + run: | + PATH="$RUNNER_TEMP/format-reference/bin:$PATH" python3 scripts/formats/conformance.py --adapter reference + { + echo '## Format migration evidence' + echo 'Pinned ijson-jcs raw-byte corpus passed. This is not application-wide conformance.' + echo 'kyq compilation, installed CLI conformance, KYAML comment preservation and runtime migration remain blocked/unverified.' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index dcc294bf..181a720a 100644 --- a/.gitignore +++ b/.gitignore @@ -247,10 +247,11 @@ vignettes/*.pdf # R Environment Variables .Renviron -## Documentation kept local except published release notes and cladistics notes +## Documentation kept local except published release, cladistics and format notes docs/* !docs/release-notes/ !docs/cladistics/ +!docs/format-migration.md # translation temp files diff --git a/config/format-tools.toml b/config/format-tools.toml new file mode 100644 index 00000000..11f3efcb --- /dev/null +++ b/config/format-tools.toml @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: MPL-2.0 +# Source identities, not claims of runtime conformance. See docs/format-migration.md. +[ijson_jcs] +repository = "https://github.com/hyperpolymath/ijson-jcs" +revision = "3a9a4ee326a968a32b2f152b4faec243a2d00765" +rust_toolchain = "1.85.1" + +[kyq] +repository = "https://github.com/metadatastician/kyq" +revision = "d448abd1f6534abf703db8a89e3d2247e16ee179" +just_version = "1.56.0" + +[affinescript] +repository = "https://github.com/hyperpolymath/affinescript" +revision = "26cdb43e141a5fdce5408e00b8baca7df6862cb8" +version = "0.1.1" diff --git a/docs/format-migration.md b/docs/format-migration.md new file mode 100644 index 00000000..0dbc7c8d --- /dev/null +++ b/docs/format-migration.md @@ -0,0 +1,155 @@ + +# JSON/KYAML migration: staged evidence and blockers + +## Status and order + +This is preparation, not completed adoption. No application parser, persisted +record, configuration file or workflow dialect is migrated by this change. + +| Stage | State | Exit evidence required | +| --- | --- | --- | +| 1. Working kyq | **BLOCKED** | Real compiler checks including imports; executable build/tests; native/WASM IO; isolated-prefix installation | +| 2. Conformance corpus | JSON raw-byte characterization harness prepared; initial pinned reference run passed; KYAML/runtime equivalence not verified | Installed-artifact tests, differential Julia/JS tests, pinned KYAML profile, comment-preservation/mutation proof, resource limits | +| 3. Shared adapters and structural gates | Not started | Explicit schemas/error mapping; replace workflow regex decisions with parser queries | +| 4. Application migration | Not started | Boundary-by-boundary tests; precision, data and old-ID compatibility | +| 5. Workflow conversion | Not started | Comment association/idempotence; actionlint, bot compatibility, real hosted execution | + +Do not treat a passing preparation workflow as permission to skip stage 1. +Kubernetes lint/schema validation is an optional layer, not a prerequisite for +parsing an ordinary MetaManifold configuration document. + +## Reproduced upstream blocker (2026-10-23) + +Source identities are recorded in `config/format-tools.toml`: + +- kyq `d448abd1f6534abf703db8a89e3d2247e16ee179`. +- AffineScript **0.1.1**, source `26cdb43e141a5fdce5408e00b8baca7df6862cb8`. +- ijson-jcs `3a9a4ee326a968a32b2f152b4faec243a2d00765`. +- Source-probe runner `just 1.56.0`. + +On clean kyq source, `just build`, `just wasm`, `just test` and `just install` +each exited **1**, reporting that the work is not implemented. This reproduces +the recipe fencing, **not** the compiler's syntax diagnosis. The available +v0.2.0 AffineScript release has no binary assets. Downloading the v0.1.1 binary +and checksum failed because the sandbox cannot reach GitHub's release-asset +host. OCaml/opam/dune are not installed. Compiler checks and runtime conformance +are **NOT RUN**; no compiler or placeholder artifact was installed. + +The source probe intentionally does **not** run installation: an isolated-prefix +contract has not been established. The manually attempted install recipe above +failed without installing anything. Clone/build dependencies outside the +application checkout; do not vendor upstream source or tool binaries here. + +Upstream implementation and acceptance criteria are tracked in +[kyq issue #11](https://github.com/metadatastician/kyq/issues/11). Fixing the +AffineScript implementation belongs there; this repository does not silently +replace it with another language or publish another tool under the kyq name. + +Reproduce using a clean clone at the pinned commit: + +```sh +PATH="/path/to/trusted/just/bin:$PATH" python3 scripts/formats/probe_kyq.py --source /path/to/kyq +``` + +Probe exit codes: 0 means only the three source recipes returned success; +1 means a recipe failed; 2 means a missing tool, wrong pin, dirty source or +infrastructure failure. Even 0 does not establish parser/CLI/installation +conformance. The tool prints those missing obligations explicitly. + +## Repository boundary coverage matrix + +Generate locations without printing source lines or study/credential values: + +```sh +python3 scripts/formats/inventory.py > /tmp/format-boundaries.tsv +``` + +The inventory includes tracked and non-ignored new files, hidden workflows and +lockfiles. It does not follow symlinks. Its regex hits are **lexical candidates**, +including possible comments/test strings; aliases and dynamic dispatch need +manual inspection. Routing buckets are not ownership decisions or conformance +verdicts. Do not turn this search into a regex-based YAML gate. Generated, +ignored and external files are outside this census, not automatically compliant. + +| Boundary / paths | Classification and current evidence | Migration obligation | +| --- | --- | --- | +| `src/canonical.jl` | First-party strict-input/JCS adapter, its own fixtures; Rust equivalence not established locally | Compare raw-input acceptance and exact bytes against pinned reference; bound depth/size; characterize numeric policy | +| `src/server/server.jl`: `_request_json`, `_read_json_object`, `_request_source` | First-party input; ordinary JSON3 parsing remains, including body inspection outside the route parser | Parse/validate original bytes once; preserve body/empty-body contracts; stable 400/413/422 behavior; do not classify internal faults as client errors | +| `src/server/routes/validated_statistics.jl` | Strict `ijson_read` plus separate exact-count token checks on input; `_statistics_json` still uses ordinary JSON3 output | Preserve integral-count lexical policy and decimal strings; migrate output only after number-field audit | +| Other `src/server/routes/*.jl` | Mixed request parsing, Oxygen responses, persisted documents and imported third-party data | Classify each operation, not only the module; explicit schemas and semantic tests | +| `frontend/src/api/client.ts`, `ijsonJcs.ts` | Canonical first-party request output; responses still use `res.json()` | Validate response text before duplicate keys/numeric tokens are lost; include error/config responses and 204/empty cases | +| `frontend/src/tree/model.ts`, figure/charts and component JSON | Mixed jplace/Plotly, display/cache keys and first-party state | Separate opaque imports, display-only JSON and actual serialization boundaries; preserve array order and external contracts | +| `src/core/doi.jl`, `epistemic.jl`, `provenance.jl` | Legacy identity-sensitive receipts/hashes and YAML provenance | Version encoding changes, retain old verification, never silently recompute old IDs | +| `src/analysis/derived_analyses.jl`, figure/view/report storage | First-party canonicalized storage at selected boundaries | Compare reference bytes; preserve saved-record read compatibility and corruption diagnostics | +| `src/server/jobs.jl`, `src/pipeline/phylogeny*`, QC files | First-party generated JSON/state, some permissive reads | Define schemas, valid numeric/missing-value policy, bounded reads and atomic writes | +| `src/core/config.jl`, `*_library.jl`, `databases.jl`, pipeline YAML calls | Application-owned YAML with config cascades and generated run configs | KYAML emission/validation, legacy YAML import policy, null/type distinctions, deterministic hashing and file discovery | +| `config/**`, sample `data/**`, generated user/study/run configs | First-party configuration plus user-owned legacy material | Review ownership before rewriting; extension support, safe migration/backups, old paths and merge precedence | +| `.github/**`, `codecov.yml` | Workflow/service-consumed YAML; mixed human/bot ownership | Structural readers first; preserve permissions, pin comments, multiline shell and expressions; retain required consumer extensions | +| `scripts/check-governance-contract.sh` and its mutations | Line-oriented workflow assertions; tests assume block YAML | Query parsed structure; block/flow parity, run-body/comment decoys, duplicate-key and missing-parser mutations | +| `bench/**`, `scripts/*.jl`, `docs/occupancy/*.yml` | Benchmarks, baselines, references and tooling YAML/JSON | Preserve independent oracles and measured baselines; errors must name files without silently substituting empty results | +| `test/**` | Positive/negative fixtures and independent reference readers | Preserve invalid raw fixtures; don't canonicalize away the defect a test is meant to exercise | +| `renv.lock`, `renv/settings.json`, `frontend/package.json`, `bun.lock`, `tsconfig.json` | External-tool-owned document contracts | Verify consumer support; do not mass-rewrite or claim arbitrary exact numbers are portable | +| `.gitbot-fleet/**`, K9 reports, CI artifacts | First-party governance evidence with external renderer involvement | Audit actual renderer bytes and schema; no general jq-as-JCS fallback without a constrained, tested equivalence claim | +| Unclassified hits/new files | **Untriaged**, not exempt | Add explicit ownership/schema/compatibility decisions during the relevant implementation stage | + +"Throughout the repository" includes the runtime, generated files, installers, +examples and CI—not just checked-in extensions. Third-party data and invalid test +fixtures stay explicit exceptions, never hidden automatic passes. + +## Raw-byte JSON characterization + +The 34-case corpus is TOML so it can carry invalid JSON tokens and hex-encoded +invalid UTF-8 without a JSON loader normalizing the evidence. It covers decoded +duplicate keys, nested duplicates, UTF-16 sorting, surrogate pairs/noncharacters, +escaped and raw controls, integer boundaries, beyond-uint64 counts, exponent +thresholds, negative zero, array order, trailing input and truncation. + +```sh +PATH="/path/to/trusted/reference/bin:$PATH" python3 scripts/formats/conformance.py --adapter reference +python3 -m unittest discover -s scripts/tests -p 'test_format*.py' -v +``` + +The protocol is the existing ijson-jcs `canon`: original bytes on stdin, exact +canonical bytes with no newline on stdout, exit 0 for valid input, exit 2 and +empty stdout for invalid input. Crashes, arbitrary nonzero exits, timeouts, +missing tools, changed byte order and extra newlines cannot pass. This runner +implements no parser/canonicalizer of its own. A future kyq adapter may implement +this protocol; it is **not** a claim that kyq has a `canon` command today. + +The subprocess runner selects only complete literal commands from an internal +operation table (`ijson-jcs`, a future `kyq-json-adapter`, `git`, `just`). No program +path or argument string is accepted from the CLI. The `--adapter` selector is a +closed choice of reference/candidate, not a command. A trusted PATH provides the +installed executables; symlinks to unrelated interpreters are refused. These are +trusted tools, not sandboxed plugins. Corpus paths are repository +constants/test fixtures, never CLI-supplied paths. The runner limits input/output +bytes and wall time, keeps diagnostics +out of logs. Linux subreaper supervision adopts/kills detached or double-forked +descendants as well as the original process group, with a bounded cleanup grace +period. This standalone probe refuses threaded/pre-existing-child contexts and +unsupported hosts rather than claiming portable lifecycle isolation. These harness budgets are **not** parser resource limits +in the application. Fake subprocesses in unit tests test the harness only. + +The preparation workflow builds the real pinned Rust CLI with locked dependencies +and an explicit Rust release, then runs the corpus. The initial hosted run +[39999978128](https://github.com/hyperpolymath/MetaManifold-WebUI/actions/runs/39999978128) +built the pinned CLI and passed all 34 cases at `f0a00cc`. Subsequent harness +changes must rerun that job. Local Rust execution is not claimed (Rust/cargo are +absent). Failures remain failures; there is no permissive +parser or sorted-jq fallback. Exit 0 means corpus compatibility at this pin, not +complete RFC coverage, safe installation of kyq or application-wide adoption. + +## Still unresolved before stage 3 + +- Reconcile Julia/JS/reference acceptance of integer-valued binary64 numbers, + including `1e21`, and representation changes such as `1e20` producing an + integer-looking canonical token. The initial one-pass corpus is not a proof + that every reference output can be reparsed under every adapter policy. +- Add installed-artifact Julia/JS differential tests, number-roundtrip/closure + tests, fuzz/property tests, underflow/boundary cases and large scientific data. +- Pin the exact KYAML profile and distinguish it from merely valid YAML or a + formatter's dialect. Prove comment association and idempotence with mutants. +- Establish isolated install, actual WASM/native IO, tool capability reporting, + artifact verification, parser limits and fail-closed dependency handling. +- Only then change application adapters, followed by application files and + finally workflows. Preserve existing licensing, scientific results and IDs. diff --git a/docs/ijson-jcs.md b/docs/ijson-jcs.md index 6ee08e2c..ace363b9 100644 --- a/docs/ijson-jcs.md +++ b/docs/ijson-jcs.md @@ -5,7 +5,7 @@ The reference is [hyperpolymath/ijson-jcs](https://github.com/hyperpolymath/ijso ## First-party boundary implemented here - Julia `MetaManifold.Canonical.ijson_read(raw)` validates UTF-8, repeated object keys (including escaped aliases at every depth), lone surrogates/noncharacters, non-finite numbers and integer tokens outside ±(2^53−1) **on the raw text**. It returns ordinary JSON3 values. `jcs_bytes` validates emitted values (including noncharacters) and sorts keys by UTF-16 code units, formats ECMAScript numbers and emits no newline. The existing `jcs_sha256` hashes those bytes. -- Validated-statistics request bodies and their responses, error responses, derived-analysis records, figure documents, tree **view-state sidecars**, and report basket indices use those helpers. Stored derived analyses must be canonical on read; existing correctly persisted JCS records are unchanged. Invalid first-party requests get 400 (the exact-count endpoint still rejects floating *count tokens* with 422). +- Validated-statistics request bodies use the strict input helper. Error responses, derived-analysis records, figure documents, tree **view-state sidecars**, and report basket indices use the canonical helpers. Successful validated-statistics responses still pass through `_statistics_json`/`JSON3.write`; those outputs are **not yet JCS-migrated**. Stored derived analyses must be canonical on read; existing correctly persisted JCS records are unchanged. Invalid first-party requests get 400 (the exact-count endpoint still rejects floating *count tokens* with 422). - The frontend's first-party API POST/PATCH/PUT payloads use `ijsonJcsStringify`, which sorts object keys by UTF-16, preserves array order and refuses unsafe JS numbers, non-finite values, lone surrogates, noncharacters and sparse arrays. Undefined optional object properties are omitted as before. Exact large counts remain decimal **strings**. `frontend/public/config.json` is canonical (plus the optional one-file newline). Use the upstream `ijson-jcs check frontend/public/config.json` to verify the checked-in config if the Rust CLI is installed. The Julia writer has its own JCS fixture tests, but cross-language equivalence with the Rust binary has **not** been independently verified here. @@ -15,3 +15,7 @@ Use the upstream `ijson-jcs check frontend/public/config.json` to verify the che Other Oxygen `json(...)` route responses, legacy JSON3 inputs/outputs, pipeline job/QC JSON, DOI receipts and epistemic receipt hashes still use their old serialization. The latter two are part of existing checksums/IDs: switching their byte encoding without an explicit versioned migration would make previously stored receipts unverifiable. External JSON (notably jplace, Plotly, renv metadata and imported report `.json` files) must not be rewritten just because its name ends in `.json`; the owning format and compatibility contract take precedence. Existing figure/view/index documents with merely different whitespace or key order still read; they are written canonically on their next save. Documents with duplicate keys or unsafe values need explicit migration. Follow-up: audit all remaining first-party route JSON with representative large result sets and tests; decide whether oversized legacy numeric fields become strings and how existing persisted records migrate, then replace their serializers and add an upstream CLI interoperability check to CI. Do **not** automatically JCS-encode opaque upstream/third-party formats or change an established content hash without versioning. + +## Staged follow-up + +See [format-migration.md](format-migration.md) for pinned tool identities, the reproducible kyq blocker, boundary coverage matrix and raw-byte reference harness. No application-wide or kyq runtime conformance is inferred from preparation tests. diff --git a/scripts/formats/conformance.py b/scripts/formats/conformance.py new file mode 100644 index 00000000..6107ef48 --- /dev/null +++ b/scripts/formats/conformance.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MPL-2.0 +"""Characterize an installed ijson-jcs-compatible `canon` adapter. + +Tests original bytes, exact stdout and documented exit codes. No parsing or +canonicalization is implemented in Python. Exit 0 = corpus passed, 1 = mismatch, +2 = infrastructure/configuration failure. This is NOT kyq readiness approval. +""" +import argparse +from pathlib import Path +import re +import sys +import tomllib + +from process import ToolError, run + +ROOT = Path(__file__).resolve().parents[2] + + +def _input(case): + if ("input" in case) == ("hex" in case): + raise ValueError("case needs exactly one input") + raw = case.get("input", case.get("hex")) + if not isinstance(raw, str): + raise ValueError("input must be a string or hex string") + return raw.encode("utf-8") if "input" in case else bytes.fromhex(raw) + + +def _case(case): + allowed = {"id", "input", "hex", "valid", "canonical"} + if not isinstance(case, dict) or set(case) - allowed: + raise ValueError("invalid case fields") + name = case.get("id", "") + if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9-]+", name): + raise ValueError("invalid case id") + if type(case.get("valid")) is not bool: + raise ValueError("case needs an explicit verdict") + if case["valid"] != ("canonical" in case): + raise ValueError("only valid cases must have canonical bytes") + expected = case.get("canonical", "") + if not isinstance(expected, str): + raise ValueError("canonical bytes must be a UTF-8 string") + data, encoded = _input(case), expected.encode("utf-8") + if len(data) > 131072 or len(encoded) > 131072: + raise ValueError("case exceeds the corpus budget") + return name, data, case["valid"], encoded + + +def load_cases(path): + # Paths come from this repo's constant fixture location or test fixtures, + # never a CLI-supplied filesystem path. No arbitrary-file reader CLI. + with Path(path).open("rb") as stream: + document = tomllib.load(stream) + if type(document.get("schema")) is not int or document["schema"] != 1: + raise ValueError("unsupported corpus schema") + if set(document) != {"schema", "case"}: + raise ValueError("unsupported corpus fields") + cases = document["case"] + if not isinstance(cases, list) or not cases: + raise ValueError("empty or invalid corpus") + seen = set() + for raw in cases: + case = _case(raw) + if case[0] in seen: + raise ValueError("duplicate case id") + seen.add(case[0]) + yield case + + +def check(adapter, cases, *, timeout=5.0): + operation = {"reference": "ijson-canon", "candidate": "kyq-canon"}[adapter] + passed = True + for name, data, valid, expected in cases: + result = run(operation, data, timeout=timeout) + # Rejection must be the documented invalid-input exit, not a crash, + # missing library, usage failure from a wrapper, or arbitrary nonzero. + wanted = 0 if valid else 2 + ok = result.code == wanted and result.stdout == expected + print(f"{'PASS' if ok else 'FAIL'} {name}") + passed = passed and ok + return passed + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--adapter", choices=("reference", "candidate"), default="reference", + help="named installed adapter on trusted PATH, never a command/path") + parser.add_argument("--timeout", type=float, default=5.0) + args = parser.parse_args(argv) + try: + # Validate the entire corpus before invoking the first child. + cases = list(load_cases(ROOT / "test/fixtures/formats/ijson-jcs.toml")) + ok = check(args.adapter, cases, timeout=args.timeout) + print(f"Corpus: {len(cases)} cases; runtime migration remains separately gated") + return 0 if ok else 1 + except (ToolError, ValueError, TypeError, OSError) as error: + # Do not echo paths, source text, child stderr or exception values. + print(f"INFRASTRUCTURE: {type(error).__name__}; no conformance pass", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/formats/descendants.py b/scripts/formats/descendants.py new file mode 100644 index 00000000..6cdab69c --- /dev/null +++ b/scripts/formats/descendants.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: MPL-2.0 +"""Linux-only, single-threaded supervision of trusted probe subprocesses. + +A subreaper adopts orphaned descendants even if they call setsid/double-fork. +Restore the previous process-wide setting; refuse pre-existing child contexts. +This is lifecycle management, not a sandbox for adversarial executables. +""" +import ctypes +import os +from pathlib import Path +import signal +import sys +import threading +import time + +PR_SET_CHILD_SUBREAPER = 36 +PR_GET_CHILD_SUBREAPER = 37 + + +class SupervisionError(RuntimeError): + pass + + +def _children(pid): + result = set() + try: + tasks = list(Path(f"/proc/{pid}/task").iterdir()) + except FileNotFoundError as error: + if pid == os.getpid(): + raise SupervisionError("procfs child enumeration unavailable") from error + return result + for task in tasks: + try: + result.update(int(value) for value in (task / "children").read_text().split()) + except FileNotFoundError as error: + if pid == os.getpid() and task.name == str(pid): + raise SupervisionError("procfs children interface unavailable") from error + # A different process/thread exited during enumeration. + return result + + +def _kill_tree(pid): + pending, stopped = [pid], [] + while pending: + current = pending.pop() + try: + os.kill(current, signal.SIGSTOP) + stopped.append(current) + pending.extend(_children(current)) + except ProcessLookupError: + pass + for current in reversed(stopped): + try: + os.kill(current, signal.SIGKILL) + except ProcessLookupError: + pass + + +class Supervise: + def __enter__(self): + if sys.platform != "linux" or threading.active_count() != 1: + raise SupervisionError("probe requires single-threaded Linux supervision") + self.libc = ctypes.CDLL(None, use_errno=True) + self.previous = ctypes.c_int() + if self.libc.prctl(PR_GET_CHILD_SUBREAPER, ctypes.byref(self.previous), 0, 0, 0) != 0: + raise SupervisionError("cannot inspect subreaper state") + if _children(os.getpid()): + raise SupervisionError("probe requires no pre-existing child processes") + self._set(1) + return self + + def _set(self, value): + if self.libc.prctl(PR_SET_CHILD_SUBREAPER, ctypes.c_ulong(value), 0, 0, 0) != 0: + raise SupervisionError("cannot set subreaper state") + + def _drain(self): + deadline = time.monotonic() + 0.5 + while True: + children = _children(os.getpid()) + if not children: + return + for pid in children: + _kill_tree(pid) + try: + os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + pass + if time.monotonic() >= deadline: + raise SupervisionError("descendants did not exit within cleanup budget") + time.sleep(0.005) # Bounded cleanup; also reap newly adopted orphans. + + def __exit__(self, *_): + try: + self._drain() + finally: + self._set(self.previous.value) diff --git a/scripts/formats/inventory.py b/scripts/formats/inventory.py new file mode 100644 index 00000000..e637c8fa --- /dev/null +++ b/scripts/formats/inventory.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MPL-2.0 +"""Inventory lexical JSON/YAML candidates; NEVER a compliance verdict. + +Includes tracked and non-ignored new files, including hidden workflow/hooks. +Search hits can be comments/test strings. Aliases/dynamic dispatch require human +review. Prints locations/operation names, not source lines or document contents. +""" +import argparse +import csv +from pathlib import Path +import re +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[2] +CODE = {".yaml", ".yml", ".kyaml", ".jl", ".ts", ".tsx", ".js", ".mjs", ".py", ".sh", ".R", ".r", ".rs"} +DATA = {".json", ".jsonl", ".yaml", ".yml", ".kyaml"} +PATTERNS = { + "JSON3.read": r"\bJSON3\.read\s*\(", + "JSON3.write": r"\bJSON3\.write\s*\(", + "YAML IO": r"\bYAML\.(?:load_file|load|write_file|write)\s*\(", + "strict/JCS adapter": r"\b(?:ijson_read|jcs_bytes|jcs_sha256|ijsonJcsStringify|_request_json)\s*\(", + "JSON.parse": r"\bJSON\.parse\s*\(", + "JSON.stringify": r"\bJSON\.stringify\s*\(", + "response.json candidate": r"\.json\s*\(", + "Python JSON IO": r"\bjson\.(?:load|loads|dump|dumps)\s*\(", + "Oxygen JSON candidate": r"(? self.limit: + raise ToolError("tool exceeded its output budget") + self.buffers[label].extend(chunk) + + def collect(self, deadline): + with selectors.DefaultSelector() as selector: + self.register(selector) + while selector.get_map(): + for key, _ in selector.select(_remaining(deadline)): + if key.data == "stdin": + self.write_input(selector, key.fileobj) + else: + self.read_output(selector, key.fileobj, key.data) + try: + code = self.child.wait(timeout=_remaining(deadline)) + except subprocess.TimeoutExpired as error: + raise ToolError(TIMEOUT) from error + return Result(code, bytes(self.buffers["stdout"]), bytes(self.buffers["stderr"])) + + +def _remaining(deadline): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ToolError(TIMEOUT) + return remaining + + +def _cleanup(child): + # Fast group cleanup; Supervise also adopts/kills detached descendants. + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + child.wait(timeout=0.5) + except subprocess.TimeoutExpired as error: + raise ToolError("tool did not exit within cleanup budget") from error + finally: + for stream in (child.stdin, child.stdout, child.stderr): + if not stream.closed: + stream.close() + + +def run(operation, data=b"", *, cwd=None, timeout=5.0, limit=131072): + if not math.isfinite(timeout) or timeout <= 0 or limit <= 0 or len(data) > limit: + raise ToolError("invalid process budget or oversized test input") + try: + with Supervise(): + child = subprocess.Popen( + _command(operation), cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, start_new_session=True, shell=False, + ) + try: + return _Pump(child, data, limit).collect(time.monotonic() + timeout) + finally: + _cleanup(child) + except (OSError, SupervisionError) as error: + raise ToolError("tool startup or descendant supervision failed") from error diff --git a/scripts/tests/test_format_readiness.py b/scripts/tests/test_format_readiness.py new file mode 100644 index 00000000..f6891219 --- /dev/null +++ b/scripts/tests/test_format_readiness.py @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: MPL-2.0 +"""Tests of the harness itself; fake children are NOT parser conformance evidence.""" +import contextlib +import io +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts/formats")) +from conformance import check, load_cases, main as conformance_main +from inventory import inventory +from process import ToolError, run +from probe_kyq import main as probe_main + + +def run_python(body, data=b"", **kwargs): + # A fake installed protocol adapter, never a production interpreter command. + with tempfile.TemporaryDirectory(prefix="format tool ") as tmp: + path = Path(tmp) / "ijson-jcs" + path.write_text(f"#!{sys.executable}\n{body}\n") + path.chmod(0o700) + with patch.dict(os.environ, {"PATH": tmp + os.pathsep + os.environ["PATH"]}): + return run("ijson-canon", data, **kwargs) + + +class ProcessTests(unittest.TestCase): + def test_round_trip_binary_io(self): + raw = bytes(range(256)) * 200 + result = run_python("import sys; sys.stdout.buffer.write(sys.stdin.buffer.read())", raw) + self.assertEqual((result.code, result.stdout, result.stderr), (0, raw, b"")) + + def test_stderr_separate_and_exit_preserved(self): + result = run_python("import sys; sys.stderr.write('private'); sys.exit(7)") + self.assertEqual((result.code, result.stdout, result.stderr), (7, b"", b"private")) + + def test_missing_binary_is_infrastructure_not_document_rejection(self): + with patch.dict(os.environ, {"PATH": ""}), self.assertRaises(ToolError): + run("ijson-canon") + + def test_timeout(self): + with self.assertRaisesRegex(ToolError, "time budget"): + run_python("import time; time.sleep(10)", timeout=0.15) + + def test_output_budget_covers_both_streams(self): + with self.assertRaisesRegex(ToolError, "output budget"): + run_python("import sys; sys.stderr.write('x' * 10000)", limit=100) + + def test_input_and_deadline_budgets(self): + for kwargs in ({"timeout": 0}, {"timeout": float("nan")}, {"timeout": float("inf")}, {"limit": 0}, {"data": b"xx", "limit": 1}): + with self.assertRaises(ToolError): + run_python("pass", **kwargs) + + def test_arbitrary_programs_and_arguments_are_refused(self): + for argv in ([sys.executable, "-c", "pass"], ["git", "config", "--global", "x", "y"]): + with self.subTest(argv=argv), self.assertRaises(ToolError): + run(argv) + + def test_symlink_cannot_disguise_an_interpreter_as_a_canonicalizer(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "ijson-jcs" + path.symlink_to(sys.executable) + with patch.dict(os.environ, {"PATH": tmp + os.pathsep + os.environ["PATH"]}): + with self.assertRaises(ToolError): + run("ijson-canon") + + def test_detached_double_fork_is_killed_and_reaped(self): + with tempfile.TemporaryDirectory() as tmp: + pid_file = Path(tmp) / "detached.pid" + body = f"""import os,time +if os.fork() == 0: + os.setsid() + if os.fork() == 0: + with open({str(pid_file)!r}, 'w') as stream: + stream.write(str(os.getpid())) + time.sleep(30) + os._exit(0) +time.sleep(30) +""" + with self.assertRaisesRegex(ToolError, "time budget"): + run_python(body, timeout=0.4) + pid = int(pid_file.read_text()) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_missing_procfs_fails_closed(self): + for method in ("iterdir", "read_text"): + with self.subTest(method=method): + with patch(f"descendants.Path.{method}", side_effect=FileNotFoundError): + with self.assertRaises(ToolError): + run("ijson-canon") + + def test_existing_child_context_is_refused_before_launch(self): + with patch("descendants._children", return_value={12345}): + with patch("process.subprocess.Popen") as launch: + with self.assertRaises(ToolError): + run("ijson-canon") + launch.assert_not_called() + + def test_descendant_keeping_pipe_open_is_bounded(self): + with self.assertRaisesRegex(ToolError, "time budget"): + run_python("import os,time; p=os.fork(); time.sleep(10) if p==0 else None", timeout=0.2) + + +class CorpusTests(unittest.TestCase): + def test_raw_fixtures_survive_loading(self): + cases = list(load_cases(ROOT / "test/fixtures/formats/ijson-jcs.toml")) + self.assertGreaterEqual(len(cases), 30) + by_id = {c[0]: c for c in cases} + self.assertEqual(by_id["invalid-utf8"][1], b'["\xff"]') + self.assertEqual(by_id["escaped-equivalent-key"][1], b'{"a":1,"\\u0061":2}') + self.assertEqual(by_id["exponent-thresholds"][3], b'[0.000001,1e-7,1e+21]') + + def test_bad_and_ambiguous_corpora_fail_before_running_tools(self): + for contents in ( + "schema=2\ncase=[]\n", "schema=true\ncase=[]\n", "schema=1\ncase=[]\n", + "schema=1\n[[case]]\nid='a'\ninput='{}'\nvalid=true\n", + "schema=1\n[[case]]\nid='a'\ninput='{}'\nhex='ff'\nvalid=false\n", + "schema=1\n[[case]]\nid='a'\ninput='{}'\nvalid='false'\n", + "schema=1\n[[case]]\nid='a'\ninput='{}'\nvalid=false\nunknown=1\n", + "schema=1\n[[case]]\nid='a'\nhex='not-hex'\nvalid=false\n", + ): + with self.subTest(contents=contents), tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "cases.toml" + path.write_text(contents) + with self.assertRaises((ValueError, TypeError)): + list(load_cases(path)) + + def test_duplicate_ids_fail(self): + case = "\n[[case]]\nid='a'\ninput='{}'\nvalid=false\n" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "cases.toml" + path.write_text("schema=1\n" + case * 2) + with self.assertRaises(ValueError): + list(load_cases(path)) + + @contextlib.contextmanager + def fake(self, directory, body): + folder = Path(directory) / "tool with spaces" + folder.mkdir() + path = folder / "ijson-jcs" + path.write_text(f"#!{sys.executable}\n{body}\n") + path.chmod(0o700) + with patch.dict(os.environ, {"PATH": str(folder) + os.pathsep + os.environ["PATH"]}): + yield "reference" + + def test_mutants_cannot_pass_as_canonicalizers(self): + cases = [("valid", b' {"a":1} ', True, b'{"a":1}'), + ("invalid", b'{"a":1,"a":2}', False, b'')] + # Always-success, always-invalid, crashes, and passthrough all fail. + for body in ("raise SystemExit(0)", "raise SystemExit(2)", + "raise SystemExit(1)", "import sys; sys.stdout.buffer.write(sys.stdin.buffer.read())"): + with self.subTest(body=body), tempfile.TemporaryDirectory() as tmp: + with contextlib.redirect_stdout(io.StringIO()): + with self.fake(tmp, body) as adapter: + self.assertFalse(check(adapter, cases)) + + def test_correct_adapter_protocol_passes_and_redacts_diagnostics(self): + cases = [("valid", b"true", True, b"true"), ("invalid", b"broken", False, b"")] + body = """import sys +raw = sys.stdin.buffer.read() +if raw == b'true': + sys.stdout.buffer.write(raw) +else: + sys.stderr.write('private input') + raise SystemExit(2) +""" + with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()) as out: + with self.fake(tmp, body) as adapter: + self.assertTrue(check(adapter, cases)) + self.assertNotIn("private input", out.getvalue()) + + def test_exact_bytes_no_newline_or_wrong_order(self): + cases = [("order", b'{}', True, b'{"a":1,"z":2}')] + for output in (b'{"z":2,"a":1}', b'{"a":1,"z":2}\n'): + with tempfile.TemporaryDirectory() as tmp, contextlib.redirect_stdout(io.StringIO()): + with self.fake(tmp, f"import sys; sys.stdout.buffer.write({output!r})") as adapter: + self.assertFalse(check(adapter, cases)) + + def test_missing_tool_cli_fails_closed_without_echoing_input(self): + out, err = io.StringIO(), io.StringIO() + with patch.dict(os.environ, {"PATH": ""}): + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = conformance_main([]) + self.assertEqual(code, 2) + self.assertIn("no conformance pass", err.getvalue()) + self.assertNotIn("9007199254740992", out.getvalue() + err.getvalue()) + + def test_kyq_probe_missing_source_is_not_a_pass(self): + with contextlib.redirect_stderr(io.StringIO()): + code = probe_main(["--source", "/nonexistent/kyq"]) + self.assertEqual(code, 2) + + +class InventoryTests(unittest.TestCase): + def test_inventory_covers_hidden_new_files_and_never_echoes_source(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run(["git", "init", "-q", tmp], check=True) + (root / ".github/workflows").mkdir(parents=True) + (root / ".github/workflows/probe.yml").write_text('name: "private-value"\nrun: jq -r .value\n') + (root / "client.ts").write_text('res.json();\nJSON.parse("private-value")\n') + (root / "foreign.json").symlink_to("/etc/passwd") + (root / ".gitignore").write_text('ignored.py\n') + (root / "ignored.py").write_text('json.loads("ignore")') + rows = list(inventory(root)) + self.assertTrue(any(r[2] == ".github/workflows/probe.yml" for r in rows)) + self.assertTrue(any(r[0] == "symlink" and r[2] == "foreign.json" for r in rows)) + self.assertTrue(any(r[3] == 2 and r[4] == "JSON.parse" for r in rows)) + self.assertTrue(any(r[2] == ".github/workflows/probe.yml" and r[3] == 2 + and r[4] == "shell jq/yq candidate" for r in rows)) + self.assertFalse(any(r[2] == "ignored.py" for r in rows)) + self.assertNotIn("private-value", repr(rows)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/fixtures/formats/ijson-jcs.toml b/test/fixtures/formats/ijson-jcs.toml new file mode 100644 index 00000000..0184f096 --- /dev/null +++ b/test/fixtures/formats/ijson-jcs.toml @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: MPL-2.0 +# Raw-text characterization of the pinned ijson-jcs `canon` protocol. +# Deliberately not a JSON file: malformed bytes/tokens must survive fixture loading. +schema = 1 + +[[case]] +id = "object-order" +input = '{"z":0,"a":1}' +valid = true +canonical = '{"a":1,"z":0}' + +[[case]] +id = "nested-order-array-preserved" +input = '{"b":[3,1,2],"a":{"z":false,"a":null}}' +valid = true +canonical = '{"a":{"a":null,"z":false},"b":[3,1,2]}' + +[[case]] +id = "utf16-not-utf8-order" +input = '{"Ａ":1,"😀":2}' +valid = true +canonical = '{"😀":2,"Ａ":1}' + +[[case]] +id = "surrogate-pair" +input = '{"s":"\ud83d\ude00"}' +valid = true +canonical = '{"s":"😀"}' + +[[case]] +id = "escaped-controls-are-valid" +input = '{"s":"\u0001\t\n\r"}' +valid = true +canonical = '{"s":"\u0001\t\n\r"}' + +[[case]] +id = "negative-zero" +input = '[-0.0,0.0]' +valid = true +canonical = '[0,0]' + +[[case]] +id = "exponent-thresholds" +input = '[1e-6,1e-7,1e21]' +valid = true +canonical = '[0.000001,1e-7,1e+21]' + +[[case]] +id = "integer-bounds" +input = '[-9007199254740991,9007199254740991]' +valid = true +canonical = '[-9007199254740991,9007199254740991]' + +[[case]] +id = "exact-large-count-string" +input = '{"count":"18446744073709551617"}' +valid = true +canonical = '{"count":"18446744073709551617"}' + +[[case]] +id = "scalar-is-format-valid" +input = 'true' +valid = true +canonical = 'true' + +[[case]] +id = "insignificant-whitespace" +input = " \t[1]\r\n" +valid = true +canonical = '[1]' + +[[case]] +id = "duplicate-key" +input = '{"a":1,"a":2}' +valid = false + +[[case]] +id = "escaped-equivalent-key" +input = '{"a":1,"\u0061":2}' +valid = false + +[[case]] +id = "nested-duplicate" +input = '{"a":[{"b":1,"b":2}]}' +valid = false + +[[case]] +id = "unsafe-positive-integer" +input = '[9007199254740992]' +valid = false + +[[case]] +id = "unsafe-negative-integer" +input = '[-9007199254740992]' +valid = false + +[[case]] +id = "integer-beyond-uint64" +input = '{"n":18446744073709551617}' +valid = false + +[[case]] +id = "overflow" +input = '[1e400]' +valid = false + +[[case]] +id = "nan" +input = '[NaN]' +valid = false + +[[case]] +id = "infinity" +input = '[Infinity]' +valid = false + +[[case]] +id = "lone-high-surrogate" +input = '["\ud800"]' +valid = false + +[[case]] +id = "lone-low-surrogate" +input = '["\udc00"]' +valid = false + +[[case]] +id = "noncharacter-value" +input = '["\ufdd0"]' +valid = false + +[[case]] +id = "noncharacter-key" +input = '{"\uffff":0}' +valid = false + +[[case]] +id = "invalid-utf8" +hex = '5b22ff225d' +valid = false + +[[case]] +id = "utf8-encoded-surrogate" +hex = '5b22eda080225d' +valid = false + +[[case]] +id = "raw-control" +hex = '5b2201225d' +valid = false + +[[case]] +id = "leading-zero" +input = '[01]' +valid = false + +[[case]] +id = "incomplete-number" +input = '[1.]' +valid = false + +[[case]] +id = "trailing-comma" +input = '[1,]' +valid = false + +[[case]] +id = "trailing-junk" +input = '[1] garbage' +valid = false + +[[case]] +id = "concatenated-documents" +input = '{"a":1}{"b":2}' +valid = false + +[[case]] +id = "truncated-body" +input = '{"a":' +valid = false + +[[case]] +id = "empty-body" +input = '' +valid = false